From 294239abb9d4e804699631a3eb0cec11c987ed22 Mon Sep 17 00:00:00 2001
From: wires <wires@noreply.wires.systems>
Date: Wed, 12 Mar 2025 13:08:11 -0400
Subject: [PATCH] recognize all simple tokens

still need to do delimiters but those require some special handling
---
 src/akh_lex.erl | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/akh_lex.erl b/src/akh_lex.erl
index c35dca5..059278d 100644
--- a/src/akh_lex.erl
+++ b/src/akh_lex.erl
@@ -16,13 +16,16 @@ This module contains functions for tokenizing Akhamoth source code.
 -define(is_space(C), C =:= $\s; C =:= $\t).
 
 -define(is_op2(T), T =:= <<"|>">>; T =:= <<"=>">>; T =:= <<"->">>; T =:= <<"==">>).
--define(is_op1(T),
+-define(is_single(T),
     T =:= <<"+">>;
     T =:= <<"-">>;
     T =:= <<"*">>;
     T =:= <<"/">>;
     T =:= <<"=">>;
-    T =:= <<".">>
+    T =:= <<".">>;
+    T =:= <<",">>;
+    T =:= <<";">>;
+    T =:= <<":">>
 ).
 
 -doc """
@@ -40,6 +43,7 @@ Tokens for which the category is the same as the content.
     | '->'
     | '=='
     | ','
+    | ';'
     | ':'
     | '('
     | ')'
@@ -98,7 +102,7 @@ next(#lexer{source = <<C, Rest/binary>>} = Lx) when ?is_space(C) ->
     next(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1});
 next(#lexer{source = <<T:2/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_op2(T) ->
     {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 2}};
-next(#lexer{source = <<T:1/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_op1(T) ->
+next(#lexer{source = <<T:1/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_single(T) ->
     {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 1}};
 next(#lexer{source = <<$\n, Rest/binary>>} = Lx) ->
     new_line(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1});