wires/akhamoth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

recognize all simple tokens

Browse source 
still need to do delimiters but those require some special handling
This commit is contained in:
wires 2025-03-12 13:08:11 -04:00
parent a0c03e8cfe
commit 294239abb9
Signed by: wires
SSH key fingerprint: SHA256:9GtP+M3O2IivPDlw1UY872UPUuJH2gI0yG6ExBxaaiM
1 changed files with 7 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/akh_lex.erl
View file

@ -16,13 +16,16 @@ This module contains functions for tokenizing Akhamoth source code.
-define(is_space(C), C =:= $\s; C =:= $\t). -define(is_space(C), C =:= $\s; C =:= $\t).
-define(is_op2(T), T =:= <<"|>">>; T =:= <<"=>">>; T =:= <<"->">>; T =:= <<"==">>). -define(is_op2(T), T =:= <<"|>">>; T =:= <<"=>">>; T =:= <<"->">>; T =:= <<"==">>).
-define(is_op1(T), -define(is_single(T),
T =:= <<"+">>; T =:= <<"+">>;
T =:= <<"-">>; T =:= <<"-">>;
T =:= <<"*">>; T =:= <<"*">>;
T =:= <<"/">>; T =:= <<"/">>;
T =:= <<"=">>; T =:= <<"=">>;
T =:= <<".">> T =:= <<".">>;
T =:= <<",">>;
T =:= <<";">>;
T =:= <<":">>
). ).
-doc """ -doc """
@ -40,6 +43,7 @@ Tokens for which the category is the same as the content.
| '->' | '->'
| '==' | '=='
| ',' | ','
| ';'
| ':' | ':'
| '(' | '('
| ')' | ')'
@ -98,7 +102,7 @@ next(#lexer{source = <<C, Rest/binary>>} = Lx) when ?is_space(C) ->
next(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1}); next(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1});
next(#lexer{source = <<T:2/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_op2(T) -> next(#lexer{source = <<T:2/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_op2(T) ->
{ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 2}}; {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 2}};
next(#lexer{source = <<T:1/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_op1(T) -> next(#lexer{source = <<T:1/binary, Rest/binary>>, offset = Offset} = Lx) when ?is_single(T) ->
{ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 1}}; {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 1}};
next(#lexer{source = <<$\n, Rest/binary>>} = Lx) -> next(#lexer{source = <<$\n, Rest/binary>>} = Lx) ->
new_line(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1}); new_line(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1});