diff --git a/src/akh_lex.erl b/src/akh_lex.erl index c35dca5..059278d 100644 --- a/src/akh_lex.erl +++ b/src/akh_lex.erl @@ -16,13 +16,16 @@ This module contains functions for tokenizing Akhamoth source code. -define(is_space(C), C =:= $\s; C =:= $\t). -define(is_op2(T), T =:= <<"|>">>; T =:= <<"=>">>; T =:= <<"->">>; T =:= <<"==">>). --define(is_op1(T), +-define(is_single(T), T =:= <<"+">>; T =:= <<"-">>; T =:= <<"*">>; T =:= <<"/">>; T =:= <<"=">>; - T =:= <<".">> + T =:= <<".">>; + T =:= <<",">>; + T =:= <<";">>; + T =:= <<":">> ). -doc """ @@ -40,6 +43,7 @@ Tokens for which the category is the same as the content. | '->' | '==' | ',' + | ';' | ':' | '(' | ')' @@ -98,7 +102,7 @@ next(#lexer{source = <>} = Lx) when ?is_space(C) -> next(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1}); next(#lexer{source = <>, offset = Offset} = Lx) when ?is_op2(T) -> {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 2}}; -next(#lexer{source = <>, offset = Offset} = Lx) when ?is_op1(T) -> +next(#lexer{source = <>, offset = Offset} = Lx) when ?is_single(T) -> {ok, {binary_to_atom(T), Offset}, Lx#lexer{source = Rest, offset = Offset + 1}}; next(#lexer{source = <<$\n, Rest/binary>>} = Lx) -> new_line(Lx#lexer{source = Rest, offset = Lx#lexer.offset + 1});