rewrite to iterator style

erlang_ls.config

@@ -1,3 +1,2 @@
 formatting:
     formatter: erlfmt
-    width: 80

src/akh_lexer.erl

@@ -1,10 +1,31 @@
 -module(akh_lexer).
--include("akh_source_map.hrl").
+-include("akhamoth.hrl").
--export([tokenize/1]).
+-export([
+    new/1,
+    next/1
+]).
+
+-define(open_delim(T),
+    T =:= $(;
+    T =:= $[;
+    T =:= ${
+).
+
+-define(close_delim(T),
+    T =:= $);
+    T =:= $];
+    T =:= $}
+).
+
+-define(op1(T),
+    T =:= $,;
+    T =:= $;;
+    T =:= $.;
+    T =:= $+
+).
 
 -type token_kind() ::
-    line_comment
+    comment
-    | block_comment
     | atom
     | identifier
     | integer
@@ -21,26 +42,31 @@
     | '->'
     | '+'.
 
--type token() :: {token_kind(), span()}.
+-define(LOC(Line, Column), {{line, Line}, {column, Column}}).
+-type location() :: {{line, pos_integer()}, {column, pos_integer()}} | inserted.
 
--spec tokenize(string()) -> {ok, source_map(), [token()]}.
+-type token() :: {token_kind(), location()}.
-tokenize(Text) -> tokenize(Text, 0, 1, akh_source_map:empty(), []).
 
--spec tokenize(
+-record(lexer, {
-    string(),
+    source :: string(),
-    Offset :: non_neg_integer(),
+    line = 1 :: pos_integer(),
-    Line :: pos_integer(),
+    column = 1 :: pos_integer(),
-    SourceMap :: source_map(),
+    delim_stack = [] :: [token()]
-    [token()]
+}).
-) -> {ok, source_map(), [token()]}.
+
-tokenize([], _, _, SourceMap, Tokens) ->
+-spec new(string()) -> #lexer{}.
-    {ok, SourceMap, Tokens};
+new(Source) -> #lexer{source = Source}.
-tokenize([$( | T], Offset, Line, SourceMap, Tokens) ->
+
-    tokenize(T, Offset + 1, Line, SourceMap, [{'(', {Offset, 1}} | Tokens]);
+-spec next(#lexer{}) -> none | {ok, token(), #lexer{}}.
-tokenize([$) | T], Offset, Line, SourceMap, Tokens) ->
+next(#lexer{source = [], delim_stack = []}) ->
-    tokenize(T, Offset + 1, Line, SourceMap, [{')', {Offset, 1}} | Tokens]);
+    none;
-tokenize([$\n | T], Offset, Line, SourceMap, Tokens) ->
+next(#lexer{source = [$\n | Rest], line = Line} = Lexer) ->
-    NewMap = akh_source_map:insert(Offset, Line, SourceMap),
+    next(Lexer#lexer{source = Rest, line = Line + 1, column = 1});
-    tokenize(T, Offset + 1, Line + 1, NewMap, Tokens);
+next(#lexer{source = [T | Rest], line = Line, column = Column} = Lexer) when ?op1(T) ->
-tokenize([_ | T], Offset, Line, SourceMap, Tokens) ->
+    Token = {list_to_atom([T]), ?LOC(Line, Column)},
-    tokenize(T, Offset + 1, Line, SourceMap, [{atom, {Offset, 0}} | Tokens]).
+    {ok, Token, Lexer#lexer{source = Rest, column = Column + 1}};
+next(#lexer{source = [$-, $> | Rest], line = Line, column = Column} = Lexer) ->
+    Token = {'->', ?LOC(Line, Column)},
+    {ok, Token, Lexer#lexer{source = Rest, column = Column + 2}};
+next(_) ->
+    none.

src/akhamoth.erl

@@ -6,4 +6,11 @@
 compile(Path) ->
     {ok, SrcBin} = file:read_file(Path),
     Src = unicode:characters_to_list(SrcBin),
-    akh_lexer:tokenize(Src).
+    Lexer = akh_lexer:new(Src),
+    collect(Lexer, []).
+
+collect(Lexer, Acc) ->
+    case akh_lexer:next(Lexer) of
+        none -> Acc;
+        {ok, T, L} -> collect(L, [T | Acc])
+    end.

src/akhamoth.hrl

@@ -0,0 +1 @@
+-type err() :: any().