Add list, map and command parsing

1 files changed, 25 insertions, 1 deletions

diff --git a/include/tokenizer.h b/include/tokenizer.h
index fcb9721..98491c7 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -22,14 +22,33 @@ typedef enum {
     TSL_TOKEN_RBRACE,
     TSL_TOKEN_LBRACKET,
     TSL_TOKEN_RBRACKET,
+    TSL_TOKEN_LPAREN,
+    TSL_TOKEN_RPAREN,
     TSL_TOKEN_COLON,
-    TSL_TOKEN_COMMA
+    TSL_TOKEN_COMMA,
+    TSL_TOKEN_FN,
+    TSL_TOKEN_DOLLAR_SIGN
 } TslToken;
 
+typedef enum {
+    TSL_COMMAND_TOKEN_END_OF_FILE,
+    TSL_COMMAND_TOKEN_ARG,
+    TSL_COMMAND_TOKEN_END
+} TslCommandToken;
+
+typedef struct {
+    TslToken token;
+    size_t code_index;
+    size_t prev_code_index;
+} TslTokenizerPeek;
+
 typedef struct {
     const char *code;
     size_t code_size;
     size_t code_index;
+    size_t prev_code_index;
+
+    TslTokenizerPeek peek;
 
     TslStringView identifier;
     TslStringView string;
@@ -41,6 +60,11 @@ void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
 
 TslToken tsl_tokenizer_next(TslTokenizer *self);
 int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
+TslToken tsl_tokenizer_peek(TslTokenizer *self);
+
+TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);
+
+int tsl_tokenizer_get_line_by_index(TslTokenizer *self, size_t index);
 
 #endif /* TSL_TOKENIZER_H */