aboutsummaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/tokenizer.h26
1 files changed, 25 insertions, 1 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
index fcb9721..98491c7 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -22,14 +22,33 @@ typedef enum {
TSL_TOKEN_RBRACE,
TSL_TOKEN_LBRACKET,
TSL_TOKEN_RBRACKET,
+ TSL_TOKEN_LPAREN,
+ TSL_TOKEN_RPAREN,
TSL_TOKEN_COLON,
- TSL_TOKEN_COMMA
+ TSL_TOKEN_COMMA,
+ TSL_TOKEN_FN,
+ TSL_TOKEN_DOLLAR_SIGN
} TslToken;
+typedef enum {
+ TSL_COMMAND_TOKEN_END_OF_FILE,
+ TSL_COMMAND_TOKEN_ARG,
+ TSL_COMMAND_TOKEN_END
+} TslCommandToken;
+
+typedef struct {
+ TslToken token;
+ size_t code_index;
+ size_t prev_code_index;
+} TslTokenizerPeek;
+
typedef struct {
const char *code;
size_t code_size;
size_t code_index;
+ size_t prev_code_index;
+
+ TslTokenizerPeek peek;
TslStringView identifier;
TslStringView string;
@@ -41,6 +60,11 @@ void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
TslToken tsl_tokenizer_next(TslTokenizer *self);
int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
+TslToken tsl_tokenizer_peek(TslTokenizer *self);
+
+TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);
+
+int tsl_tokenizer_get_line_by_index(TslTokenizer *self, size_t index);
#endif /* TSL_TOKENIZER_H */