diff options
Diffstat (limited to 'include/tokenizer.h')
-rw-r--r-- | include/tokenizer.h | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h index fac61e7..c76cd52 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -19,9 +19,12 @@ typedef enum { TOK_EQUALS, TOK_OPEN_PAREN, TOK_CLOSING_PAREN, + TOK_COMMA, TOK_OPEN_BRACE, TOK_CLOSING_BRACE, - TOK_IMPORT + TOK_IMPORT, + TOK_NUMBER, + TOK_DOT } Token; typedef struct { @@ -40,9 +43,17 @@ typedef struct { union { BufferView identifier; BufferView string; + i64 integer; + f64 floating; } value; + bool number_is_integer; } Tokenizer; +typedef struct { + int index; + const char* str; +} TokenizerError; + CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name); CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); /* @@ -51,5 +62,7 @@ CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); */ CHECK_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result); void tokenizer_print_error(Tokenizer *self, const char *fmt, ...); +void tokenizer_print_error_object(Tokenizer *self, TokenizerError *error); +TokenizerError tokenizer_create_error(Tokenizer *tokenizer, const char *err_str); #endif |