diff options
author | dec05eba <dec05eba@protonmail.com> | 2019-03-02 21:20:33 +0100 |
---|---|---|
committer | dec05eba <dec05eba@protonmail.com> | 2020-07-25 14:36:46 +0200 |
commit | 5e240bdab90c45f935e7d2b33181de13295e7e6b (patch) | |
tree | 0718d7ca2386292c5b3646d0cb1ae499bb7ba818 /include/tokenizer.h | |
parent | 2a17f5225a09c01eb04225d0241c686ea553f912 (diff) |
Add string, variable and number. Fix identifier match against const and var
Diffstat (limited to 'include/tokenizer.h')
-rw-r--r-- | include/tokenizer.h | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h index fac61e7..c76cd52 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -19,9 +19,12 @@ typedef enum { TOK_EQUALS, TOK_OPEN_PAREN, TOK_CLOSING_PAREN, + TOK_COMMA, TOK_OPEN_BRACE, TOK_CLOSING_BRACE, - TOK_IMPORT + TOK_IMPORT, + TOK_NUMBER, + TOK_DOT } Token; typedef struct { @@ -40,9 +43,17 @@ typedef struct { union { BufferView identifier; BufferView string; + i64 integer; + f64 floating; } value; + bool number_is_integer; } Tokenizer; +typedef struct { + int index; + const char* str; +} TokenizerError; + CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name); CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); /* @@ -51,5 +62,7 @@ CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); */ CHECK_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result); void tokenizer_print_error(Tokenizer *self, const char *fmt, ...); +void tokenizer_print_error_object(Tokenizer *self, TokenizerError *error); +TokenizerError tokenizer_create_error(Tokenizer *tokenizer, const char *err_str); #endif |