diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/tokenizer.h | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h index e79f070..fac61e7 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -29,6 +29,12 @@ typedef struct { int index; int prev_index; int line; + Token token; + /* + @needs_update is an optimization when running tokenizer_consume_if. If expected_token is wrong and tokenizer_consume_if is called again, + then do not rollback to previous token and instead reuse the already parsed token + */ + bool needs_update; BufferView code_name; union { @@ -38,8 +44,6 @@ typedef struct { } Tokenizer; CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name); - -CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token); CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); /* @result is set to 0 if the next token is equal to @expected_token, |