Fix crash in parser import, optimize tokenizer_consume_if to not reparse if already parsed

1 files changed, 6 insertions, 2 deletions

diff --git a/include/tokenizer.h b/include/tokenizer.h
index e79f070..fac61e7 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -29,6 +29,12 @@ typedef struct {
     int index;
     int prev_index;
     int line;
+    Token token;
+    /*
+    @needs_update is an optimization when running tokenizer_consume_if. If expected_token is wrong and tokenizer_consume_if is called again,
+    then do not rollback to previous token and instead reuse the already parsed token
+    */
+    bool needs_update;
     BufferView code_name;
     
     union {
@@ -38,8 +44,6 @@ typedef struct {
 } Tokenizer;
 
 CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
-
-CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 /* 
    @result is set to 0 if the next token is equal to @expected_token,