From 00ab5c3488c02beab5c3f4e371f5196404334e3c Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Sat, 2 Mar 2019 00:40:08 +0100
Subject: Fix crash in parser import, optimize tokenizer_consume_if to not
 reparse if already parsed

---
 include/tokenizer.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'include')

diff --git a/include/tokenizer.h b/include/tokenizer.h
index e79f070..fac61e7 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -29,6 +29,12 @@ typedef struct {
     int index;
     int prev_index;
     int line;
+    Token token;
+    /*
+    @needs_update is an optimization when running tokenizer_consume_if. If expected_token is wrong and tokenizer_consume_if is called again,
+    then do not rollback to previous token and instead reuse the already parsed token
+    */
+    bool needs_update;
     BufferView code_name;
     
     union {
@@ -38,8 +44,6 @@ typedef struct {
 } Tokenizer;
 
 CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
-
-CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 /* 
    @result is set to 0 if the next token is equal to @expected_token,
-- 
cgit v1.2.3