1 files changed, 7 insertions, 3 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 21e93ee..b38cc58 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -3,6 +3,7 @@
 
 #include "std/buffer_view.h"
 #include "std/misc.h"
+#include "std/defs.h"
 #include "binop_type.h"
 
 #define TOKENIZER_OK 0
@@ -52,14 +53,15 @@ typedef struct {
         BinopType binop_type;
     } value;
     bool number_is_integer;
+    ScopedAllocator *allocator; /* borrowed */
 } Tokenizer;
 
 typedef struct {
     int index;
-    const char* str;
+    char* str;
 } TokenizerError;
 
-CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
+CHECK_RESULT int tokenizer_init(Tokenizer *self, ScopedAllocator *allocator, BufferView code, BufferView code_name);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 /* 
    @result is set to 0 if the next token is equal to @expected_token,
@@ -68,6 +70,8 @@ CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 CHECK_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result);
 void tokenizer_print_error(Tokenizer *self, const char *fmt, ...);
 void tokenizer_print_error_object(Tokenizer *self, TokenizerError *error);
-TokenizerError tokenizer_create_error(Tokenizer *tokenizer, const char *err_str);
+TokenizerError tokenizer_create_error(Tokenizer *self, int index, const char *fmt, ...);
+int tokenizer_get_error_index(Tokenizer *self);
+int tokenizer_get_code_reference_index(Tokenizer *self, const char *ref);
 
 #endif