Use multiple threads to parse

1 files changed, 11 insertions, 5 deletions

diff --git a/include/tokenizer.h b/include/tokenizer.h
index 9584542..e79f070 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -1,11 +1,13 @@
 #ifndef AMALGAM_TOKENIZER_H
 #define AMALGAM_TOKENIZER_H
 
-#include "buffer_view.h"
-#include "misc.h"
+#include "std/buffer_view.h"
+#include "std/misc.h"
 
 #define TOKENIZER_OK 0
-#define TOKENIZER_UNEXPECTED_TOKEN -1
+/* General error */
+#define TOKENIZER_ERR -1
+#define TOKENIZER_UNEXPECTED_TOKEN -2
 
 typedef enum {
     TOK_NONE,
@@ -13,11 +15,13 @@ typedef enum {
     TOK_IDENTIFIER,
     TOK_CONST,
     TOK_VAR,
+    TOK_STRING,
     TOK_EQUALS,
     TOK_OPEN_PAREN,
     TOK_CLOSING_PAREN,
     TOK_OPEN_BRACE,
-    TOK_CLOSING_BRACE
+    TOK_CLOSING_BRACE,
+    TOK_IMPORT
 } Token;
 
 typedef struct {
@@ -25,13 +29,15 @@ typedef struct {
     int index;
     int prev_index;
     int line;
+    BufferView code_name;
     
     union {
         BufferView identifier;
+        BufferView string;
     } value;
 } Tokenizer;
 
-CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code);
+CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
 
 CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);