From 5e240bdab90c45f935e7d2b33181de13295e7e6b Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Sat, 2 Mar 2019 21:20:33 +0100
Subject: Add string, variable and number. Fix identifier match against const
 and var

---
 include/tokenizer.h | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

(limited to 'include/tokenizer.h')

diff --git a/include/tokenizer.h b/include/tokenizer.h
index fac61e7..c76cd52 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -19,9 +19,12 @@ typedef enum {
     TOK_EQUALS,
     TOK_OPEN_PAREN,
     TOK_CLOSING_PAREN,
+    TOK_COMMA,
     TOK_OPEN_BRACE,
     TOK_CLOSING_BRACE,
-    TOK_IMPORT
+    TOK_IMPORT,
+    TOK_NUMBER,
+    TOK_DOT
 } Token;
 
 typedef struct {
@@ -40,9 +43,17 @@ typedef struct {
     union {
         BufferView identifier;
         BufferView string;
+        i64 integer;
+        f64 floating;
     } value;
+    bool number_is_integer;
 } Tokenizer;
 
+typedef struct {
+    int index;
+    const char* str;
+} TokenizerError;
+
 CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 /* 
@@ -51,5 +62,7 @@ CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
 */
 CHECK_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result);
 void tokenizer_print_error(Tokenizer *self, const char *fmt, ...);
+void tokenizer_print_error_object(Tokenizer *self, TokenizerError *error);
+TokenizerError tokenizer_create_error(Tokenizer *tokenizer, const char *err_str);
 
 #endif
-- 
cgit v1.2.3