1 files changed, 43 insertions, 6 deletions

diff --git a/src/tokenizer.c b/src/tokenizer.c
index 97673f5..efc9eee 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -9,6 +9,8 @@ void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size)
 
     self->identifier.data = NULL;
     self->identifier.size = 0;
+    self->string.data = NULL;
+    self->string.size = 0;
     self->bool_value = 0;
     self->number_value = 0;
 }
@@ -84,6 +86,27 @@ static int64_t string_to_int(TslStringView *str) {
     return num;
 }
 
+static int tsl_tokenizer_goto_end_of_string(TslTokenizer *self, char string_start_symbol) {
+    int string_escape = 0;
+    for(;;) {
+        char c = tsl_tokenizer_get_char(self);
+        if(c == string_start_symbol) {
+            ++self->code_index;
+            if(!string_escape)
+                return 1;
+            string_escape = 0;
+        } else if(c == '\\') {
+            ++self->code_index;
+            string_escape = !string_escape;
+        } else if(c == '\0') {
+            return 0;
+        } else {
+            ++self->code_index;
+            string_escape = 0;
+        }
+    }
+}
+
 TslToken tsl_tokenizer_next(TslTokenizer *self) {
     char c;
     tsl_tokenizer_skip_whitespace(self);
@@ -102,16 +125,12 @@ TslToken tsl_tokenizer_next(TslTokenizer *self) {
         self->identifier.size = self->code_index - identifier_start;
 
         switch(self->identifier.size) {
-            case 3: {
-                if(memcmp(self->identifier.data, "null", 3) == 0) {
-                    return TSL_TOKEN_NULL;
-                }
-                break;
-            }
             case 4: {
                 if(memcmp(self->identifier.data, "true", 4) == 0) {
                     self->bool_value = 1;
                     return TSL_TOKEN_BOOL;
+                } else if(memcmp(self->identifier.data, "null", 4) == 0) {
+                    return TSL_TOKEN_NULL;
                 }
                 break;
             }
@@ -137,9 +156,27 @@ TslToken tsl_tokenizer_next(TslTokenizer *self) {
         self->identifier.size = self->code_index - num_start;
         self->number_value = string_to_int(&self->identifier);
         return TSL_TOKEN_NUM;
+    } else if(c == '"') {
+        char string_start_symbol = c;
+        size_t string_start;
+        ++self->code_index;
+        string_start = self->code_index;
+        if(tsl_tokenizer_goto_end_of_string(self, string_start_symbol)) {
+            self->string.data = self->code + string_start;
+            self->string.size = self->code_index - 1 - string_start;
+            return TSL_TOKEN_STRING;
+        } else {
+            return TSL_TOKEN_END_OF_FILE;
+        }
     } else if(c == '=') {
         ++self->code_index;
         return TSL_TOKEN_EQUAL;
+    } else if(c == '{') {
+        ++self->code_index;
+        return TSL_TOKEN_LBRACE;
+    } else if(c == '}') {
+        ++self->code_index;
+        return TSL_TOKEN_RBRACE;
     } else if(c == '\0') {
         return TSL_TOKEN_END_OF_FILE;
     } else {