1 files changed, 28 insertions, 16 deletions

diff --git a/src/tokenizer.c b/src/tokenizer.c
index 89c40cb..889152b 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -2,6 +2,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <assert.h>
+#include <stdint.h>
 
 void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size) {
     self->code = code;
@@ -19,6 +20,7 @@ void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size)
     self->string.size = 0;
     self->bool_value = 0;
     self->number_value = 0;
+    self->arithmetic_symbol = '\0';
 }
 
 static char tsl_tokenizer_get_char(TslTokenizer *self) {
@@ -173,6 +175,7 @@ static TslToken tsl_tokenizer_next_internal(TslTokenizer *self) {
         }
         self->identifier.data = self->code + num_start;
         self->identifier.size = self->code_index - num_start;
+        /* TODO: Check if the result of string_to_int is too large to fit into double */
         self->number_value = string_to_int(&self->identifier);
         return TSL_TOKEN_NUM;
     } else if(c == '"') {
@@ -217,6 +220,10 @@ static TslToken tsl_tokenizer_next_internal(TslTokenizer *self) {
     } else if(c == '$') {
         ++self->code_index;
         return TSL_TOKEN_DOLLAR_SIGN;
+    } else if(c == '+' || c == '-' || c == '*' || c == '/') {
+        self->arithmetic_symbol = c;
+        ++self->code_index;
+        return TSL_TOKEN_ARITHMETIC;
     } else if(c == '\0') {
         return TSL_TOKEN_END_OF_FILE;
     } else {
@@ -225,14 +232,6 @@ static TslToken tsl_tokenizer_next_internal(TslTokenizer *self) {
     }
 }
 
-static TslToken tsl_tokenizer_consume_peek(TslTokenizer *self) {
-    TslToken token = self->peek.token;
-    self->code_index = self->peek.code_index;
-    self->prev_code_index = self->peek.prev_code_index;
-    self->peek.token = -1;
-    return token;
-}
-
 TslToken tsl_tokenizer_next(TslTokenizer *self) {
     if((int)self->peek.token == -1) {
         return tsl_tokenizer_next_internal(self);
@@ -256,16 +255,29 @@ int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token) {
 }
 
 TslToken tsl_tokenizer_peek(TslTokenizer *self) {
-    size_t p_prev_code_index = self->prev_code_index;
-    size_t p_code_index = self->code_index;
+    if((int)self->peek.token == -1) {
+        size_t p_prev_code_index = self->prev_code_index;
+        size_t p_code_index = self->code_index;
 
-    self->peek.token = tsl_tokenizer_next_internal(self);
-    self->peek.code_index = self->code_index;
-    self->peek.prev_code_index = self->prev_code_index;
+        self->peek.token = tsl_tokenizer_next_internal(self);
+        self->peek.code_index = self->code_index;
+        self->peek.prev_code_index = self->prev_code_index;
 
-    self->prev_code_index = p_prev_code_index;
-    self->code_index = p_code_index;
-    return self->peek.token;
+        self->prev_code_index = p_prev_code_index;
+        self->code_index = p_code_index;
+        return self->peek.token;
+    } else {
+        return self->peek.token;
+    }
+}
+
+TslToken tsl_tokenizer_consume_peek(TslTokenizer *self) {
+    TslToken token = self->peek.token;
+    assert((int)token != -1);
+    self->code_index = self->peek.code_index;
+    self->prev_code_index = self->peek.prev_code_index;
+    self->peek.token = -1;
+    return token;
 }
 
 TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg) {