1 files changed, 10 insertions, 9 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 98491c7..2e7d42b 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -1,13 +1,7 @@
 #ifndef TSL_TOKENIZER_H
 #define TSL_TOKENIZER_H
 
-#include <stddef.h>
-#include <stdint.h>
-
-typedef struct {
-    const char *data;
-    size_t size;
-} TslStringView;
+#include "std/string_view.h"
 
 typedef enum {
     TSL_TOKEN_END_OF_FILE,
@@ -27,7 +21,8 @@ typedef enum {
     TSL_TOKEN_COLON,
     TSL_TOKEN_COMMA,
     TSL_TOKEN_FN,
-    TSL_TOKEN_DOLLAR_SIGN
+    TSL_TOKEN_DOLLAR_SIGN,
+    TSL_TOKEN_ARITHMETIC
 } TslToken;
 
 typedef enum {
@@ -53,14 +48,20 @@ typedef struct {
     TslStringView identifier;
     TslStringView string;
     int bool_value;
-    int64_t number_value;
+    double number_value;
+    char arithmetic_symbol;
 } TslTokenizer;
 
 void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
 
 TslToken tsl_tokenizer_next(TslTokenizer *self);
 int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
+/*
+    If peek was previously called without consuming the token, then the previous value peek token is returned.
+    In other words, calling tsl_tokenizer_peek twice in a row will return the same token without progressing.
+*/
 TslToken tsl_tokenizer_peek(TslTokenizer *self);
+TslToken tsl_tokenizer_consume_peek(TslTokenizer *self);
 
 TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);