aboutsummaryrefslogtreecommitdiff
path: root/include/tokenizer.h
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2020-01-20 23:00:39 +0100
committerdec05eba <dec05eba@protonmail.com>2020-01-20 23:00:39 +0100
commit108018e3e7326dabbbef568ab08bc5cebf5d427b (patch)
tree7c9a522276aea7015638492592eba02615b78d43 /include/tokenizer.h
parent50c928d224bff0af322f23a7d2b842cd54aa2e68 (diff)
Add arithmetic, implement hash map
Diffstat (limited to 'include/tokenizer.h')
-rw-r--r--include/tokenizer.h19
1 files changed, 10 insertions, 9 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 98491c7..2e7d42b 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -1,13 +1,7 @@
#ifndef TSL_TOKENIZER_H
#define TSL_TOKENIZER_H
-#include <stddef.h>
-#include <stdint.h>
-
-typedef struct {
- const char *data;
- size_t size;
-} TslStringView;
+#include "std/string_view.h"
typedef enum {
TSL_TOKEN_END_OF_FILE,
@@ -27,7 +21,8 @@ typedef enum {
TSL_TOKEN_COLON,
TSL_TOKEN_COMMA,
TSL_TOKEN_FN,
- TSL_TOKEN_DOLLAR_SIGN
+ TSL_TOKEN_DOLLAR_SIGN,
+ TSL_TOKEN_ARITHMETIC
} TslToken;
typedef enum {
@@ -53,14 +48,20 @@ typedef struct {
TslStringView identifier;
TslStringView string;
int bool_value;
- int64_t number_value;
+ double number_value;
+ char arithmetic_symbol;
} TslTokenizer;
void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
TslToken tsl_tokenizer_next(TslTokenizer *self);
int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
+/*
+ If peek was previously called without consuming the token, then the previous value peek token is returned.
+ In other words, calling tsl_tokenizer_peek twice in a row will return the same token without progressing.
+*/
TslToken tsl_tokenizer_peek(TslTokenizer *self);
+TslToken tsl_tokenizer_consume_peek(TslTokenizer *self);
TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);