aboutsummaryrefslogtreecommitdiff
path: root/include/tokenizer.h
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2020-01-14 07:27:47 +0100
committerdec05eba <dec05eba@protonmail.com>2020-01-14 07:27:47 +0100
commite27bd78c8211532bf0d39d87d2051222f7e86e26 (patch)
tree631c050014070e52519798b228770f578b4c0a59 /include/tokenizer.h
start
Diffstat (limited to 'include/tokenizer.h')
-rw-r--r--include/tokenizer.h37
1 files changed, 37 insertions, 0 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
new file mode 100644
index 0000000..decdae4
--- /dev/null
+++ b/include/tokenizer.h
@@ -0,0 +1,37 @@
+#ifndef TSL_TOKENIZER_H
+#define TSL_TOKENIZER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+ const char *data;
+ size_t size;
+} TslStringView;
+
+typedef enum {
+ TSL_TOKEN_END_OF_FILE,
+ TSL_TOKEN_UNEXPECTED_SYMBOL,
+ TSL_TOKEN_IDENTIFIER,
+ TSL_TOKEN_NUM,
+ TSL_TOKEN_BOOL,
+ TSL_TOKEN_NULL,
+ TSL_TOKEN_EQUAL
+} TslToken;
+
+typedef struct {
+ const char *code;
+ size_t code_size;
+ size_t code_index;
+
+ TslStringView identifier;
+ int bool_value;
+ int64_t number_value;
+} TslTokenizer;
+
+void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
+
+TslToken tsl_tokenizer_next(TslTokenizer *self);
+
+#endif /* TSL_TOKENIZER_H */
+