From e27bd78c8211532bf0d39d87d2051222f7e86e26 Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Tue, 14 Jan 2020 07:27:47 +0100
Subject: start

---
 include/tokenizer.h | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 include/tokenizer.h

(limited to 'include')

diff --git a/include/tokenizer.h b/include/tokenizer.h
new file mode 100644
index 0000000..decdae4
--- /dev/null
+++ b/include/tokenizer.h
@@ -0,0 +1,37 @@
+#ifndef TSL_TOKENIZER_H
+#define TSL_TOKENIZER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+    const char *data;
+    size_t size;
+} TslStringView;
+
+typedef enum {
+    TSL_TOKEN_END_OF_FILE,
+    TSL_TOKEN_UNEXPECTED_SYMBOL,
+    TSL_TOKEN_IDENTIFIER,
+    TSL_TOKEN_NUM,
+    TSL_TOKEN_BOOL,
+    TSL_TOKEN_NULL,
+    TSL_TOKEN_EQUAL
+} TslToken;
+
+typedef struct {
+    const char *code;
+    size_t code_size;
+    size_t code_index;
+
+    TslStringView identifier;
+    int bool_value;
+    int64_t number_value;
+} TslTokenizer;
+
+void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
+
+TslToken tsl_tokenizer_next(TslTokenizer *self);
+
+#endif /* TSL_TOKENIZER_H */
+
-- 
cgit v1.2.3