1 files changed, 37 insertions, 0 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
new file mode 100644
index 0000000..decdae4
--- /dev/null
+++ b/include/tokenizer.h
@@ -0,0 +1,37 @@
+#ifndef TSL_TOKENIZER_H
+#define TSL_TOKENIZER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+    const char *data;
+    size_t size;
+} TslStringView;
+
+typedef enum {
+    TSL_TOKEN_END_OF_FILE,
+    TSL_TOKEN_UNEXPECTED_SYMBOL,
+    TSL_TOKEN_IDENTIFIER,
+    TSL_TOKEN_NUM,
+    TSL_TOKEN_BOOL,
+    TSL_TOKEN_NULL,
+    TSL_TOKEN_EQUAL
+} TslToken;
+
+typedef struct {
+    const char *code;
+    size_t code_size;
+    size_t code_index;
+
+    TslStringView identifier;
+    int bool_value;
+    int64_t number_value;
+} TslTokenizer;
+
+void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
+
+TslToken tsl_tokenizer_next(TslTokenizer *self);
+
+#endif /* TSL_TOKENIZER_H */
+