1 files changed, 46 insertions, 0 deletions
diff --git a/include/tokenizer.h b/include/tokenizer.h
new file mode 100644
index 0000000..7dd377f
--- /dev/null
+++ b/include/tokenizer.h
@@ -0,0 +1,46 @@
+#ifndef AMALGAM_TOKENIZER_H
+#define AMALGAM_TOKENIZER_H
+
+#include "buffer_view.h"
+#include "misc.h"
+
+#define TOKENIZER_OK 0
+#define TOKENIZER_UNEXPECTED_TOKEN -1
+
+typedef enum {
+    TOK_NONE,
+    TOK_END_OF_FILE,
+    TOK_IDENTIFIER,
+    TOK_CONST,
+    TOK_VAR,
+    TOK_EQUALS,
+    TOK_OPEN_PAREN,
+    TOK_CLOSING_PAREN,
+    TOK_OPEN_BRACE,
+    TOK_CLOSING_BRACE
+} Token;
+
+typedef struct {
+    BufferView code;
+    int index;
+    int prev_index;
+    int line;
+    
+    union {
+        BufferView identifier;
+    } value;
+} Tokenizer;
+
+WARN_UNUSED_RESULT int tokenizer_init(Tokenizer *self, BufferView code);
+void tokenizer_deinit(Tokenizer *self);
+
+WARN_UNUSED_RESULT int tokenizer_next(Tokenizer *self, Token *token);
+WARN_UNUSED_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
+/* 
+   @result is set to 0 if the next token is equal to @expected_token,
+   otherwise @result is set to 1
+*/
+WARN_UNUSED_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result);
+void tokenizer_print_error(Tokenizer *self, const char *fmt, ...);
+
+#endif