From 11dc4b81935e3dfee997c421d8d6fa166edd7a05 Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Sun, 24 Feb 2019 02:10:58 +0100
Subject: Initial commit, Function declaration work somewhat

---
 include/tokenizer.h | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 include/tokenizer.h

(limited to 'include/tokenizer.h')

diff --git a/include/tokenizer.h b/include/tokenizer.h
new file mode 100644
index 0000000..7dd377f
--- /dev/null
+++ b/include/tokenizer.h
@@ -0,0 +1,46 @@
+#ifndef AMALGAM_TOKENIZER_H
+#define AMALGAM_TOKENIZER_H
+
+#include "buffer_view.h"
+#include "misc.h"
+
+#define TOKENIZER_OK 0
+#define TOKENIZER_UNEXPECTED_TOKEN -1
+
+typedef enum {
+    TOK_NONE,
+    TOK_END_OF_FILE,
+    TOK_IDENTIFIER,
+    TOK_CONST,
+    TOK_VAR,
+    TOK_EQUALS,
+    TOK_OPEN_PAREN,
+    TOK_CLOSING_PAREN,
+    TOK_OPEN_BRACE,
+    TOK_CLOSING_BRACE
+} Token;
+
+typedef struct {
+    BufferView code;
+    int index;
+    int prev_index;
+    int line;
+    
+    union {
+        BufferView identifier;
+    } value;
+} Tokenizer;
+
+WARN_UNUSED_RESULT int tokenizer_init(Tokenizer *self, BufferView code);
+void tokenizer_deinit(Tokenizer *self);
+
+WARN_UNUSED_RESULT int tokenizer_next(Tokenizer *self, Token *token);
+WARN_UNUSED_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);
+/* 
+   @result is set to 0 if the next token is equal to @expected_token,
+   otherwise @result is set to 1
+*/
+WARN_UNUSED_RESULT int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result);
+void tokenizer_print_error(Tokenizer *self, const char *fmt, ...);
+
+#endif
-- 
cgit v1.2.3