Use multiple threads to parse

16 files changed, 201 insertions, 17 deletions

diff --git a/include/ast.h b/include/ast.h
index f3580c0..529b3c8 100644
--- a/include/ast.h
+++ b/include/ast.h
@@ -1,26 +1,29 @@
 #ifndef AMALGAM_AST_H
 #define AMALGAM_AST_H
 
-#include "buffer_view.h"
-#include "buffer.h"
-#include "misc.h"
-#include "scoped_allocator.h"
+#include "std/buffer_view.h"
+#include "std/buffer.h"
+#include "std/misc.h"
+#include "std/scoped_allocator.h"
 
 typedef struct FunctionDecl FunctionDecl;
 typedef struct FunctionCall FunctionCall;
 typedef struct LhsExpr LhsExpr;
+typedef struct Import Import;
 
 typedef union {
     FunctionDecl *func_decl;
     FunctionCall *func_call;
     LhsExpr *lhs_expr;
+    Import *import;
 } AstValue;
 
 typedef enum {
     AST_NONE,
     AST_FUNCTION_DECL,
     AST_FUNCTION_CALL,
-    AST_LHS
+    AST_LHS,
+    AST_IMPORT
 } AstType;
 
 typedef struct {
@@ -43,6 +46,10 @@ struct LhsExpr {
     Ast rhs_expr;
 };
 
+struct Import {
+    BufferView path;
+};
+
 Ast ast_none();
 
 CHECK_RESULT int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator);
@@ -52,4 +59,6 @@ void funccall_init(FunctionCall *self, BufferView name);
 
 void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name);
 
+void import_init(Import *self, BufferView path);
+
 #endif
diff --git a/include/compiler.h b/include/compiler.h
new file mode 100644
index 0000000..691263a
--- /dev/null
+++ b/include/compiler.h
@@ -0,0 +1,31 @@
+#ifndef AMALGAM_COMPILER_H
+#define AMALGAM_COMPILER_H
+
+#include "std/misc.h"
+#include "std/buffer.h"
+#include "std/buffer_view.h"
+#include "std/scoped_allocator.h"
+#include "std/thread.h"
+#include "defs.h"
+
+#define AMAL_COMPILER_OK 0
+/* General error */
+#define AMAL_COMPILER_ERR -1
+
+struct amal_compiler {
+    ScopedAllocator allocator;
+    ScopedAllocator main_thread_allocator;
+    Buffer parsers;
+    Buffer queued_files;
+    ParserThreadData *threads;
+    int usable_thread_count;
+    bool started;
+    amal_mutex mutex;
+};
+
+CHECK_RESULT int amal_compiler_init(amal_compiler *self);
+CHECK_RESULT int amal_compiler_deinit(amal_compiler *self);
+CHECK_RESULT int amal_compiler_load_file(amal_compiler *self, BufferView filepath);
+/* TODO: amal_compiler_unload_file */
+
+#endif
\ No newline at end of file
diff --git a/include/defs.h b/include/defs.h
new file mode 100644
index 0000000..9b754d8
--- /dev/null
+++ b/include/defs.h
@@ -0,0 +1,7 @@
+#ifndef AMALGAM_DEFS_H
+#define AMALGAM_DEFS_H
+
+typedef struct ParserThreadData ParserThreadData;
+typedef struct amal_compiler amal_compiler;
+
+#endif
\ No newline at end of file
diff --git a/include/parser.h b/include/parser.h
index e90871f..9b41e5e 100644
--- a/include/parser.h
+++ b/include/parser.h
@@ -1,25 +1,51 @@
 #ifndef AMALGAM_PARSER_H
 #define AMALGAM_PARSER_H
 
-#include "buffer.h"
-#include "buffer_view.h"
+#include "std/buffer.h"
+#include "std/buffer_view.h"
+#include "std/scoped_allocator.h"
+#include "std/thread.h"
 #include "tokenizer.h"
-#include "scoped_allocator.h"
+#include "defs.h"
 
 #define PARSER_OK 0
 /* General error */
 #define PARSER_ERR -1
 #define PARSER_UNEXPECTED_TOKEN -2
 
+typedef enum {
+    PARSER_THREAD_STATUS_NEW,
+    PARSER_THREAD_STATUS_IDLE,
+    PARSER_THREAD_STATUS_RUNNING
+} ParserThreadStatus;
+
+struct ParserThreadData {
+    amal_thread thread;
+    ParserThreadStatus status;
+    ScopedAllocator allocator;
+};
+
 typedef struct {
     Tokenizer tokenizer;
-    ScopedAllocator allocator;
     Buffer ast_objects;
+    ScopedAllocator *allocator; /* borrowed. Copied from @compiler for faster access to allocator */
+    amal_compiler *compiler;
+    bool started;
 } Parser;
 
-CHECK_RESULT int parser_init(Parser *self);
-void parser_deinit(Parser *self);
+CHECK_RESULT int parser_thread_data_init(ParserThreadData *self);
+CHECK_RESULT int parser_thread_data_deinit(ParserThreadData *self);
+CHECK_RESULT int parser_thread_data_start(ParserThreadData *self, AmalThreadCallbackFunc callback_func, void *userdata);
+CHECK_RESULT int parser_thread_data_join(ParserThreadData *self, void **result);
+
+CHECK_RESULT int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocator);
 
-CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer);
+/*
+@buffer_name will be the path to the file when using parser_parse_file and when parsing a buffer
+you can name the buffer anything  you want to identify it.
+*/
+CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_name);
+/* Parses a file and the dependencies that are included using @import */
+CHECK_RESULT int parser_parse_file(Parser *self, BufferView filepath);
 
 #endif
diff --git a/include/alloc.h b/include/std/alloc.h
index 6809287..6809287 100644
--- a/include/alloc.h
+++ b/include/std/alloc.h
diff --git a/include/buffer.h b/include/std/buffer.h
index 5339108..ed87f29 100644
--- a/include/buffer.h
+++ b/include/std/buffer.h
@@ -18,5 +18,6 @@ CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator);
 
 CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size);
 void* buffer_get(Buffer *self, usize index, usize type_size);
+CHECK_RESULT int buffer_pop(Buffer *self, void *data, usize size);
 
 #endif
\ No newline at end of file
diff --git a/include/buffer_view.h b/include/std/buffer_view.h
index 4993dc2..4993dc2 100644
--- a/include/buffer_view.h
+++ b/include/std/buffer_view.h
diff --git a/include/std/defs.h b/include/std/defs.h
new file mode 100644
index 0000000..1376e16
--- /dev/null
+++ b/include/std/defs.h
@@ -0,0 +1,7 @@
+#ifndef AMALGAM_STD_DEFS_H
+#define AMALGAM_STD_DEFS_H
+
+typedef struct amal_thread amal_thread;
+typedef struct amal_mutex amal_mutex;
+
+#endif
\ No newline at end of file
diff --git a/include/std/file.h b/include/std/file.h
new file mode 100644
index 0000000..ac2abec
--- /dev/null
+++ b/include/std/file.h
@@ -0,0 +1,27 @@
+#ifndef AMALGAM_FILE_H
+#define AMALGAM_FILE_H
+
+#include "misc.h"
+#include "types.h"
+
+/* Return bool_true if you want to continue scanning, otherwise return bool_false */
+typedef bool (*scan_dir_callback_func)(const char *filepath, int filepath_length, void *userdata);
+
+typedef struct {
+    const char *file_data;
+    usize file_size;
+    int fd;
+} MappedFile;
+
+typedef enum {
+    MAPPED_FILE_READ,
+    MAPPED_FILE_WRITE,
+    MAPPED_FILE_READ_WRITE
+} MappedFileMode;
+
+/* Hidden files (files starting with a dot) are skipped */
+CHECK_RESULT int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata);
+CHECK_RESULT int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode);
+CHECK_RESULT int mapped_file_deinit(MappedFile *self);
+
+#endif
\ No newline at end of file
diff --git a/include/std/log.h b/include/std/log.h
new file mode 100644
index 0000000..6ce9e4c
--- /dev/null
+++ b/include/std/log.h
@@ -0,0 +1,13 @@
+#ifndef AMALGAM_LOG_H
+#define AMALGAM_LOG_H
+
+#include "defs.h"
+
+amal_mutex* amal_log_get_mutex();
+void amal_log_debug(const char *fmt, ...);
+void amal_log_error(const char *fmt, ...);
+void amal_log_info(const char *fmt, ...);
+void amal_log_warning(const char *fmt, ...);
+void amal_log_perror(const char *prefix);
+
+#endif
\ No newline at end of file
diff --git a/include/mem.h b/include/std/mem.h
index bad6353..a5fe9b4 100644
--- a/include/mem.h
+++ b/include/std/mem.h
@@ -6,5 +6,6 @@
 
 void am_memcpy(void *dest, const void *src, usize size);
 bool am_memeql(const void *lhs, const void *rhs, usize size);
+void am_memset(void *dest, int value, usize size);
 
 #endif
\ No newline at end of file
diff --git a/include/misc.h b/include/std/misc.h
index 9cb2dde..3ac524a 100644
--- a/include/misc.h
+++ b/include/std/misc.h
@@ -19,9 +19,15 @@ do { \
 #define CHECK_RESULT
 #endif
 
+#define ignore_result_int(expr) (void)((expr)+1)
+
 typedef enum {
     bool_false,
     bool_true
 } bool;
 
+#ifndef NULL
+#define NULL ((void*)0)
+#endif
+
 #endif
\ No newline at end of file
diff --git a/include/scoped_allocator.h b/include/std/scoped_allocator.h
index fdaee2a..fdaee2a 100644
--- a/include/scoped_allocator.h
+++ b/include/std/scoped_allocator.h
diff --git a/include/std/thread.h b/include/std/thread.h
new file mode 100644
index 0000000..dd09039
--- /dev/null
+++ b/include/std/thread.h
@@ -0,0 +1,50 @@
+#ifndef AMALGAM_THREAD_H
+#define AMALGAM_THREAD_H
+
+#include "misc.h"
+#include "types.h"
+#include "defs.h"
+#include <pthread.h>
+
+typedef void* (AmalThreadCallbackFunc)(void *userdata);
+
+#define AMAL_THREAD_OK 0
+/* General error */
+#define AMAL_THREAD_ERR -1
+#define AMAL_THREAD_NOT_JOINABLE -23
+
+struct amal_thread {
+    pthread_t thread_id;
+    pthread_attr_t thread_attr;
+    const char *name;
+    bool cancellable;
+    bool destroyable;
+};
+
+typedef enum {
+    AMAL_THREAD_JOINABLE,
+    AMAL_THREAD_DETACHED
+} amal_thread_type;
+
+struct amal_mutex {
+    pthread_mutex_t mutex;
+    const char *lock_identifier;
+};
+
+CHECK_RESULT int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata);
+/* Safe to call multiple times */
+CHECK_RESULT int amal_thread_deinit(amal_thread *self);
+CHECK_RESULT int amal_thread_detach(amal_thread *self);
+CHECK_RESULT int amal_thread_join(amal_thread *self, void **result);
+
+void amal_mutex_init(amal_mutex *self);
+CHECK_RESULT int amal_mutex_lock(amal_mutex *self, const char *lock_identifier);
+CHECK_RESULT int amal_mutex_unlock(amal_mutex *self);
+void amal_mutex_tryunlock(amal_mutex *self);
+
+bool amal_thread_is_main();
+
+/* Returns 0 if the number of usable threads is unknown */
+int amal_get_usable_thread_count();
+
+#endif
\ No newline at end of file
diff --git a/include/types.h b/include/std/types.h
index 68e2d0f..68e2d0f 100644
--- a/include/types.h
+++ b/include/std/types.h
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 9584542..e79f070 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -1,11 +1,13 @@
 #ifndef AMALGAM_TOKENIZER_H
 #define AMALGAM_TOKENIZER_H
 
-#include "buffer_view.h"
-#include "misc.h"
+#include "std/buffer_view.h"
+#include "std/misc.h"
 
 #define TOKENIZER_OK 0
-#define TOKENIZER_UNEXPECTED_TOKEN -1
+/* General error */
+#define TOKENIZER_ERR -1
+#define TOKENIZER_UNEXPECTED_TOKEN -2
 
 typedef enum {
     TOK_NONE,
@@ -13,11 +15,13 @@ typedef enum {
     TOK_IDENTIFIER,
     TOK_CONST,
     TOK_VAR,
+    TOK_STRING,
     TOK_EQUALS,
     TOK_OPEN_PAREN,
     TOK_CLOSING_PAREN,
     TOK_OPEN_BRACE,
-    TOK_CLOSING_BRACE
+    TOK_CLOSING_BRACE,
+    TOK_IMPORT
 } Token;
 
 typedef struct {
@@ -25,13 +29,15 @@ typedef struct {
     int index;
     int prev_index;
     int line;
+    BufferView code_name;
     
     union {
         BufferView identifier;
+        BufferView string;
     } value;
 } Tokenizer;
 
-CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code);
+CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name);
 
 CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token);
 CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token);