Add ast resolving using multiple threads

Fix issue where not all files are parsed

9 files changed, 155 insertions, 45 deletions

diff --git a/doc/IMPLEMENTATION.md b/doc/IMPLEMENTATION.md
index 0da1c1a..5dc6ef5 100644
--- a/doc/IMPLEMENTATION.md
+++ b/doc/IMPLEMENTATION.md
@@ -12,9 +12,9 @@ before the SSA is created.
 is optimized before creating the bytecode.
 5. If optimization is enabled then the bytecode is optimized.
 
-# Currently implemented
+# Progress
 1. Parsing using multiple threads is done, but the parser is not finished.
-2. Not started.
+2. Resolving ast using multiple threads is done, but the ast resolver is not finished.
 3. Not started.
 4. Not started.
 5. Not started.
\ No newline at end of file
diff --git a/include/ast.h b/include/ast.h
index 29a0b64..7911a59 100644
--- a/include/ast.h
+++ b/include/ast.h
@@ -15,6 +15,7 @@ typedef struct String String;
 typedef struct Variable Variable;
 typedef struct Number Number;
 typedef struct Binop Binop;
+typedef struct Scope Scope;
 
 typedef union {
     FunctionDecl *func_decl;
@@ -89,15 +90,20 @@ struct Binop {
     bool grouped;
 };
 
+struct Scope {
+    Buffer ast_objects;
+};
+
 Ast ast_none();
 
 CHECK_RESULT int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator);
-CHECK_RESULT int funcdecl_add_to_body(FunctionDecl *self, Ast ast);
 CHECK_RESULT int funccall_init(FunctionCall *self, BufferView name, ScopedAllocator *allocator);
 void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name);
 void import_init(Import *self, BufferView path);
 CHECK_RESULT int string_init(String *self, BufferView str);
 void number_init(Number *self, i64 value, bool is_integer);
 void binop_init(Binop *self);
+CHECK_RESULT int scope_init(Scope *self, ScopedAllocator *allocator);
+void scope_resolve(Scope *self);
 
 #endif
diff --git a/include/compiler.h b/include/compiler.h
index d01c756..d7314cc 100644
--- a/include/compiler.h
+++ b/include/compiler.h
@@ -21,10 +21,12 @@ struct amal_compiler {
     int usable_thread_count;
     bool started;
     amal_mutex mutex;
+    int resolve_ast_index;
 };
 
 CHECK_RESULT int amal_compiler_init(amal_compiler *self);
 CHECK_RESULT int amal_compiler_deinit(amal_compiler *self);
+/* Not thread-safe */
 CHECK_RESULT int amal_compiler_load_file(amal_compiler *self, BufferView filepath);
 /* TODO: amal_compiler_unload_file */
 
diff --git a/include/parser.h b/include/parser.h
index b9fb3b7..5c055d9 100644
--- a/include/parser.h
+++ b/include/parser.h
@@ -1,11 +1,11 @@
 #ifndef AMALGAM_PARSER_H
 #define AMALGAM_PARSER_H
 
-#include "std/buffer.h"
 #include "std/buffer_view.h"
 #include "std/scoped_allocator.h"
 #include "std/thread.h"
 #include "tokenizer.h"
+#include "ast.h"
 #include "defs.h"
 
 #include <setjmp.h>
@@ -34,7 +34,7 @@ typedef enum {
 
 typedef struct {
     Tokenizer tokenizer;
-    Buffer ast_objects;
+    Scope scope;
     ScopedAllocator *allocator; /* borrowed. Copied from @compiler for faster access to allocator */
     amal_compiler *compiler;
     bool started;
diff --git a/include/std/buffer.h b/include/std/buffer.h
index 02cc20a..a60def9 100644
--- a/include/std/buffer.h
+++ b/include/std/buffer.h
@@ -19,5 +19,9 @@ CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator);
 CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size);
 void* buffer_get(Buffer *self, usize index, usize type_size);
 CHECK_RESULT int buffer_pop(Buffer *self, void *data, usize size);
+void* buffer_start(Buffer *self);
+void *buffer_end(Buffer *self);
+usize __buffer_get_size(Buffer *self, usize type_size);
+#define buffer_get_size(self, type) __buffer_get_size((self), sizeof(type))
 
 #endif
diff --git a/src/ast.c b/src/ast.c
index 993cf95..1db114a 100644
--- a/src/ast.c
+++ b/src/ast.c
@@ -12,11 +12,6 @@ int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator) {
     return buffer_init(&self->body, allocator);
 }
 
-int funcdecl_add_to_body(FunctionDecl *self, Ast ast) {
-    return_if_error(buffer_append(&self->body, &ast, sizeof(ast)));
-    return BUFFER_OK;
-}
-
 int funccall_init(FunctionCall *self, BufferView name, ScopedAllocator *allocator) {
     self->name = name;
     return buffer_init(&self->args, allocator);
@@ -49,4 +44,13 @@ void binop_init(Binop *self) {
     self->rhs = ast_none();
     self->type = BINOP_ADD;
     self->grouped = bool_false;
+}
+
+int scope_init(Scope *self, ScopedAllocator *allocator) {
+    return buffer_init(&self->ast_objects, allocator);
+}
+
+void scope_resolve(Scope *self) {
+    /* TODO: Implement. Also use longjmp to jump back to compiler on error */
+    (void)self;
 }
\ No newline at end of file
diff --git a/src/compiler.c b/src/compiler.c
index b8bca1c..e44ba6a 100644
--- a/src/compiler.c
+++ b/src/compiler.c
@@ -38,6 +38,7 @@ int amal_compiler_init(amal_compiler *self) {
     am_memset(&self->allocator, 0, sizeof(self->allocator));
     am_memset(&self->main_thread_allocator, 0, sizeof(self->main_thread_allocator));
     self->started = bool_false;
+    self->resolve_ast_index = 0;
     amal_mutex_init(&self->mutex);
 
     return_if_error(scoped_allocator_init(&self->allocator));
@@ -80,7 +81,26 @@ typedef struct {
     BufferView filepath;
 } CompilerParserThreadUserData;
 
-static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) {
+typedef struct {
+    amal_compiler *compiler;
+    ParserThreadData *parser_thread_data;
+    Parser *parser;
+} CompilerAstResolverThreadUserData;
+
+typedef enum {
+    THREAD_WORK_PARSE,
+    THREAD_WORK_RESOLVE_AST
+} ThreadWorkType;
+
+typedef struct {
+    union {
+        BufferView filepath;
+        Parser *parser;
+    } value;
+    ThreadWorkType type;
+} ThreadWorkData;
+
+static CHECK_RESULT int amal_compiler_load_in_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) {
     Parser parser;
     int result;
     result = AMAL_COMPILER_ERR;
@@ -88,7 +108,7 @@ static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self
 
     return_if_error(parser_init(&parser, self, allocator));
     cleanup_if_error(parser_parse_file(&parser, filepath));
-    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_first_this_thread"));
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_in_this_thread"));
     cleanup_if_error(buffer_append(&self->parsers, &parser, sizeof(parser)));
     result = AMAL_COMPILER_OK;
 
@@ -99,54 +119,92 @@ static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self
 
 /* TODO: Handle errors (stop parsing in all other threads and report errors/warnings) */
 static void* thread_callback_parse_file(void *userdata) {
-    int _;
     BufferView next_file;
     CompilerParserThreadUserData compiler_parser_userdata;
-    (void)_;
-    next_file = create_buffer_view_null();
     assert(!amal_thread_is_main());
     
     am_memcpy(&compiler_parser_userdata, userdata, sizeof(compiler_parser_userdata));
     am_free(userdata);
-    cleanup_if_error(amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, 
-                                                          compiler_parser_userdata.filepath,
-                                                          &compiler_parser_userdata.parser_thread_data->allocator));
-    cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file"));
-    if(compiler_parser_userdata.compiler->queued_files.size > 0)
-        _ = buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file));
+    next_file = compiler_parser_userdata.filepath;
+    for(;;) {
+        cleanup_if_error(amal_compiler_load_in_this_thread(compiler_parser_userdata.compiler, 
+                                                           next_file,
+                                                           &compiler_parser_userdata.parser_thread_data->allocator));
+        cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file"));
+        cleanup_if_error(buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file)));
+        amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex);
+    }
 
     cleanup:
-    if(!next_file.data)
-        compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE;
+    compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE;
     amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex);
-    if(next_file.data) {
-        _ = amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler,
-                                                 next_file,
-                                                 &compiler_parser_userdata.parser_thread_data->allocator);
+    return NULL;
+}
+
+/* TODO: Handle errors (stop resolving ast in all other threads and report errors/warnings) */
+static void* thread_callback_resolve_ast(void *userdata) {
+    Parser *parser;
+    CompilerAstResolverThreadUserData compiler_ast_resolver_userdata;
+    assert(!amal_thread_is_main());
+    
+    am_memcpy(&compiler_ast_resolver_userdata, userdata, sizeof(compiler_ast_resolver_userdata));
+    am_free(userdata);
+    parser = compiler_ast_resolver_userdata.parser;
+    for(;;) {
+        amal_log_debug("Resolving AST for file: %.*s", parser->tokenizer.code_name.size, parser->tokenizer.code_name.data);
+        scope_resolve(&parser->scope);
+        cleanup_if_error(amal_mutex_lock(&compiler_ast_resolver_userdata.compiler->mutex, "thread_callback_resolve_ast"));
+        if(compiler_ast_resolver_userdata.compiler->resolve_ast_index + 1 >= (int)buffer_get_size(&compiler_ast_resolver_userdata.compiler->parsers, Parser))
+            break;
+        ++compiler_ast_resolver_userdata.compiler->resolve_ast_index;
+        parser = buffer_get(&compiler_ast_resolver_userdata.compiler->parsers, compiler_ast_resolver_userdata.compiler->resolve_ast_index, sizeof(Parser));
+        amal_mutex_tryunlock(&compiler_ast_resolver_userdata.compiler->mutex);
     }
+
+    cleanup:
+    compiler_ast_resolver_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE;
+    amal_mutex_tryunlock(&compiler_ast_resolver_userdata.compiler->mutex);
     return NULL;
 }
 
-static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *self, BufferView filepath, ParserThreadData **thread_selected) {
+static CHECK_RESULT int amal_compiler_select_thread_for_work(amal_compiler *self, ThreadWorkData work_data, ParserThreadData **thread_selected) {
     int i;
     int result;
     ParserThreadData *parser_thread_data;
-    CompilerParserThreadUserData *thread_user_data;
+    void *thread_user_data;
     thread_user_data = NULL;
     *thread_selected = NULL;
     result = AMAL_COMPILER_ERR;
 
-    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_select_thread"));
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_select_thread_for_work"));
     for(i = 0; i < self->usable_thread_count; ++i) {
         parser_thread_data = &self->threads[i];
         if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING)
             continue;
 
-        cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&thread_user_data));
-        thread_user_data->compiler = self;
-        thread_user_data->parser_thread_data = parser_thread_data;
-        thread_user_data->filepath = filepath;
-        result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, thread_user_data);
+        switch(work_data.type) {
+            case THREAD_WORK_PARSE: {
+                CompilerParserThreadUserData *userdata;
+                cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&userdata));
+                thread_user_data = userdata;
+                userdata->compiler = self;
+                userdata->parser_thread_data = parser_thread_data;
+                userdata->filepath = work_data.value.filepath;
+                result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, userdata);
+                break;
+            }
+            case THREAD_WORK_RESOLVE_AST: {
+                CompilerAstResolverThreadUserData *userdata;
+                cleanup_if_error(am_malloc(sizeof(CompilerAstResolverThreadUserData), (void**)&userdata));
+                thread_user_data = userdata;
+                userdata->compiler = self;
+                userdata->parser_thread_data = parser_thread_data;
+                userdata->parser = work_data.value.parser;
+                ++self->resolve_ast_index;
+                result = parser_thread_data_start(parser_thread_data, thread_callback_resolve_ast, userdata);
+                break;
+            }
+        }
         *thread_selected = parser_thread_data;
         break;
     }
@@ -158,13 +216,13 @@ static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *sel
     return result;
 }
 
-static CHECK_RESULT int amal_compiler_all_threads_done(amal_compiler *self, bool *done) {
+static CHECK_RESULT int amal_compiler_check_all_threads_done(amal_compiler *self, bool *done) {
     int i;
     int result;
     result = AMAL_COMPILER_OK;
     *done = bool_false;
 
-    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_all_threads_done"));
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_check_all_threads_done"));
     for(i = 0; i < self->usable_thread_count; ++i) {
         ParserThreadData *parser_thread_data;
         parser_thread_data = &self->threads[i];
@@ -201,11 +259,10 @@ static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self
             amal_mutex_tryunlock(&self->mutex);
             if(result != 0)
                 goto cleanup;
-            amal_log_debug("Joining thread %d, status %d", i, parser_thread_data->status);
             ignore_result_int(parser_thread_data_join(parser_thread_data, &thread_return_data));
         }
 
-        cleanup_if_error(amal_compiler_all_threads_done(self, &done));
+        cleanup_if_error(amal_compiler_check_all_threads_done(self, &done));
         if(done)
             break;
     }
@@ -215,12 +272,33 @@ static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self
     return result;
 }
 
+static CHECK_RESULT int amal_compiler_resolve_ast(amal_compiler *self) {
+    Parser *parser;
+    Parser *parser_end;
+    parser = buffer_start(&self->parsers);
+    parser_end = buffer_end(&self->parsers);
+    for(; parser != parser_end; ++parser) {
+        ParserThreadData *thread_selected;
+        ThreadWorkData thread_work_data;
+        thread_work_data.type = THREAD_WORK_RESOLVE_AST;
+        thread_work_data.value.parser = parser;
+        return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &thread_selected));
+        /* After all threads have been used, they will handle using the remaining parsers */
+        if(!thread_selected)
+            break;
+    }
+    return amal_compiler_load_file_join_threads(self);
+}
+
 int amal_compiler_load_file(amal_compiler *self, BufferView filepath) {
     int result;
     ParserThreadData *parser_thread_data;
+    ThreadWorkData thread_work_data;
     result = AMAL_COMPILER_ERR;
+    thread_work_data.type = THREAD_WORK_PARSE;
+    thread_work_data.value.filepath = filepath;
 
-    return_if_error(amal_compiler_load_file_select_thread(self, filepath, &parser_thread_data));
+    return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &parser_thread_data));
 
     /*
     amal_compiler_load_file is called by the user for the first file to compile
@@ -230,7 +308,10 @@ int amal_compiler_load_file(amal_compiler *self, BufferView filepath) {
         self->started = bool_true;
         /*amal_log_info("Parsing %.*s using %d thread(s)", (int)filepath.size, filepath.data, self->usable_thread_count);*/
         /*return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator));*/
-        return amal_compiler_load_file_join_threads(self);
+        return_if_error(amal_compiler_load_file_join_threads(self));
+        amal_log_debug("Finished parsing all files, resolving AST");
+        return_if_error(amal_compiler_resolve_ast(self));
+        return AMAL_COMPILER_OK;
     }
 
     if(parser_thread_data)
diff --git a/src/parser.c b/src/parser.c
index fb18685..6ee273b 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -63,7 +63,7 @@ int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocato
     self->error.index = 0;
     self->error.str = NULL;
     self->error_context = ERROR_CONTEXT_NONE;
-    return buffer_init(&self->ast_objects, self->allocator);
+    return scope_init(&self->scope, self->allocator);
 }
 
 /*
@@ -116,7 +116,6 @@ static THROWABLE parser_parse_lhs(Parser *self, LhsExpr **result, bool *assignme
 
     throw_if_error(tokenizer_accept(&self->tokenizer, TOK_IDENTIFIER));
     var_name = self->tokenizer.value.identifier;
-    amal_log_debug("var name: %.*s", (int)var_name.size, var_name.data);
     throw_if_error(scoped_allocator_alloc(self->allocator, sizeof(LhsExpr), (void**)result));
     lhsexpr_init(*result, is_const, var_name);
 
@@ -379,7 +378,7 @@ int parser_parse_rhs_start(Parser *self, Ast *rhs_expr) {
 }
 
 /*
-BODY_SEMICOLON
+BODY_SEMICOLON = ';'
 
 Note: Semicolon is not required for closures, structs and tables
 */
@@ -441,7 +440,7 @@ int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_
     throw_if_error(tokenizer_init(&self->tokenizer, code_buffer, buffer_name));
     result = setjmp(self->parse_env);
     if(result == 0)
-        try(parser_parse_body_loop(self, &self->ast_objects, TOK_END_OF_FILE));
+        try(parser_parse_body_loop(self, &self->scope.ast_objects, TOK_END_OF_FILE));
     else if(self->error.str != NULL) {
         switch(self->error_context) {
             case ERROR_CONTEXT_NONE:
@@ -487,6 +486,7 @@ the path separator is a dot, otherwise the path separator is forward slash '/'
 int parser_queue_file(Parser *self, BufferView path) {
     /* TODO: Do not load same path twice or the compile will fail (and it can have recursive import) also for performance reasons */
     /* TODO: Parse special path (to include library path with dots) */
+    /* TODO: Path should be relative to the file that uses @import */
     throw_if_error(amal_compiler_load_file(self->compiler, path));
     return PARSER_OK;
 }
diff --git a/src/std/buffer.c b/src/std/buffer.c
index c4c9845..1fdcfd4 100644
--- a/src/std/buffer.c
+++ b/src/std/buffer.c
@@ -60,3 +60,16 @@ int buffer_pop(Buffer *self, void *data, usize size) {
     self->size -= size;
     return 0;
 }
+
+void* buffer_start(Buffer *self) {
+    return self->data;
+}
+
+void *buffer_end(Buffer *self) {
+    return self->data + self->size;
+}
+
+usize __buffer_get_size(Buffer *self, usize type_size) {
+    assert(type_size != 0);
+    return self->size / type_size;
+}
\ No newline at end of file