13 files changed, 848 insertions, 71 deletions

diff --git a/src/ast.c b/src/ast.c
index 3ea4fe1..0acc69c 100644
--- a/src/ast.c
+++ b/src/ast.c
@@ -25,4 +25,8 @@ void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name) {
     self->isConst = isConst;
     self->var_name = var_name;
     self->rhs_expr = ast_none();
+}
+
+void import_init(Import *self, BufferView path) {
+    self->path = path;
 }
\ No newline at end of file
diff --git a/src/compiler.c b/src/compiler.c
new file mode 100644
index 0000000..fba18d3
--- /dev/null
+++ b/src/compiler.c
@@ -0,0 +1,225 @@
+#include "../include/compiler.h"
+#include "../include/parser.h"
+#include "../include/std/log.h"
+#include "../include/std/mem.h"
+#include "../include/std/alloc.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+static CHECK_RESULT int get_thread_count_env_var(int *thread_count) {
+    char *threads;
+    threads = getenv("THREADS");
+    if(!threads)
+        return -1;
+    *thread_count = atoi(threads);
+    return 0;
+}
+
+int amal_compiler_init(amal_compiler *self) {
+    int i;
+    int result;
+
+    result = get_thread_count_env_var(&self->usable_thread_count);
+    if(result != 0) {
+        self->usable_thread_count = amal_get_usable_thread_count();
+        if(self->usable_thread_count == 0) {
+            amal_log_info("Unable to get the number of threads available on the system, using 1 thread.");
+            amal_log_info("You can override the number of threads using by setting the environment variable THREADS");
+            self->usable_thread_count = 1;
+        }
+    } else if(self->usable_thread_count == 0) {
+        amal_log_info("Environment variable THREADS contains invalid number for threads. THREADS has to be at least 1.");
+        return AMAL_COMPILER_ERR;
+    }
+    /* Exclude thread count because we also use the main thread */
+    --self->usable_thread_count;
+
+    am_memset(&self->allocator, 0, sizeof(self->allocator));
+    am_memset(&self->main_thread_allocator, 0, sizeof(self->main_thread_allocator));
+    self->started = bool_false;
+    amal_mutex_init(&self->mutex);
+
+    return_if_error(scoped_allocator_init(&self->allocator));
+    cleanup_if_error(scoped_allocator_init(&self->main_thread_allocator));
+    cleanup_if_error(buffer_init(&self->parsers, &self->allocator));
+    cleanup_if_error(buffer_init(&self->queued_files, &self->allocator));
+    cleanup_if_error(scoped_allocator_alloc(&self->allocator,
+                                            self->usable_thread_count * sizeof(ParserThreadData),
+                                            (void**)&self->threads));
+    for(i = 0; i < self->usable_thread_count; ++i)
+        cleanup_if_error(parser_thread_data_init(&self->threads[i]));
+    return AMAL_COMPILER_OK;
+
+    cleanup:
+    /* Ignore result */
+    result = amal_compiler_deinit(self);
+    return AMAL_COMPILER_ERR;
+}
+
+int amal_compiler_deinit(amal_compiler *self) {
+    int i;
+    int result;
+    result = AMAL_COMPILER_OK;
+
+    for(i = 0; i < self->usable_thread_count; ++i) {
+        int r;
+        r = parser_thread_data_deinit(&self->threads[i]);
+        if(r != 0)
+            result = r;
+    }
+
+    scoped_allocator_deinit(&self->allocator);
+    scoped_allocator_deinit(&self->main_thread_allocator);
+    return result;
+}
+
+typedef struct {
+    amal_compiler *compiler;
+    ParserThreadData *parser_thread_data;
+    BufferView filepath;
+} CompilerParserThreadUserData;
+
+static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) {
+    Parser parser;
+    int result;
+    result = AMAL_COMPILER_ERR;
+    am_memset(&parser, 0, sizeof(parser));
+
+    return_if_error(parser_init(&parser, self, allocator));
+    cleanup_if_error(parser_parse_file(&parser, filepath));
+    amal_log_debug("Finished parsing in thread");
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_first_this_thread"));
+    cleanup_if_error(buffer_append(&self->parsers, &parser, sizeof(parser)));
+    result = AMAL_COMPILER_OK;
+
+    cleanup:
+    amal_mutex_tryunlock(&self->mutex);
+    return result;
+}
+
+/* TODO: Handle errors (stop parsing in all other threads and report errors/warnings) */
+static void* thread_callback_parse_file(void *userdata) {
+    int _;
+    BufferView next_file;
+    CompilerParserThreadUserData compiler_parser_userdata;
+    (void)_;
+    next_file = create_buffer_view_null();
+    assert(!amal_thread_is_main());
+    
+    am_memcpy(&compiler_parser_userdata, userdata, sizeof(compiler_parser_userdata));
+    am_free(userdata);
+    amal_log_debug("Thread start parse file");
+    cleanup_if_error(amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, 
+                                                          compiler_parser_userdata.filepath,
+                                                          &compiler_parser_userdata.parser_thread_data->allocator));
+    cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file"));
+    if(compiler_parser_userdata.compiler->queued_files.size > 0)
+        _ = buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file));
+
+    cleanup:
+    amal_log_debug("Thread finished, next file: %s", next_file.data ? "yes" : "no");
+    if(!next_file.data)
+        compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE;
+    amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex);
+    if(next_file.data) {
+        _ = amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler,
+                                                 next_file,
+                                                 &compiler_parser_userdata.parser_thread_data->allocator);
+    }
+    return NULL;
+}
+
+static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *self, BufferView filepath, ParserThreadData **thread_selected) {
+    int i;
+    int result;
+    ParserThreadData *parser_thread_data;
+    CompilerParserThreadUserData *thread_user_data;
+    thread_user_data = NULL;
+    *thread_selected = NULL;
+    result = AMAL_COMPILER_ERR;
+    assert(amal_thread_is_main());
+
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_select_thread"));
+    for(i = 0; i < self->usable_thread_count; ++i) {
+        parser_thread_data = &self->threads[i];
+        if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING)
+            continue;
+
+        cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&thread_user_data));
+        thread_user_data->compiler = self;
+        thread_user_data->parser_thread_data = parser_thread_data;
+        thread_user_data->filepath = filepath;
+        result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, thread_user_data);
+        *thread_selected = parser_thread_data;
+        break;
+    }
+
+    cleanup:
+    if(result != 0)
+        am_free(thread_user_data);
+    amal_mutex_tryunlock(&self->mutex);
+    return result;
+}
+
+static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self) {
+    int i;
+    int _;
+    int result;
+    void *thread_return_data;
+    ParserThreadData *parser_thread_data;
+    result = AMAL_COMPILER_ERR;
+    (void)_;
+
+    while(self->queued_files.size > 0) {
+        BufferView filepath;
+        cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_join_threads remaining files"));
+        _ = buffer_pop(&self->queued_files, &filepath, sizeof(filepath));
+        amal_mutex_tryunlock(&self->mutex);
+        if(filepath.data)
+            return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator));
+    }
+
+    for(i = 0; i < self->usable_thread_count; ++i) {
+        cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_join_threads, waiting for workers"));
+        parser_thread_data = &self->threads[i];
+        amal_mutex_tryunlock(&self->mutex);
+        amal_log_debug("Joining thread %d, status %d", i, parser_thread_data->status);
+        _ = parser_thread_data_join(parser_thread_data, &thread_return_data);
+    }
+
+    result = AMAL_COMPILER_OK;
+    cleanup:
+    return result;
+}
+
+int amal_compiler_load_file(amal_compiler *self, BufferView filepath) {
+    int result;
+    ParserThreadData *parser_thread_data;
+    result = AMAL_COMPILER_ERR;
+
+    /*
+    amal_compiler_load_file is called by the user for the first file to compile
+    but also called by the parser when it sees @import
+    */
+    if(!self->started) {
+        self->started = bool_true;
+        /*amal_log_info("Parsing %.*s using %d thread(s)", (int)filepath.size, filepath.data, self->usable_thread_count);*/
+        return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator));
+        return amal_compiler_load_file_join_threads(self);
+    }
+
+    return_if_error(amal_compiler_load_file_select_thread(self, filepath, &parser_thread_data));
+    if(parser_thread_data)
+        return AMAL_COMPILER_OK;
+
+    cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file"));
+    cleanup_if_error(buffer_append(&self->queued_files, &filepath, sizeof(filepath)));
+    result = AMAL_COMPILER_OK;
+
+    cleanup:
+    amal_mutex_tryunlock(&self->mutex);
+    return result;
+}
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index f3147bb..c016892 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1,37 +1,27 @@
 #include <stdio.h>
 #include <string.h>
-#include "../include/parser.h"
+#include "../include/compiler.h"
 
 int main() {
-    const char *code;
-    Parser parser;
-    BufferView code_view;
+    amal_compiler compiler;
     int result;
+    const char *filepath;
+    filepath = "tests/main.amal";
 
-    code = 
-    "const main = () {\n"
-    "   var hello = () {\n"
-    "       \n"
-    "   }\n"
-    "   hello()\n"
-    "}\n"
-    "const print = () {\n"
-    "   \n"
-    "}";
-    result = parser_init(&parser);
-    if(result != PARSER_OK) {
-        fprintf(stderr, "Failed to initialize parser\n");
+    result = amal_compiler_init(&compiler);
+    if(result != AMAL_COMPILER_OK) {
+        fprintf(stderr, "Failed to initialize compiler, error code: %d\n", result);
         return 1;
     }
 
-    code_view = create_buffer_view(code, strlen(code));
-    result = parser_parse_buffer(&parser, code_view);
-    if(result != PARSER_OK) {
-        fprintf(stderr, "Failed to parse\n");
+    result = amal_compiler_load_file(&compiler, create_buffer_view(filepath, strlen(filepath)));
+    if(result != AMAL_COMPILER_OK) {
+        fprintf(stderr, "Failed to load file, error code: %d\n", result);
         return 1;
     }
 
-    /* No need to do this here as the program is exiting */
-    /* parser_deinit(&parser); */
+#ifdef DEBUG
+    return amal_compiler_deinit(&compiler);
+#endif
     return 0;
 }
diff --git a/src/parser.c b/src/parser.c
index 78b0018..da85292 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -1,19 +1,51 @@
 #include "../include/parser.h"
 #include "../include/ast.h"
-#include "../include/misc.h"
-#include "../include/alloc.h"
+#include "../include/compiler.h"
+#include "../include/std/misc.h"
+#include "../include/std/file.h"
+#include "../include/std/mem.h"
+#include "../include/std/log.h"
 #include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
 
+static CHECK_RESULT int parser_queue_file(Parser *self, BufferView path);
 static CHECK_RESULT int parser_parse_body(Parser *self, Ast *ast);
 
-int parser_init(Parser *self) {
-    return_if_error(scoped_allocator_init(&self->allocator));
-    return_if_error(buffer_init(&self->ast_objects, &self->allocator));
-    return PARSER_OK;
+int parser_thread_data_init(ParserThreadData *self) {
+    am_memset(&self->allocator, 0, sizeof(self->allocator));
+    am_memset(&self->thread, 0, sizeof(self->thread));
+    self->status = PARSER_THREAD_STATUS_NEW;
+    return scoped_allocator_init(&self->allocator);
 }
 
-void parser_deinit(Parser *self) {
+int parser_thread_data_deinit(ParserThreadData *self) {
     scoped_allocator_deinit(&self->allocator);
+    return amal_thread_deinit(&self->thread);
+}
+
+int parser_thread_data_start(ParserThreadData *self, AmalThreadCallbackFunc callback_func, void *userdata) {
+    return_if_error(amal_thread_deinit(&self->thread));
+    return_if_error(amal_thread_create(&self->thread, AMAL_THREAD_JOINABLE, "Parser", callback_func, userdata));
+    self->status = PARSER_THREAD_STATUS_RUNNING;
+    return 0;
+}
+
+int parser_thread_data_join(ParserThreadData *self, void **result) {
+    int retval;
+    if(self->status == PARSER_THREAD_STATUS_NEW)
+        return 0;
+    retval = amal_thread_join(&self->thread, result);
+    if(retval == 0 || retval == AMAL_THREAD_NOT_JOINABLE)
+        self->status = PARSER_THREAD_STATUS_IDLE;
+    return retval;
+}
+
+int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocator) {
+    self->allocator = allocator;
+    self->compiler = compiler;
+    self->started = bool_false;
+    return buffer_init(&self->ast_objects, self->allocator);
 }
 
 /*
@@ -34,8 +66,8 @@ static CHECK_RESULT int parser_parse_lhs(Parser *self, LhsExpr **result) {
 
     return_if_error(tokenizer_accept(&self->tokenizer, TOK_IDENTIFIER));
     var_name = self->tokenizer.value.identifier;
-    fprintf(stderr, "var name: %.*s\n", (int)var_name.size, var_name.data);
-    return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(LhsExpr), (void**)result));
+    amal_log_debug("var name: %.*s", (int)var_name.size, var_name.data);
+    return_if_error(scoped_allocator_alloc(self->allocator, sizeof(LhsExpr), (void**)result));
     lhsexpr_init(*result, isConst, var_name);
     return PARSER_OK;
 }
@@ -56,23 +88,19 @@ static CHECK_RESULT int parser_parse_function_decl(Parser *self, FunctionDecl **
     /* TODO: Parse return types */
     return_if_error(tokenizer_accept(&self->tokenizer, TOK_OPEN_BRACE));
 
-    return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(FunctionDecl), (void**)func_decl));
-    cleanup_if_error(funcdecl_init(*func_decl, &self->allocator));
+    return_if_error(scoped_allocator_alloc(self->allocator, sizeof(FunctionDecl), (void**)func_decl));
+    return_if_error(funcdecl_init(*func_decl, self->allocator));
 
     for(;;) {
         Ast body_obj;
-        cleanup_if_error(tokenizer_consume_if(&self->tokenizer, TOK_CLOSING_BRACE, &result));
+        return_if_error(tokenizer_consume_if(&self->tokenizer, TOK_CLOSING_BRACE, &result));
         if(result)
             break;
         
-        cleanup_if_error(parser_parse_body(self, &body_obj));
-        cleanup_if_error(funcdecl_add_to_body(*func_decl, body_obj));
+        return_if_error(parser_parse_body(self, &body_obj));
+        return_if_error(funcdecl_add_to_body(*func_decl, body_obj));
     }
     return PARSER_OK;
-
-    cleanup:
-    *func_decl = NULL;
-    return PARSER_ERR;
 }
 
 /*
@@ -92,20 +120,33 @@ static CHECK_RESULT int parser_parse_function_call(Parser *self, FunctionCall **
     /* TODO: Parse arguments */
     return_if_error(tokenizer_accept(&self->tokenizer, TOK_CLOSING_PAREN));
 
-    return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(FunctionCall), (void**)func_call));
+    return_if_error(scoped_allocator_alloc(self->allocator, sizeof(FunctionCall), (void**)func_call));
     funccall_init(*func_call, func_name);
     return PARSER_OK;
 }
 
 /*
-RHS = FUNC_DECL | FUNC_CALL
+IMPORT = IMPORT_SYMBOL
+*/
+static CHECK_RESULT int parser_parse_import(Parser *self, Import **import) {
+    bool result;
+
+    return_if_error(tokenizer_consume_if(&self->tokenizer, TOK_IMPORT, &result));
+    if(!result)
+        return PARSER_OK;
+    
+    return_if_error(scoped_allocator_alloc(self->allocator, sizeof(Import), (void**)import));
+    import_init(*import, self->tokenizer.value.string);
+    return PARSER_OK;
+}
+
+/*
+RHS = FUNC_DECL | FUNC_CALL | IMPORT
 */
 static CHECK_RESULT int parser_parse_rhs(Parser *self, Ast *rhs_expr) {
     FunctionDecl *func_decl;
     FunctionCall *func_call;
-    Token token;
-    func_decl = NULL;
-    func_call = NULL;
+    Import *import;
 
     return_if_error(parser_parse_function_decl(self, &func_decl));
     if(func_decl) {
@@ -121,9 +162,16 @@ static CHECK_RESULT int parser_parse_rhs(Parser *self, Ast *rhs_expr) {
         return PARSER_OK;
     }
 
-    return_if_error(tokenizer_next(&self->tokenizer, &token));
+    return_if_error(parser_parse_import(self, &import));
+    if(import) {
+        rhs_expr->type = AST_IMPORT;
+        rhs_expr->value.import = import;
+        return_if_error(parser_queue_file(self, import->path));
+        return PARSER_OK;
+    }
+
     /* TODO: Convert token to string */
-    tokenizer_print_error(&self->tokenizer, "Expected function declaration or function call, got token: %d");
+    tokenizer_print_error(&self->tokenizer, "Expected function declaration or function call");
     return PARSER_UNEXPECTED_TOKEN;
 }
 
@@ -153,9 +201,9 @@ int parser_parse_body(Parser *self, Ast *ast) {
 /*
 ROOT = BODY*
 */
-int parser_parse_buffer(Parser *self, BufferView code_buffer) {
+int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_name) {
     Ast ast;
-    return_if_error(tokenizer_init(&self->tokenizer, code_buffer));
+    return_if_error(tokenizer_init(&self->tokenizer, code_buffer, buffer_name));
 
     for(;;) {
         bool isEof;
@@ -169,3 +217,35 @@ int parser_parse_buffer(Parser *self, BufferView code_buffer) {
 
     return PARSER_OK;
 }
+
+int parser_parse_file(Parser *self, BufferView filepath) {
+    int result;
+    int mapped_file_deinit_result;
+    MappedFile mapped_file;
+    char *filepath_tmp;
+
+    amal_log_debug("Parsing %.*s", (int)filepath.size, filepath.data);
+    assert(!self->started && "Parser can't be reused. Create a new parser.");
+    self->started = bool_true;
+    /* TODO: Somehow free this... */
+    /*return_if_error(scoped_allocator_alloc(self->allocator, filepath.size + 1, (void**)&filepath_tmp));*/
+    filepath_tmp = malloc(filepath.size + 1);
+    am_memcpy(filepath_tmp, filepath.data, filepath.size);
+    filepath_tmp[filepath.size] = '\0';
+    result = mapped_file_init(&mapped_file, filepath_tmp, MAPPED_FILE_READ);
+    if(result != 0) return result;
+    result = parser_parse_buffer(self, create_buffer_view(mapped_file.file_data, mapped_file.file_size), filepath);
+    mapped_file_deinit_result = mapped_file_deinit(&mapped_file);
+    return result != 0 ? result : mapped_file_deinit_result;
+}
+
+/*
+Path can be path to included library path (or system library path) in which case
+the path separator is a dot, otherwise the path separator is forward slash '/'
+*/
+int parser_queue_file(Parser *self, BufferView path) {
+    /* TODO: Do not load same path twice or the compile will fail (and it can have recursive import) also for performance reasons */
+    /* TODO: Parse special path (to include library path with dots) */
+    return_if_error(amal_compiler_load_file(self->compiler, path));
+    return PARSER_OK;
+}
\ No newline at end of file
diff --git a/src/alloc.c b/src/std/alloc.c
index c9ca7c3..93dcb98 100644
--- a/src/alloc.c
+++ b/src/std/alloc.c
@@ -1,4 +1,4 @@
-#include "../include/alloc.h"
+#include "../../include/std/alloc.h"
 #include <stdlib.h>
 
 int am_malloc(usize size, void **mem) {
diff --git a/src/buffer.c b/src/std/buffer.c
index a097cbf..f32c515 100644
--- a/src/buffer.c
+++ b/src/std/buffer.c
@@ -1,7 +1,7 @@
-#include "../include/buffer.h"
-#include "../include/alloc.h"
-#include "../include/mem.h"
-#include "../include/scoped_allocator.h"
+#include "../../include/std/buffer.h"
+#include "../../include/std/alloc.h"
+#include "../../include/std/mem.h"
+#include "../../include/std/scoped_allocator.h"
 #include <assert.h>
 
 int buffer_init(Buffer *self, struct ScopedAllocator *allocator) {
@@ -42,6 +42,7 @@ static CHECK_RESULT int buffer_ensure_capacity(Buffer *self, usize new_capacity)
 int buffer_append(Buffer *self, void *data, usize size) {
     return_if_error(buffer_ensure_capacity(self, self->size + size));
     am_memcpy(self->data + self->size, data, size);
+    self->size += size;
     return BUFFER_OK; 
 }
 
@@ -50,4 +51,12 @@ void* buffer_get(Buffer *self, usize index, usize type_size) {
     real_index = index * type_size;
     assert(real_index < self->size);
     return &self->data[real_index];
+}
+
+int buffer_pop(Buffer *self, void *data, usize size) {
+    if(size > self->size)
+        return -1;
+    am_memcpy(data, &self->data[self->size - size], size);
+    self->size -= size;
+    return 0;
 }
\ No newline at end of file
diff --git a/src/buffer_view.c b/src/std/buffer_view.c
index 96b0dd7..977626c 100644
--- a/src/buffer_view.c
+++ b/src/std/buffer_view.c
@@ -1,4 +1,4 @@
-#include "../include/buffer_view.h"
+#include "../../include/std/buffer_view.h"
 
 BufferView create_buffer_view_null() {
     BufferView buffer_view;
diff --git a/src/std/file.c b/src/std/file.c
new file mode 100644
index 0000000..7701be4
--- /dev/null
+++ b/src/std/file.c
@@ -0,0 +1,152 @@
+#include "../../include/std/file.h"
+#include "../../include/std/mem.h"
+#include "../../include/std/log.h"
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#define SCANDIR_PATH_LENGTH 4096
+
+static int scan_dir_recursive_internal(char *dir_path, int path_length, scan_dir_callback_func callback_func, void *userdata) {
+    DIR *dir;
+    struct dirent *entry;
+    struct stat file_stats;
+    int filename_length;
+    int path_length_start;
+
+    path_length_start = path_length;
+    if((dir = opendir(dir_path)) == NULL) {
+        amal_log_perror("scan_dir_recursive");
+        return errno;
+    }
+
+    while((entry = readdir(dir)) != NULL) {
+        /*
+        Skip hidden files. Name will always at least include null terminator
+        so it's ok to not check filename length. And file names are never empty, right???
+        */
+        if(entry->d_name[0] == '.')
+            continue;
+
+        /* Reset path concat, because it's overwritten by recursive call to scan dir */
+        path_length = path_length_start;
+        filename_length = strlen(entry->d_name);
+        /* current path '/' filename '\0' */
+        if(path_length + 1 + filename_length + 1 > SCANDIR_PATH_LENGTH) {
+            amal_log_error("Filepath too long. Trying to append %s to %s", entry->d_name, dir_path);
+            return -1;
+        }
+
+        /* Replace '\0' with '/' */
+        dir_path[path_length] = '/';
+        ++path_length;
+        am_memcpy(&dir_path[path_length], entry->d_name, filename_length);
+        path_length += filename_length;
+        dir_path[path_length] = '\0';
+        
+        if(stat(dir_path, &file_stats) == -1) {
+            amal_log_perror("scan_dir_recursive_internal");
+            goto cleanup;
+        }
+
+        if(S_ISDIR(file_stats.st_mode)) {
+            cleanup_if_error(scan_dir_recursive_internal(dir_path, path_length, callback_func, userdata));
+        } else if(S_ISREG(file_stats.st_mode)) {
+            if(!callback_func(dir_path, path_length, userdata))
+                goto cleanup_noerr;
+        } else if(S_ISLNK(file_stats.st_mode)) {
+            amal_log_warning("Symlink ignored: %s", entry->d_name);
+        } else {
+            /* Ignore block devices and all other types of files */
+        }
+    }
+
+    cleanup_noerr:
+    closedir(dir);
+    return 0;
+
+    cleanup:
+    closedir(dir);
+    return -1;
+}
+
+int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata) {
+    char scandir_path[SCANDIR_PATH_LENGTH];
+    int dir_path_length = strlen(dir_path) + 1; /* include null terminator */
+    am_memcpy(scandir_path, dir_path, dir_path_length);
+    return scan_dir_recursive_internal(scandir_path, dir_path_length - 1, callback_func, userdata);
+}
+
+static int mapped_file_mode_to_system_file_mode(MappedFileMode file_mode) {
+    switch(file_mode) {
+        case MAPPED_FILE_READ:          return PROT_READ;
+        case MAPPED_FILE_WRITE:         return PROT_WRITE;
+        case MAPPED_FILE_READ_WRITE:    return PROT_READ | PROT_WRITE;
+        default:
+            assert(bool_false);
+            return PROT_READ;
+    }
+}
+
+int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode) {
+    struct stat file_stats;
+    int fd;
+    const char *memblock;
+    int map_file_system_mode;
+
+    self->file_data = NULL;
+    self->file_size = 0;
+    self->fd = -1;
+
+    fd = open(filepath, O_RDONLY);
+    if(fd == -1) return errno;
+    if(fstat(fd, &file_stats) == -1) {
+        amal_log_perror("map_file_read");
+        goto cleanup;
+    }
+
+    map_file_system_mode = mapped_file_mode_to_system_file_mode(file_mode);
+    memblock = mmap(NULL, file_stats.st_size, map_file_system_mode, MAP_PRIVATE, fd, 0);
+    if(memblock == MAP_FAILED) {
+        amal_log_perror("map_file_read");
+        goto cleanup;
+    }
+
+    self->file_data = memblock;
+    self->file_size = file_stats.st_size;
+    self->fd = fd;
+    return 0;
+
+    cleanup:
+    close(fd);
+    return -1;
+}
+
+int mapped_file_deinit(MappedFile *self) {
+    int result_munmap;
+    int result_close;
+
+    if(!self->file_data)
+        return 0;
+    
+    result_munmap = munmap((void*)self->file_data, self->file_size);
+    if(result_munmap == -1)
+        amal_log_perror("mapped_file_deinit");
+    result_close = close(self->fd);
+    if(result_close == -1)
+        amal_log_perror("mapped_file_deinit");
+    self->file_data = NULL;
+    self->file_size = 0;
+    self->fd = 0;
+    if(result_munmap == -1 || result_close == -1)
+        return -1;
+    else
+        return 0;
+}
\ No newline at end of file
diff --git a/src/std/log.c b/src/std/log.c
new file mode 100644
index 0000000..1fd0e4e
--- /dev/null
+++ b/src/std/log.c
@@ -0,0 +1,76 @@
+#include "../../include/std/log.h"
+#include "../../include/std/thread.h"
+#include "../../include/std/misc.h"
+#include <stdio.h>
+#include <stdarg.h>
+
+static amal_mutex mutex;
+static bool mutex_initialized = bool_false;
+
+/* Safe to call multiple times */
+static void mutex_init() {
+    if(!mutex_initialized) {
+        amal_mutex_init(&mutex);
+        mutex_initialized = bool_true;
+    }
+}
+
+amal_mutex* amal_log_get_mutex() {
+    mutex_init();
+    return &mutex;
+}
+
+void amal_log_debug(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Debug: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_error(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Error: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_info(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Info: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_warning(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Warning: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_perror(const char *prefix) {
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    perror(prefix);
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
\ No newline at end of file
diff --git a/src/mem.c b/src/std/mem.c
index acd2ebd..5bdb73b 100644
--- a/src/mem.c
+++ b/src/std/mem.c
@@ -1,4 +1,4 @@
-#include "../include/mem.h"
+#include "../../include/std/mem.h"
 #include <string.h>
 
 void am_memcpy(void *dest, const void *src, usize size) {
@@ -7,4 +7,8 @@ void am_memcpy(void *dest, const void *src, usize size) {
 
 bool am_memeql(const void *lhs, const void *rhs, usize size) {
     return memcmp(lhs, rhs, size) == 0;
+}
+
+void am_memset(void *dest, int value, usize size) {
+    memset(dest, value, size);
 }
\ No newline at end of file
diff --git a/src/scoped_allocator.c b/src/std/scoped_allocator.c
index 4d40740..d53edad 100644
--- a/src/scoped_allocator.c
+++ b/src/std/scoped_allocator.c
@@ -1,5 +1,5 @@
-#include "../include/scoped_allocator.h"
-#include "../include/alloc.h"
+#include "../../include/std/scoped_allocator.h"
+#include "../../include/std/alloc.h"
 #include <assert.h>
 
 #define ALLOC_NODE_SIZE 4096
@@ -41,7 +41,7 @@ void scoped_allocator_deinit(ScopedAllocator *self) {
 
     scoped_allocator_node_deinit(&self->head);
     self->current = NULL;
-    buffer = (Buffer*)&self->buffers.data[0];
+    buffer = (Buffer*)self->buffers.data;
     buffer_end = buffer + self->buffers.size / sizeof(Buffer);
     while(buffer != buffer_end) {
         buffer_deinit(buffer);
@@ -60,23 +60,43 @@ static CHECK_RESULT int scoped_allocator_ensure_capacity_for(ScopedAllocator *se
         self->current->next = new_node;
         self->current = new_node;
     }
-    return 0;
+    return ALLOC_OK;
 
     cleanup:
     if(new_node)
         am_free(new_node);
-    return -1;
+    return ALLOC_FAIL;
+}
+
+static void* align_ptr_ceil(void *ptr, uintptr_t alignment) {
+    uintptr_t ptrval;
+    ptrval = (uintptr_t)ptr;
+    return (void*)((ptrval + alignment + 1) & ~(alignment - 1));
+}
+
+static usize align_ptr_ceil_offset(void *ptr, uintptr_t alignment) {
+    return (uintptr_t)align_ptr_ceil(ptr, alignment) - (uintptr_t)ptr;
 }
 
 int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem) {
+    ScopedAllocatorNode *current;
+    usize alloc_size;
     assert(self->current);
     assert(size <= ALLOC_NODE_SIZE);
-    return_if_error(scoped_allocator_ensure_capacity_for(self, size));
-    *mem = &self->current->data[self->current->size];
-    self->current->size += size;
+    current = self->current;
+
+    alloc_size = size + align_ptr_ceil_offset(self->current->data + self->current->size, 16);
+    return_if_error(scoped_allocator_ensure_capacity_for(self, alloc_size));
+    /* Reallocated (new node created) */
+    if(self->current != current) {
+        *mem = self->current->data;
+    } else {
+        *mem = align_ptr_ceil(self->current->data + self->current->size, 16);
+    }
+    self->current->size = size;
     return 0;
 }
 
 int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer) {
-    return buffer_append(&self->buffers, buffer, sizeof(buffer));
+    return buffer_append(&self->buffers, buffer, sizeof(Buffer));
 }
\ No newline at end of file
diff --git a/src/std/thread.c b/src/std/thread.c
new file mode 100644
index 0000000..f717865
--- /dev/null
+++ b/src/std/thread.c
@@ -0,0 +1,140 @@
+#include "../../include/std/thread.h"
+#include "../../include/std/log.h"
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/sysinfo.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+static int thread_type_to_system_thread_type(amal_thread_type thread_type) {
+    switch(thread_type) {
+        case AMAL_THREAD_JOINABLE:    return PTHREAD_CREATE_JOINABLE;
+        case AMAL_THREAD_DETACHED:    return PTHREAD_CREATE_DETACHED;
+    }
+    assert(bool_false);
+    return PTHREAD_CREATE_JOINABLE;
+}
+
+int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata) {
+    int result;
+    self->name = name;
+    self->thread_id = 0;
+    self->cancellable = bool_false;
+    self->destroyable = bool_false;
+    if((result = pthread_attr_init(&self->thread_attr)) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    self->destroyable = bool_true;
+    if((result = pthread_attr_setdetachstate(&self->thread_attr, thread_type_to_system_thread_type(thread_type))) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    if((result = pthread_create(&self->thread_id, NULL, callback_func, userdata)) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    self->cancellable = bool_true;
+    return 0;
+}
+
+int amal_thread_deinit(amal_thread *self) {
+    int r1;
+    int r2;
+    r1 = 0;
+    r2 = 0;
+
+    if(self->cancellable) {
+        r1 = pthread_cancel(self->thread_id);
+        self->cancellable = bool_false;
+    }
+    if(self->destroyable) {
+        r2 = pthread_attr_destroy(&self->thread_attr);
+        self->destroyable = bool_false;
+    }
+    return r1 != 0 ? r1 : r2;
+}
+
+int amal_thread_detach(amal_thread *self) {
+    int result_err;
+    int thread_type;
+    if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0)
+        return result_err;
+    if(thread_type != PTHREAD_CREATE_DETACHED)
+        return AMAL_THREAD_ERR;
+    if((result_err = pthread_detach(self->thread_id)) != 0)
+        return result_err;
+    self->cancellable = bool_false;
+    return 0;
+}
+
+int amal_thread_join(amal_thread *self, void **result) {
+    int result_err;
+    int thread_type;
+    if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0)
+        return result_err;
+    if(thread_type != PTHREAD_CREATE_JOINABLE)
+        return AMAL_THREAD_NOT_JOINABLE;
+    if((result_err = pthread_join(self->thread_id, result)) != 0)
+        return result_err == EINVAL ? AMAL_THREAD_NOT_JOINABLE : result_err;
+    self->cancellable = bool_false;
+    return 0;
+}
+
+void amal_mutex_init(amal_mutex *self) {
+    pthread_mutex_init(&self->mutex, NULL);
+    /* TODO: pthread_mutex_destroy in amal_mutex_deinit */
+    self->lock_identifier = NULL;
+}
+
+static long amal_process_get_id() {
+    return getpid();
+}
+
+static long amal_thread_get_id() {
+    return syscall(SYS_gettid);
+}
+
+int amal_mutex_lock(amal_mutex *self, const char *lock_identifier) {
+    int result;
+    result = pthread_mutex_lock(&self->mutex);
+    self->lock_identifier = lock_identifier;
+    if(result == 0 && self->lock_identifier) {
+        amal_log_debug("amal_mutex_lock: mutex locked by thread %lu (%s), identification: %s", 
+            amal_thread_get_id(),
+            amal_thread_is_main() ? "main" : "not main",
+            self->lock_identifier ? self->lock_identifier : "none");
+    }
+    return result;
+}
+
+int amal_mutex_unlock(amal_mutex *self) {
+    int result;
+    const char *identifier;
+    identifier = self->lock_identifier;
+    result = pthread_mutex_unlock(&self->mutex);
+    if(result == 0 && identifier) {
+        amal_log_debug("amal_mutex_unlock: mutex unlocked by thread %lu (%s), identification: %s", 
+            amal_thread_get_id(),
+            amal_thread_is_main() ? "main" : "not main",
+            identifier ? identifier : "none");
+    }
+    return result;
+}
+
+void amal_mutex_tryunlock(amal_mutex *self) {
+    int _;
+    (void)_;
+    _ = amal_mutex_unlock(self);
+}
+
+bool amal_thread_is_main() {
+    /* TODO: This only works for linux, use equivalent functions on other platforms */
+    return amal_thread_get_id() == amal_process_get_id();
+}
+
+int amal_get_usable_thread_count() {
+    return get_nprocs();
+}
\ No newline at end of file
diff --git a/src/tokenizer.c b/src/tokenizer.c
index fda3e40..e85c952 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -1,5 +1,7 @@
 #include "../include/tokenizer.h"
-#include "../include/mem.h"
+#include "../include/std/mem.h"
+#include "../include/std/log.h"
+#include "../include/std/thread.h"
 #include <assert.h>
 #include <limits.h>
 #include <stdio.h>
@@ -17,12 +19,13 @@ static int isAlphaDigit(int c) {
     return isAlpha(c) || isDigit(c);
 }
 
-int tokenizer_init(Tokenizer *self, BufferView code) {
+int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name) {
     assert(code.size <= INT_MAX);
     self->code = code;
     self->index = 0;
     self->prev_index = 0;
     self->line = 1;
+    self->code_name = code_name.data ? code_name : create_buffer_view("<buffer>", 8);
     return 0;
 }
 
@@ -52,9 +55,28 @@ static Token tokenizer_skip_whitespace(Tokenizer *self) {
     }
 }
 
+/* Returns -1 if end of string can't be found */
+static int find_end_of_string(BufferView buf, int index) {
+    int c;
+    bool escape_quote;
+    escape_quote = bool_false;
+
+    for(; index < (int)buf.size; ++index) {
+        c = buf.data[index];
+        if(c == '\\')
+            escape_quote = !escape_quote;
+        else if(!escape_quote && c == '"')
+            return index;
+        else
+            escape_quote = bool_false;
+    }
+    return -1;
+}
+
 int tokenizer_next(Tokenizer *self, Token *token) {
     Token last_token;
     int c;
+    int result;
 
     last_token = tokenizer_skip_whitespace(self);
     if(last_token == TOK_END_OF_FILE) {
@@ -85,6 +107,20 @@ int tokenizer_next(Tokenizer *self, Token *token) {
             *token = TOK_VAR;
         else
             *token = TOK_IDENTIFIER;
+    } else if(c == '"') {
+        int string_end;
+        ++self->index;
+        string_end = find_end_of_string(self->code, self->index);
+        if(string_end == -1) {
+            tokenizer_print_error(self, "String end not found. Did you forget '\"' or did you have a mismatch of number of '\"'?");
+            return TOKENIZER_ERR;
+        }
+
+        self->value.string.data = &self->code.data[self->index];
+        self->value.string.size = string_end - self->index;
+        self->index = string_end + 1;
+        *token = TOK_STRING;
+        return TOKENIZER_OK;
     } else if(c == '=') {
         ++self->index;
         *token = TOK_EQUALS;
@@ -100,8 +136,45 @@ int tokenizer_next(Tokenizer *self, Token *token) {
     } else if(c == '}') {
         ++self->index;
         *token = TOK_CLOSING_BRACE;
+    } else if(c == '@') {
+        const char *err_msg;
+        ++self->index;
+        if(self->index + 6 >= (int)self->code.size || !am_memeql(self->code.data + self->index, "import", 6)) {
+            err_msg = "Expected '@import(path)'";
+            goto import_error;
+        }
+        self->index += 6;
+
+        result = tokenizer_next(self, &last_token);
+        if(result != 0 || last_token != TOK_OPEN_PAREN) {
+            err_msg = "Expected '(' after @import";
+            goto import_error;
+        }
+
+        result = tokenizer_next(self, &last_token);
+        if(result != 0 || last_token != TOK_STRING) {
+            err_msg = "Expected string after @import(";
+            goto import_error;
+        }
+
+        if(self->value.string.size == 0) {
+            err_msg = "Path in @import can't be empty";
+            goto import_error;
+        }
+
+        result = tokenizer_next(self, &last_token);
+        if(result != 0 || last_token != TOK_CLOSING_PAREN) {
+            err_msg = "Expected ')' after @import(path";
+            goto import_error;
+        }
+        
+        *token = TOK_IMPORT;
+        return TOKENIZER_OK;
+        
+        import_error:
+        tokenizer_print_error(self, err_msg);
+        return TOKENIZER_ERR;
     } else {
-        /*self.printError("Unexpected symbol '{c}'", c);*/
         tokenizer_print_error(self, "Unexpected symbol '%c'", c);
         return TOKENIZER_UNEXPECTED_TOKEN;
     }
@@ -130,8 +203,8 @@ int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result) {
     if(actual_token == expected_token) {
         *result = bool_true;
     } else {
-        /* No need to restore self.prev_index as it's updated on the next call to tokenizer_next */
         self->index = index;
+        self->prev_index = index;
         self->line = line;
         *result = bool_false;
     }
@@ -167,16 +240,20 @@ void tokenizer_print_error(Tokenizer *self, const char *fmt, ...) {
     int line_end;
     int prev_column;
     int i;
+    amal_mutex *mutex;
 
+    mutex = amal_log_get_mutex();
+    ignore_result_int(amal_mutex_lock(mutex, "tokenizer_print_error"));
     va_start(args, fmt);
     line_start = tokenizer_get_start_of_line_from_index(self, self->prev_index);
     line_end = tokenizer_get_end_of_line_from_index(self, self->prev_index);
     prev_column = self->prev_index - line_start;
-    fprintf(stderr, "\x1b[1;37m%s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", "file.am", self->line, 1 + prev_column);
+    fprintf(stderr, "\x1b[1;37m%.*s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", (int)self->code_name.size, self->code_name.data, self->line, 1 + prev_column);
     vfprintf(stderr, fmt, args);
     fprintf(stderr, "\n%.*s\n", line_end - line_start, self->code.data + line_start);
     for(i = 0; i < prev_column; ++i)
         fprintf(stderr, " ");
     fprintf(stderr, "\x1b[1;32m^\x1b[0m\n");
     va_end(args);
+    ignore_result_int(amal_mutex_unlock(mutex));
 }
\ No newline at end of file