diff options
| -rw-r--r-- | doc/IMPLEMENTATION.md | 4 | ||||
| -rw-r--r-- | include/ast.h | 8 | ||||
| -rw-r--r-- | include/compiler.h | 2 | ||||
| -rw-r--r-- | include/parser.h | 4 | ||||
| -rw-r--r-- | include/std/buffer.h | 4 | ||||
| -rw-r--r-- | src/ast.c | 14 | ||||
| -rw-r--r-- | src/compiler.c | 143 | ||||
| -rw-r--r-- | src/parser.c | 8 | ||||
| -rw-r--r-- | src/std/buffer.c | 13 |
9 files changed, 155 insertions, 45 deletions
diff --git a/doc/IMPLEMENTATION.md b/doc/IMPLEMENTATION.md index 0da1c1a..5dc6ef5 100644 --- a/doc/IMPLEMENTATION.md +++ b/doc/IMPLEMENTATION.md @@ -12,9 +12,9 @@ before the SSA is created. is optimized before creating the bytecode. 5. If optimization is enabled then the bytecode is optimized. -# Currently implemented +# Progress 1. Parsing using multiple threads is done, but the parser is not finished. -2. Not started. +2. Resolving ast using multiple threads is done, but the ast resolver is not finished. 3. Not started. 4. Not started. 5. Not started.
\ No newline at end of file diff --git a/include/ast.h b/include/ast.h index 29a0b64..7911a59 100644 --- a/include/ast.h +++ b/include/ast.h @@ -15,6 +15,7 @@ typedef struct String String; typedef struct Variable Variable; typedef struct Number Number; typedef struct Binop Binop; +typedef struct Scope Scope; typedef union { FunctionDecl *func_decl; @@ -89,15 +90,20 @@ struct Binop { bool grouped; }; +struct Scope { + Buffer ast_objects; +}; + Ast ast_none(); CHECK_RESULT int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator); -CHECK_RESULT int funcdecl_add_to_body(FunctionDecl *self, Ast ast); CHECK_RESULT int funccall_init(FunctionCall *self, BufferView name, ScopedAllocator *allocator); void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name); void import_init(Import *self, BufferView path); CHECK_RESULT int string_init(String *self, BufferView str); void number_init(Number *self, i64 value, bool is_integer); void binop_init(Binop *self); +CHECK_RESULT int scope_init(Scope *self, ScopedAllocator *allocator); +void scope_resolve(Scope *self); #endif diff --git a/include/compiler.h b/include/compiler.h index d01c756..d7314cc 100644 --- a/include/compiler.h +++ b/include/compiler.h @@ -21,10 +21,12 @@ struct amal_compiler { int usable_thread_count; bool started; amal_mutex mutex; + int resolve_ast_index; }; CHECK_RESULT int amal_compiler_init(amal_compiler *self); CHECK_RESULT int amal_compiler_deinit(amal_compiler *self); +/* Not thread-safe */ CHECK_RESULT int amal_compiler_load_file(amal_compiler *self, BufferView filepath); /* TODO: amal_compiler_unload_file */ diff --git a/include/parser.h b/include/parser.h index b9fb3b7..5c055d9 100644 --- a/include/parser.h +++ b/include/parser.h @@ -1,11 +1,11 @@ #ifndef AMALGAM_PARSER_H #define AMALGAM_PARSER_H -#include "std/buffer.h" #include "std/buffer_view.h" #include "std/scoped_allocator.h" #include "std/thread.h" #include "tokenizer.h" +#include "ast.h" #include "defs.h" #include <setjmp.h> @@ -34,7 +34,7 @@ typedef enum { typedef struct { Tokenizer tokenizer; - Buffer ast_objects; + Scope scope; ScopedAllocator *allocator; /* borrowed. Copied from @compiler for faster access to allocator */ amal_compiler *compiler; bool started; diff --git a/include/std/buffer.h b/include/std/buffer.h index 02cc20a..a60def9 100644 --- a/include/std/buffer.h +++ b/include/std/buffer.h @@ -19,5 +19,9 @@ CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator); CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size); void* buffer_get(Buffer *self, usize index, usize type_size); CHECK_RESULT int buffer_pop(Buffer *self, void *data, usize size); +void* buffer_start(Buffer *self); +void *buffer_end(Buffer *self); +usize __buffer_get_size(Buffer *self, usize type_size); +#define buffer_get_size(self, type) __buffer_get_size((self), sizeof(type)) #endif @@ -12,11 +12,6 @@ int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator) { return buffer_init(&self->body, allocator); } -int funcdecl_add_to_body(FunctionDecl *self, Ast ast) { - return_if_error(buffer_append(&self->body, &ast, sizeof(ast))); - return BUFFER_OK; -} - int funccall_init(FunctionCall *self, BufferView name, ScopedAllocator *allocator) { self->name = name; return buffer_init(&self->args, allocator); @@ -49,4 +44,13 @@ void binop_init(Binop *self) { self->rhs = ast_none(); self->type = BINOP_ADD; self->grouped = bool_false; +} + +int scope_init(Scope *self, ScopedAllocator *allocator) { + return buffer_init(&self->ast_objects, allocator); +} + +void scope_resolve(Scope *self) { + /* TODO: Implement. Also use longjmp to jump back to compiler on error */ + (void)self; }
\ No newline at end of file diff --git a/src/compiler.c b/src/compiler.c index b8bca1c..e44ba6a 100644 --- a/src/compiler.c +++ b/src/compiler.c @@ -38,6 +38,7 @@ int amal_compiler_init(amal_compiler *self) { am_memset(&self->allocator, 0, sizeof(self->allocator)); am_memset(&self->main_thread_allocator, 0, sizeof(self->main_thread_allocator)); self->started = bool_false; + self->resolve_ast_index = 0; amal_mutex_init(&self->mutex); return_if_error(scoped_allocator_init(&self->allocator)); @@ -80,7 +81,26 @@ typedef struct { BufferView filepath; } CompilerParserThreadUserData; -static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) { +typedef struct { + amal_compiler *compiler; + ParserThreadData *parser_thread_data; + Parser *parser; +} CompilerAstResolverThreadUserData; + +typedef enum { + THREAD_WORK_PARSE, + THREAD_WORK_RESOLVE_AST +} ThreadWorkType; + +typedef struct { + union { + BufferView filepath; + Parser *parser; + } value; + ThreadWorkType type; +} ThreadWorkData; + +static CHECK_RESULT int amal_compiler_load_in_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) { Parser parser; int result; result = AMAL_COMPILER_ERR; @@ -88,7 +108,7 @@ static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self return_if_error(parser_init(&parser, self, allocator)); cleanup_if_error(parser_parse_file(&parser, filepath)); - cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_first_this_thread")); + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_in_this_thread")); cleanup_if_error(buffer_append(&self->parsers, &parser, sizeof(parser))); result = AMAL_COMPILER_OK; @@ -99,54 +119,92 @@ static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self /* TODO: Handle errors (stop parsing in all other threads and report errors/warnings) */ static void* thread_callback_parse_file(void *userdata) { - int _; BufferView next_file; CompilerParserThreadUserData compiler_parser_userdata; - (void)_; - next_file = create_buffer_view_null(); assert(!amal_thread_is_main()); am_memcpy(&compiler_parser_userdata, userdata, sizeof(compiler_parser_userdata)); am_free(userdata); - cleanup_if_error(amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, - compiler_parser_userdata.filepath, - &compiler_parser_userdata.parser_thread_data->allocator)); - cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file")); - if(compiler_parser_userdata.compiler->queued_files.size > 0) - _ = buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file)); + next_file = compiler_parser_userdata.filepath; + for(;;) { + cleanup_if_error(amal_compiler_load_in_this_thread(compiler_parser_userdata.compiler, + next_file, + &compiler_parser_userdata.parser_thread_data->allocator)); + cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file")); + cleanup_if_error(buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file))); + amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex); + } cleanup: - if(!next_file.data) - compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; + compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex); - if(next_file.data) { - _ = amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, - next_file, - &compiler_parser_userdata.parser_thread_data->allocator); + return NULL; +} + +/* TODO: Handle errors (stop resolving ast in all other threads and report errors/warnings) */ +static void* thread_callback_resolve_ast(void *userdata) { + Parser *parser; + CompilerAstResolverThreadUserData compiler_ast_resolver_userdata; + assert(!amal_thread_is_main()); + + am_memcpy(&compiler_ast_resolver_userdata, userdata, sizeof(compiler_ast_resolver_userdata)); + am_free(userdata); + parser = compiler_ast_resolver_userdata.parser; + for(;;) { + amal_log_debug("Resolving AST for file: %.*s", parser->tokenizer.code_name.size, parser->tokenizer.code_name.data); + scope_resolve(&parser->scope); + cleanup_if_error(amal_mutex_lock(&compiler_ast_resolver_userdata.compiler->mutex, "thread_callback_resolve_ast")); + if(compiler_ast_resolver_userdata.compiler->resolve_ast_index + 1 >= (int)buffer_get_size(&compiler_ast_resolver_userdata.compiler->parsers, Parser)) + break; + ++compiler_ast_resolver_userdata.compiler->resolve_ast_index; + parser = buffer_get(&compiler_ast_resolver_userdata.compiler->parsers, compiler_ast_resolver_userdata.compiler->resolve_ast_index, sizeof(Parser)); + amal_mutex_tryunlock(&compiler_ast_resolver_userdata.compiler->mutex); } + + cleanup: + compiler_ast_resolver_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; + amal_mutex_tryunlock(&compiler_ast_resolver_userdata.compiler->mutex); return NULL; } -static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *self, BufferView filepath, ParserThreadData **thread_selected) { +static CHECK_RESULT int amal_compiler_select_thread_for_work(amal_compiler *self, ThreadWorkData work_data, ParserThreadData **thread_selected) { int i; int result; ParserThreadData *parser_thread_data; - CompilerParserThreadUserData *thread_user_data; + void *thread_user_data; thread_user_data = NULL; *thread_selected = NULL; result = AMAL_COMPILER_ERR; - cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_select_thread")); + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_select_thread_for_work")); for(i = 0; i < self->usable_thread_count; ++i) { parser_thread_data = &self->threads[i]; if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING) continue; - cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&thread_user_data)); - thread_user_data->compiler = self; - thread_user_data->parser_thread_data = parser_thread_data; - thread_user_data->filepath = filepath; - result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, thread_user_data); + switch(work_data.type) { + case THREAD_WORK_PARSE: { + CompilerParserThreadUserData *userdata; + cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&userdata)); + thread_user_data = userdata; + userdata->compiler = self; + userdata->parser_thread_data = parser_thread_data; + userdata->filepath = work_data.value.filepath; + result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, userdata); + break; + } + case THREAD_WORK_RESOLVE_AST: { + CompilerAstResolverThreadUserData *userdata; + cleanup_if_error(am_malloc(sizeof(CompilerAstResolverThreadUserData), (void**)&userdata)); + thread_user_data = userdata; + userdata->compiler = self; + userdata->parser_thread_data = parser_thread_data; + userdata->parser = work_data.value.parser; + ++self->resolve_ast_index; + result = parser_thread_data_start(parser_thread_data, thread_callback_resolve_ast, userdata); + break; + } + } *thread_selected = parser_thread_data; break; } @@ -158,13 +216,13 @@ static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *sel return result; } -static CHECK_RESULT int amal_compiler_all_threads_done(amal_compiler *self, bool *done) { +static CHECK_RESULT int amal_compiler_check_all_threads_done(amal_compiler *self, bool *done) { int i; int result; result = AMAL_COMPILER_OK; *done = bool_false; - cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_all_threads_done")); + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_check_all_threads_done")); for(i = 0; i < self->usable_thread_count; ++i) { ParserThreadData *parser_thread_data; parser_thread_data = &self->threads[i]; @@ -201,11 +259,10 @@ static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self amal_mutex_tryunlock(&self->mutex); if(result != 0) goto cleanup; - amal_log_debug("Joining thread %d, status %d", i, parser_thread_data->status); ignore_result_int(parser_thread_data_join(parser_thread_data, &thread_return_data)); } - cleanup_if_error(amal_compiler_all_threads_done(self, &done)); + cleanup_if_error(amal_compiler_check_all_threads_done(self, &done)); if(done) break; } @@ -215,12 +272,33 @@ static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self return result; } +static CHECK_RESULT int amal_compiler_resolve_ast(amal_compiler *self) { + Parser *parser; + Parser *parser_end; + parser = buffer_start(&self->parsers); + parser_end = buffer_end(&self->parsers); + for(; parser != parser_end; ++parser) { + ParserThreadData *thread_selected; + ThreadWorkData thread_work_data; + thread_work_data.type = THREAD_WORK_RESOLVE_AST; + thread_work_data.value.parser = parser; + return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &thread_selected)); + /* After all threads have been used, they will handle using the remaining parsers */ + if(!thread_selected) + break; + } + return amal_compiler_load_file_join_threads(self); +} + int amal_compiler_load_file(amal_compiler *self, BufferView filepath) { int result; ParserThreadData *parser_thread_data; + ThreadWorkData thread_work_data; result = AMAL_COMPILER_ERR; + thread_work_data.type = THREAD_WORK_PARSE; + thread_work_data.value.filepath = filepath; - return_if_error(amal_compiler_load_file_select_thread(self, filepath, &parser_thread_data)); + return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &parser_thread_data)); /* amal_compiler_load_file is called by the user for the first file to compile @@ -230,7 +308,10 @@ int amal_compiler_load_file(amal_compiler *self, BufferView filepath) { self->started = bool_true; /*amal_log_info("Parsing %.*s using %d thread(s)", (int)filepath.size, filepath.data, self->usable_thread_count);*/ /*return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator));*/ - return amal_compiler_load_file_join_threads(self); + return_if_error(amal_compiler_load_file_join_threads(self)); + amal_log_debug("Finished parsing all files, resolving AST"); + return_if_error(amal_compiler_resolve_ast(self)); + return AMAL_COMPILER_OK; } if(parser_thread_data) diff --git a/src/parser.c b/src/parser.c index fb18685..6ee273b 100644 --- a/src/parser.c +++ b/src/parser.c @@ -63,7 +63,7 @@ int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocato self->error.index = 0; self->error.str = NULL; self->error_context = ERROR_CONTEXT_NONE; - return buffer_init(&self->ast_objects, self->allocator); + return scope_init(&self->scope, self->allocator); } /* @@ -116,7 +116,6 @@ static THROWABLE parser_parse_lhs(Parser *self, LhsExpr **result, bool *assignme throw_if_error(tokenizer_accept(&self->tokenizer, TOK_IDENTIFIER)); var_name = self->tokenizer.value.identifier; - amal_log_debug("var name: %.*s", (int)var_name.size, var_name.data); throw_if_error(scoped_allocator_alloc(self->allocator, sizeof(LhsExpr), (void**)result)); lhsexpr_init(*result, is_const, var_name); @@ -379,7 +378,7 @@ int parser_parse_rhs_start(Parser *self, Ast *rhs_expr) { } /* -BODY_SEMICOLON +BODY_SEMICOLON = ';' Note: Semicolon is not required for closures, structs and tables */ @@ -441,7 +440,7 @@ int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_ throw_if_error(tokenizer_init(&self->tokenizer, code_buffer, buffer_name)); result = setjmp(self->parse_env); if(result == 0) - try(parser_parse_body_loop(self, &self->ast_objects, TOK_END_OF_FILE)); + try(parser_parse_body_loop(self, &self->scope.ast_objects, TOK_END_OF_FILE)); else if(self->error.str != NULL) { switch(self->error_context) { case ERROR_CONTEXT_NONE: @@ -487,6 +486,7 @@ the path separator is a dot, otherwise the path separator is forward slash '/' int parser_queue_file(Parser *self, BufferView path) { /* TODO: Do not load same path twice or the compile will fail (and it can have recursive import) also for performance reasons */ /* TODO: Parse special path (to include library path with dots) */ + /* TODO: Path should be relative to the file that uses @import */ throw_if_error(amal_compiler_load_file(self->compiler, path)); return PARSER_OK; } diff --git a/src/std/buffer.c b/src/std/buffer.c index c4c9845..1fdcfd4 100644 --- a/src/std/buffer.c +++ b/src/std/buffer.c @@ -60,3 +60,16 @@ int buffer_pop(Buffer *self, void *data, usize size) { self->size -= size; return 0; } + +void* buffer_start(Buffer *self) { + return self->data; +} + +void *buffer_end(Buffer *self) { + return self->data + self->size; +} + +usize __buffer_get_size(Buffer *self, usize type_size) { + assert(type_size != 0); + return self->size / type_size; +}
\ No newline at end of file |