#include "../include/compiler.h" #include "../include/parser.h" #include "../include/ssa/ssa.h" #include "../include/std/log.h" #include "../include/std/mem.h" #include "../include/std/alloc.h" #include #include #include #define MIN(a, b) ((a) < (b) ? (a) : (b)) static CHECK_RESULT int get_thread_count_env_var(int *thread_count) { char *threads; threads = getenv("THREADS"); if(!threads) return -1; *thread_count = atoi(threads); return 0; } int amal_compiler_init(amal_compiler *self) { int i; int result; result = get_thread_count_env_var(&self->usable_thread_count); if(result != 0) { self->usable_thread_count = amal_get_usable_thread_count(); if(self->usable_thread_count == 0) { amal_log_warning("Unable to get the number of threads available on the system, using 1 thread."); amal_log_warning("You can override the number of threads using by setting the environment variable THREADS"); self->usable_thread_count = 1; } } else if(self->usable_thread_count <= 0) { amal_log_error("Environment variable THREADS contains invalid number for threads. THREADS has to be at least 1."); return AMAL_COMPILER_ERR; } am_memset(&self->allocator, 0, sizeof(self->allocator)); am_memset(&self->main_thread_allocator, 0, sizeof(self->main_thread_allocator)); self->started = bool_false; self->generic_work_object_index = 0; amal_mutex_init(&self->mutex); return_if_error(scoped_allocator_init(&self->allocator)); cleanup_if_error(scoped_allocator_init(&self->main_thread_allocator)); cleanup_if_error(buffer_init(&self->parsers, &self->allocator)); cleanup_if_error(buffer_init(&self->queued_files, &self->allocator)); cleanup_if_error(scoped_allocator_alloc(&self->allocator, self->usable_thread_count * sizeof(ParserThreadData), (void**)&self->threads)); for(i = 0; i < self->usable_thread_count; ++i) cleanup_if_error(parser_thread_data_init(&self->threads[i])); return AMAL_COMPILER_OK; cleanup: ignore_result_int(amal_compiler_deinit(self)); return AMAL_COMPILER_ERR; } int amal_compiler_deinit(amal_compiler *self) { int i; int result; result = AMAL_COMPILER_OK; for(i = 0; i < self->usable_thread_count; ++i) { int r; r = parser_thread_data_deinit(&self->threads[i]); if(r != 0) result = r; } amal_mutex_deinit(&self->mutex); scoped_allocator_deinit(&self->allocator); scoped_allocator_deinit(&self->main_thread_allocator); return result; } typedef enum { THREAD_WORK_PARSE, THREAD_WORK_RESOLVE_AST, THREAD_WORK_GENERATE_SSA } ThreadWorkType; typedef struct { amal_compiler *compiler; ParserThreadData *parser_thread_data; BufferView filepath; } CompilerParserThreadUserData; typedef struct { amal_compiler *compiler; ParserThreadData *parser_thread_data; Parser *parser; ThreadWorkType work_type; } CompilerGenericThreadUserData; typedef struct { union { BufferView filepath; Parser *parser; } value; ThreadWorkType type; } ThreadWorkData; static CHECK_RESULT int amal_compiler_load_in_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) { Parser *parser; int result; result = AMAL_COMPILER_ERR; return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(Parser), (void**)&parser)); return_if_error(parser_init(parser, self, allocator)); return_if_error(parser_parse_file(parser, filepath)); cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_in_this_thread")); cleanup_if_error(buffer_append(&self->parsers, &parser, sizeof(parser))); result = AMAL_COMPILER_OK; cleanup: amal_mutex_tryunlock(&self->mutex); return result; } /* TODO: Handle errors (stop parsing in all other threads and report errors/warnings) */ static void* thread_callback_parse_file(void *userdata) { BufferView next_file; CompilerParserThreadUserData compiler_parser_userdata; void *result; assert(!amal_thread_is_main()); am_memcpy(&compiler_parser_userdata, userdata, sizeof(compiler_parser_userdata)); am_free(userdata); next_file = compiler_parser_userdata.filepath; result = (void*)AMAL_COMPILER_ERR; for(;;) { int has_next; cleanup_if_error(amal_compiler_load_in_this_thread(compiler_parser_userdata.compiler, next_file, &compiler_parser_userdata.parser_thread_data->allocator)); cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file")); has_next = buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file)); amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex); if(has_next != 0) break; } result = NULL; cleanup: compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex); return result; } static CHECK_RESULT int thread_resolve_ast(Parser *parser) { AstCompilerContext compiler_context; int result; compiler_context.parser = parser; compiler_context.scope = NULL; result = setjmp(compiler_context.env); assert(!parser->scope.parent); if(result == 0) { amal_log_debug("Resolving AST for file: %.*s", parser->tokenizer.code_name.size, parser->tokenizer.code_name.data); scope_resolve(&parser->scope, &compiler_context); } if(result == 0) assert(!compiler_context.scope); return result; } static CHECK_RESULT int thread_generate_ssa(Parser *parser) { SsaCompilerContext compiler_context; int result; compiler_context.parser = parser; result = setjmp(compiler_context.env); if(result == 0) { return_if_error(ssa_init(&compiler_context.ssa, parser->allocator)); amal_log_debug("Generating SSA for file: %.*s", parser->tokenizer.code_name.size, parser->tokenizer.code_name.data); scope_generate_ssa(&parser->scope, &compiler_context); } return result; } /* TODO: Handle errors (stop work in all other threads and report errors/warnings) */ static void* thread_callback_generic(void *userdata) { CompilerGenericThreadUserData compiler_userdata; Parser *parser; void *result; assert(!amal_thread_is_main()); am_memcpy(&compiler_userdata, userdata, sizeof(compiler_userdata)); am_free(userdata); parser = compiler_userdata.parser; result = (void*)AMAL_COMPILER_ERR; for(;;) { /* TODO: stop work in all other threads on failure */ switch(compiler_userdata.work_type) { case THREAD_WORK_PARSE: { assert(bool_false && "Thread work type can't ge 'parse' for generic work"); break; } case THREAD_WORK_RESOLVE_AST: cleanup_if_error(thread_resolve_ast(parser)); break; case THREAD_WORK_GENERATE_SSA: cleanup_if_error(thread_generate_ssa(parser)); break; } cleanup_if_error(amal_mutex_lock(&compiler_userdata.compiler->mutex, "thread_callback_generic")); if(compiler_userdata.compiler->generic_work_object_index + 1 >= (int)buffer_get_size(&compiler_userdata.compiler->parsers, Parser*)) break; ++compiler_userdata.compiler->generic_work_object_index; parser = *(Parser**)buffer_get(&compiler_userdata.compiler->parsers, compiler_userdata.compiler->generic_work_object_index, sizeof(parser)); amal_mutex_tryunlock(&compiler_userdata.compiler->mutex); } result = NULL; cleanup: compiler_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; amal_mutex_tryunlock(&compiler_userdata.compiler->mutex); return result; } static CHECK_RESULT int amal_compiler_select_thread_for_work(amal_compiler *self, ThreadWorkData work_data, ParserThreadData **thread_selected) { int i; int result; ParserThreadData *parser_thread_data; void *thread_user_data; thread_user_data = NULL; *thread_selected = NULL; result = AMAL_COMPILER_ERR; cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_select_thread_for_work")); for(i = 0; i < self->usable_thread_count; ++i) { parser_thread_data = &self->threads[i]; if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING) continue; switch(work_data.type) { case THREAD_WORK_PARSE: { CompilerParserThreadUserData *userdata; cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&userdata)); thread_user_data = userdata; userdata->compiler = self; userdata->parser_thread_data = parser_thread_data; userdata->filepath = work_data.value.filepath; result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, userdata); break; } case THREAD_WORK_RESOLVE_AST: case THREAD_WORK_GENERATE_SSA: { CompilerGenericThreadUserData *userdata; cleanup_if_error(am_malloc(sizeof(CompilerGenericThreadUserData), (void**)&userdata)); thread_user_data = userdata; userdata->compiler = self; userdata->parser_thread_data = parser_thread_data; userdata->parser = work_data.value.parser; userdata->work_type = work_data.type; ++self->generic_work_object_index; result = parser_thread_data_start(parser_thread_data, thread_callback_generic, userdata); break; } } *thread_selected = parser_thread_data; break; } cleanup: if(result != 0) am_free(thread_user_data); amal_mutex_tryunlock(&self->mutex); return result; } static CHECK_RESULT int amal_compiler_check_all_threads_done(amal_compiler *self, bool *done) { int i; int result; result = AMAL_COMPILER_OK; *done = bool_false; cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_check_all_threads_done")); for(i = 0; i < self->usable_thread_count; ++i) { ParserThreadData *parser_thread_data; parser_thread_data = &self->threads[i]; if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING) { result = AMAL_COMPILER_ERR; goto cleanup; } } *done = bool_true; cleanup: amal_mutex_tryunlock(&self->mutex); return result; } static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self) { int i; int result; void *thread_return_data; ParserThreadData *parser_thread_data; bool work_failed; result = AMAL_COMPILER_ERR; assert(amal_thread_is_main()); thread_return_data = NULL; work_failed = bool_false; for(;;) { bool done; /* Joining running threads. After checking one running thread another one might start up, so this is mostly to wait for threads to finish and to sleep without doing work. The check after that (amal_compiler_all_threads_done) check that all threads are done correctly */ for(i = 0; i < self->usable_thread_count; ++i) { result = amal_mutex_lock(&self->mutex, "amal_compiler_load_file_join_threads, waiting for workers"); parser_thread_data = &self->threads[i]; amal_mutex_tryunlock(&self->mutex); if(result != 0) goto cleanup; ignore_result_int(parser_thread_data_join(parser_thread_data, &thread_return_data)); if(thread_return_data != NULL) { /* TODO: Somehow exit running jobs */ amal_log_error("Failed, waiting for jobs to finish"); work_failed = bool_true; } } cleanup_if_error(amal_compiler_check_all_threads_done(self, &done)); if(done) break; } result = AMAL_COMPILER_OK; cleanup: if(work_failed) result = AMAL_COMPILER_ERR; return result; } static CHECK_RESULT int amal_compiler_dispatch_generic(amal_compiler *self, ThreadWorkType work_type) { Parser **parser; Parser **parser_end; parser = buffer_start(&self->parsers); parser_end = buffer_end(&self->parsers); self->generic_work_object_index = 0; for(; parser != parser_end; ++parser) { ParserThreadData *thread_selected; ThreadWorkData thread_work_data; thread_work_data.type = work_type; thread_work_data.value.parser = *parser; return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &thread_selected)); /* After all threads have been used, they will handle using the remaining parsers or stop if there is an error */ if(!thread_selected) break; } return amal_compiler_load_file_join_threads(self); } /* amal_compiler_load_file is called by the user for the first file to compile but also called by the parser when it sees @import */ int amal_compiler_load_file(amal_compiler *self, BufferView filepath) { int result; ParserThreadData *parser_thread_data; ThreadWorkData thread_work_data; bool main_job; result = AMAL_COMPILER_ERR; thread_work_data.type = THREAD_WORK_PARSE; thread_work_data.value.filepath = filepath; main_job = bool_false; /* The first time we get here, this will run single-threaded so this part doesn't need mutex */ if(!self->started) { self->started = bool_true; main_job = bool_true; } return_if_error(amal_compiler_select_thread_for_work(self, thread_work_data, &parser_thread_data)); if(main_job) { /*amal_log_info("Parsing %.*s using %d thread(s)", (int)filepath.size, filepath.data, self->usable_thread_count);*/ /*return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator));*/ return_if_error(amal_compiler_load_file_join_threads(self)); amal_log_debug("Finished parsing all files, resolving AST"); return_if_error(amal_compiler_dispatch_generic(self, THREAD_WORK_RESOLVE_AST)); amal_log_debug("Finished resolving AST, generating SSA"); return_if_error(amal_compiler_dispatch_generic(self, THREAD_WORK_GENERATE_SSA)); return AMAL_COMPILER_OK; } if(parser_thread_data) return AMAL_COMPILER_OK; cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file")); cleanup_if_error(buffer_append(&self->queued_files, &filepath, sizeof(filepath))); result = AMAL_COMPILER_OK; cleanup: amal_mutex_tryunlock(&self->mutex); return result; }