From 76d85a10f6cda93eba29dad5372e8160af7289c8 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 27 Feb 2019 22:26:35 +0100 Subject: Use multiple threads to parse --- README.md | 6 +- build.sh | 6 +- doc/IMPLEMENTATION.md | 13 +++ doc/IMPLEMENTED.md | 2 - include/alloc.h | 14 --- include/ast.h | 19 +++- include/buffer.h | 22 ---- include/buffer_view.h | 14 --- include/compiler.h | 31 ++++++ include/defs.h | 7 ++ include/mem.h | 10 -- include/misc.h | 27 ----- include/parser.h | 40 ++++++-- include/scoped_allocator.h | 31 ------ include/std/alloc.h | 14 +++ include/std/buffer.h | 23 +++++ include/std/buffer_view.h | 14 +++ include/std/defs.h | 7 ++ include/std/file.h | 27 +++++ include/std/log.h | 13 +++ include/std/mem.h | 11 ++ include/std/misc.h | 33 ++++++ include/std/scoped_allocator.h | 31 ++++++ include/std/thread.h | 50 +++++++++ include/std/types.h | 20 ++++ include/tokenizer.h | 16 ++- include/types.h | 20 ---- src/alloc.c | 25 ----- src/ast.c | 4 + src/buffer.c | 53 ---------- src/buffer_view.c | 15 --- src/compiler.c | 225 +++++++++++++++++++++++++++++++++++++++++ src/main.c | 36 +++---- src/mem.c | 10 -- src/parser.c | 134 +++++++++++++++++++----- src/scoped_allocator.c | 82 --------------- src/std/alloc.c | 25 +++++ src/std/buffer.c | 62 ++++++++++++ src/std/buffer_view.c | 15 +++ src/std/file.c | 152 ++++++++++++++++++++++++++++ src/std/log.c | 76 ++++++++++++++ src/std/mem.c | 14 +++ src/std/scoped_allocator.c | 102 +++++++++++++++++++ src/std/thread.c | 140 +++++++++++++++++++++++++ src/tokenizer.c | 87 +++++++++++++++- tests/io.amal | 3 + tests/main.amal | 12 +++ 47 files changed, 1392 insertions(+), 401 deletions(-) create mode 100644 doc/IMPLEMENTATION.md delete mode 100644 doc/IMPLEMENTED.md delete mode 100644 include/alloc.h delete mode 100644 include/buffer.h delete mode 100644 include/buffer_view.h create mode 100644 include/compiler.h create mode 100644 include/defs.h delete mode 100644 include/mem.h delete mode 100644 include/misc.h delete mode 100644 include/scoped_allocator.h create mode 100644 include/std/alloc.h create mode 100644 include/std/buffer.h create mode 100644 include/std/buffer_view.h create mode 100644 include/std/defs.h create mode 100644 include/std/file.h create mode 100644 include/std/log.h create mode 100644 include/std/mem.h create mode 100644 include/std/misc.h create mode 100644 include/std/scoped_allocator.h create mode 100644 include/std/thread.h create mode 100644 include/std/types.h delete mode 100644 include/types.h delete mode 100644 src/alloc.c delete mode 100644 src/buffer.c delete mode 100644 src/buffer_view.c create mode 100644 src/compiler.c delete mode 100644 src/mem.c delete mode 100644 src/scoped_allocator.c create mode 100644 src/std/alloc.c create mode 100644 src/std/buffer.c create mode 100644 src/std/buffer_view.c create mode 100644 src/std/file.c create mode 100644 src/std/log.c create mode 100644 src/std/mem.c create mode 100644 src/std/scoped_allocator.c create mode 100644 src/std/thread.c create mode 100644 tests/io.amal create mode 100644 tests/main.amal diff --git a/README.md b/README.md index ead55c9..d89a2d5 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -Amalgam is written in c89 C standard to work on as many devices as possible and with many different compilers, which would allow you to compile amalgam with a compiler that generates smaller (static) binaries than gcc +Amalgam is written in c89 C standard to work on as many devices as possible and with many different compilers, +which would allow you to compile amalgam with a compiler that generates smaller (static) binaries than gcc. + +Amalgam is not meant to be a replacement for any other language but rather a new unique language for programming +with gpu without writing an external gpu program (glsl/hlsl). diff --git a/build.sh b/build.sh index 8307dfe..47fdc04 100755 --- a/build.sh +++ b/build.sh @@ -4,14 +4,14 @@ set -e this_script_path=$(readlink -f "$0") this_script_dir=$(dirname "$this_script_path") -source_files=$(readlink -f "$this_script_dir/src/"*) +source_files=$(readlink -f $(find "$this_script_dir/src" -name "*.c")) if [ -z "$CC" ]; then CC=cc fi -CFLAGS="-Wall -Wextra -Werror -g -O0 -DDEBUG -std=c89 -pedantic" -LIBS="-pthread" +CFLAGS="-Wall -Wextra -Werror -g -O0 -DDEBUG -std=c89 -pedantic -fsanitize=address -D_GNU_SOURCE" +LIBS="-pthread -lasan" set -x time "$CC" $source_files $CFLAGS $LIBS -o amalgam diff --git a/doc/IMPLEMENTATION.md b/doc/IMPLEMENTATION.md new file mode 100644 index 0000000..9a0fb4a --- /dev/null +++ b/doc/IMPLEMENTATION.md @@ -0,0 +1,13 @@ +1. In the first stage the parser parses multiple files at the same time using multiple threads. +2. In the second stage the ast is handled using multiple threads. In this stage, variables, parameters +and types are defined and resolved and if a type is defined after there is a reference to it, +then the compiler first resolves that type. There are flags set to make sure there aren't recursive dependencies. +3. In the third stage the resolved ast is used to create SSA form (static single assignment form). If optimization is +enabled then then some inlining for ast is done by copying ast from functions to the places they are called from +before the SSA is created. +4. In the fourth stage the SSA form is used to create the bytecode. If optimization is enabled then the SSA form +is optimized before creating the bytecode. +5. If optimization is enabled then the bytecode is optimized. + +Currently implemented: +None \ No newline at end of file diff --git a/doc/IMPLEMENTED.md b/doc/IMPLEMENTED.md deleted file mode 100644 index 2583774..0000000 --- a/doc/IMPLEMENTED.md +++ /dev/null @@ -1,2 +0,0 @@ -const main = () { -} \ No newline at end of file diff --git a/include/alloc.h b/include/alloc.h deleted file mode 100644 index 6809287..0000000 --- a/include/alloc.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef AMALGAM_ALLOC_H -#define AMALGAM_ALLOC_H - -#include "types.h" -#include "misc.h" - -#define ALLOC_OK 0 -#define ALLOC_FAIL -1 - -CHECK_RESULT int am_malloc(usize size, void **mem); -CHECK_RESULT int am_realloc(void *mem, usize new_size, void **new_mem); -void am_free(void *mem); - -#endif diff --git a/include/ast.h b/include/ast.h index f3580c0..529b3c8 100644 --- a/include/ast.h +++ b/include/ast.h @@ -1,26 +1,29 @@ #ifndef AMALGAM_AST_H #define AMALGAM_AST_H -#include "buffer_view.h" -#include "buffer.h" -#include "misc.h" -#include "scoped_allocator.h" +#include "std/buffer_view.h" +#include "std/buffer.h" +#include "std/misc.h" +#include "std/scoped_allocator.h" typedef struct FunctionDecl FunctionDecl; typedef struct FunctionCall FunctionCall; typedef struct LhsExpr LhsExpr; +typedef struct Import Import; typedef union { FunctionDecl *func_decl; FunctionCall *func_call; LhsExpr *lhs_expr; + Import *import; } AstValue; typedef enum { AST_NONE, AST_FUNCTION_DECL, AST_FUNCTION_CALL, - AST_LHS + AST_LHS, + AST_IMPORT } AstType; typedef struct { @@ -43,6 +46,10 @@ struct LhsExpr { Ast rhs_expr; }; +struct Import { + BufferView path; +}; + Ast ast_none(); CHECK_RESULT int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator); @@ -52,4 +59,6 @@ void funccall_init(FunctionCall *self, BufferView name); void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name); +void import_init(Import *self, BufferView path); + #endif diff --git a/include/buffer.h b/include/buffer.h deleted file mode 100644 index 5339108..0000000 --- a/include/buffer.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef AMALGAM_BUFFER_H -#define AMALGAM_BUFFER_H - -#include "types.h" -#include "misc.h" - -#define BUFFER_OK 0 -#define BUFFER_ALLOC_FAIL -1 - -typedef struct { - char* data; - usize size; - usize capacity; -} Buffer; - -struct ScopedAllocator; -CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator); - -CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size); -void* buffer_get(Buffer *self, usize index, usize type_size); - -#endif \ No newline at end of file diff --git a/include/buffer_view.h b/include/buffer_view.h deleted file mode 100644 index 4993dc2..0000000 --- a/include/buffer_view.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef AMALGAM_BUFFER_VIEW_H -#define AMALGAM_BUFFER_VIEW_H - -#include "types.h" - -typedef struct { - const char* data; - usize size; -} BufferView; - -BufferView create_buffer_view_null(); -BufferView create_buffer_view(const char *data, usize size); - -#endif diff --git a/include/compiler.h b/include/compiler.h new file mode 100644 index 0000000..691263a --- /dev/null +++ b/include/compiler.h @@ -0,0 +1,31 @@ +#ifndef AMALGAM_COMPILER_H +#define AMALGAM_COMPILER_H + +#include "std/misc.h" +#include "std/buffer.h" +#include "std/buffer_view.h" +#include "std/scoped_allocator.h" +#include "std/thread.h" +#include "defs.h" + +#define AMAL_COMPILER_OK 0 +/* General error */ +#define AMAL_COMPILER_ERR -1 + +struct amal_compiler { + ScopedAllocator allocator; + ScopedAllocator main_thread_allocator; + Buffer parsers; + Buffer queued_files; + ParserThreadData *threads; + int usable_thread_count; + bool started; + amal_mutex mutex; +}; + +CHECK_RESULT int amal_compiler_init(amal_compiler *self); +CHECK_RESULT int amal_compiler_deinit(amal_compiler *self); +CHECK_RESULT int amal_compiler_load_file(amal_compiler *self, BufferView filepath); +/* TODO: amal_compiler_unload_file */ + +#endif \ No newline at end of file diff --git a/include/defs.h b/include/defs.h new file mode 100644 index 0000000..9b754d8 --- /dev/null +++ b/include/defs.h @@ -0,0 +1,7 @@ +#ifndef AMALGAM_DEFS_H +#define AMALGAM_DEFS_H + +typedef struct ParserThreadData ParserThreadData; +typedef struct amal_compiler amal_compiler; + +#endif \ No newline at end of file diff --git a/include/mem.h b/include/mem.h deleted file mode 100644 index bad6353..0000000 --- a/include/mem.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef AMALGAM_MEM_H -#define AMALGAM_MEM_H - -#include "types.h" -#include "misc.h" - -void am_memcpy(void *dest, const void *src, usize size); -bool am_memeql(const void *lhs, const void *rhs, usize size); - -#endif \ No newline at end of file diff --git a/include/misc.h b/include/misc.h deleted file mode 100644 index 9cb2dde..0000000 --- a/include/misc.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef AMALGAM_MISC_H -#define AMALGAM_MISC_H - -#define return_if_error(result) \ -do { \ - int return_if_result; \ - return_if_result = (result); \ - if((return_if_result) != 0) \ - return return_if_result; \ -} while(0) - -#define cleanup_if_error(result) do { if((result) != 0) goto cleanup; } while(0) - -#if defined(__GNUC__) && __GNUC__ >= 4 -#define CHECK_RESULT __attribute__ ((warn_unused_result)) -#elif defined(_MSC_VER) && _MSC_VER >= 1700 -#define CHECK_RESULT _Check_return_ -#else -#define CHECK_RESULT -#endif - -typedef enum { - bool_false, - bool_true -} bool; - -#endif \ No newline at end of file diff --git a/include/parser.h b/include/parser.h index e90871f..9b41e5e 100644 --- a/include/parser.h +++ b/include/parser.h @@ -1,25 +1,51 @@ #ifndef AMALGAM_PARSER_H #define AMALGAM_PARSER_H -#include "buffer.h" -#include "buffer_view.h" +#include "std/buffer.h" +#include "std/buffer_view.h" +#include "std/scoped_allocator.h" +#include "std/thread.h" #include "tokenizer.h" -#include "scoped_allocator.h" +#include "defs.h" #define PARSER_OK 0 /* General error */ #define PARSER_ERR -1 #define PARSER_UNEXPECTED_TOKEN -2 +typedef enum { + PARSER_THREAD_STATUS_NEW, + PARSER_THREAD_STATUS_IDLE, + PARSER_THREAD_STATUS_RUNNING +} ParserThreadStatus; + +struct ParserThreadData { + amal_thread thread; + ParserThreadStatus status; + ScopedAllocator allocator; +}; + typedef struct { Tokenizer tokenizer; - ScopedAllocator allocator; Buffer ast_objects; + ScopedAllocator *allocator; /* borrowed. Copied from @compiler for faster access to allocator */ + amal_compiler *compiler; + bool started; } Parser; -CHECK_RESULT int parser_init(Parser *self); -void parser_deinit(Parser *self); +CHECK_RESULT int parser_thread_data_init(ParserThreadData *self); +CHECK_RESULT int parser_thread_data_deinit(ParserThreadData *self); +CHECK_RESULT int parser_thread_data_start(ParserThreadData *self, AmalThreadCallbackFunc callback_func, void *userdata); +CHECK_RESULT int parser_thread_data_join(ParserThreadData *self, void **result); + +CHECK_RESULT int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocator); -CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer); +/* +@buffer_name will be the path to the file when using parser_parse_file and when parsing a buffer +you can name the buffer anything you want to identify it. +*/ +CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_name); +/* Parses a file and the dependencies that are included using @import */ +CHECK_RESULT int parser_parse_file(Parser *self, BufferView filepath); #endif diff --git a/include/scoped_allocator.h b/include/scoped_allocator.h deleted file mode 100644 index fdaee2a..0000000 --- a/include/scoped_allocator.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef AMALGAM_SCOPED_ALLOCATOR_H -#define AMALGAM_SCOPED_ALLOCATOR_H - -#include "misc.h" -#include "types.h" -#include "buffer.h" - -typedef struct ScopedAllocatorNode ScopedAllocatorNode; -typedef struct ScopedAllocator ScopedAllocator; - -struct ScopedAllocatorNode { - char *data; - usize size; - ScopedAllocatorNode *next; -}; - -struct ScopedAllocator { - ScopedAllocatorNode head; - ScopedAllocatorNode *current; - Buffer buffers; -}; - -CHECK_RESULT int scoped_allocator_node_init(ScopedAllocatorNode *self); -void scoped_allocator_node_deinit(ScopedAllocatorNode *self); - -CHECK_RESULT int scoped_allocator_init(ScopedAllocator *self); -void scoped_allocator_deinit(ScopedAllocator *self); -CHECK_RESULT int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem); -CHECK_RESULT int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer); - -#endif \ No newline at end of file diff --git a/include/std/alloc.h b/include/std/alloc.h new file mode 100644 index 0000000..6809287 --- /dev/null +++ b/include/std/alloc.h @@ -0,0 +1,14 @@ +#ifndef AMALGAM_ALLOC_H +#define AMALGAM_ALLOC_H + +#include "types.h" +#include "misc.h" + +#define ALLOC_OK 0 +#define ALLOC_FAIL -1 + +CHECK_RESULT int am_malloc(usize size, void **mem); +CHECK_RESULT int am_realloc(void *mem, usize new_size, void **new_mem); +void am_free(void *mem); + +#endif diff --git a/include/std/buffer.h b/include/std/buffer.h new file mode 100644 index 0000000..ed87f29 --- /dev/null +++ b/include/std/buffer.h @@ -0,0 +1,23 @@ +#ifndef AMALGAM_BUFFER_H +#define AMALGAM_BUFFER_H + +#include "types.h" +#include "misc.h" + +#define BUFFER_OK 0 +#define BUFFER_ALLOC_FAIL -1 + +typedef struct { + char* data; + usize size; + usize capacity; +} Buffer; + +struct ScopedAllocator; +CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator); + +CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size); +void* buffer_get(Buffer *self, usize index, usize type_size); +CHECK_RESULT int buffer_pop(Buffer *self, void *data, usize size); + +#endif \ No newline at end of file diff --git a/include/std/buffer_view.h b/include/std/buffer_view.h new file mode 100644 index 0000000..4993dc2 --- /dev/null +++ b/include/std/buffer_view.h @@ -0,0 +1,14 @@ +#ifndef AMALGAM_BUFFER_VIEW_H +#define AMALGAM_BUFFER_VIEW_H + +#include "types.h" + +typedef struct { + const char* data; + usize size; +} BufferView; + +BufferView create_buffer_view_null(); +BufferView create_buffer_view(const char *data, usize size); + +#endif diff --git a/include/std/defs.h b/include/std/defs.h new file mode 100644 index 0000000..1376e16 --- /dev/null +++ b/include/std/defs.h @@ -0,0 +1,7 @@ +#ifndef AMALGAM_STD_DEFS_H +#define AMALGAM_STD_DEFS_H + +typedef struct amal_thread amal_thread; +typedef struct amal_mutex amal_mutex; + +#endif \ No newline at end of file diff --git a/include/std/file.h b/include/std/file.h new file mode 100644 index 0000000..ac2abec --- /dev/null +++ b/include/std/file.h @@ -0,0 +1,27 @@ +#ifndef AMALGAM_FILE_H +#define AMALGAM_FILE_H + +#include "misc.h" +#include "types.h" + +/* Return bool_true if you want to continue scanning, otherwise return bool_false */ +typedef bool (*scan_dir_callback_func)(const char *filepath, int filepath_length, void *userdata); + +typedef struct { + const char *file_data; + usize file_size; + int fd; +} MappedFile; + +typedef enum { + MAPPED_FILE_READ, + MAPPED_FILE_WRITE, + MAPPED_FILE_READ_WRITE +} MappedFileMode; + +/* Hidden files (files starting with a dot) are skipped */ +CHECK_RESULT int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata); +CHECK_RESULT int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode); +CHECK_RESULT int mapped_file_deinit(MappedFile *self); + +#endif \ No newline at end of file diff --git a/include/std/log.h b/include/std/log.h new file mode 100644 index 0000000..6ce9e4c --- /dev/null +++ b/include/std/log.h @@ -0,0 +1,13 @@ +#ifndef AMALGAM_LOG_H +#define AMALGAM_LOG_H + +#include "defs.h" + +amal_mutex* amal_log_get_mutex(); +void amal_log_debug(const char *fmt, ...); +void amal_log_error(const char *fmt, ...); +void amal_log_info(const char *fmt, ...); +void amal_log_warning(const char *fmt, ...); +void amal_log_perror(const char *prefix); + +#endif \ No newline at end of file diff --git a/include/std/mem.h b/include/std/mem.h new file mode 100644 index 0000000..a5fe9b4 --- /dev/null +++ b/include/std/mem.h @@ -0,0 +1,11 @@ +#ifndef AMALGAM_MEM_H +#define AMALGAM_MEM_H + +#include "types.h" +#include "misc.h" + +void am_memcpy(void *dest, const void *src, usize size); +bool am_memeql(const void *lhs, const void *rhs, usize size); +void am_memset(void *dest, int value, usize size); + +#endif \ No newline at end of file diff --git a/include/std/misc.h b/include/std/misc.h new file mode 100644 index 0000000..3ac524a --- /dev/null +++ b/include/std/misc.h @@ -0,0 +1,33 @@ +#ifndef AMALGAM_MISC_H +#define AMALGAM_MISC_H + +#define return_if_error(result) \ +do { \ + int return_if_result; \ + return_if_result = (result); \ + if((return_if_result) != 0) \ + return return_if_result; \ +} while(0) + +#define cleanup_if_error(result) do { if((result) != 0) goto cleanup; } while(0) + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define CHECK_RESULT __attribute__ ((warn_unused_result)) +#elif defined(_MSC_VER) && _MSC_VER >= 1700 +#define CHECK_RESULT _Check_return_ +#else +#define CHECK_RESULT +#endif + +#define ignore_result_int(expr) (void)((expr)+1) + +typedef enum { + bool_false, + bool_true +} bool; + +#ifndef NULL +#define NULL ((void*)0) +#endif + +#endif \ No newline at end of file diff --git a/include/std/scoped_allocator.h b/include/std/scoped_allocator.h new file mode 100644 index 0000000..fdaee2a --- /dev/null +++ b/include/std/scoped_allocator.h @@ -0,0 +1,31 @@ +#ifndef AMALGAM_SCOPED_ALLOCATOR_H +#define AMALGAM_SCOPED_ALLOCATOR_H + +#include "misc.h" +#include "types.h" +#include "buffer.h" + +typedef struct ScopedAllocatorNode ScopedAllocatorNode; +typedef struct ScopedAllocator ScopedAllocator; + +struct ScopedAllocatorNode { + char *data; + usize size; + ScopedAllocatorNode *next; +}; + +struct ScopedAllocator { + ScopedAllocatorNode head; + ScopedAllocatorNode *current; + Buffer buffers; +}; + +CHECK_RESULT int scoped_allocator_node_init(ScopedAllocatorNode *self); +void scoped_allocator_node_deinit(ScopedAllocatorNode *self); + +CHECK_RESULT int scoped_allocator_init(ScopedAllocator *self); +void scoped_allocator_deinit(ScopedAllocator *self); +CHECK_RESULT int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem); +CHECK_RESULT int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer); + +#endif \ No newline at end of file diff --git a/include/std/thread.h b/include/std/thread.h new file mode 100644 index 0000000..dd09039 --- /dev/null +++ b/include/std/thread.h @@ -0,0 +1,50 @@ +#ifndef AMALGAM_THREAD_H +#define AMALGAM_THREAD_H + +#include "misc.h" +#include "types.h" +#include "defs.h" +#include + +typedef void* (AmalThreadCallbackFunc)(void *userdata); + +#define AMAL_THREAD_OK 0 +/* General error */ +#define AMAL_THREAD_ERR -1 +#define AMAL_THREAD_NOT_JOINABLE -23 + +struct amal_thread { + pthread_t thread_id; + pthread_attr_t thread_attr; + const char *name; + bool cancellable; + bool destroyable; +}; + +typedef enum { + AMAL_THREAD_JOINABLE, + AMAL_THREAD_DETACHED +} amal_thread_type; + +struct amal_mutex { + pthread_mutex_t mutex; + const char *lock_identifier; +}; + +CHECK_RESULT int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata); +/* Safe to call multiple times */ +CHECK_RESULT int amal_thread_deinit(amal_thread *self); +CHECK_RESULT int amal_thread_detach(amal_thread *self); +CHECK_RESULT int amal_thread_join(amal_thread *self, void **result); + +void amal_mutex_init(amal_mutex *self); +CHECK_RESULT int amal_mutex_lock(amal_mutex *self, const char *lock_identifier); +CHECK_RESULT int amal_mutex_unlock(amal_mutex *self); +void amal_mutex_tryunlock(amal_mutex *self); + +bool amal_thread_is_main(); + +/* Returns 0 if the number of usable threads is unknown */ +int amal_get_usable_thread_count(); + +#endif \ No newline at end of file diff --git a/include/std/types.h b/include/std/types.h new file mode 100644 index 0000000..68e2d0f --- /dev/null +++ b/include/std/types.h @@ -0,0 +1,20 @@ +#ifndef AMALGAM_TYPES_H +#define AMALGAM_TYPES_H + +#include +#include + +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +typedef ptrdiff_t isize; +typedef size_t usize; + +#endif diff --git a/include/tokenizer.h b/include/tokenizer.h index 9584542..e79f070 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -1,11 +1,13 @@ #ifndef AMALGAM_TOKENIZER_H #define AMALGAM_TOKENIZER_H -#include "buffer_view.h" -#include "misc.h" +#include "std/buffer_view.h" +#include "std/misc.h" #define TOKENIZER_OK 0 -#define TOKENIZER_UNEXPECTED_TOKEN -1 +/* General error */ +#define TOKENIZER_ERR -1 +#define TOKENIZER_UNEXPECTED_TOKEN -2 typedef enum { TOK_NONE, @@ -13,11 +15,13 @@ typedef enum { TOK_IDENTIFIER, TOK_CONST, TOK_VAR, + TOK_STRING, TOK_EQUALS, TOK_OPEN_PAREN, TOK_CLOSING_PAREN, TOK_OPEN_BRACE, - TOK_CLOSING_BRACE + TOK_CLOSING_BRACE, + TOK_IMPORT } Token; typedef struct { @@ -25,13 +29,15 @@ typedef struct { int index; int prev_index; int line; + BufferView code_name; union { BufferView identifier; + BufferView string; } value; } Tokenizer; -CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code); +CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name); CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token); CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); diff --git a/include/types.h b/include/types.h deleted file mode 100644 index 68e2d0f..0000000 --- a/include/types.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef AMALGAM_TYPES_H -#define AMALGAM_TYPES_H - -#include -#include - -typedef int8_t i8; -typedef int16_t i16; -typedef int32_t i32; -typedef int64_t i64; - -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -typedef ptrdiff_t isize; -typedef size_t usize; - -#endif diff --git a/src/alloc.c b/src/alloc.c deleted file mode 100644 index c9ca7c3..0000000 --- a/src/alloc.c +++ /dev/null @@ -1,25 +0,0 @@ -#include "../include/alloc.h" -#include - -int am_malloc(usize size, void **mem) { - void *allocated_data = malloc(size); - if(!allocated_data) - return ALLOC_FAIL; - - *mem = allocated_data; - return ALLOC_OK; -} - -int am_realloc(void *mem, usize new_size, void **new_mem) { - void *new_allocated_data = realloc(mem, new_size); - if(!new_allocated_data) - return ALLOC_FAIL; - - *new_mem = new_allocated_data; - return ALLOC_OK; -} - -void am_free(void *mem) { - free(mem); -} - diff --git a/src/ast.c b/src/ast.c index 3ea4fe1..0acc69c 100644 --- a/src/ast.c +++ b/src/ast.c @@ -25,4 +25,8 @@ void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name) { self->isConst = isConst; self->var_name = var_name; self->rhs_expr = ast_none(); +} + +void import_init(Import *self, BufferView path) { + self->path = path; } \ No newline at end of file diff --git a/src/buffer.c b/src/buffer.c deleted file mode 100644 index a097cbf..0000000 --- a/src/buffer.c +++ /dev/null @@ -1,53 +0,0 @@ -#include "../include/buffer.h" -#include "../include/alloc.h" -#include "../include/mem.h" -#include "../include/scoped_allocator.h" -#include - -int buffer_init(Buffer *self, struct ScopedAllocator *allocator) { - self->data = NULL; - self->size = 0; - self->capacity = 0; - if(allocator) - return scoped_allocator_add_buffer(allocator, self); - return 0; -} - -static CHECK_RESULT int buffer_ensure_capacity(Buffer *self, usize new_capacity) { - usize capacity; - void *new_mem; - int alloc_result; - - if(self->capacity >= new_capacity) - return BUFFER_OK; - - capacity = self->capacity; - if(capacity == 0) { - capacity = new_capacity; - } else { - while(capacity < new_capacity) { - capacity *= 1.5; - } - } - - alloc_result = am_realloc(self->data, capacity, &new_mem); - if(alloc_result != ALLOC_OK) - return BUFFER_ALLOC_FAIL; - - self->data = new_mem; - self->capacity = capacity; - return BUFFER_OK; -} - -int buffer_append(Buffer *self, void *data, usize size) { - return_if_error(buffer_ensure_capacity(self, self->size + size)); - am_memcpy(self->data + self->size, data, size); - return BUFFER_OK; -} - -void* buffer_get(Buffer *self, usize index, usize type_size) { - usize real_index; - real_index = index * type_size; - assert(real_index < self->size); - return &self->data[real_index]; -} \ No newline at end of file diff --git a/src/buffer_view.c b/src/buffer_view.c deleted file mode 100644 index 96b0dd7..0000000 --- a/src/buffer_view.c +++ /dev/null @@ -1,15 +0,0 @@ -#include "../include/buffer_view.h" - -BufferView create_buffer_view_null() { - BufferView buffer_view; - buffer_view.data = NULL; - buffer_view.size = 0; - return buffer_view; -} - -BufferView create_buffer_view(const char *data, usize size) { - BufferView buffer_view; - buffer_view.data = data; - buffer_view.size = size; - return buffer_view; -} \ No newline at end of file diff --git a/src/compiler.c b/src/compiler.c new file mode 100644 index 0000000..fba18d3 --- /dev/null +++ b/src/compiler.c @@ -0,0 +1,225 @@ +#include "../include/compiler.h" +#include "../include/parser.h" +#include "../include/std/log.h" +#include "../include/std/mem.h" +#include "../include/std/alloc.h" +#include +#include +#include + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +static CHECK_RESULT int get_thread_count_env_var(int *thread_count) { + char *threads; + threads = getenv("THREADS"); + if(!threads) + return -1; + *thread_count = atoi(threads); + return 0; +} + +int amal_compiler_init(amal_compiler *self) { + int i; + int result; + + result = get_thread_count_env_var(&self->usable_thread_count); + if(result != 0) { + self->usable_thread_count = amal_get_usable_thread_count(); + if(self->usable_thread_count == 0) { + amal_log_info("Unable to get the number of threads available on the system, using 1 thread."); + amal_log_info("You can override the number of threads using by setting the environment variable THREADS"); + self->usable_thread_count = 1; + } + } else if(self->usable_thread_count == 0) { + amal_log_info("Environment variable THREADS contains invalid number for threads. THREADS has to be at least 1."); + return AMAL_COMPILER_ERR; + } + /* Exclude thread count because we also use the main thread */ + --self->usable_thread_count; + + am_memset(&self->allocator, 0, sizeof(self->allocator)); + am_memset(&self->main_thread_allocator, 0, sizeof(self->main_thread_allocator)); + self->started = bool_false; + amal_mutex_init(&self->mutex); + + return_if_error(scoped_allocator_init(&self->allocator)); + cleanup_if_error(scoped_allocator_init(&self->main_thread_allocator)); + cleanup_if_error(buffer_init(&self->parsers, &self->allocator)); + cleanup_if_error(buffer_init(&self->queued_files, &self->allocator)); + cleanup_if_error(scoped_allocator_alloc(&self->allocator, + self->usable_thread_count * sizeof(ParserThreadData), + (void**)&self->threads)); + for(i = 0; i < self->usable_thread_count; ++i) + cleanup_if_error(parser_thread_data_init(&self->threads[i])); + return AMAL_COMPILER_OK; + + cleanup: + /* Ignore result */ + result = amal_compiler_deinit(self); + return AMAL_COMPILER_ERR; +} + +int amal_compiler_deinit(amal_compiler *self) { + int i; + int result; + result = AMAL_COMPILER_OK; + + for(i = 0; i < self->usable_thread_count; ++i) { + int r; + r = parser_thread_data_deinit(&self->threads[i]); + if(r != 0) + result = r; + } + + scoped_allocator_deinit(&self->allocator); + scoped_allocator_deinit(&self->main_thread_allocator); + return result; +} + +typedef struct { + amal_compiler *compiler; + ParserThreadData *parser_thread_data; + BufferView filepath; +} CompilerParserThreadUserData; + +static CHECK_RESULT int amal_compiler_load_first_this_thread(amal_compiler *self, BufferView filepath, ScopedAllocator *allocator) { + Parser parser; + int result; + result = AMAL_COMPILER_ERR; + am_memset(&parser, 0, sizeof(parser)); + + return_if_error(parser_init(&parser, self, allocator)); + cleanup_if_error(parser_parse_file(&parser, filepath)); + amal_log_debug("Finished parsing in thread"); + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_first_this_thread")); + cleanup_if_error(buffer_append(&self->parsers, &parser, sizeof(parser))); + result = AMAL_COMPILER_OK; + + cleanup: + amal_mutex_tryunlock(&self->mutex); + return result; +} + +/* TODO: Handle errors (stop parsing in all other threads and report errors/warnings) */ +static void* thread_callback_parse_file(void *userdata) { + int _; + BufferView next_file; + CompilerParserThreadUserData compiler_parser_userdata; + (void)_; + next_file = create_buffer_view_null(); + assert(!amal_thread_is_main()); + + am_memcpy(&compiler_parser_userdata, userdata, sizeof(compiler_parser_userdata)); + am_free(userdata); + amal_log_debug("Thread start parse file"); + cleanup_if_error(amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, + compiler_parser_userdata.filepath, + &compiler_parser_userdata.parser_thread_data->allocator)); + cleanup_if_error(amal_mutex_lock(&compiler_parser_userdata.compiler->mutex, "thread_callback_parse_file")); + if(compiler_parser_userdata.compiler->queued_files.size > 0) + _ = buffer_pop(&compiler_parser_userdata.compiler->queued_files, &next_file, sizeof(next_file)); + + cleanup: + amal_log_debug("Thread finished, next file: %s", next_file.data ? "yes" : "no"); + if(!next_file.data) + compiler_parser_userdata.parser_thread_data->status = PARSER_THREAD_STATUS_IDLE; + amal_mutex_tryunlock(&compiler_parser_userdata.compiler->mutex); + if(next_file.data) { + _ = amal_compiler_load_first_this_thread(compiler_parser_userdata.compiler, + next_file, + &compiler_parser_userdata.parser_thread_data->allocator); + } + return NULL; +} + +static CHECK_RESULT int amal_compiler_load_file_select_thread(amal_compiler *self, BufferView filepath, ParserThreadData **thread_selected) { + int i; + int result; + ParserThreadData *parser_thread_data; + CompilerParserThreadUserData *thread_user_data; + thread_user_data = NULL; + *thread_selected = NULL; + result = AMAL_COMPILER_ERR; + assert(amal_thread_is_main()); + + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_select_thread")); + for(i = 0; i < self->usable_thread_count; ++i) { + parser_thread_data = &self->threads[i]; + if(parser_thread_data->status == PARSER_THREAD_STATUS_RUNNING) + continue; + + cleanup_if_error(am_malloc(sizeof(CompilerParserThreadUserData), (void**)&thread_user_data)); + thread_user_data->compiler = self; + thread_user_data->parser_thread_data = parser_thread_data; + thread_user_data->filepath = filepath; + result = parser_thread_data_start(parser_thread_data, thread_callback_parse_file, thread_user_data); + *thread_selected = parser_thread_data; + break; + } + + cleanup: + if(result != 0) + am_free(thread_user_data); + amal_mutex_tryunlock(&self->mutex); + return result; +} + +static CHECK_RESULT int amal_compiler_load_file_join_threads(amal_compiler *self) { + int i; + int _; + int result; + void *thread_return_data; + ParserThreadData *parser_thread_data; + result = AMAL_COMPILER_ERR; + (void)_; + + while(self->queued_files.size > 0) { + BufferView filepath; + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_join_threads remaining files")); + _ = buffer_pop(&self->queued_files, &filepath, sizeof(filepath)); + amal_mutex_tryunlock(&self->mutex); + if(filepath.data) + return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator)); + } + + for(i = 0; i < self->usable_thread_count; ++i) { + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file_join_threads, waiting for workers")); + parser_thread_data = &self->threads[i]; + amal_mutex_tryunlock(&self->mutex); + amal_log_debug("Joining thread %d, status %d", i, parser_thread_data->status); + _ = parser_thread_data_join(parser_thread_data, &thread_return_data); + } + + result = AMAL_COMPILER_OK; + cleanup: + return result; +} + +int amal_compiler_load_file(amal_compiler *self, BufferView filepath) { + int result; + ParserThreadData *parser_thread_data; + result = AMAL_COMPILER_ERR; + + /* + amal_compiler_load_file is called by the user for the first file to compile + but also called by the parser when it sees @import + */ + if(!self->started) { + self->started = bool_true; + /*amal_log_info("Parsing %.*s using %d thread(s)", (int)filepath.size, filepath.data, self->usable_thread_count);*/ + return_if_error(amal_compiler_load_first_this_thread(self, filepath, &self->main_thread_allocator)); + return amal_compiler_load_file_join_threads(self); + } + + return_if_error(amal_compiler_load_file_select_thread(self, filepath, &parser_thread_data)); + if(parser_thread_data) + return AMAL_COMPILER_OK; + + cleanup_if_error(amal_mutex_lock(&self->mutex, "amal_compiler_load_file")); + cleanup_if_error(buffer_append(&self->queued_files, &filepath, sizeof(filepath))); + result = AMAL_COMPILER_OK; + + cleanup: + amal_mutex_tryunlock(&self->mutex); + return result; +} \ No newline at end of file diff --git a/src/main.c b/src/main.c index f3147bb..c016892 100644 --- a/src/main.c +++ b/src/main.c @@ -1,37 +1,27 @@ #include #include -#include "../include/parser.h" +#include "../include/compiler.h" int main() { - const char *code; - Parser parser; - BufferView code_view; + amal_compiler compiler; int result; + const char *filepath; + filepath = "tests/main.amal"; - code = - "const main = () {\n" - " var hello = () {\n" - " \n" - " }\n" - " hello()\n" - "}\n" - "const print = () {\n" - " \n" - "}"; - result = parser_init(&parser); - if(result != PARSER_OK) { - fprintf(stderr, "Failed to initialize parser\n"); + result = amal_compiler_init(&compiler); + if(result != AMAL_COMPILER_OK) { + fprintf(stderr, "Failed to initialize compiler, error code: %d\n", result); return 1; } - code_view = create_buffer_view(code, strlen(code)); - result = parser_parse_buffer(&parser, code_view); - if(result != PARSER_OK) { - fprintf(stderr, "Failed to parse\n"); + result = amal_compiler_load_file(&compiler, create_buffer_view(filepath, strlen(filepath))); + if(result != AMAL_COMPILER_OK) { + fprintf(stderr, "Failed to load file, error code: %d\n", result); return 1; } - /* No need to do this here as the program is exiting */ - /* parser_deinit(&parser); */ +#ifdef DEBUG + return amal_compiler_deinit(&compiler); +#endif return 0; } diff --git a/src/mem.c b/src/mem.c deleted file mode 100644 index acd2ebd..0000000 --- a/src/mem.c +++ /dev/null @@ -1,10 +0,0 @@ -#include "../include/mem.h" -#include - -void am_memcpy(void *dest, const void *src, usize size) { - memcpy(dest, src, size); -} - -bool am_memeql(const void *lhs, const void *rhs, usize size) { - return memcmp(lhs, rhs, size) == 0; -} \ No newline at end of file diff --git a/src/parser.c b/src/parser.c index 78b0018..da85292 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,19 +1,51 @@ #include "../include/parser.h" #include "../include/ast.h" -#include "../include/misc.h" -#include "../include/alloc.h" +#include "../include/compiler.h" +#include "../include/std/misc.h" +#include "../include/std/file.h" +#include "../include/std/mem.h" +#include "../include/std/log.h" #include +#include +#include +static CHECK_RESULT int parser_queue_file(Parser *self, BufferView path); static CHECK_RESULT int parser_parse_body(Parser *self, Ast *ast); -int parser_init(Parser *self) { - return_if_error(scoped_allocator_init(&self->allocator)); - return_if_error(buffer_init(&self->ast_objects, &self->allocator)); - return PARSER_OK; +int parser_thread_data_init(ParserThreadData *self) { + am_memset(&self->allocator, 0, sizeof(self->allocator)); + am_memset(&self->thread, 0, sizeof(self->thread)); + self->status = PARSER_THREAD_STATUS_NEW; + return scoped_allocator_init(&self->allocator); } -void parser_deinit(Parser *self) { +int parser_thread_data_deinit(ParserThreadData *self) { scoped_allocator_deinit(&self->allocator); + return amal_thread_deinit(&self->thread); +} + +int parser_thread_data_start(ParserThreadData *self, AmalThreadCallbackFunc callback_func, void *userdata) { + return_if_error(amal_thread_deinit(&self->thread)); + return_if_error(amal_thread_create(&self->thread, AMAL_THREAD_JOINABLE, "Parser", callback_func, userdata)); + self->status = PARSER_THREAD_STATUS_RUNNING; + return 0; +} + +int parser_thread_data_join(ParserThreadData *self, void **result) { + int retval; + if(self->status == PARSER_THREAD_STATUS_NEW) + return 0; + retval = amal_thread_join(&self->thread, result); + if(retval == 0 || retval == AMAL_THREAD_NOT_JOINABLE) + self->status = PARSER_THREAD_STATUS_IDLE; + return retval; +} + +int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocator) { + self->allocator = allocator; + self->compiler = compiler; + self->started = bool_false; + return buffer_init(&self->ast_objects, self->allocator); } /* @@ -34,8 +66,8 @@ static CHECK_RESULT int parser_parse_lhs(Parser *self, LhsExpr **result) { return_if_error(tokenizer_accept(&self->tokenizer, TOK_IDENTIFIER)); var_name = self->tokenizer.value.identifier; - fprintf(stderr, "var name: %.*s\n", (int)var_name.size, var_name.data); - return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(LhsExpr), (void**)result)); + amal_log_debug("var name: %.*s", (int)var_name.size, var_name.data); + return_if_error(scoped_allocator_alloc(self->allocator, sizeof(LhsExpr), (void**)result)); lhsexpr_init(*result, isConst, var_name); return PARSER_OK; } @@ -56,23 +88,19 @@ static CHECK_RESULT int parser_parse_function_decl(Parser *self, FunctionDecl ** /* TODO: Parse return types */ return_if_error(tokenizer_accept(&self->tokenizer, TOK_OPEN_BRACE)); - return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(FunctionDecl), (void**)func_decl)); - cleanup_if_error(funcdecl_init(*func_decl, &self->allocator)); + return_if_error(scoped_allocator_alloc(self->allocator, sizeof(FunctionDecl), (void**)func_decl)); + return_if_error(funcdecl_init(*func_decl, self->allocator)); for(;;) { Ast body_obj; - cleanup_if_error(tokenizer_consume_if(&self->tokenizer, TOK_CLOSING_BRACE, &result)); + return_if_error(tokenizer_consume_if(&self->tokenizer, TOK_CLOSING_BRACE, &result)); if(result) break; - cleanup_if_error(parser_parse_body(self, &body_obj)); - cleanup_if_error(funcdecl_add_to_body(*func_decl, body_obj)); + return_if_error(parser_parse_body(self, &body_obj)); + return_if_error(funcdecl_add_to_body(*func_decl, body_obj)); } return PARSER_OK; - - cleanup: - *func_decl = NULL; - return PARSER_ERR; } /* @@ -92,20 +120,33 @@ static CHECK_RESULT int parser_parse_function_call(Parser *self, FunctionCall ** /* TODO: Parse arguments */ return_if_error(tokenizer_accept(&self->tokenizer, TOK_CLOSING_PAREN)); - return_if_error(scoped_allocator_alloc(&self->allocator, sizeof(FunctionCall), (void**)func_call)); + return_if_error(scoped_allocator_alloc(self->allocator, sizeof(FunctionCall), (void**)func_call)); funccall_init(*func_call, func_name); return PARSER_OK; } /* -RHS = FUNC_DECL | FUNC_CALL +IMPORT = IMPORT_SYMBOL +*/ +static CHECK_RESULT int parser_parse_import(Parser *self, Import **import) { + bool result; + + return_if_error(tokenizer_consume_if(&self->tokenizer, TOK_IMPORT, &result)); + if(!result) + return PARSER_OK; + + return_if_error(scoped_allocator_alloc(self->allocator, sizeof(Import), (void**)import)); + import_init(*import, self->tokenizer.value.string); + return PARSER_OK; +} + +/* +RHS = FUNC_DECL | FUNC_CALL | IMPORT */ static CHECK_RESULT int parser_parse_rhs(Parser *self, Ast *rhs_expr) { FunctionDecl *func_decl; FunctionCall *func_call; - Token token; - func_decl = NULL; - func_call = NULL; + Import *import; return_if_error(parser_parse_function_decl(self, &func_decl)); if(func_decl) { @@ -121,9 +162,16 @@ static CHECK_RESULT int parser_parse_rhs(Parser *self, Ast *rhs_expr) { return PARSER_OK; } - return_if_error(tokenizer_next(&self->tokenizer, &token)); + return_if_error(parser_parse_import(self, &import)); + if(import) { + rhs_expr->type = AST_IMPORT; + rhs_expr->value.import = import; + return_if_error(parser_queue_file(self, import->path)); + return PARSER_OK; + } + /* TODO: Convert token to string */ - tokenizer_print_error(&self->tokenizer, "Expected function declaration or function call, got token: %d"); + tokenizer_print_error(&self->tokenizer, "Expected function declaration or function call"); return PARSER_UNEXPECTED_TOKEN; } @@ -153,9 +201,9 @@ int parser_parse_body(Parser *self, Ast *ast) { /* ROOT = BODY* */ -int parser_parse_buffer(Parser *self, BufferView code_buffer) { +int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_name) { Ast ast; - return_if_error(tokenizer_init(&self->tokenizer, code_buffer)); + return_if_error(tokenizer_init(&self->tokenizer, code_buffer, buffer_name)); for(;;) { bool isEof; @@ -169,3 +217,35 @@ int parser_parse_buffer(Parser *self, BufferView code_buffer) { return PARSER_OK; } + +int parser_parse_file(Parser *self, BufferView filepath) { + int result; + int mapped_file_deinit_result; + MappedFile mapped_file; + char *filepath_tmp; + + amal_log_debug("Parsing %.*s", (int)filepath.size, filepath.data); + assert(!self->started && "Parser can't be reused. Create a new parser."); + self->started = bool_true; + /* TODO: Somehow free this... */ + /*return_if_error(scoped_allocator_alloc(self->allocator, filepath.size + 1, (void**)&filepath_tmp));*/ + filepath_tmp = malloc(filepath.size + 1); + am_memcpy(filepath_tmp, filepath.data, filepath.size); + filepath_tmp[filepath.size] = '\0'; + result = mapped_file_init(&mapped_file, filepath_tmp, MAPPED_FILE_READ); + if(result != 0) return result; + result = parser_parse_buffer(self, create_buffer_view(mapped_file.file_data, mapped_file.file_size), filepath); + mapped_file_deinit_result = mapped_file_deinit(&mapped_file); + return result != 0 ? result : mapped_file_deinit_result; +} + +/* +Path can be path to included library path (or system library path) in which case +the path separator is a dot, otherwise the path separator is forward slash '/' +*/ +int parser_queue_file(Parser *self, BufferView path) { + /* TODO: Do not load same path twice or the compile will fail (and it can have recursive import) also for performance reasons */ + /* TODO: Parse special path (to include library path with dots) */ + return_if_error(amal_compiler_load_file(self->compiler, path)); + return PARSER_OK; +} \ No newline at end of file diff --git a/src/scoped_allocator.c b/src/scoped_allocator.c deleted file mode 100644 index 4d40740..0000000 --- a/src/scoped_allocator.c +++ /dev/null @@ -1,82 +0,0 @@ -#include "../include/scoped_allocator.h" -#include "../include/alloc.h" -#include - -#define ALLOC_NODE_SIZE 4096 - -int scoped_allocator_node_init(ScopedAllocatorNode *self) { - self->data = NULL; - self->size = 0; - self->next = NULL; - return am_malloc(ALLOC_NODE_SIZE, (void**)&self->data); -} - -void scoped_allocator_node_deinit(ScopedAllocatorNode *self) { - am_free(self->data); - self->data = NULL; - self->size = 0; - if(self->next) { - scoped_allocator_node_deinit(self->next); - am_free(self->next); - self->next = NULL; - } -} - -int scoped_allocator_init(ScopedAllocator *self) { - return_if_error(scoped_allocator_node_init(&self->head)); - self->current = &self->head; - return buffer_init(&self->buffers, NULL); -} - -static void buffer_deinit(Buffer *self) { - am_free(self->data); - self->data = NULL; - self->size = 0; - self->capacity = 0; -} - -void scoped_allocator_deinit(ScopedAllocator *self) { - Buffer *buffer; - Buffer *buffer_end; - - scoped_allocator_node_deinit(&self->head); - self->current = NULL; - buffer = (Buffer*)&self->buffers.data[0]; - buffer_end = buffer + self->buffers.size / sizeof(Buffer); - while(buffer != buffer_end) { - buffer_deinit(buffer); - ++buffer; - } - buffer_deinit(&self->buffers); -} - -static CHECK_RESULT int scoped_allocator_ensure_capacity_for(ScopedAllocator *self, usize size) { - void *new_node; - new_node = NULL; - - if(self->current->size + size > ALLOC_NODE_SIZE) { - return_if_error(am_malloc(sizeof(ScopedAllocatorNode), &new_node)); - cleanup_if_error(scoped_allocator_node_init(new_node)); - self->current->next = new_node; - self->current = new_node; - } - return 0; - - cleanup: - if(new_node) - am_free(new_node); - return -1; -} - -int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem) { - assert(self->current); - assert(size <= ALLOC_NODE_SIZE); - return_if_error(scoped_allocator_ensure_capacity_for(self, size)); - *mem = &self->current->data[self->current->size]; - self->current->size += size; - return 0; -} - -int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer) { - return buffer_append(&self->buffers, buffer, sizeof(buffer)); -} \ No newline at end of file diff --git a/src/std/alloc.c b/src/std/alloc.c new file mode 100644 index 0000000..93dcb98 --- /dev/null +++ b/src/std/alloc.c @@ -0,0 +1,25 @@ +#include "../../include/std/alloc.h" +#include + +int am_malloc(usize size, void **mem) { + void *allocated_data = malloc(size); + if(!allocated_data) + return ALLOC_FAIL; + + *mem = allocated_data; + return ALLOC_OK; +} + +int am_realloc(void *mem, usize new_size, void **new_mem) { + void *new_allocated_data = realloc(mem, new_size); + if(!new_allocated_data) + return ALLOC_FAIL; + + *new_mem = new_allocated_data; + return ALLOC_OK; +} + +void am_free(void *mem) { + free(mem); +} + diff --git a/src/std/buffer.c b/src/std/buffer.c new file mode 100644 index 0000000..f32c515 --- /dev/null +++ b/src/std/buffer.c @@ -0,0 +1,62 @@ +#include "../../include/std/buffer.h" +#include "../../include/std/alloc.h" +#include "../../include/std/mem.h" +#include "../../include/std/scoped_allocator.h" +#include + +int buffer_init(Buffer *self, struct ScopedAllocator *allocator) { + self->data = NULL; + self->size = 0; + self->capacity = 0; + if(allocator) + return scoped_allocator_add_buffer(allocator, self); + return 0; +} + +static CHECK_RESULT int buffer_ensure_capacity(Buffer *self, usize new_capacity) { + usize capacity; + void *new_mem; + int alloc_result; + + if(self->capacity >= new_capacity) + return BUFFER_OK; + + capacity = self->capacity; + if(capacity == 0) { + capacity = new_capacity; + } else { + while(capacity < new_capacity) { + capacity *= 1.5; + } + } + + alloc_result = am_realloc(self->data, capacity, &new_mem); + if(alloc_result != ALLOC_OK) + return BUFFER_ALLOC_FAIL; + + self->data = new_mem; + self->capacity = capacity; + return BUFFER_OK; +} + +int buffer_append(Buffer *self, void *data, usize size) { + return_if_error(buffer_ensure_capacity(self, self->size + size)); + am_memcpy(self->data + self->size, data, size); + self->size += size; + return BUFFER_OK; +} + +void* buffer_get(Buffer *self, usize index, usize type_size) { + usize real_index; + real_index = index * type_size; + assert(real_index < self->size); + return &self->data[real_index]; +} + +int buffer_pop(Buffer *self, void *data, usize size) { + if(size > self->size) + return -1; + am_memcpy(data, &self->data[self->size - size], size); + self->size -= size; + return 0; +} \ No newline at end of file diff --git a/src/std/buffer_view.c b/src/std/buffer_view.c new file mode 100644 index 0000000..977626c --- /dev/null +++ b/src/std/buffer_view.c @@ -0,0 +1,15 @@ +#include "../../include/std/buffer_view.h" + +BufferView create_buffer_view_null() { + BufferView buffer_view; + buffer_view.data = NULL; + buffer_view.size = 0; + return buffer_view; +} + +BufferView create_buffer_view(const char *data, usize size) { + BufferView buffer_view; + buffer_view.data = data; + buffer_view.size = size; + return buffer_view; +} \ No newline at end of file diff --git a/src/std/file.c b/src/std/file.c new file mode 100644 index 0000000..7701be4 --- /dev/null +++ b/src/std/file.c @@ -0,0 +1,152 @@ +#include "../../include/std/file.h" +#include "../../include/std/mem.h" +#include "../../include/std/log.h" +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define SCANDIR_PATH_LENGTH 4096 + +static int scan_dir_recursive_internal(char *dir_path, int path_length, scan_dir_callback_func callback_func, void *userdata) { + DIR *dir; + struct dirent *entry; + struct stat file_stats; + int filename_length; + int path_length_start; + + path_length_start = path_length; + if((dir = opendir(dir_path)) == NULL) { + amal_log_perror("scan_dir_recursive"); + return errno; + } + + while((entry = readdir(dir)) != NULL) { + /* + Skip hidden files. Name will always at least include null terminator + so it's ok to not check filename length. And file names are never empty, right??? + */ + if(entry->d_name[0] == '.') + continue; + + /* Reset path concat, because it's overwritten by recursive call to scan dir */ + path_length = path_length_start; + filename_length = strlen(entry->d_name); + /* current path '/' filename '\0' */ + if(path_length + 1 + filename_length + 1 > SCANDIR_PATH_LENGTH) { + amal_log_error("Filepath too long. Trying to append %s to %s", entry->d_name, dir_path); + return -1; + } + + /* Replace '\0' with '/' */ + dir_path[path_length] = '/'; + ++path_length; + am_memcpy(&dir_path[path_length], entry->d_name, filename_length); + path_length += filename_length; + dir_path[path_length] = '\0'; + + if(stat(dir_path, &file_stats) == -1) { + amal_log_perror("scan_dir_recursive_internal"); + goto cleanup; + } + + if(S_ISDIR(file_stats.st_mode)) { + cleanup_if_error(scan_dir_recursive_internal(dir_path, path_length, callback_func, userdata)); + } else if(S_ISREG(file_stats.st_mode)) { + if(!callback_func(dir_path, path_length, userdata)) + goto cleanup_noerr; + } else if(S_ISLNK(file_stats.st_mode)) { + amal_log_warning("Symlink ignored: %s", entry->d_name); + } else { + /* Ignore block devices and all other types of files */ + } + } + + cleanup_noerr: + closedir(dir); + return 0; + + cleanup: + closedir(dir); + return -1; +} + +int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata) { + char scandir_path[SCANDIR_PATH_LENGTH]; + int dir_path_length = strlen(dir_path) + 1; /* include null terminator */ + am_memcpy(scandir_path, dir_path, dir_path_length); + return scan_dir_recursive_internal(scandir_path, dir_path_length - 1, callback_func, userdata); +} + +static int mapped_file_mode_to_system_file_mode(MappedFileMode file_mode) { + switch(file_mode) { + case MAPPED_FILE_READ: return PROT_READ; + case MAPPED_FILE_WRITE: return PROT_WRITE; + case MAPPED_FILE_READ_WRITE: return PROT_READ | PROT_WRITE; + default: + assert(bool_false); + return PROT_READ; + } +} + +int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode) { + struct stat file_stats; + int fd; + const char *memblock; + int map_file_system_mode; + + self->file_data = NULL; + self->file_size = 0; + self->fd = -1; + + fd = open(filepath, O_RDONLY); + if(fd == -1) return errno; + if(fstat(fd, &file_stats) == -1) { + amal_log_perror("map_file_read"); + goto cleanup; + } + + map_file_system_mode = mapped_file_mode_to_system_file_mode(file_mode); + memblock = mmap(NULL, file_stats.st_size, map_file_system_mode, MAP_PRIVATE, fd, 0); + if(memblock == MAP_FAILED) { + amal_log_perror("map_file_read"); + goto cleanup; + } + + self->file_data = memblock; + self->file_size = file_stats.st_size; + self->fd = fd; + return 0; + + cleanup: + close(fd); + return -1; +} + +int mapped_file_deinit(MappedFile *self) { + int result_munmap; + int result_close; + + if(!self->file_data) + return 0; + + result_munmap = munmap((void*)self->file_data, self->file_size); + if(result_munmap == -1) + amal_log_perror("mapped_file_deinit"); + result_close = close(self->fd); + if(result_close == -1) + amal_log_perror("mapped_file_deinit"); + self->file_data = NULL; + self->file_size = 0; + self->fd = 0; + if(result_munmap == -1 || result_close == -1) + return -1; + else + return 0; +} \ No newline at end of file diff --git a/src/std/log.c b/src/std/log.c new file mode 100644 index 0000000..1fd0e4e --- /dev/null +++ b/src/std/log.c @@ -0,0 +1,76 @@ +#include "../../include/std/log.h" +#include "../../include/std/thread.h" +#include "../../include/std/misc.h" +#include +#include + +static amal_mutex mutex; +static bool mutex_initialized = bool_false; + +/* Safe to call multiple times */ +static void mutex_init() { + if(!mutex_initialized) { + amal_mutex_init(&mutex); + mutex_initialized = bool_true; + } +} + +amal_mutex* amal_log_get_mutex() { + mutex_init(); + return &mutex; +} + +void amal_log_debug(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Debug: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_error(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Error: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_info(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Info: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_warning(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Warning: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_perror(const char *prefix) { + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + perror(prefix); + ignore_result_int(amal_mutex_unlock(&mutex)); +} \ No newline at end of file diff --git a/src/std/mem.c b/src/std/mem.c new file mode 100644 index 0000000..5bdb73b --- /dev/null +++ b/src/std/mem.c @@ -0,0 +1,14 @@ +#include "../../include/std/mem.h" +#include + +void am_memcpy(void *dest, const void *src, usize size) { + memcpy(dest, src, size); +} + +bool am_memeql(const void *lhs, const void *rhs, usize size) { + return memcmp(lhs, rhs, size) == 0; +} + +void am_memset(void *dest, int value, usize size) { + memset(dest, value, size); +} \ No newline at end of file diff --git a/src/std/scoped_allocator.c b/src/std/scoped_allocator.c new file mode 100644 index 0000000..d53edad --- /dev/null +++ b/src/std/scoped_allocator.c @@ -0,0 +1,102 @@ +#include "../../include/std/scoped_allocator.h" +#include "../../include/std/alloc.h" +#include + +#define ALLOC_NODE_SIZE 4096 + +int scoped_allocator_node_init(ScopedAllocatorNode *self) { + self->data = NULL; + self->size = 0; + self->next = NULL; + return am_malloc(ALLOC_NODE_SIZE, (void**)&self->data); +} + +void scoped_allocator_node_deinit(ScopedAllocatorNode *self) { + am_free(self->data); + self->data = NULL; + self->size = 0; + if(self->next) { + scoped_allocator_node_deinit(self->next); + am_free(self->next); + self->next = NULL; + } +} + +int scoped_allocator_init(ScopedAllocator *self) { + return_if_error(scoped_allocator_node_init(&self->head)); + self->current = &self->head; + return buffer_init(&self->buffers, NULL); +} + +static void buffer_deinit(Buffer *self) { + am_free(self->data); + self->data = NULL; + self->size = 0; + self->capacity = 0; +} + +void scoped_allocator_deinit(ScopedAllocator *self) { + Buffer *buffer; + Buffer *buffer_end; + + scoped_allocator_node_deinit(&self->head); + self->current = NULL; + buffer = (Buffer*)self->buffers.data; + buffer_end = buffer + self->buffers.size / sizeof(Buffer); + while(buffer != buffer_end) { + buffer_deinit(buffer); + ++buffer; + } + buffer_deinit(&self->buffers); +} + +static CHECK_RESULT int scoped_allocator_ensure_capacity_for(ScopedAllocator *self, usize size) { + void *new_node; + new_node = NULL; + + if(self->current->size + size > ALLOC_NODE_SIZE) { + return_if_error(am_malloc(sizeof(ScopedAllocatorNode), &new_node)); + cleanup_if_error(scoped_allocator_node_init(new_node)); + self->current->next = new_node; + self->current = new_node; + } + return ALLOC_OK; + + cleanup: + if(new_node) + am_free(new_node); + return ALLOC_FAIL; +} + +static void* align_ptr_ceil(void *ptr, uintptr_t alignment) { + uintptr_t ptrval; + ptrval = (uintptr_t)ptr; + return (void*)((ptrval + alignment + 1) & ~(alignment - 1)); +} + +static usize align_ptr_ceil_offset(void *ptr, uintptr_t alignment) { + return (uintptr_t)align_ptr_ceil(ptr, alignment) - (uintptr_t)ptr; +} + +int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem) { + ScopedAllocatorNode *current; + usize alloc_size; + assert(self->current); + assert(size <= ALLOC_NODE_SIZE); + current = self->current; + + alloc_size = size + align_ptr_ceil_offset(self->current->data + self->current->size, 16); + return_if_error(scoped_allocator_ensure_capacity_for(self, alloc_size)); + /* Reallocated (new node created) */ + if(self->current != current) { + *mem = self->current->data; + } else { + *mem = align_ptr_ceil(self->current->data + self->current->size, 16); + } + self->current->size = size; + return 0; +} + +int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer) { + return buffer_append(&self->buffers, buffer, sizeof(Buffer)); +} \ No newline at end of file diff --git a/src/std/thread.c b/src/std/thread.c new file mode 100644 index 0000000..f717865 --- /dev/null +++ b/src/std/thread.c @@ -0,0 +1,140 @@ +#include "../../include/std/thread.h" +#include "../../include/std/log.h" +#include +#include +#include +#include +#include +#include +#include + +static int thread_type_to_system_thread_type(amal_thread_type thread_type) { + switch(thread_type) { + case AMAL_THREAD_JOINABLE: return PTHREAD_CREATE_JOINABLE; + case AMAL_THREAD_DETACHED: return PTHREAD_CREATE_DETACHED; + } + assert(bool_false); + return PTHREAD_CREATE_JOINABLE; +} + +int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata) { + int result; + self->name = name; + self->thread_id = 0; + self->cancellable = bool_false; + self->destroyable = bool_false; + if((result = pthread_attr_init(&self->thread_attr)) != 0) { + perror("amal_thread_create"); + return result; + } + self->destroyable = bool_true; + if((result = pthread_attr_setdetachstate(&self->thread_attr, thread_type_to_system_thread_type(thread_type))) != 0) { + perror("amal_thread_create"); + return result; + } + if((result = pthread_create(&self->thread_id, NULL, callback_func, userdata)) != 0) { + perror("amal_thread_create"); + return result; + } + self->cancellable = bool_true; + return 0; +} + +int amal_thread_deinit(amal_thread *self) { + int r1; + int r2; + r1 = 0; + r2 = 0; + + if(self->cancellable) { + r1 = pthread_cancel(self->thread_id); + self->cancellable = bool_false; + } + if(self->destroyable) { + r2 = pthread_attr_destroy(&self->thread_attr); + self->destroyable = bool_false; + } + return r1 != 0 ? r1 : r2; +} + +int amal_thread_detach(amal_thread *self) { + int result_err; + int thread_type; + if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0) + return result_err; + if(thread_type != PTHREAD_CREATE_DETACHED) + return AMAL_THREAD_ERR; + if((result_err = pthread_detach(self->thread_id)) != 0) + return result_err; + self->cancellable = bool_false; + return 0; +} + +int amal_thread_join(amal_thread *self, void **result) { + int result_err; + int thread_type; + if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0) + return result_err; + if(thread_type != PTHREAD_CREATE_JOINABLE) + return AMAL_THREAD_NOT_JOINABLE; + if((result_err = pthread_join(self->thread_id, result)) != 0) + return result_err == EINVAL ? AMAL_THREAD_NOT_JOINABLE : result_err; + self->cancellable = bool_false; + return 0; +} + +void amal_mutex_init(amal_mutex *self) { + pthread_mutex_init(&self->mutex, NULL); + /* TODO: pthread_mutex_destroy in amal_mutex_deinit */ + self->lock_identifier = NULL; +} + +static long amal_process_get_id() { + return getpid(); +} + +static long amal_thread_get_id() { + return syscall(SYS_gettid); +} + +int amal_mutex_lock(amal_mutex *self, const char *lock_identifier) { + int result; + result = pthread_mutex_lock(&self->mutex); + self->lock_identifier = lock_identifier; + if(result == 0 && self->lock_identifier) { + amal_log_debug("amal_mutex_lock: mutex locked by thread %lu (%s), identification: %s", + amal_thread_get_id(), + amal_thread_is_main() ? "main" : "not main", + self->lock_identifier ? self->lock_identifier : "none"); + } + return result; +} + +int amal_mutex_unlock(amal_mutex *self) { + int result; + const char *identifier; + identifier = self->lock_identifier; + result = pthread_mutex_unlock(&self->mutex); + if(result == 0 && identifier) { + amal_log_debug("amal_mutex_unlock: mutex unlocked by thread %lu (%s), identification: %s", + amal_thread_get_id(), + amal_thread_is_main() ? "main" : "not main", + identifier ? identifier : "none"); + } + return result; +} + +void amal_mutex_tryunlock(amal_mutex *self) { + int _; + (void)_; + _ = amal_mutex_unlock(self); +} + +bool amal_thread_is_main() { + /* TODO: This only works for linux, use equivalent functions on other platforms */ + return amal_thread_get_id() == amal_process_get_id(); +} + +int amal_get_usable_thread_count() { + return get_nprocs(); +} \ No newline at end of file diff --git a/src/tokenizer.c b/src/tokenizer.c index fda3e40..e85c952 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -1,5 +1,7 @@ #include "../include/tokenizer.h" -#include "../include/mem.h" +#include "../include/std/mem.h" +#include "../include/std/log.h" +#include "../include/std/thread.h" #include #include #include @@ -17,12 +19,13 @@ static int isAlphaDigit(int c) { return isAlpha(c) || isDigit(c); } -int tokenizer_init(Tokenizer *self, BufferView code) { +int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name) { assert(code.size <= INT_MAX); self->code = code; self->index = 0; self->prev_index = 0; self->line = 1; + self->code_name = code_name.data ? code_name : create_buffer_view("", 8); return 0; } @@ -52,9 +55,28 @@ static Token tokenizer_skip_whitespace(Tokenizer *self) { } } +/* Returns -1 if end of string can't be found */ +static int find_end_of_string(BufferView buf, int index) { + int c; + bool escape_quote; + escape_quote = bool_false; + + for(; index < (int)buf.size; ++index) { + c = buf.data[index]; + if(c == '\\') + escape_quote = !escape_quote; + else if(!escape_quote && c == '"') + return index; + else + escape_quote = bool_false; + } + return -1; +} + int tokenizer_next(Tokenizer *self, Token *token) { Token last_token; int c; + int result; last_token = tokenizer_skip_whitespace(self); if(last_token == TOK_END_OF_FILE) { @@ -85,6 +107,20 @@ int tokenizer_next(Tokenizer *self, Token *token) { *token = TOK_VAR; else *token = TOK_IDENTIFIER; + } else if(c == '"') { + int string_end; + ++self->index; + string_end = find_end_of_string(self->code, self->index); + if(string_end == -1) { + tokenizer_print_error(self, "String end not found. Did you forget '\"' or did you have a mismatch of number of '\"'?"); + return TOKENIZER_ERR; + } + + self->value.string.data = &self->code.data[self->index]; + self->value.string.size = string_end - self->index; + self->index = string_end + 1; + *token = TOK_STRING; + return TOKENIZER_OK; } else if(c == '=') { ++self->index; *token = TOK_EQUALS; @@ -100,8 +136,45 @@ int tokenizer_next(Tokenizer *self, Token *token) { } else if(c == '}') { ++self->index; *token = TOK_CLOSING_BRACE; + } else if(c == '@') { + const char *err_msg; + ++self->index; + if(self->index + 6 >= (int)self->code.size || !am_memeql(self->code.data + self->index, "import", 6)) { + err_msg = "Expected '@import(path)'"; + goto import_error; + } + self->index += 6; + + result = tokenizer_next(self, &last_token); + if(result != 0 || last_token != TOK_OPEN_PAREN) { + err_msg = "Expected '(' after @import"; + goto import_error; + } + + result = tokenizer_next(self, &last_token); + if(result != 0 || last_token != TOK_STRING) { + err_msg = "Expected string after @import("; + goto import_error; + } + + if(self->value.string.size == 0) { + err_msg = "Path in @import can't be empty"; + goto import_error; + } + + result = tokenizer_next(self, &last_token); + if(result != 0 || last_token != TOK_CLOSING_PAREN) { + err_msg = "Expected ')' after @import(path"; + goto import_error; + } + + *token = TOK_IMPORT; + return TOKENIZER_OK; + + import_error: + tokenizer_print_error(self, err_msg); + return TOKENIZER_ERR; } else { - /*self.printError("Unexpected symbol '{c}'", c);*/ tokenizer_print_error(self, "Unexpected symbol '%c'", c); return TOKENIZER_UNEXPECTED_TOKEN; } @@ -130,8 +203,8 @@ int tokenizer_consume_if(Tokenizer *self, Token expected_token, bool *result) { if(actual_token == expected_token) { *result = bool_true; } else { - /* No need to restore self.prev_index as it's updated on the next call to tokenizer_next */ self->index = index; + self->prev_index = index; self->line = line; *result = bool_false; } @@ -167,16 +240,20 @@ void tokenizer_print_error(Tokenizer *self, const char *fmt, ...) { int line_end; int prev_column; int i; + amal_mutex *mutex; + mutex = amal_log_get_mutex(); + ignore_result_int(amal_mutex_lock(mutex, "tokenizer_print_error")); va_start(args, fmt); line_start = tokenizer_get_start_of_line_from_index(self, self->prev_index); line_end = tokenizer_get_end_of_line_from_index(self, self->prev_index); prev_column = self->prev_index - line_start; - fprintf(stderr, "\x1b[1;37m%s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", "file.am", self->line, 1 + prev_column); + fprintf(stderr, "\x1b[1;37m%.*s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", (int)self->code_name.size, self->code_name.data, self->line, 1 + prev_column); vfprintf(stderr, fmt, args); fprintf(stderr, "\n%.*s\n", line_end - line_start, self->code.data + line_start); for(i = 0; i < prev_column; ++i) fprintf(stderr, " "); fprintf(stderr, "\x1b[1;32m^\x1b[0m\n"); va_end(args); + ignore_result_int(amal_mutex_unlock(mutex)); } \ No newline at end of file diff --git a/tests/io.amal b/tests/io.amal new file mode 100644 index 0000000..c75fa2d --- /dev/null +++ b/tests/io.amal @@ -0,0 +1,3 @@ +const puts = () { + +} \ No newline at end of file diff --git a/tests/main.amal b/tests/main.amal new file mode 100644 index 0000000..dde97f7 --- /dev/null +++ b/tests/main.amal @@ -0,0 +1,12 @@ +const io = @import("tests/io.amal") + +const main = () { + var hello = () { + + } + hello() +} + +const print = () { + +} -- cgit v1.2.3