From 76d85a10f6cda93eba29dad5372e8160af7289c8 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 27 Feb 2019 22:26:35 +0100 Subject: Use multiple threads to parse --- include/alloc.h | 14 ------------ include/ast.h | 19 +++++++++++----- include/buffer.h | 22 ------------------- include/buffer_view.h | 14 ------------ include/compiler.h | 31 ++++++++++++++++++++++++++ include/defs.h | 7 ++++++ include/mem.h | 10 --------- include/misc.h | 27 ----------------------- include/parser.h | 40 +++++++++++++++++++++++++++------ include/scoped_allocator.h | 31 -------------------------- include/std/alloc.h | 14 ++++++++++++ include/std/buffer.h | 23 +++++++++++++++++++ include/std/buffer_view.h | 14 ++++++++++++ include/std/defs.h | 7 ++++++ include/std/file.h | 27 +++++++++++++++++++++++ include/std/log.h | 13 +++++++++++ include/std/mem.h | 11 ++++++++++ include/std/misc.h | 33 ++++++++++++++++++++++++++++ include/std/scoped_allocator.h | 31 ++++++++++++++++++++++++++ include/std/thread.h | 50 ++++++++++++++++++++++++++++++++++++++++++ include/std/types.h | 20 +++++++++++++++++ include/tokenizer.h | 16 +++++++++----- include/types.h | 20 ----------------- 23 files changed, 339 insertions(+), 155 deletions(-) delete mode 100644 include/alloc.h delete mode 100644 include/buffer.h delete mode 100644 include/buffer_view.h create mode 100644 include/compiler.h create mode 100644 include/defs.h delete mode 100644 include/mem.h delete mode 100644 include/misc.h delete mode 100644 include/scoped_allocator.h create mode 100644 include/std/alloc.h create mode 100644 include/std/buffer.h create mode 100644 include/std/buffer_view.h create mode 100644 include/std/defs.h create mode 100644 include/std/file.h create mode 100644 include/std/log.h create mode 100644 include/std/mem.h create mode 100644 include/std/misc.h create mode 100644 include/std/scoped_allocator.h create mode 100644 include/std/thread.h create mode 100644 include/std/types.h delete mode 100644 include/types.h (limited to 'include') diff --git a/include/alloc.h b/include/alloc.h deleted file mode 100644 index 6809287..0000000 --- a/include/alloc.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef AMALGAM_ALLOC_H -#define AMALGAM_ALLOC_H - -#include "types.h" -#include "misc.h" - -#define ALLOC_OK 0 -#define ALLOC_FAIL -1 - -CHECK_RESULT int am_malloc(usize size, void **mem); -CHECK_RESULT int am_realloc(void *mem, usize new_size, void **new_mem); -void am_free(void *mem); - -#endif diff --git a/include/ast.h b/include/ast.h index f3580c0..529b3c8 100644 --- a/include/ast.h +++ b/include/ast.h @@ -1,26 +1,29 @@ #ifndef AMALGAM_AST_H #define AMALGAM_AST_H -#include "buffer_view.h" -#include "buffer.h" -#include "misc.h" -#include "scoped_allocator.h" +#include "std/buffer_view.h" +#include "std/buffer.h" +#include "std/misc.h" +#include "std/scoped_allocator.h" typedef struct FunctionDecl FunctionDecl; typedef struct FunctionCall FunctionCall; typedef struct LhsExpr LhsExpr; +typedef struct Import Import; typedef union { FunctionDecl *func_decl; FunctionCall *func_call; LhsExpr *lhs_expr; + Import *import; } AstValue; typedef enum { AST_NONE, AST_FUNCTION_DECL, AST_FUNCTION_CALL, - AST_LHS + AST_LHS, + AST_IMPORT } AstType; typedef struct { @@ -43,6 +46,10 @@ struct LhsExpr { Ast rhs_expr; }; +struct Import { + BufferView path; +}; + Ast ast_none(); CHECK_RESULT int funcdecl_init(FunctionDecl *self, ScopedAllocator *allocator); @@ -52,4 +59,6 @@ void funccall_init(FunctionCall *self, BufferView name); void lhsexpr_init(LhsExpr *self, int isConst, BufferView var_name); +void import_init(Import *self, BufferView path); + #endif diff --git a/include/buffer.h b/include/buffer.h deleted file mode 100644 index 5339108..0000000 --- a/include/buffer.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef AMALGAM_BUFFER_H -#define AMALGAM_BUFFER_H - -#include "types.h" -#include "misc.h" - -#define BUFFER_OK 0 -#define BUFFER_ALLOC_FAIL -1 - -typedef struct { - char* data; - usize size; - usize capacity; -} Buffer; - -struct ScopedAllocator; -CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator); - -CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size); -void* buffer_get(Buffer *self, usize index, usize type_size); - -#endif \ No newline at end of file diff --git a/include/buffer_view.h b/include/buffer_view.h deleted file mode 100644 index 4993dc2..0000000 --- a/include/buffer_view.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef AMALGAM_BUFFER_VIEW_H -#define AMALGAM_BUFFER_VIEW_H - -#include "types.h" - -typedef struct { - const char* data; - usize size; -} BufferView; - -BufferView create_buffer_view_null(); -BufferView create_buffer_view(const char *data, usize size); - -#endif diff --git a/include/compiler.h b/include/compiler.h new file mode 100644 index 0000000..691263a --- /dev/null +++ b/include/compiler.h @@ -0,0 +1,31 @@ +#ifndef AMALGAM_COMPILER_H +#define AMALGAM_COMPILER_H + +#include "std/misc.h" +#include "std/buffer.h" +#include "std/buffer_view.h" +#include "std/scoped_allocator.h" +#include "std/thread.h" +#include "defs.h" + +#define AMAL_COMPILER_OK 0 +/* General error */ +#define AMAL_COMPILER_ERR -1 + +struct amal_compiler { + ScopedAllocator allocator; + ScopedAllocator main_thread_allocator; + Buffer parsers; + Buffer queued_files; + ParserThreadData *threads; + int usable_thread_count; + bool started; + amal_mutex mutex; +}; + +CHECK_RESULT int amal_compiler_init(amal_compiler *self); +CHECK_RESULT int amal_compiler_deinit(amal_compiler *self); +CHECK_RESULT int amal_compiler_load_file(amal_compiler *self, BufferView filepath); +/* TODO: amal_compiler_unload_file */ + +#endif \ No newline at end of file diff --git a/include/defs.h b/include/defs.h new file mode 100644 index 0000000..9b754d8 --- /dev/null +++ b/include/defs.h @@ -0,0 +1,7 @@ +#ifndef AMALGAM_DEFS_H +#define AMALGAM_DEFS_H + +typedef struct ParserThreadData ParserThreadData; +typedef struct amal_compiler amal_compiler; + +#endif \ No newline at end of file diff --git a/include/mem.h b/include/mem.h deleted file mode 100644 index bad6353..0000000 --- a/include/mem.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef AMALGAM_MEM_H -#define AMALGAM_MEM_H - -#include "types.h" -#include "misc.h" - -void am_memcpy(void *dest, const void *src, usize size); -bool am_memeql(const void *lhs, const void *rhs, usize size); - -#endif \ No newline at end of file diff --git a/include/misc.h b/include/misc.h deleted file mode 100644 index 9cb2dde..0000000 --- a/include/misc.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef AMALGAM_MISC_H -#define AMALGAM_MISC_H - -#define return_if_error(result) \ -do { \ - int return_if_result; \ - return_if_result = (result); \ - if((return_if_result) != 0) \ - return return_if_result; \ -} while(0) - -#define cleanup_if_error(result) do { if((result) != 0) goto cleanup; } while(0) - -#if defined(__GNUC__) && __GNUC__ >= 4 -#define CHECK_RESULT __attribute__ ((warn_unused_result)) -#elif defined(_MSC_VER) && _MSC_VER >= 1700 -#define CHECK_RESULT _Check_return_ -#else -#define CHECK_RESULT -#endif - -typedef enum { - bool_false, - bool_true -} bool; - -#endif \ No newline at end of file diff --git a/include/parser.h b/include/parser.h index e90871f..9b41e5e 100644 --- a/include/parser.h +++ b/include/parser.h @@ -1,25 +1,51 @@ #ifndef AMALGAM_PARSER_H #define AMALGAM_PARSER_H -#include "buffer.h" -#include "buffer_view.h" +#include "std/buffer.h" +#include "std/buffer_view.h" +#include "std/scoped_allocator.h" +#include "std/thread.h" #include "tokenizer.h" -#include "scoped_allocator.h" +#include "defs.h" #define PARSER_OK 0 /* General error */ #define PARSER_ERR -1 #define PARSER_UNEXPECTED_TOKEN -2 +typedef enum { + PARSER_THREAD_STATUS_NEW, + PARSER_THREAD_STATUS_IDLE, + PARSER_THREAD_STATUS_RUNNING +} ParserThreadStatus; + +struct ParserThreadData { + amal_thread thread; + ParserThreadStatus status; + ScopedAllocator allocator; +}; + typedef struct { Tokenizer tokenizer; - ScopedAllocator allocator; Buffer ast_objects; + ScopedAllocator *allocator; /* borrowed. Copied from @compiler for faster access to allocator */ + amal_compiler *compiler; + bool started; } Parser; -CHECK_RESULT int parser_init(Parser *self); -void parser_deinit(Parser *self); +CHECK_RESULT int parser_thread_data_init(ParserThreadData *self); +CHECK_RESULT int parser_thread_data_deinit(ParserThreadData *self); +CHECK_RESULT int parser_thread_data_start(ParserThreadData *self, AmalThreadCallbackFunc callback_func, void *userdata); +CHECK_RESULT int parser_thread_data_join(ParserThreadData *self, void **result); + +CHECK_RESULT int parser_init(Parser *self, amal_compiler *compiler, ScopedAllocator *allocator); -CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer); +/* +@buffer_name will be the path to the file when using parser_parse_file and when parsing a buffer +you can name the buffer anything you want to identify it. +*/ +CHECK_RESULT int parser_parse_buffer(Parser *self, BufferView code_buffer, BufferView buffer_name); +/* Parses a file and the dependencies that are included using @import */ +CHECK_RESULT int parser_parse_file(Parser *self, BufferView filepath); #endif diff --git a/include/scoped_allocator.h b/include/scoped_allocator.h deleted file mode 100644 index fdaee2a..0000000 --- a/include/scoped_allocator.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef AMALGAM_SCOPED_ALLOCATOR_H -#define AMALGAM_SCOPED_ALLOCATOR_H - -#include "misc.h" -#include "types.h" -#include "buffer.h" - -typedef struct ScopedAllocatorNode ScopedAllocatorNode; -typedef struct ScopedAllocator ScopedAllocator; - -struct ScopedAllocatorNode { - char *data; - usize size; - ScopedAllocatorNode *next; -}; - -struct ScopedAllocator { - ScopedAllocatorNode head; - ScopedAllocatorNode *current; - Buffer buffers; -}; - -CHECK_RESULT int scoped_allocator_node_init(ScopedAllocatorNode *self); -void scoped_allocator_node_deinit(ScopedAllocatorNode *self); - -CHECK_RESULT int scoped_allocator_init(ScopedAllocator *self); -void scoped_allocator_deinit(ScopedAllocator *self); -CHECK_RESULT int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem); -CHECK_RESULT int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer); - -#endif \ No newline at end of file diff --git a/include/std/alloc.h b/include/std/alloc.h new file mode 100644 index 0000000..6809287 --- /dev/null +++ b/include/std/alloc.h @@ -0,0 +1,14 @@ +#ifndef AMALGAM_ALLOC_H +#define AMALGAM_ALLOC_H + +#include "types.h" +#include "misc.h" + +#define ALLOC_OK 0 +#define ALLOC_FAIL -1 + +CHECK_RESULT int am_malloc(usize size, void **mem); +CHECK_RESULT int am_realloc(void *mem, usize new_size, void **new_mem); +void am_free(void *mem); + +#endif diff --git a/include/std/buffer.h b/include/std/buffer.h new file mode 100644 index 0000000..ed87f29 --- /dev/null +++ b/include/std/buffer.h @@ -0,0 +1,23 @@ +#ifndef AMALGAM_BUFFER_H +#define AMALGAM_BUFFER_H + +#include "types.h" +#include "misc.h" + +#define BUFFER_OK 0 +#define BUFFER_ALLOC_FAIL -1 + +typedef struct { + char* data; + usize size; + usize capacity; +} Buffer; + +struct ScopedAllocator; +CHECK_RESULT int buffer_init(Buffer *self, struct ScopedAllocator *allocator); + +CHECK_RESULT int buffer_append(Buffer *self, void *data, usize size); +void* buffer_get(Buffer *self, usize index, usize type_size); +CHECK_RESULT int buffer_pop(Buffer *self, void *data, usize size); + +#endif \ No newline at end of file diff --git a/include/std/buffer_view.h b/include/std/buffer_view.h new file mode 100644 index 0000000..4993dc2 --- /dev/null +++ b/include/std/buffer_view.h @@ -0,0 +1,14 @@ +#ifndef AMALGAM_BUFFER_VIEW_H +#define AMALGAM_BUFFER_VIEW_H + +#include "types.h" + +typedef struct { + const char* data; + usize size; +} BufferView; + +BufferView create_buffer_view_null(); +BufferView create_buffer_view(const char *data, usize size); + +#endif diff --git a/include/std/defs.h b/include/std/defs.h new file mode 100644 index 0000000..1376e16 --- /dev/null +++ b/include/std/defs.h @@ -0,0 +1,7 @@ +#ifndef AMALGAM_STD_DEFS_H +#define AMALGAM_STD_DEFS_H + +typedef struct amal_thread amal_thread; +typedef struct amal_mutex amal_mutex; + +#endif \ No newline at end of file diff --git a/include/std/file.h b/include/std/file.h new file mode 100644 index 0000000..ac2abec --- /dev/null +++ b/include/std/file.h @@ -0,0 +1,27 @@ +#ifndef AMALGAM_FILE_H +#define AMALGAM_FILE_H + +#include "misc.h" +#include "types.h" + +/* Return bool_true if you want to continue scanning, otherwise return bool_false */ +typedef bool (*scan_dir_callback_func)(const char *filepath, int filepath_length, void *userdata); + +typedef struct { + const char *file_data; + usize file_size; + int fd; +} MappedFile; + +typedef enum { + MAPPED_FILE_READ, + MAPPED_FILE_WRITE, + MAPPED_FILE_READ_WRITE +} MappedFileMode; + +/* Hidden files (files starting with a dot) are skipped */ +CHECK_RESULT int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata); +CHECK_RESULT int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode); +CHECK_RESULT int mapped_file_deinit(MappedFile *self); + +#endif \ No newline at end of file diff --git a/include/std/log.h b/include/std/log.h new file mode 100644 index 0000000..6ce9e4c --- /dev/null +++ b/include/std/log.h @@ -0,0 +1,13 @@ +#ifndef AMALGAM_LOG_H +#define AMALGAM_LOG_H + +#include "defs.h" + +amal_mutex* amal_log_get_mutex(); +void amal_log_debug(const char *fmt, ...); +void amal_log_error(const char *fmt, ...); +void amal_log_info(const char *fmt, ...); +void amal_log_warning(const char *fmt, ...); +void amal_log_perror(const char *prefix); + +#endif \ No newline at end of file diff --git a/include/std/mem.h b/include/std/mem.h new file mode 100644 index 0000000..a5fe9b4 --- /dev/null +++ b/include/std/mem.h @@ -0,0 +1,11 @@ +#ifndef AMALGAM_MEM_H +#define AMALGAM_MEM_H + +#include "types.h" +#include "misc.h" + +void am_memcpy(void *dest, const void *src, usize size); +bool am_memeql(const void *lhs, const void *rhs, usize size); +void am_memset(void *dest, int value, usize size); + +#endif \ No newline at end of file diff --git a/include/std/misc.h b/include/std/misc.h new file mode 100644 index 0000000..3ac524a --- /dev/null +++ b/include/std/misc.h @@ -0,0 +1,33 @@ +#ifndef AMALGAM_MISC_H +#define AMALGAM_MISC_H + +#define return_if_error(result) \ +do { \ + int return_if_result; \ + return_if_result = (result); \ + if((return_if_result) != 0) \ + return return_if_result; \ +} while(0) + +#define cleanup_if_error(result) do { if((result) != 0) goto cleanup; } while(0) + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define CHECK_RESULT __attribute__ ((warn_unused_result)) +#elif defined(_MSC_VER) && _MSC_VER >= 1700 +#define CHECK_RESULT _Check_return_ +#else +#define CHECK_RESULT +#endif + +#define ignore_result_int(expr) (void)((expr)+1) + +typedef enum { + bool_false, + bool_true +} bool; + +#ifndef NULL +#define NULL ((void*)0) +#endif + +#endif \ No newline at end of file diff --git a/include/std/scoped_allocator.h b/include/std/scoped_allocator.h new file mode 100644 index 0000000..fdaee2a --- /dev/null +++ b/include/std/scoped_allocator.h @@ -0,0 +1,31 @@ +#ifndef AMALGAM_SCOPED_ALLOCATOR_H +#define AMALGAM_SCOPED_ALLOCATOR_H + +#include "misc.h" +#include "types.h" +#include "buffer.h" + +typedef struct ScopedAllocatorNode ScopedAllocatorNode; +typedef struct ScopedAllocator ScopedAllocator; + +struct ScopedAllocatorNode { + char *data; + usize size; + ScopedAllocatorNode *next; +}; + +struct ScopedAllocator { + ScopedAllocatorNode head; + ScopedAllocatorNode *current; + Buffer buffers; +}; + +CHECK_RESULT int scoped_allocator_node_init(ScopedAllocatorNode *self); +void scoped_allocator_node_deinit(ScopedAllocatorNode *self); + +CHECK_RESULT int scoped_allocator_init(ScopedAllocator *self); +void scoped_allocator_deinit(ScopedAllocator *self); +CHECK_RESULT int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem); +CHECK_RESULT int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer); + +#endif \ No newline at end of file diff --git a/include/std/thread.h b/include/std/thread.h new file mode 100644 index 0000000..dd09039 --- /dev/null +++ b/include/std/thread.h @@ -0,0 +1,50 @@ +#ifndef AMALGAM_THREAD_H +#define AMALGAM_THREAD_H + +#include "misc.h" +#include "types.h" +#include "defs.h" +#include + +typedef void* (AmalThreadCallbackFunc)(void *userdata); + +#define AMAL_THREAD_OK 0 +/* General error */ +#define AMAL_THREAD_ERR -1 +#define AMAL_THREAD_NOT_JOINABLE -23 + +struct amal_thread { + pthread_t thread_id; + pthread_attr_t thread_attr; + const char *name; + bool cancellable; + bool destroyable; +}; + +typedef enum { + AMAL_THREAD_JOINABLE, + AMAL_THREAD_DETACHED +} amal_thread_type; + +struct amal_mutex { + pthread_mutex_t mutex; + const char *lock_identifier; +}; + +CHECK_RESULT int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata); +/* Safe to call multiple times */ +CHECK_RESULT int amal_thread_deinit(amal_thread *self); +CHECK_RESULT int amal_thread_detach(amal_thread *self); +CHECK_RESULT int amal_thread_join(amal_thread *self, void **result); + +void amal_mutex_init(amal_mutex *self); +CHECK_RESULT int amal_mutex_lock(amal_mutex *self, const char *lock_identifier); +CHECK_RESULT int amal_mutex_unlock(amal_mutex *self); +void amal_mutex_tryunlock(amal_mutex *self); + +bool amal_thread_is_main(); + +/* Returns 0 if the number of usable threads is unknown */ +int amal_get_usable_thread_count(); + +#endif \ No newline at end of file diff --git a/include/std/types.h b/include/std/types.h new file mode 100644 index 0000000..68e2d0f --- /dev/null +++ b/include/std/types.h @@ -0,0 +1,20 @@ +#ifndef AMALGAM_TYPES_H +#define AMALGAM_TYPES_H + +#include +#include + +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +typedef ptrdiff_t isize; +typedef size_t usize; + +#endif diff --git a/include/tokenizer.h b/include/tokenizer.h index 9584542..e79f070 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -1,11 +1,13 @@ #ifndef AMALGAM_TOKENIZER_H #define AMALGAM_TOKENIZER_H -#include "buffer_view.h" -#include "misc.h" +#include "std/buffer_view.h" +#include "std/misc.h" #define TOKENIZER_OK 0 -#define TOKENIZER_UNEXPECTED_TOKEN -1 +/* General error */ +#define TOKENIZER_ERR -1 +#define TOKENIZER_UNEXPECTED_TOKEN -2 typedef enum { TOK_NONE, @@ -13,11 +15,13 @@ typedef enum { TOK_IDENTIFIER, TOK_CONST, TOK_VAR, + TOK_STRING, TOK_EQUALS, TOK_OPEN_PAREN, TOK_CLOSING_PAREN, TOK_OPEN_BRACE, - TOK_CLOSING_BRACE + TOK_CLOSING_BRACE, + TOK_IMPORT } Token; typedef struct { @@ -25,13 +29,15 @@ typedef struct { int index; int prev_index; int line; + BufferView code_name; union { BufferView identifier; + BufferView string; } value; } Tokenizer; -CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code); +CHECK_RESULT int tokenizer_init(Tokenizer *self, BufferView code, BufferView code_name); CHECK_RESULT int tokenizer_next(Tokenizer *self, Token *token); CHECK_RESULT int tokenizer_accept(Tokenizer *self, Token expected_token); diff --git a/include/types.h b/include/types.h deleted file mode 100644 index 68e2d0f..0000000 --- a/include/types.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef AMALGAM_TYPES_H -#define AMALGAM_TYPES_H - -#include -#include - -typedef int8_t i8; -typedef int16_t i16; -typedef int32_t i32; -typedef int64_t i64; - -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -typedef ptrdiff_t isize; -typedef size_t usize; - -#endif -- cgit v1.2.3