From 76d85a10f6cda93eba29dad5372e8160af7289c8 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 27 Feb 2019 22:26:35 +0100 Subject: Use multiple threads to parse --- src/std/alloc.c | 25 ++++++++ src/std/buffer.c | 62 ++++++++++++++++++ src/std/buffer_view.c | 15 +++++ src/std/file.c | 152 +++++++++++++++++++++++++++++++++++++++++++++ src/std/log.c | 76 +++++++++++++++++++++++ src/std/mem.c | 14 +++++ src/std/scoped_allocator.c | 102 ++++++++++++++++++++++++++++++ src/std/thread.c | 140 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 586 insertions(+) create mode 100644 src/std/alloc.c create mode 100644 src/std/buffer.c create mode 100644 src/std/buffer_view.c create mode 100644 src/std/file.c create mode 100644 src/std/log.c create mode 100644 src/std/mem.c create mode 100644 src/std/scoped_allocator.c create mode 100644 src/std/thread.c (limited to 'src/std') diff --git a/src/std/alloc.c b/src/std/alloc.c new file mode 100644 index 0000000..93dcb98 --- /dev/null +++ b/src/std/alloc.c @@ -0,0 +1,25 @@ +#include "../../include/std/alloc.h" +#include + +int am_malloc(usize size, void **mem) { + void *allocated_data = malloc(size); + if(!allocated_data) + return ALLOC_FAIL; + + *mem = allocated_data; + return ALLOC_OK; +} + +int am_realloc(void *mem, usize new_size, void **new_mem) { + void *new_allocated_data = realloc(mem, new_size); + if(!new_allocated_data) + return ALLOC_FAIL; + + *new_mem = new_allocated_data; + return ALLOC_OK; +} + +void am_free(void *mem) { + free(mem); +} + diff --git a/src/std/buffer.c b/src/std/buffer.c new file mode 100644 index 0000000..f32c515 --- /dev/null +++ b/src/std/buffer.c @@ -0,0 +1,62 @@ +#include "../../include/std/buffer.h" +#include "../../include/std/alloc.h" +#include "../../include/std/mem.h" +#include "../../include/std/scoped_allocator.h" +#include + +int buffer_init(Buffer *self, struct ScopedAllocator *allocator) { + self->data = NULL; + self->size = 0; + self->capacity = 0; + if(allocator) + return scoped_allocator_add_buffer(allocator, self); + return 0; +} + +static CHECK_RESULT int buffer_ensure_capacity(Buffer *self, usize new_capacity) { + usize capacity; + void *new_mem; + int alloc_result; + + if(self->capacity >= new_capacity) + return BUFFER_OK; + + capacity = self->capacity; + if(capacity == 0) { + capacity = new_capacity; + } else { + while(capacity < new_capacity) { + capacity *= 1.5; + } + } + + alloc_result = am_realloc(self->data, capacity, &new_mem); + if(alloc_result != ALLOC_OK) + return BUFFER_ALLOC_FAIL; + + self->data = new_mem; + self->capacity = capacity; + return BUFFER_OK; +} + +int buffer_append(Buffer *self, void *data, usize size) { + return_if_error(buffer_ensure_capacity(self, self->size + size)); + am_memcpy(self->data + self->size, data, size); + self->size += size; + return BUFFER_OK; +} + +void* buffer_get(Buffer *self, usize index, usize type_size) { + usize real_index; + real_index = index * type_size; + assert(real_index < self->size); + return &self->data[real_index]; +} + +int buffer_pop(Buffer *self, void *data, usize size) { + if(size > self->size) + return -1; + am_memcpy(data, &self->data[self->size - size], size); + self->size -= size; + return 0; +} \ No newline at end of file diff --git a/src/std/buffer_view.c b/src/std/buffer_view.c new file mode 100644 index 0000000..977626c --- /dev/null +++ b/src/std/buffer_view.c @@ -0,0 +1,15 @@ +#include "../../include/std/buffer_view.h" + +BufferView create_buffer_view_null() { + BufferView buffer_view; + buffer_view.data = NULL; + buffer_view.size = 0; + return buffer_view; +} + +BufferView create_buffer_view(const char *data, usize size) { + BufferView buffer_view; + buffer_view.data = data; + buffer_view.size = size; + return buffer_view; +} \ No newline at end of file diff --git a/src/std/file.c b/src/std/file.c new file mode 100644 index 0000000..7701be4 --- /dev/null +++ b/src/std/file.c @@ -0,0 +1,152 @@ +#include "../../include/std/file.h" +#include "../../include/std/mem.h" +#include "../../include/std/log.h" +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define SCANDIR_PATH_LENGTH 4096 + +static int scan_dir_recursive_internal(char *dir_path, int path_length, scan_dir_callback_func callback_func, void *userdata) { + DIR *dir; + struct dirent *entry; + struct stat file_stats; + int filename_length; + int path_length_start; + + path_length_start = path_length; + if((dir = opendir(dir_path)) == NULL) { + amal_log_perror("scan_dir_recursive"); + return errno; + } + + while((entry = readdir(dir)) != NULL) { + /* + Skip hidden files. Name will always at least include null terminator + so it's ok to not check filename length. And file names are never empty, right??? + */ + if(entry->d_name[0] == '.') + continue; + + /* Reset path concat, because it's overwritten by recursive call to scan dir */ + path_length = path_length_start; + filename_length = strlen(entry->d_name); + /* current path '/' filename '\0' */ + if(path_length + 1 + filename_length + 1 > SCANDIR_PATH_LENGTH) { + amal_log_error("Filepath too long. Trying to append %s to %s", entry->d_name, dir_path); + return -1; + } + + /* Replace '\0' with '/' */ + dir_path[path_length] = '/'; + ++path_length; + am_memcpy(&dir_path[path_length], entry->d_name, filename_length); + path_length += filename_length; + dir_path[path_length] = '\0'; + + if(stat(dir_path, &file_stats) == -1) { + amal_log_perror("scan_dir_recursive_internal"); + goto cleanup; + } + + if(S_ISDIR(file_stats.st_mode)) { + cleanup_if_error(scan_dir_recursive_internal(dir_path, path_length, callback_func, userdata)); + } else if(S_ISREG(file_stats.st_mode)) { + if(!callback_func(dir_path, path_length, userdata)) + goto cleanup_noerr; + } else if(S_ISLNK(file_stats.st_mode)) { + amal_log_warning("Symlink ignored: %s", entry->d_name); + } else { + /* Ignore block devices and all other types of files */ + } + } + + cleanup_noerr: + closedir(dir); + return 0; + + cleanup: + closedir(dir); + return -1; +} + +int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata) { + char scandir_path[SCANDIR_PATH_LENGTH]; + int dir_path_length = strlen(dir_path) + 1; /* include null terminator */ + am_memcpy(scandir_path, dir_path, dir_path_length); + return scan_dir_recursive_internal(scandir_path, dir_path_length - 1, callback_func, userdata); +} + +static int mapped_file_mode_to_system_file_mode(MappedFileMode file_mode) { + switch(file_mode) { + case MAPPED_FILE_READ: return PROT_READ; + case MAPPED_FILE_WRITE: return PROT_WRITE; + case MAPPED_FILE_READ_WRITE: return PROT_READ | PROT_WRITE; + default: + assert(bool_false); + return PROT_READ; + } +} + +int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode) { + struct stat file_stats; + int fd; + const char *memblock; + int map_file_system_mode; + + self->file_data = NULL; + self->file_size = 0; + self->fd = -1; + + fd = open(filepath, O_RDONLY); + if(fd == -1) return errno; + if(fstat(fd, &file_stats) == -1) { + amal_log_perror("map_file_read"); + goto cleanup; + } + + map_file_system_mode = mapped_file_mode_to_system_file_mode(file_mode); + memblock = mmap(NULL, file_stats.st_size, map_file_system_mode, MAP_PRIVATE, fd, 0); + if(memblock == MAP_FAILED) { + amal_log_perror("map_file_read"); + goto cleanup; + } + + self->file_data = memblock; + self->file_size = file_stats.st_size; + self->fd = fd; + return 0; + + cleanup: + close(fd); + return -1; +} + +int mapped_file_deinit(MappedFile *self) { + int result_munmap; + int result_close; + + if(!self->file_data) + return 0; + + result_munmap = munmap((void*)self->file_data, self->file_size); + if(result_munmap == -1) + amal_log_perror("mapped_file_deinit"); + result_close = close(self->fd); + if(result_close == -1) + amal_log_perror("mapped_file_deinit"); + self->file_data = NULL; + self->file_size = 0; + self->fd = 0; + if(result_munmap == -1 || result_close == -1) + return -1; + else + return 0; +} \ No newline at end of file diff --git a/src/std/log.c b/src/std/log.c new file mode 100644 index 0000000..1fd0e4e --- /dev/null +++ b/src/std/log.c @@ -0,0 +1,76 @@ +#include "../../include/std/log.h" +#include "../../include/std/thread.h" +#include "../../include/std/misc.h" +#include +#include + +static amal_mutex mutex; +static bool mutex_initialized = bool_false; + +/* Safe to call multiple times */ +static void mutex_init() { + if(!mutex_initialized) { + amal_mutex_init(&mutex); + mutex_initialized = bool_true; + } +} + +amal_mutex* amal_log_get_mutex() { + mutex_init(); + return &mutex; +} + +void amal_log_debug(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Debug: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_error(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Error: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_info(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Info: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_warning(const char *fmt, ...) { + va_list args; + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + fprintf(stderr, "Warning: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + ignore_result_int(amal_mutex_unlock(&mutex)); +} + +void amal_log_perror(const char *prefix) { + mutex_init(); + ignore_result_int(amal_mutex_lock(&mutex, NULL)); + perror(prefix); + ignore_result_int(amal_mutex_unlock(&mutex)); +} \ No newline at end of file diff --git a/src/std/mem.c b/src/std/mem.c new file mode 100644 index 0000000..5bdb73b --- /dev/null +++ b/src/std/mem.c @@ -0,0 +1,14 @@ +#include "../../include/std/mem.h" +#include + +void am_memcpy(void *dest, const void *src, usize size) { + memcpy(dest, src, size); +} + +bool am_memeql(const void *lhs, const void *rhs, usize size) { + return memcmp(lhs, rhs, size) == 0; +} + +void am_memset(void *dest, int value, usize size) { + memset(dest, value, size); +} \ No newline at end of file diff --git a/src/std/scoped_allocator.c b/src/std/scoped_allocator.c new file mode 100644 index 0000000..d53edad --- /dev/null +++ b/src/std/scoped_allocator.c @@ -0,0 +1,102 @@ +#include "../../include/std/scoped_allocator.h" +#include "../../include/std/alloc.h" +#include + +#define ALLOC_NODE_SIZE 4096 + +int scoped_allocator_node_init(ScopedAllocatorNode *self) { + self->data = NULL; + self->size = 0; + self->next = NULL; + return am_malloc(ALLOC_NODE_SIZE, (void**)&self->data); +} + +void scoped_allocator_node_deinit(ScopedAllocatorNode *self) { + am_free(self->data); + self->data = NULL; + self->size = 0; + if(self->next) { + scoped_allocator_node_deinit(self->next); + am_free(self->next); + self->next = NULL; + } +} + +int scoped_allocator_init(ScopedAllocator *self) { + return_if_error(scoped_allocator_node_init(&self->head)); + self->current = &self->head; + return buffer_init(&self->buffers, NULL); +} + +static void buffer_deinit(Buffer *self) { + am_free(self->data); + self->data = NULL; + self->size = 0; + self->capacity = 0; +} + +void scoped_allocator_deinit(ScopedAllocator *self) { + Buffer *buffer; + Buffer *buffer_end; + + scoped_allocator_node_deinit(&self->head); + self->current = NULL; + buffer = (Buffer*)self->buffers.data; + buffer_end = buffer + self->buffers.size / sizeof(Buffer); + while(buffer != buffer_end) { + buffer_deinit(buffer); + ++buffer; + } + buffer_deinit(&self->buffers); +} + +static CHECK_RESULT int scoped_allocator_ensure_capacity_for(ScopedAllocator *self, usize size) { + void *new_node; + new_node = NULL; + + if(self->current->size + size > ALLOC_NODE_SIZE) { + return_if_error(am_malloc(sizeof(ScopedAllocatorNode), &new_node)); + cleanup_if_error(scoped_allocator_node_init(new_node)); + self->current->next = new_node; + self->current = new_node; + } + return ALLOC_OK; + + cleanup: + if(new_node) + am_free(new_node); + return ALLOC_FAIL; +} + +static void* align_ptr_ceil(void *ptr, uintptr_t alignment) { + uintptr_t ptrval; + ptrval = (uintptr_t)ptr; + return (void*)((ptrval + alignment + 1) & ~(alignment - 1)); +} + +static usize align_ptr_ceil_offset(void *ptr, uintptr_t alignment) { + return (uintptr_t)align_ptr_ceil(ptr, alignment) - (uintptr_t)ptr; +} + +int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem) { + ScopedAllocatorNode *current; + usize alloc_size; + assert(self->current); + assert(size <= ALLOC_NODE_SIZE); + current = self->current; + + alloc_size = size + align_ptr_ceil_offset(self->current->data + self->current->size, 16); + return_if_error(scoped_allocator_ensure_capacity_for(self, alloc_size)); + /* Reallocated (new node created) */ + if(self->current != current) { + *mem = self->current->data; + } else { + *mem = align_ptr_ceil(self->current->data + self->current->size, 16); + } + self->current->size = size; + return 0; +} + +int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer) { + return buffer_append(&self->buffers, buffer, sizeof(Buffer)); +} \ No newline at end of file diff --git a/src/std/thread.c b/src/std/thread.c new file mode 100644 index 0000000..f717865 --- /dev/null +++ b/src/std/thread.c @@ -0,0 +1,140 @@ +#include "../../include/std/thread.h" +#include "../../include/std/log.h" +#include +#include +#include +#include +#include +#include +#include + +static int thread_type_to_system_thread_type(amal_thread_type thread_type) { + switch(thread_type) { + case AMAL_THREAD_JOINABLE: return PTHREAD_CREATE_JOINABLE; + case AMAL_THREAD_DETACHED: return PTHREAD_CREATE_DETACHED; + } + assert(bool_false); + return PTHREAD_CREATE_JOINABLE; +} + +int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata) { + int result; + self->name = name; + self->thread_id = 0; + self->cancellable = bool_false; + self->destroyable = bool_false; + if((result = pthread_attr_init(&self->thread_attr)) != 0) { + perror("amal_thread_create"); + return result; + } + self->destroyable = bool_true; + if((result = pthread_attr_setdetachstate(&self->thread_attr, thread_type_to_system_thread_type(thread_type))) != 0) { + perror("amal_thread_create"); + return result; + } + if((result = pthread_create(&self->thread_id, NULL, callback_func, userdata)) != 0) { + perror("amal_thread_create"); + return result; + } + self->cancellable = bool_true; + return 0; +} + +int amal_thread_deinit(amal_thread *self) { + int r1; + int r2; + r1 = 0; + r2 = 0; + + if(self->cancellable) { + r1 = pthread_cancel(self->thread_id); + self->cancellable = bool_false; + } + if(self->destroyable) { + r2 = pthread_attr_destroy(&self->thread_attr); + self->destroyable = bool_false; + } + return r1 != 0 ? r1 : r2; +} + +int amal_thread_detach(amal_thread *self) { + int result_err; + int thread_type; + if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0) + return result_err; + if(thread_type != PTHREAD_CREATE_DETACHED) + return AMAL_THREAD_ERR; + if((result_err = pthread_detach(self->thread_id)) != 0) + return result_err; + self->cancellable = bool_false; + return 0; +} + +int amal_thread_join(amal_thread *self, void **result) { + int result_err; + int thread_type; + if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0) + return result_err; + if(thread_type != PTHREAD_CREATE_JOINABLE) + return AMAL_THREAD_NOT_JOINABLE; + if((result_err = pthread_join(self->thread_id, result)) != 0) + return result_err == EINVAL ? AMAL_THREAD_NOT_JOINABLE : result_err; + self->cancellable = bool_false; + return 0; +} + +void amal_mutex_init(amal_mutex *self) { + pthread_mutex_init(&self->mutex, NULL); + /* TODO: pthread_mutex_destroy in amal_mutex_deinit */ + self->lock_identifier = NULL; +} + +static long amal_process_get_id() { + return getpid(); +} + +static long amal_thread_get_id() { + return syscall(SYS_gettid); +} + +int amal_mutex_lock(amal_mutex *self, const char *lock_identifier) { + int result; + result = pthread_mutex_lock(&self->mutex); + self->lock_identifier = lock_identifier; + if(result == 0 && self->lock_identifier) { + amal_log_debug("amal_mutex_lock: mutex locked by thread %lu (%s), identification: %s", + amal_thread_get_id(), + amal_thread_is_main() ? "main" : "not main", + self->lock_identifier ? self->lock_identifier : "none"); + } + return result; +} + +int amal_mutex_unlock(amal_mutex *self) { + int result; + const char *identifier; + identifier = self->lock_identifier; + result = pthread_mutex_unlock(&self->mutex); + if(result == 0 && identifier) { + amal_log_debug("amal_mutex_unlock: mutex unlocked by thread %lu (%s), identification: %s", + amal_thread_get_id(), + amal_thread_is_main() ? "main" : "not main", + identifier ? identifier : "none"); + } + return result; +} + +void amal_mutex_tryunlock(amal_mutex *self) { + int _; + (void)_; + _ = amal_mutex_unlock(self); +} + +bool amal_thread_is_main() { + /* TODO: This only works for linux, use equivalent functions on other platforms */ + return amal_thread_get_id() == amal_process_get_id(); +} + +int amal_get_usable_thread_count() { + return get_nprocs(); +} \ No newline at end of file -- cgit v1.2.3