Use multiple threads to parse

8 files changed, 586 insertions, 0 deletions

diff --git a/src/std/alloc.c b/src/std/alloc.c
new file mode 100644
index 0000000..93dcb98
--- /dev/null
+++ b/src/std/alloc.c
@@ -0,0 +1,25 @@
+#include "../../include/std/alloc.h"
+#include <stdlib.h>
+
+int am_malloc(usize size, void **mem) {
+    void *allocated_data = malloc(size);
+    if(!allocated_data)
+        return ALLOC_FAIL;
+    
+    *mem = allocated_data;
+    return ALLOC_OK;
+}
+
+int am_realloc(void *mem, usize new_size, void **new_mem) {
+    void *new_allocated_data = realloc(mem, new_size);
+    if(!new_allocated_data)
+        return ALLOC_FAIL;
+    
+    *new_mem = new_allocated_data;
+    return ALLOC_OK;
+}
+
+void am_free(void *mem) {
+    free(mem);
+}
+
diff --git a/src/std/buffer.c b/src/std/buffer.c
new file mode 100644
index 0000000..f32c515
--- /dev/null
+++ b/src/std/buffer.c
@@ -0,0 +1,62 @@
+#include "../../include/std/buffer.h"
+#include "../../include/std/alloc.h"
+#include "../../include/std/mem.h"
+#include "../../include/std/scoped_allocator.h"
+#include <assert.h>
+
+int buffer_init(Buffer *self, struct ScopedAllocator *allocator) {
+    self->data = NULL;
+    self->size = 0;
+    self->capacity = 0;
+    if(allocator)
+        return scoped_allocator_add_buffer(allocator, self);
+    return 0;
+}
+
+static CHECK_RESULT int buffer_ensure_capacity(Buffer *self, usize new_capacity) {
+    usize capacity;
+    void *new_mem;
+    int alloc_result;
+
+    if(self->capacity >= new_capacity)
+        return BUFFER_OK;
+    
+    capacity = self->capacity;
+    if(capacity == 0) {
+        capacity = new_capacity;
+    } else {
+        while(capacity < new_capacity) {
+            capacity *= 1.5;
+        }
+    }
+
+    alloc_result = am_realloc(self->data, capacity, &new_mem);
+    if(alloc_result != ALLOC_OK)
+        return BUFFER_ALLOC_FAIL;
+    
+    self->data = new_mem;
+    self->capacity = capacity;
+    return BUFFER_OK;
+}
+
+int buffer_append(Buffer *self, void *data, usize size) {
+    return_if_error(buffer_ensure_capacity(self, self->size + size));
+    am_memcpy(self->data + self->size, data, size);
+    self->size += size;
+    return BUFFER_OK; 
+}
+
+void* buffer_get(Buffer *self, usize index, usize type_size) {
+    usize real_index;
+    real_index = index * type_size;
+    assert(real_index < self->size);
+    return &self->data[real_index];
+}
+
+int buffer_pop(Buffer *self, void *data, usize size) {
+    if(size > self->size)
+        return -1;
+    am_memcpy(data, &self->data[self->size - size], size);
+    self->size -= size;
+    return 0;
+}
\ No newline at end of file
diff --git a/src/std/buffer_view.c b/src/std/buffer_view.c
new file mode 100644
index 0000000..977626c
--- /dev/null
+++ b/src/std/buffer_view.c
@@ -0,0 +1,15 @@
+#include "../../include/std/buffer_view.h"
+
+BufferView create_buffer_view_null() {
+    BufferView buffer_view;
+    buffer_view.data = NULL;
+    buffer_view.size = 0;
+    return buffer_view;
+}
+
+BufferView create_buffer_view(const char *data, usize size) {
+    BufferView buffer_view;
+    buffer_view.data = data;
+    buffer_view.size = size;
+    return buffer_view;
+}
\ No newline at end of file
diff --git a/src/std/file.c b/src/std/file.c
new file mode 100644
index 0000000..7701be4
--- /dev/null
+++ b/src/std/file.c
@@ -0,0 +1,152 @@
+#include "../../include/std/file.h"
+#include "../../include/std/mem.h"
+#include "../../include/std/log.h"
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#define SCANDIR_PATH_LENGTH 4096
+
+static int scan_dir_recursive_internal(char *dir_path, int path_length, scan_dir_callback_func callback_func, void *userdata) {
+    DIR *dir;
+    struct dirent *entry;
+    struct stat file_stats;
+    int filename_length;
+    int path_length_start;
+
+    path_length_start = path_length;
+    if((dir = opendir(dir_path)) == NULL) {
+        amal_log_perror("scan_dir_recursive");
+        return errno;
+    }
+
+    while((entry = readdir(dir)) != NULL) {
+        /*
+        Skip hidden files. Name will always at least include null terminator
+        so it's ok to not check filename length. And file names are never empty, right???
+        */
+        if(entry->d_name[0] == '.')
+            continue;
+
+        /* Reset path concat, because it's overwritten by recursive call to scan dir */
+        path_length = path_length_start;
+        filename_length = strlen(entry->d_name);
+        /* current path '/' filename '\0' */
+        if(path_length + 1 + filename_length + 1 > SCANDIR_PATH_LENGTH) {
+            amal_log_error("Filepath too long. Trying to append %s to %s", entry->d_name, dir_path);
+            return -1;
+        }
+
+        /* Replace '\0' with '/' */
+        dir_path[path_length] = '/';
+        ++path_length;
+        am_memcpy(&dir_path[path_length], entry->d_name, filename_length);
+        path_length += filename_length;
+        dir_path[path_length] = '\0';
+        
+        if(stat(dir_path, &file_stats) == -1) {
+            amal_log_perror("scan_dir_recursive_internal");
+            goto cleanup;
+        }
+
+        if(S_ISDIR(file_stats.st_mode)) {
+            cleanup_if_error(scan_dir_recursive_internal(dir_path, path_length, callback_func, userdata));
+        } else if(S_ISREG(file_stats.st_mode)) {
+            if(!callback_func(dir_path, path_length, userdata))
+                goto cleanup_noerr;
+        } else if(S_ISLNK(file_stats.st_mode)) {
+            amal_log_warning("Symlink ignored: %s", entry->d_name);
+        } else {
+            /* Ignore block devices and all other types of files */
+        }
+    }
+
+    cleanup_noerr:
+    closedir(dir);
+    return 0;
+
+    cleanup:
+    closedir(dir);
+    return -1;
+}
+
+int scan_dir_recursive(const char *dir_path, scan_dir_callback_func callback_func, void *userdata) {
+    char scandir_path[SCANDIR_PATH_LENGTH];
+    int dir_path_length = strlen(dir_path) + 1; /* include null terminator */
+    am_memcpy(scandir_path, dir_path, dir_path_length);
+    return scan_dir_recursive_internal(scandir_path, dir_path_length - 1, callback_func, userdata);
+}
+
+static int mapped_file_mode_to_system_file_mode(MappedFileMode file_mode) {
+    switch(file_mode) {
+        case MAPPED_FILE_READ:          return PROT_READ;
+        case MAPPED_FILE_WRITE:         return PROT_WRITE;
+        case MAPPED_FILE_READ_WRITE:    return PROT_READ | PROT_WRITE;
+        default:
+            assert(bool_false);
+            return PROT_READ;
+    }
+}
+
+int mapped_file_init(MappedFile *self, const char *filepath, MappedFileMode file_mode) {
+    struct stat file_stats;
+    int fd;
+    const char *memblock;
+    int map_file_system_mode;
+
+    self->file_data = NULL;
+    self->file_size = 0;
+    self->fd = -1;
+
+    fd = open(filepath, O_RDONLY);
+    if(fd == -1) return errno;
+    if(fstat(fd, &file_stats) == -1) {
+        amal_log_perror("map_file_read");
+        goto cleanup;
+    }
+
+    map_file_system_mode = mapped_file_mode_to_system_file_mode(file_mode);
+    memblock = mmap(NULL, file_stats.st_size, map_file_system_mode, MAP_PRIVATE, fd, 0);
+    if(memblock == MAP_FAILED) {
+        amal_log_perror("map_file_read");
+        goto cleanup;
+    }
+
+    self->file_data = memblock;
+    self->file_size = file_stats.st_size;
+    self->fd = fd;
+    return 0;
+
+    cleanup:
+    close(fd);
+    return -1;
+}
+
+int mapped_file_deinit(MappedFile *self) {
+    int result_munmap;
+    int result_close;
+
+    if(!self->file_data)
+        return 0;
+    
+    result_munmap = munmap((void*)self->file_data, self->file_size);
+    if(result_munmap == -1)
+        amal_log_perror("mapped_file_deinit");
+    result_close = close(self->fd);
+    if(result_close == -1)
+        amal_log_perror("mapped_file_deinit");
+    self->file_data = NULL;
+    self->file_size = 0;
+    self->fd = 0;
+    if(result_munmap == -1 || result_close == -1)
+        return -1;
+    else
+        return 0;
+}
\ No newline at end of file
diff --git a/src/std/log.c b/src/std/log.c
new file mode 100644
index 0000000..1fd0e4e
--- /dev/null
+++ b/src/std/log.c
@@ -0,0 +1,76 @@
+#include "../../include/std/log.h"
+#include "../../include/std/thread.h"
+#include "../../include/std/misc.h"
+#include <stdio.h>
+#include <stdarg.h>
+
+static amal_mutex mutex;
+static bool mutex_initialized = bool_false;
+
+/* Safe to call multiple times */
+static void mutex_init() {
+    if(!mutex_initialized) {
+        amal_mutex_init(&mutex);
+        mutex_initialized = bool_true;
+    }
+}
+
+amal_mutex* amal_log_get_mutex() {
+    mutex_init();
+    return &mutex;
+}
+
+void amal_log_debug(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Debug: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_error(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Error: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_info(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Info: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_warning(const char *fmt, ...) {
+    va_list args;
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    fprintf(stderr, "Warning: ");
+    va_start(args, fmt);
+    vfprintf(stderr, fmt, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
+
+void amal_log_perror(const char *prefix) {
+    mutex_init();
+    ignore_result_int(amal_mutex_lock(&mutex, NULL));
+    perror(prefix);
+    ignore_result_int(amal_mutex_unlock(&mutex));
+}
\ No newline at end of file
diff --git a/src/std/mem.c b/src/std/mem.c
new file mode 100644
index 0000000..5bdb73b
--- /dev/null
+++ b/src/std/mem.c
@@ -0,0 +1,14 @@
+#include "../../include/std/mem.h"
+#include <string.h>
+
+void am_memcpy(void *dest, const void *src, usize size) {
+    memcpy(dest, src, size);
+}
+
+bool am_memeql(const void *lhs, const void *rhs, usize size) {
+    return memcmp(lhs, rhs, size) == 0;
+}
+
+void am_memset(void *dest, int value, usize size) {
+    memset(dest, value, size);
+}
\ No newline at end of file
diff --git a/src/std/scoped_allocator.c b/src/std/scoped_allocator.c
new file mode 100644
index 0000000..d53edad
--- /dev/null
+++ b/src/std/scoped_allocator.c
@@ -0,0 +1,102 @@
+#include "../../include/std/scoped_allocator.h"
+#include "../../include/std/alloc.h"
+#include <assert.h>
+
+#define ALLOC_NODE_SIZE 4096
+
+int scoped_allocator_node_init(ScopedAllocatorNode *self) {
+    self->data = NULL;
+    self->size = 0;
+    self->next = NULL;
+    return am_malloc(ALLOC_NODE_SIZE, (void**)&self->data);
+}
+
+void scoped_allocator_node_deinit(ScopedAllocatorNode *self) {
+    am_free(self->data);
+    self->data = NULL;
+    self->size = 0;
+    if(self->next) {
+        scoped_allocator_node_deinit(self->next);
+        am_free(self->next);
+        self->next = NULL;
+    }
+}
+
+int scoped_allocator_init(ScopedAllocator *self) {
+    return_if_error(scoped_allocator_node_init(&self->head));
+    self->current = &self->head;
+    return buffer_init(&self->buffers, NULL);
+}
+
+static void buffer_deinit(Buffer *self) {
+    am_free(self->data);
+    self->data = NULL;
+    self->size = 0;
+    self->capacity = 0;
+}
+
+void scoped_allocator_deinit(ScopedAllocator *self) {
+    Buffer *buffer;
+    Buffer *buffer_end;
+
+    scoped_allocator_node_deinit(&self->head);
+    self->current = NULL;
+    buffer = (Buffer*)self->buffers.data;
+    buffer_end = buffer + self->buffers.size / sizeof(Buffer);
+    while(buffer != buffer_end) {
+        buffer_deinit(buffer);
+        ++buffer;
+    }
+    buffer_deinit(&self->buffers);
+}
+
+static CHECK_RESULT int scoped_allocator_ensure_capacity_for(ScopedAllocator *self, usize size) {
+    void *new_node;
+    new_node = NULL;
+    
+    if(self->current->size + size > ALLOC_NODE_SIZE) {
+        return_if_error(am_malloc(sizeof(ScopedAllocatorNode), &new_node));
+        cleanup_if_error(scoped_allocator_node_init(new_node));
+        self->current->next = new_node;
+        self->current = new_node;
+    }
+    return ALLOC_OK;
+
+    cleanup:
+    if(new_node)
+        am_free(new_node);
+    return ALLOC_FAIL;
+}
+
+static void* align_ptr_ceil(void *ptr, uintptr_t alignment) {
+    uintptr_t ptrval;
+    ptrval = (uintptr_t)ptr;
+    return (void*)((ptrval + alignment + 1) & ~(alignment - 1));
+}
+
+static usize align_ptr_ceil_offset(void *ptr, uintptr_t alignment) {
+    return (uintptr_t)align_ptr_ceil(ptr, alignment) - (uintptr_t)ptr;
+}
+
+int scoped_allocator_alloc(ScopedAllocator *self, usize size, void **mem) {
+    ScopedAllocatorNode *current;
+    usize alloc_size;
+    assert(self->current);
+    assert(size <= ALLOC_NODE_SIZE);
+    current = self->current;
+
+    alloc_size = size + align_ptr_ceil_offset(self->current->data + self->current->size, 16);
+    return_if_error(scoped_allocator_ensure_capacity_for(self, alloc_size));
+    /* Reallocated (new node created) */
+    if(self->current != current) {
+        *mem = self->current->data;
+    } else {
+        *mem = align_ptr_ceil(self->current->data + self->current->size, 16);
+    }
+    self->current->size = size;
+    return 0;
+}
+
+int scoped_allocator_add_buffer(ScopedAllocator *self, Buffer *buffer) {
+    return buffer_append(&self->buffers, buffer, sizeof(Buffer));
+}
\ No newline at end of file
diff --git a/src/std/thread.c b/src/std/thread.c
new file mode 100644
index 0000000..f717865
--- /dev/null
+++ b/src/std/thread.c
@@ -0,0 +1,140 @@
+#include "../../include/std/thread.h"
+#include "../../include/std/log.h"
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/sysinfo.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+static int thread_type_to_system_thread_type(amal_thread_type thread_type) {
+    switch(thread_type) {
+        case AMAL_THREAD_JOINABLE:    return PTHREAD_CREATE_JOINABLE;
+        case AMAL_THREAD_DETACHED:    return PTHREAD_CREATE_DETACHED;
+    }
+    assert(bool_false);
+    return PTHREAD_CREATE_JOINABLE;
+}
+
+int amal_thread_create(amal_thread *self, amal_thread_type thread_type, const char *name, AmalThreadCallbackFunc callback_func, void *userdata) {
+    int result;
+    self->name = name;
+    self->thread_id = 0;
+    self->cancellable = bool_false;
+    self->destroyable = bool_false;
+    if((result = pthread_attr_init(&self->thread_attr)) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    self->destroyable = bool_true;
+    if((result = pthread_attr_setdetachstate(&self->thread_attr, thread_type_to_system_thread_type(thread_type))) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    if((result = pthread_create(&self->thread_id, NULL, callback_func, userdata)) != 0) {
+        perror("amal_thread_create");
+        return result;
+    }
+    self->cancellable = bool_true;
+    return 0;
+}
+
+int amal_thread_deinit(amal_thread *self) {
+    int r1;
+    int r2;
+    r1 = 0;
+    r2 = 0;
+
+    if(self->cancellable) {
+        r1 = pthread_cancel(self->thread_id);
+        self->cancellable = bool_false;
+    }
+    if(self->destroyable) {
+        r2 = pthread_attr_destroy(&self->thread_attr);
+        self->destroyable = bool_false;
+    }
+    return r1 != 0 ? r1 : r2;
+}
+
+int amal_thread_detach(amal_thread *self) {
+    int result_err;
+    int thread_type;
+    if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0)
+        return result_err;
+    if(thread_type != PTHREAD_CREATE_DETACHED)
+        return AMAL_THREAD_ERR;
+    if((result_err = pthread_detach(self->thread_id)) != 0)
+        return result_err;
+    self->cancellable = bool_false;
+    return 0;
+}
+
+int amal_thread_join(amal_thread *self, void **result) {
+    int result_err;
+    int thread_type;
+    if((result_err = pthread_attr_getdetachstate(&self->thread_attr, &thread_type)) != 0)
+        return result_err;
+    if(thread_type != PTHREAD_CREATE_JOINABLE)
+        return AMAL_THREAD_NOT_JOINABLE;
+    if((result_err = pthread_join(self->thread_id, result)) != 0)
+        return result_err == EINVAL ? AMAL_THREAD_NOT_JOINABLE : result_err;
+    self->cancellable = bool_false;
+    return 0;
+}
+
+void amal_mutex_init(amal_mutex *self) {
+    pthread_mutex_init(&self->mutex, NULL);
+    /* TODO: pthread_mutex_destroy in amal_mutex_deinit */
+    self->lock_identifier = NULL;
+}
+
+static long amal_process_get_id() {
+    return getpid();
+}
+
+static long amal_thread_get_id() {
+    return syscall(SYS_gettid);
+}
+
+int amal_mutex_lock(amal_mutex *self, const char *lock_identifier) {
+    int result;
+    result = pthread_mutex_lock(&self->mutex);
+    self->lock_identifier = lock_identifier;
+    if(result == 0 && self->lock_identifier) {
+        amal_log_debug("amal_mutex_lock: mutex locked by thread %lu (%s), identification: %s", 
+            amal_thread_get_id(),
+            amal_thread_is_main() ? "main" : "not main",
+            self->lock_identifier ? self->lock_identifier : "none");
+    }
+    return result;
+}
+
+int amal_mutex_unlock(amal_mutex *self) {
+    int result;
+    const char *identifier;
+    identifier = self->lock_identifier;
+    result = pthread_mutex_unlock(&self->mutex);
+    if(result == 0 && identifier) {
+        amal_log_debug("amal_mutex_unlock: mutex unlocked by thread %lu (%s), identification: %s", 
+            amal_thread_get_id(),
+            amal_thread_is_main() ? "main" : "not main",
+            identifier ? identifier : "none");
+    }
+    return result;
+}
+
+void amal_mutex_tryunlock(amal_mutex *self) {
+    int _;
+    (void)_;
+    _ = amal_mutex_unlock(self);
+}
+
+bool amal_thread_is_main() {
+    /* TODO: This only works for linux, use equivalent functions on other platforms */
+    return amal_thread_get_id() == amal_process_get_id();
+}
+
+int amal_get_usable_thread_count() {
+    return get_nprocs();
+}
\ No newline at end of file