From a52fdf470aa2c164108aeccc2c83bad62208913c Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 9 Mar 2019 14:14:09 +0100 Subject: Start on resolving ast. Add recursive declaration check --- src/tokenizer.c | 63 +++++++++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 33 deletions(-) (limited to 'src/tokenizer.c') diff --git a/src/tokenizer.c b/src/tokenizer.c index b2bd6c5..4f1871b 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -30,7 +30,6 @@ int tokenizer_init(Tokenizer *self, ScopedAllocator *allocator, BufferView code, self->code = code; self->index = 0; self->prev_index = 0; - self->line = 1; self->token = TOK_NONE; self->needs_update = bool_true; self->code_name = code_name.data ? code_name : create_buffer_view("", 8); @@ -53,8 +52,6 @@ static Token tokenizer_skip_whitespace(Tokenizer *self) { c = self->code.data[self->index]; switch(c) { case '\n': - ++self->line; - /* fallthrough */ case ' ': case '\t': break; @@ -183,7 +180,7 @@ static CHECK_RESULT int __tokenizer_next(Tokenizer *self, Token *token) { ++self->index; string_end = find_end_of_string(self->code, self->index); if(string_end == -1) { - tokenizer_print_error(self, "String end not found. Did you forget '\"' or did you have a mismatch of number of '\"'?"); + tokenizer_print_error(self, self->prev_index, "String end not found. Did you forget '\"' or did you have a mismatch of number of '\"'?"); return TOKENIZER_ERR; } @@ -218,7 +215,7 @@ static CHECK_RESULT int __tokenizer_next(Tokenizer *self, Token *token) { result = string_to_integer_unchecked(number_str, &self->value.integer); if(result != 0) { /* TODO */ - tokenizer_print_error(self, "Integer value %.*s is too large to fit in signed 64-bit. Support for large integers is not supported yet.", number_str.size, number_str.data); + tokenizer_print_error(self, self->prev_index, "Integer value %.*s is too large to fit in signed 64-bit. Support for large integers is not supported yet.", number_str.size, number_str.data); return TOKENIZER_ERR; } self->number_is_integer = bool_true; @@ -227,7 +224,7 @@ static CHECK_RESULT int __tokenizer_next(Tokenizer *self, Token *token) { result = string_to_float_unchecked(number_str, &self->value.floating); if(result != 0) { /* TODO */ - tokenizer_print_error(self, "Float value %.*s is too large to fit in 64-bit. Support for large floating numbers is not supported yet.", number_str.size, number_str.data); + tokenizer_print_error(self, self->prev_index, "Float value %.*s is too large to fit in 64-bit. Support for large floating numbers is not supported yet.", number_str.size, number_str.data); return TOKENIZER_ERR; } self->number_is_integer = bool_false; @@ -258,7 +255,7 @@ static CHECK_RESULT int __tokenizer_next(Tokenizer *self, Token *token) { ++self->index; self->index = tokenizer_get_end_of_multiline_comment(self, self->index); if(self->index == -1) { - tokenizer_print_error(self, "End of multiline comment not found"); + tokenizer_print_error(self, self->prev_index, "End of multiline comment not found"); return TOKENIZER_ERR; } return __tokenizer_next(self, token); @@ -325,10 +322,10 @@ static CHECK_RESULT int __tokenizer_next(Tokenizer *self, Token *token) { return TOKENIZER_OK; import_error: - tokenizer_print_error(self, err_msg); + tokenizer_print_error(self, self->prev_index, err_msg); return TOKENIZER_ERR; } else { - tokenizer_print_error(self, "Unexpected symbol '%c'", c); + tokenizer_print_error(self, self->prev_index, "Unexpected symbol '%c'", c); return TOKENIZER_UNEXPECTED_TOKEN; } return TOKENIZER_OK; @@ -506,7 +503,7 @@ int tokenizer_accept(Tokenizer *self, Token expected_token) { expected_token_str = tokenizer_expected_token_as_string(expected_token); actual_token_str = tokenizer_actual_token_as_string(self); - tokenizer_print_error(self, "Expected \"%.*s\", got \"%.*s\"", expected_token_str.size, expected_token_str.data, actual_token_str.size, actual_token_str.data); + tokenizer_print_error(self, self->prev_index, "Expected \"%.*s\", got \"%.*s\"", expected_token_str.size, expected_token_str.data, actual_token_str.size, actual_token_str.data); return TOKENIZER_UNEXPECTED_TOKEN; } @@ -571,8 +568,24 @@ int tokenizer_get_end_of_multiline_comment(Tokenizer *self, int index) { return -1; } -void tokenizer_print_error(Tokenizer *self, const char *fmt, ...) { +/* TODO: Optimize */ +static int tokenizer_get_line_by_index(Tokenizer *self, int index) { + int i; + int line; + if(index < 0 || index >= (int)self->code.size) + return -1; + + line = 1; + for(i = 0; i < index; ++i) { + if(self->code.data[i] == '\n') + ++line; + } + return line; +} + +void tokenizer_print_error(Tokenizer *self, int index, const char *fmt, ...) { va_list args; + int line; int line_start; int line_end; int prev_column; @@ -582,10 +595,11 @@ void tokenizer_print_error(Tokenizer *self, const char *fmt, ...) { mutex = amal_log_get_mutex(); ignore_result_int(amal_mutex_lock(mutex, "tokenizer_print_error")); va_start(args, fmt); - line_start = tokenizer_get_start_of_line_from_index(self, self->prev_index); - line_end = tokenizer_get_end_of_line_from_index(self, self->prev_index); - prev_column = self->prev_index - line_start; - fprintf(stderr, "\x1b[1;37m%.*s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", (int)self->code_name.size, self->code_name.data, self->line, 1 + prev_column); + line = tokenizer_get_line_by_index(self, index); + line_start = tokenizer_get_start_of_line_from_index(self, index); + line_end = tokenizer_get_end_of_line_from_index(self, index); + prev_column = index - line_start; + fprintf(stderr, "\x1b[1;37m%.*s:%d:%d:\x1b[0m \x1b[1;31merror:\x1b[0m ", (int)self->code_name.size, self->code_name.data, line, 1 + prev_column); vfprintf(stderr, fmt, args); fprintf(stderr, "\n%.*s\n", line_end - line_start, self->code.data + line_start); for(i = 0; i < prev_column; ++i) @@ -595,25 +609,8 @@ void tokenizer_print_error(Tokenizer *self, const char *fmt, ...) { ignore_result_int(amal_mutex_unlock(mutex)); } -/* TODO: Optimize */ -static int tokenizer_get_line_by_index(Tokenizer *self, int index) { - int i; - int line; - if(index < 0 || index >= (int)self->code.size) - return -1; - - line = 1; - for(i = 0; i < index; ++i) { - if(self->code.data[i] == '\n') - ++line; - } - return line; -} - void tokenizer_print_error_object(Tokenizer *self, TokenizerError *error) { - self->prev_index = error->index; - self->line = tokenizer_get_line_by_index(self, self->prev_index); - tokenizer_print_error(self, "%s", error->str); + tokenizer_print_error(self, error->index, "%s", error->str); } TokenizerError tokenizer_create_error(Tokenizer *self, int index, const char *fmt, ...) { -- cgit v1.2.3