diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/tokenizer.c | 19 |
1 files changed, 12 insertions, 7 deletions
diff --git a/src/tokenizer.c b/src/tokenizer.c index de467c3..9ce1bba 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -29,11 +29,6 @@ static int tokenizer_get_end_of_multiline_comment(Tokenizer *self, int index); int tokenizer_init(Tokenizer *self, ArenaAllocator *allocator, BufferView code, BufferView code_name, const amal_compiler_options *compiler_options) { assert(code.size <= INT_MAX); assert(compiler_options); - /* Skip UTF-8 BOM */ - if(code.size >= 3 && (u8)code.data[0] == 0xEF && (u8)code.data[1] == 0xBB && (u8)code.data[2] == 0xBF) { - code.data += 3; - code.size -= 3; - } self->code = code; self->index = 0; self->prev_index = 0; @@ -43,6 +38,17 @@ int tokenizer_init(Tokenizer *self, ArenaAllocator *allocator, BufferView code, self->number_is_integer = bool_false; self->allocator = allocator; self->compiler_options = compiler_options; + + /* Skip UTF-8 BOM */ + if(self->code.size >= 3 && am_memcmp(self->code.data, "\xEF\xBB\xBF", 3) == 0) { + self->code.data += 3; + self->code.size -= 3; + } + /* Skip shebang */ + if(self->code.size >= 2 && am_memcmp(self->code.data, "#!", 2) == 0) { + self->index = tokenizer_get_end_of_line_from_index(self, self->index + 2); + } + return 0; } @@ -608,9 +614,8 @@ int tokenizer_get_start_of_line_from_index(Tokenizer *self, int index) { } int tokenizer_get_end_of_line_from_index(Tokenizer *self, int index) { - char c; while(index < (int)self->code.size) { - c = self->code.data[index]; + char c = self->code.data[index]; if(c == '\n' || c == '\r') break; ++index; |