aboutsummaryrefslogtreecommitdiff
path: root/include/tokenizer.h
blob: 875396adc7f52d10dc5330c3bf0797b9f5b76011 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#ifndef TSL_TOKENIZER_H
#define TSL_TOKENIZER_H

#include "string_view.h"

typedef enum {
    TSL_TOKEN_END_OF_FILE,
    TSL_TOKEN_UNEXPECTED_SYMBOL,
    TSL_TOKEN_IDENTIFIER,
    TSL_TOKEN_STRING,
    TSL_TOKEN_NUM,
    TSL_TOKEN_BOOL,
    TSL_TOKEN_NULL,
    TSL_TOKEN_EQUAL,
    TSL_TOKEN_LBRACE,
    TSL_TOKEN_RBRACE,
    TSL_TOKEN_LBRACKET,
    TSL_TOKEN_RBRACKET,
    TSL_TOKEN_LPAREN,
    TSL_TOKEN_RPAREN,
    TSL_TOKEN_COLON,
    TSL_TOKEN_COMMA,
    TSL_TOKEN_FN,
    TSL_TOKEN_DOLLAR_SIGN,
    TSL_TOKEN_ARITHMETIC
} TslToken;

typedef enum {
    TSL_COMMAND_TOKEN_END_OF_FILE,
    TSL_COMMAND_TOKEN_ARG,
    TSL_COMMAND_TOKEN_END
} TslCommandToken;

typedef struct {
    TslToken token;
    size_t code_index;
    size_t prev_code_index;
} TslTokenizerPeek;

typedef struct {
    char *code;
    size_t code_size;
    size_t code_index;
    size_t prev_code_index;

    TslTokenizerPeek peek;

    TslStringView identifier;
    TslStringView string;
    int bool_value;
    double number_value;
    char arithmetic_symbol;
} TslTokenizer;

void tsl_tokenizer_init(TslTokenizer *self, char *code, size_t code_size);

TslToken tsl_tokenizer_next(TslTokenizer *self);
int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
/*
    If peek was previously called without consuming the token, then the previous value peek token is returned.
    In other words, calling tsl_tokenizer_peek twice in a row will return the same token without progressing.
*/
TslToken tsl_tokenizer_peek(TslTokenizer *self);
TslToken tsl_tokenizer_consume_peek(TslTokenizer *self);

TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);

int tsl_tokenizer_get_line_by_index(TslTokenizer *self, size_t index);

#endif /* TSL_TOKENIZER_H */