1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
#ifndef TSL_TOKENIZER_H
#define TSL_TOKENIZER_H
#include <stddef.h>
#include <stdint.h>
typedef struct {
const char *data;
size_t size;
} TslStringView;
typedef enum {
TSL_TOKEN_END_OF_FILE,
TSL_TOKEN_UNEXPECTED_SYMBOL,
TSL_TOKEN_IDENTIFIER,
TSL_TOKEN_STRING,
TSL_TOKEN_NUM,
TSL_TOKEN_BOOL,
TSL_TOKEN_NULL,
TSL_TOKEN_EQUAL,
TSL_TOKEN_LBRACE,
TSL_TOKEN_RBRACE,
TSL_TOKEN_LBRACKET,
TSL_TOKEN_RBRACKET,
TSL_TOKEN_LPAREN,
TSL_TOKEN_RPAREN,
TSL_TOKEN_COLON,
TSL_TOKEN_COMMA,
TSL_TOKEN_FN,
TSL_TOKEN_DOLLAR_SIGN
} TslToken;
typedef enum {
TSL_COMMAND_TOKEN_END_OF_FILE,
TSL_COMMAND_TOKEN_ARG,
TSL_COMMAND_TOKEN_END
} TslCommandToken;
typedef struct {
TslToken token;
size_t code_index;
size_t prev_code_index;
} TslTokenizerPeek;
typedef struct {
const char *code;
size_t code_size;
size_t code_index;
size_t prev_code_index;
TslTokenizerPeek peek;
TslStringView identifier;
TslStringView string;
int bool_value;
int64_t number_value;
} TslTokenizer;
void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
TslToken tsl_tokenizer_next(TslTokenizer *self);
int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
TslToken tsl_tokenizer_peek(TslTokenizer *self);
TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);
int tsl_tokenizer_get_line_by_index(TslTokenizer *self, size_t index);
#endif /* TSL_TOKENIZER_H */
|
