1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
#ifndef TSL_TOKENIZER_H
#define TSL_TOKENIZER_H
#include "std/string_view.h"
typedef enum {
TSL_TOKEN_END_OF_FILE,
TSL_TOKEN_UNEXPECTED_SYMBOL,
TSL_TOKEN_IDENTIFIER,
TSL_TOKEN_STRING,
TSL_TOKEN_NUM,
TSL_TOKEN_BOOL,
TSL_TOKEN_NULL,
TSL_TOKEN_EQUAL,
TSL_TOKEN_LBRACE,
TSL_TOKEN_RBRACE,
TSL_TOKEN_LBRACKET,
TSL_TOKEN_RBRACKET,
TSL_TOKEN_LPAREN,
TSL_TOKEN_RPAREN,
TSL_TOKEN_COLON,
TSL_TOKEN_COMMA,
TSL_TOKEN_FN,
TSL_TOKEN_DOLLAR_SIGN,
TSL_TOKEN_ARITHMETIC
} TslToken;
typedef enum {
TSL_COMMAND_TOKEN_END_OF_FILE,
TSL_COMMAND_TOKEN_ARG,
TSL_COMMAND_TOKEN_END
} TslCommandToken;
typedef struct {
TslToken token;
size_t code_index;
size_t prev_code_index;
} TslTokenizerPeek;
typedef struct {
const char *code;
size_t code_size;
size_t code_index;
size_t prev_code_index;
TslTokenizerPeek peek;
TslStringView identifier;
TslStringView string;
int bool_value;
double number_value;
char arithmetic_symbol;
} TslTokenizer;
void tsl_tokenizer_init(TslTokenizer *self, const char *code, size_t code_size);
TslToken tsl_tokenizer_next(TslTokenizer *self);
int tsl_tokenizer_accept(TslTokenizer *self, TslToken expected_token);
/*
If peek was previously called without consuming the token, then the previous value peek token is returned.
In other words, calling tsl_tokenizer_peek twice in a row will return the same token without progressing.
*/
TslToken tsl_tokenizer_peek(TslTokenizer *self);
TslToken tsl_tokenizer_consume_peek(TslTokenizer *self);
TslCommandToken tsl_tokenizer_next_command_arg(TslTokenizer *self, TslStringView *arg);
int tsl_tokenizer_get_line_by_index(TslTokenizer *self, size_t index);
#endif /* TSL_TOKENIZER_H */
|
