lexer/
token.rs

1//! Token model
2
3use std::fmt;
4
5use span::Span;
6
7/// Models all the types of token
8#[derive(Clone, Copy, Debug, PartialEq)]
9pub enum TokenKind {
10    /// "("
11    LeftParen,
12
13    /// ")"
14    RightParen,
15
16    /// "{"
17    LeftBrace,
18
19    /// "}"
20    RightBrace,
21
22    /// "["
23    LeftBracket,
24
25    /// "]"
26    RightBracket,
27
28    /// ","
29    Comma,
30
31    /// "."
32    Dot,
33
34    /// "-"
35    Minus,
36
37    /// "+"
38    Plus,
39
40    /// ";"
41    Semicolon,
42
43    /// ":"
44    Colon,
45
46    /// "::"
47    DoubleColon,
48
49    /// "/"
50    Slash,
51
52    /// "*"
53    Star,
54
55    /// "?"
56    Question,
57
58    /// "!"
59    Bang,
60
61    /// "!="
62    BangEqual,
63
64    /// "="
65    Equal,
66
67    /// "=="
68    EqualEqual,
69
70    /// "\>"
71    Greater,
72
73    /// "\>="
74    GreaterEqual,
75
76    /// "<"
77    Less,
78
79    /// "->"
80    Arrow,
81
82    /// "<="
83    LessEqual,
84
85    /// [a-zA-Z_]\[a-zA-Z_0-9\]*
86    Identifier,
87
88    /// "(.|\\\\.)*?"
89    String,
90
91    /// [0-9]+
92    IntLiteral,
93
94    /// [0-9]+\.[0-9]+
95    FloatLiteral,
96
97    /// '\\?.'
98    CharLiteral,
99
100    /* ==== Primitive Types ==== */
101    /// "i8"
102    I8,
103    /// "i16"
104    I16,
105    /// "i32"
106    I32,
107    /// "i64"
108    I64,
109    /// "u8"
110    U8,
111    /// "u16"
112    U16,
113    /// "u32"
114    U32,
115    /// "u64"
116    U64,
117    /// "f32"
118    F32,
119    /// "f64"
120    F64,
121    /// "char"
122    Char,
123    /// "bool"
124    Bool,
125    /* ========================= */
126
127    /// "struct"
128    Struct,
129
130    /// "else"
131    Else,
132
133    /// "false"
134    False,
135
136    /// "fn"
137    Fn,
138
139    /// "for"
140    For,
141
142    /// "if"
143    If,
144
145    /// "return"
146    Return,
147
148    /// "true"
149    True,
150
151    /// "let"
152    Let,
153
154    /// "const"
155    Const,
156
157    /// "while"
158    While,
159
160    /// "break"
161    Break,
162
163    /// "continue"
164    Continue,
165
166    /// "use"
167    Use,
168
169    /// "as"
170    As,
171
172    /// "mod"
173    Mod,
174
175    /// "&"
176    Ampersand,
177
178    /// "|"
179    VerticalPipe,
180
181    /// "&&"
182    And,
183
184    /// "||"
185    Or,
186
187    /// '%'
188    Modulo,
189
190    /// "extern"
191    Extern,
192
193    /// "..."
194    ThreeDot,
195}
196
197/// A token consist on a discriminator [`TokenKind`] and a [Span]
198#[derive(Debug, Clone)]
199pub struct Token {
200    pub kind: TokenKind,
201    pub span: Span,
202}
203
204impl fmt::Display for TokenKind {
205    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{self:?}") }
206}
lexer/token.rs

lexer/
token.rs
