-
-
Notifications
You must be signed in to change notification settings - Fork 2.2k
/
token.v
53 lines (50 loc) · 1.38 KB
/
token.v
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module token
// Token holds information about the current scan of bytes.
pub struct Token {
pub:
kind Kind // the token number/enum; for quick comparisons
lit string // literal representation of the token
col int // the column in the source where the token occurred
line_nr int // the line number in the source where the token occurred
pos int // the position of the token in scanner text
len int // length of the literal
}
// Kind represents a logical type of entity found in any given TOML document.
pub enum Kind {
unknown
eof
bare // user
boolean // true or false
number // 123
quoted // 'foo', "foo", """foo""" or '''foo'''
plus // +
minus // -
underscore // _
comma // ,
colon // :
hash // # comment
assign // =
lcbr // {
rcbr // }
lsbr // [
rsbr // ]
nl // \n linefeed / newline character
cr // \r carriage return
tab // \t character
whitespace // ` `
period // .
_end_
}
// pos returns the exact position of a token in the input.
@[inline]
pub fn (tok &Token) pos() Pos {
return Pos{
len: tok.len
line_nr: tok.line_nr - 1
pos: tok.pos
col: tok.col - 1
}
}