-
-
Notifications
You must be signed in to change notification settings - Fork 991
/
tokenizer.hpp
73 lines (58 loc) · 1.62 KB
/
tokenizer.hpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
/*
Copyright (C) 2003 - 2018 by David White <dave@whitevine.net>
Part of the Battle for Wesnoth Project http://www.wesnoth.org/
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY.
See the COPYING file for more details.
*/
#pragma once
#include <string>
namespace wfl
{
namespace tokenizer
{
typedef std::string::const_iterator iterator;
/// TOKEN_TYPE is already defined in a Winnt.h (a windows header which is included under some conditions.)
enum TOKEN_TYPE { TOKEN_OPERATOR, TOKEN_STRING_LITERAL,
TOKEN_IDENTIFIER, TOKEN_INTEGER, TOKEN_DECIMAL,
TOKEN_LPARENS, TOKEN_RPARENS,
TOKEN_LSQUARE, TOKEN_RSQUARE,
TOKEN_COMMA, TOKEN_SEMICOLON,
TOKEN_WHITESPACE, TOKEN_EOL, TOKEN_KEYWORD,
TOKEN_COMMENT, TOKEN_POINTER };
struct token {
token() :
type(TOKEN_COMMENT),
begin(),
end(),
line_number(1),
filename()
{
}
token(iterator& i1, iterator i2, TOKEN_TYPE type) :
type(type),
begin(i1),
end(i2),
line_number(1),
filename()
{
}
TOKEN_TYPE type;
iterator begin, end;
int line_number;
const std::string* filename;
};
token get_token(iterator& i1, iterator i2);
struct token_error
{
token_error() : description_(), formula_() {}
token_error(const std::string& dsc, const std::string& formula) : description_(dsc), formula_(formula) {}
std::string description_;
std::string formula_;
};
}
}