-
Notifications
You must be signed in to change notification settings - Fork 14
/
TokenStream.h
124 lines (99 loc) · 2.33 KB
/
TokenStream.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#ifndef _TOKENSTREAMH_GUARD
#define _TOKENSTREAMH_GUARD
#include <string>
#include <fstream>
#include <map>
#include <vector>
#include <boost/any.hpp>
/**
* Display an error message and throw an exception.
*/
void error(const std::string& msg, int line);
enum class TokenType {
Unkown,
Begin, Declaration,
SetVar, ValueOf,
Error, End,
Article, Or, And,
To, KnownAs,
Identifier, String,
Number, Operator, Dot, // 14, 15, 16
Plus, Minus, Times, // These are only used in TokenStream
If, Else,
Equals, NotEquals, BlockEnd,
BlockBegin, Is, FuncName,
FuncResult, On, Of,
While, //BlockBeginW,
WhileCondition, WhileBody,
Comment, Argument, When,
Calling
};
class Token {
boost::any value;
public:
TokenType type;
int line;
Token()
: value(), type(TokenType::Unkown), line(0) { }
Token(TokenType type)
: value(), type(type), line(0) { }
template<class T>
Token(const T& v, TokenType type)
: value(v), type(type), line(0) { }
template<class T>
void setValue(const T& v)
{
value = v;
}
template<class T>
T getValue() const
{
return boost::any_cast<T>(value);
}
};
/**
* Maps words (as std::string) to ::TokenType objects.
*/
class TokenTable {
std::map<std::string, TokenType> table;
public:
TokenTable() = default;
void add(TokenType t, const std::string& word);
template<class... Rest>
void add(TokenType t, const std::string& word, const Rest&... words)
{
add(t, word);
add(t, words...);
}
TokenType operator[](const std::string& word);
};
typedef std::vector<Token> TokenStream;
class Lexer {
std::string filepath;
std::ifstream ifs;
TokenTable type_table;
TokenStream tokens;
char readChar(bool ws);
void readString(std::string& str);
double readNumber();
void skipSentence();
/**
* Makes a comparions operator ::Token.
*/
Token makeComparison(std::string& text);
/**
* Makes a function call ::Token.
*/
Token makeFunctionCall(std::string& text);
/**
* Extracts a ::Token beginning with a word from the ::Lexer
*/
Token getTxt();
Token get();
void open();
public:
int line;
Lexer(const std::string& filename);
TokenStream tokenize();
};
#endif