forked from davidsiaw/coco-r-cpp
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Scanner.h
executable file
·279 lines (232 loc) · 8.15 KB
/
Scanner.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
/*----------------------------------------------------------------------
Compiler Generator Coco/R,
Copyright (c) 1990, 2004 Hanspeter Moessenboeck, University of Linz
extended by M. Loeberbauer & A. Woess, Univ. of Linz
ported to C++ by Csaba Balazs, University of Szeged
with improvements by Pat Terry, Rhodes University
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
As an exception, it is allowed to write an extension of Coco/R that is
used as a plugin in non-free software.
If not otherwise stated, any source code generated by Coco/R (other than
Coco/R itself) does not fall under the GNU General Public License.
-----------------------------------------------------------------------*/
#if !defined(Coco_COCO_SCANNER_H__)
#define Coco_COCO_SCANNER_H__
#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// io.h and fcntl are used to ensure binary read from streams on windows
#if _MSC_VER >= 1300
#include <io.h>
#include <fcntl.h>
#endif
#if _MSC_VER >= 1400
#define coco_sprintf sprintf_s
#elif _MSC_VER >= 1300
#define coco_sprintf _snprintf
#elif defined __MINGW32__
#define coco_sprintf _snprintf
#else
// assume every other compiler knows swprintf
#define coco_sprintf swprintf
#endif
#define COCO_CHAR_MAX 255
#define COCO_MIN_BUFFER_LENGTH 1024
#define COCO_MAX_BUFFER_LENGTH (64*COCO_MIN_BUFFER_LENGTH)
#define COCO_HEAP_BLOCK_SIZE (64*1024)
#define COCO_CPP_NAMESPACE_SEPARATOR ':'
namespace Coco {
// string handling, wide character
char* coco_string_create(const char *value);
char* coco_string_create(const char *value, int startIndex);
char* coco_string_create(const char *value, int startIndex, int length);
char* coco_string_create_upper(const char* data);
char* coco_string_create_lower(const char* data);
char* coco_string_create_lower(const char* data, int startIndex, int dataLen);
char* coco_string_create_append(const char* data1, const char* data2);
char* coco_string_create_append(const char* data, const char value);
void coco_string_delete(char* &data);
int coco_string_length(const char* data);
bool coco_string_endswith(const char* data, const char *value);
int coco_string_indexof(const char* data, const char value);
int coco_string_lastindexof(const char* data, const char value);
void coco_string_merge(char* &data, const char* value);
bool coco_string_equal(const char* data1, const char* data2);
int coco_string_compareto(const char* data1, const char* data2);
int coco_string_hash(const char* data);
class Token
{
public:
int kind; // token kind
int pos; // token position in bytes in the source text (starting at 0)
int charPos; // token position in characters in the source text (starting at 0)
int col; // token column (starting at 1)
int line; // token line (starting at 1)
char* val; // token value
Token *next; // ML 2005-03-11 Peek tokens are kept in linked list
Token();
~Token();
};
class Buffer {
// This Buffer supports the following cases:
// 1) seekable stream (file)
// a) whole stream in buffer
// b) part of stream in buffer
// 2) non seekable stream (network, console)
private:
unsigned char *buf; // input buffer
int bufCapacity; // capacity of buf
int bufStart; // position of first byte in buffer relative to input stream
int bufLen; // length of buffer
int fileLen; // length of input stream (may change if the stream is no file)
int bufPos; // current position in buffer
FILE* stream; // input stream (seekable)
bool isUserStream; // was the stream opened by the user?
int ReadNextStreamChunk();
bool CanSeek(); // true if stream can be seeked otherwise false
public:
static const int EoF = COCO_CHAR_MAX + 1;
Buffer(FILE* s, bool isUserStream);
Buffer(const char* buf, int len);
Buffer(Buffer *b);
virtual ~Buffer();
virtual void Close();
virtual int Read();
virtual int Peek();
virtual char* GetString(int beg, int end);
virtual int GetPos();
virtual void SetPos(int value);
};
//-----------------------------------------------------------------------------------
// StartStates -- maps characters to start states of tokens
//-----------------------------------------------------------------------------------
class StartStates {
private:
class Elem {
public:
int key, val;
Elem *next;
Elem(int key, int val) { this->key = key; this->val = val; next = NULL; }
};
Elem **tab;
public:
StartStates() { tab = new Elem*[128]; memset(tab, 0, 128 * sizeof(Elem*)); }
virtual ~StartStates() {
for (int i = 0; i < 128; ++i) {
Elem *e = tab[i];
while (e != NULL) {
Elem *next = e->next;
delete e;
e = next;
}
}
delete [] tab;
}
void set(int key, int val) {
Elem *e = new Elem(key, val);
int k = ((unsigned int) key) % 128;
e->next = tab[k]; tab[k] = e;
}
int state(int key) {
Elem *e = tab[((unsigned int) key) % 128];
while (e != NULL && e->key != key) e = e->next;
return e == NULL ? 0 : e->val;
}
};
//-------------------------------------------------------------------------------------------
// KeywordMap -- maps strings to integers (identifiers to keyword kinds)
//-------------------------------------------------------------------------------------------
class KeywordMap {
private:
class Elem {
public:
char *key;
int val;
Elem *next;
Elem(const char *key, int val) { this->key = coco_string_create(key); this->val = val; next = NULL; }
virtual ~Elem() { coco_string_delete(key); }
};
Elem **tab;
public:
KeywordMap() { tab = new Elem*[128]; memset(tab, 0, 128 * sizeof(Elem*)); }
virtual ~KeywordMap() {
for (int i = 0; i < 128; ++i) {
Elem *e = tab[i];
while (e != NULL) {
Elem *next = e->next;
delete e;
e = next;
}
}
delete [] tab;
}
void set(const char *key, int val) {
Elem *e = new Elem(key, val);
int k = coco_string_hash(key) % 128;
e->next = tab[k]; tab[k] = e;
}
int get(const char *key, int defaultVal) {
Elem *e = tab[coco_string_hash(key) % 128];
while (e != NULL && !coco_string_equal(e->key, key)) e = e->next;
return e == NULL ? defaultVal : e->val;
}
};
class Scanner {
private:
void *firstHeap;
void *heap;
void *heapTop;
void **heapEnd;
char EOL;
int eofSym;
int noSym;
int maxT;
int charSetSize;
StartStates start;
KeywordMap keywords;
Token *t; // current token
char *tval; // text of current token
int tvalLength; // length of text of current token
int tlen; // length of current token
Token *tokens; // list of tokens already peeked (first token is a dummy)
Token *pt; // current peek token
int ch; // current input character
int pos; // byte position of current character
int charPos; // position by unicode characters starting with 0
int line; // line number of current character
int col; // column number of current character
int oldEols; // EOLs that appeared in a comment;
void CreateHeapBlock();
Token* CreateToken();
void AppendVal(Token *t);
void SetScannerBehindT();
void Init();
void NextCh();
void AddCh();
bool Comment0();
bool Comment1();
Token* NextToken();
public:
Buffer *buffer; // scanner buffer
Scanner(const char* buf, int len);
Scanner(const char* fileName);
Scanner(FILE* s);
~Scanner();
Token* Scan();
Token* Peek();
void ResetPeek();
}; // end Scanner
} // namespace
#endif