-
Notifications
You must be signed in to change notification settings - Fork 1
/
jstokens.py
executable file
·107 lines (92 loc) · 2.46 KB
/
jstokens.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import ply.lex as lex
tokens = (
'ANDAND', # &&
'COMMA', # ,
'DIVIDE', # /
'ELSE', # else
'EQUAL', # =
'EQUALEQUAL', # ==
'FALSE', # false
'FUNCTION', # function
'GE', # >=
'GT', # >
'IDENTIFIER', # factorial
'IF', # if
'LBRACE', # {
'LE', # <=
'LPAREN', # (
'LT', # <
'MINUS', # -
'MOD', # %
'NOT', # !
'NUMBER', # 1234 5.678
'OROR', # ||
'PLUS', # +
'RBRACE', # }
'RETURN', # return
'RPAREN', # )
'SEMICOLON', # ;
'STRING', # "this is a \"tricky\" string"
'TIMES', # *
'TRUE', # TRUE
'VAR', # var
'WHILE', # while
)
states = (
('comment', 'exclusive'), # /* ... */
)
def t_comment(t):
r'\/\*'
t.lexer.begin('comment')
def t_comment_end(t):
r'\*\/'
t.lexer.lineno += t.value.count('\n')
t.lexer.begin('INITIAL')
pass
def t_comment_error(t):
t.lexer.skip(1)
def t_eolcomment(t):
r'//.*'
pass
reserved = [ 'function', 'if', 'var', 'return', 'else', 'true', 'false', 'while' ]
def t_IDENTIFIER(t):
r'[A-Za-z][A-Za-z_]*'
if t.value in reserved:
t.type = t.value.upper()
return t
def t_NUMBER(t):
r'-?[0-9]+(\.[0-9]*)?'
t.value = float(t.value)
return t
def t_STRING(t):
r'"([^"\\]|(\\.))*"'
t.value = t.value[1:-1] # strip off "
return t
t_ANDAND = r'&&'
t_COMMA = r','
t_DIVIDE = r'/'
t_EQUALEQUAL = r'=='
t_EQUAL = r'='
t_LPAREN = r'\('
t_LBRACE = r'{'
t_RBRACE = r'}'
t_SEMICOLON = r';'
t_MINUS = r'-'
t_MOD = r'%'
t_NOT = r'!'
t_OROR = r'\|\|'
t_PLUS = r'\+'
t_RPAREN = r'\)'
t_TIMES = r'\*'
t_LE = r'<='
t_LT = r'<'
t_GT = r'>'
t_GE = r'>='
t_ignore = ' \t\v\r'
t_comment_ignore = ' \t\v\r'
def t_newline(t):
r'\n'
t.lexer.lineno += 1
def t_error(t):
print "JavaScript Lexer: Illegal character " + t.value[0]
t.lexer.skip(1)