forked from mseld2/ProgrammerCalculator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokenizer_tests.py
133 lines (116 loc) · 5.85 KB
/
tokenizer_tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
import unittest
from number import Number
from token_type import TokenType
from tokenizer import Tokenizer
class ToeknizerTests(unittest.TestCase):
def test_can_tokenize_arithmetic_operators_with_mixed_positive_numbers(self):
expr = '3 + xe5 - b101 * o16 // 8'
tokenizer = Tokenizer(expr)
tokens = tokenizer.tokenize()
tokenList = []
for token in tokens:
tokenList.append(token)
self.assertEqual(9, len(tokenList))
self.assertEqual(TokenType.DECIMAL, tokenList[0].type)
self.assertEqual('3', tokenList[0].value)
self.assertEqual(TokenType.ADD, tokenList[1].type)
self.assertEqual('+', tokenList[1].value)
self.assertEqual(TokenType.HEX, tokenList[2].type)
self.assertEqual('0xe5', tokenList[2].value)
self.assertEqual(TokenType.MINUS, tokenList[3].type)
self.assertEqual('-', tokenList[3].value)
self.assertEqual(TokenType.BINARY, tokenList[4].type)
self.assertEqual('0b101', tokenList[4].value)
self.assertEqual(TokenType.MULTIPLY, tokenList[5].type)
self.assertEqual('*', tokenList[5].value)
self.assertEqual(TokenType.OCTAL, tokenList[6].type)
self.assertEqual('0o16', tokenList[6].value)
self.assertEqual(TokenType.DIVIDE, tokenList[7].type)
self.assertEqual('//', tokenList[7].value)
self.assertEqual(TokenType.DECIMAL, tokenList[8].type)
self.assertEqual('8', tokenList[8].value)
def test_can_tokenize_arithmetic_operators_with_mixed_negative_numbers(self):
expr = '-3 + -xe5 - -b101 * -o16 // -8'
tokenizer = Tokenizer(expr)
tokens = tokenizer.tokenize()
tokenList = []
for token in tokens:
tokenList.append(token)
self.assertEqual(14, len(tokenList))
self.assertEqual(TokenType.MINUS, tokenList[0].type)
self.assertEqual('-', tokenList[0].value)
self.assertEqual(TokenType.DECIMAL, tokenList[1].type)
self.assertEqual('3', tokenList[1].value)
self.assertEqual(TokenType.ADD, tokenList[2].type)
self.assertEqual('+', tokenList[2].value)
self.assertEqual(TokenType.MINUS, tokenList[3].type)
self.assertEqual('-', tokenList[3].value)
self.assertEqual(TokenType.HEX, tokenList[4].type)
self.assertEqual('0xe5', tokenList[4].value)
self.assertEqual(TokenType.MINUS, tokenList[5].type)
self.assertEqual('-', tokenList[5].value)
self.assertEqual(TokenType.MINUS, tokenList[6].type)
self.assertEqual('-', tokenList[6].value)
self.assertEqual(TokenType.BINARY, tokenList[7].type)
self.assertEqual('0b101', tokenList[7].value)
self.assertEqual(TokenType.MULTIPLY, tokenList[8].type)
self.assertEqual('*', tokenList[8].value)
self.assertEqual(TokenType.MINUS, tokenList[9].type)
self.assertEqual('-', tokenList[9].value)
self.assertEqual(TokenType.OCTAL, tokenList[10].type)
self.assertEqual('0o16', tokenList[10].value)
self.assertEqual(TokenType.DIVIDE, tokenList[11].type)
self.assertEqual('//', tokenList[11].value)
self.assertEqual(TokenType.MINUS, tokenList[12].type)
self.assertEqual('-', tokenList[12].value)
self.assertEqual(TokenType.DECIMAL, tokenList[13].type)
self.assertEqual('8', tokenList[13].value)
def test_can_tokenize_group_lshift_or_operators(self):
expr = '(b11010101 << 4) | x0F'
tokenizer = Tokenizer(expr)
tokens = tokenizer.tokenize()
tokenList = []
for token in tokens:
tokenList.append(token)
self.assertEqual(7, len(tokenList))
self.assertEqual(TokenType.LPAREN, tokenList[0].type)
self.assertEqual('(', tokenList[0].value)
self.assertEqual(TokenType.BINARY, tokenList[1].type)
self.assertEqual('0b11010101', tokenList[1].value)
self.assertEqual(TokenType.LSHIFT, tokenList[2].type)
self.assertEqual('<<', tokenList[2].value)
self.assertEqual(TokenType.DECIMAL, tokenList[3].type)
self.assertEqual('4', tokenList[3].value)
self.assertEqual(TokenType.RPAREN, tokenList[4].type)
self.assertEqual(')', tokenList[4].value)
self.assertEqual(TokenType.OR, tokenList[5].type)
self.assertEqual('|', tokenList[5].value)
self.assertEqual(TokenType.HEX, tokenList[6].type)
self.assertEqual('0x0F', tokenList[6].value)
def test_can_tokenize_group_rshift_and_operators(self):
expr = '(b11010101 >> 4) & xF0'
tokenizer = Tokenizer(expr)
tokens = tokenizer.tokenize()
tokenList = []
for token in tokens:
tokenList.append(token)
self.assertEqual(7, len(tokenList))
self.assertEqual(TokenType.LPAREN, tokenList[0].type)
self.assertEqual('(', tokenList[0].value)
self.assertEqual(TokenType.BINARY, tokenList[1].type)
self.assertEqual('0b11010101', tokenList[1].value)
self.assertEqual(TokenType.RSHIFT, tokenList[2].type)
self.assertEqual('>>', tokenList[2].value)
self.assertEqual(TokenType.DECIMAL, tokenList[3].type)
self.assertEqual('4', tokenList[3].value)
self.assertEqual(TokenType.RPAREN, tokenList[4].type)
self.assertEqual(')', tokenList[4].value)
self.assertEqual(TokenType.AND, tokenList[5].type)
self.assertEqual('&', tokenList[5].value)
self.assertEqual(TokenType.HEX, tokenList[6].type)
self.assertEqual('0xF0', tokenList[6].value)
@unittest.expectedFailure
def test_fails_when_unexpected_value_found(self):
expr = '(13 % 8) w'
tokenizer = Tokenizer(expr)
tokenizer.tokenize()