-
Notifications
You must be signed in to change notification settings - Fork 0
/
Tokenizer.scala
90 lines (62 loc) · 1.75 KB
/
Tokenizer.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
package lib
import scala.collection.mutable.ListBuffer
import scala.util.control.Breaks._
class Token(val _type:String, val value:String){
override def toString: String = {
"{'type':'%s', 'value':'%s'}".format(_type, value)
}
}
/*
(add 2 (subtract 4 2)) => [{ type: 'paren', value: '(' }, ...]
*/
class Tokenizer {
private final val NUMBERS = "[0-9]+"
private final val LETTERS = "[a-z]+"
private final val WHITESPACE = "\\s"
def tokenize(input: String): List[Token] = {
var current: Int = 0
var tokens: ListBuffer[Token] = ListBuffer() //Stack is deprecated so using list with `::` to push and `tail` to pop
while (current < input.length) {
var char = input(current)
breakable {
if (char == '(') {
tokens += new Token("paren", "(")
current += 1
break
}
if (char == ')') {
tokens += new Token("paren", ")")
current += 1
break
}
if (char.toString.matches(WHITESPACE)) {
current += 1
break
}
if (char.toString.matches(NUMBERS)) {
var value = " "
while (char.toString.matches(NUMBERS)) {
value += char
current += 1
char = input(current)
}
tokens += new Token("number", value)
break
}
if (char.toString.matches(LETTERS)) {
var value = ""
while (char.toString.matches(LETTERS)) {
value += char
current += 1
char = input(current)
}
tokens += new Token("name", value)
break
}
}
//TODO: solve
//throw new Exception(s"Char is unknown: $char")
}
tokens.toList
}
}