-
Notifications
You must be signed in to change notification settings - Fork 7
/
tokenizer.ts
130 lines (105 loc) · 3.13 KB
/
tokenizer.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import analyze from "./js.ts";
export type TokenType = "string" | "tag" | "filter" | "comment";
export type Token = [TokenType, string, number?];
export interface TokenizeResult {
tokens: Token[];
position: number;
error: Error | undefined;
}
export default function tokenize(source: string): TokenizeResult {
const tokens: Token[] = [];
let type: TokenType = "string";
let position = 0;
try {
while (source.length > 0) {
if (type === "string") {
const index = source.indexOf("{{");
const code = index === -1 ? source : source.slice(0, index);
tokens.push([type, code, position]);
if (index === -1) {
break;
}
position += index;
source = source.slice(index);
type = source.startsWith("{{#") ? "comment" : "tag";
continue;
}
if (type === "comment") {
source = source.slice(3);
const index = source.indexOf("#}}");
const comment = index === -1 ? source : source.slice(0, index);
tokens.push([type, comment, position]);
if (index === -1) {
break;
}
position += index + 3;
source = source.slice(index + 3);
type = "string";
continue;
}
if (type === "tag") {
const indexes = parseTag(source);
const lastIndex = indexes.length - 1;
let tag: Token | undefined;
indexes.reduce((prev, curr, index) => {
const code = source.slice(prev, curr - 2);
// Tag
if (index === 1) {
tag = [type, code, position];
tokens.push(tag);
return curr;
}
// Filters
tokens.push(["filter", code]);
return curr;
});
position += indexes[lastIndex];
source = source.slice(indexes[lastIndex]);
type = "string";
// Search the closing echo tag {{ /echo }}
if (tag?.[1].match(/^\-?\s*echo\s*\-?$/)) {
const end = source.match(/{{\-?\s*\/echo\s*\-?}}/);
if (!end) {
throw new Error("Unclosed echo tag");
}
const rawCode = source.slice(0, end.index);
tag[1] = `echo ${JSON.stringify(rawCode)}`;
const length = Number(end.index) + end[0].length;
source = source.slice(length);
position += length;
}
continue;
}
}
} catch (error) {
return { tokens, position, error };
}
return { tokens, position, error: undefined };
}
type status =
| "single-quote"
| "double-quote"
| "regex"
| "literal"
| "bracket"
| "comment"
| "line-comment";
/**
* Parse a tag and return the indexes of the start and end brackets, and the filters between.
* For example: {{ tag |> filter1 |> filter2 }} => [2, 9, 20, 31]
*/
export function parseTag(source: string): number[] {
const indexes: number[] = [2];
analyze(source, (type, index) => {
if (type === "close") {
indexes.push(index);
return false;
}
if (type === "new-filter") {
indexes.push(index);
} else if (type === "unclosed") {
throw new Error("Unclosed tag");
}
});
return indexes;
}