Skip to content

Commit 6b31181

Browse files
mysticateamysticatea
committed
Chore: refactor to mingle parsing of expressions to HTML parser
1 parent 217c698 commit 6b31181

File tree

5 files changed

+409
-412
lines changed

5 files changed

+409
-412
lines changed

‎src/html/intermediate-tokenizer.ts‎

Lines changed: 125 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,26 @@
44
* See LICENSE file in root directory for full license.
55
*/
66
import assert from "assert"
7+
import * as lodash from "lodash"
78
import {ErrorCode, HasLocation, Namespace, ParseError, Token, VAttribute} from "../ast"
89
import {debug} from "../common/debug"
910
import {Tokenizer, TokenizerState, TokenType} from "./tokenizer"
1011

1112
const DUMMY_PARENT: any = Object.freeze({})
1213

14+
/**
15+
* Concatenate token values.
16+
* @param text Concatenated text.
17+
* @param token The token to concatenate.
18+
*/
19+
function concat(text: string, token: Token): string {
20+
return text + token.value
21+
}
22+
1323
/**
1424
* The type of intermediate tokens.
1525
*/
16-
export type IntermediateToken = StartTag | EndTag | Text
26+
export type IntermediateToken = StartTag | EndTag | Text | Mustache
1727

1828
/**
1929
* The type of start tags.
@@ -41,17 +51,36 @@ export interface Text extends HasLocation {
4151
value: string
4252
}
4353

54+
/**
55+
* The type of text chunks of an expression container.
56+
*/
57+
export interface Mustache extends HasLocation {
58+
type: "Mustache"
59+
value: string
60+
startToken: Token
61+
endToken: Token
62+
}
63+
4464
/**
4565
* The class to create HTML tokens from ESTree-like tokens which are created by a Tokenizer.
4666
*/
4767
export class IntermediateTokenizer {
4868
private tokenizer: Tokenizer
4969
private currentToken: IntermediateToken | null
50-
private currentAttribute: VAttribute | null
70+
private attribute: VAttribute | null
71+
private expressionStartToken: Token | null
72+
private expressionTokens: Token[]
5173

5274
public readonly tokens: Token[]
5375
public readonly comments: Token[]
5476

77+
/**
78+
* The source code text.
79+
*/
80+
get text(): string {
81+
return this.tokenizer.text
82+
}
83+
5584
/**
5685
* The parse errors.
5786
*/
@@ -96,7 +125,9 @@ export class IntermediateTokenizer {
96125
constructor(tokenizer: Tokenizer) {
97126
this.tokenizer = tokenizer
98127
this.currentToken = null
99-
this.currentAttribute = null
128+
this.attribute = null
129+
this.expressionStartToken = null
130+
this.expressionTokens = []
100131
this.tokens = []
101132
this.comments = []
102133
}
@@ -124,11 +155,38 @@ export class IntermediateTokenizer {
124155
* Commit the current token.
125156
*/
126157
private commit(): IntermediateToken {
127-
assert(this.currentToken != null)
158+
assert(this.currentToken != null || this.expressionStartToken != null)
128159

129-
const token = this.currentToken
160+
let token = this.currentToken
130161
this.currentToken = null
131-
this.currentAttribute = null
162+
this.attribute = null
163+
164+
if (this.expressionStartToken != null) {
165+
// VExpressionEnd was not found.
166+
// Concatenate the deferred tokens to the committed token.
167+
const start = this.expressionStartToken
168+
const end = lodash.last(this.expressionTokens) || start
169+
const value = this.expressionTokens.reduce(concat, start.value) + (start !== end ? end.value : "")
170+
this.expressionStartToken = null
171+
this.expressionTokens = []
172+
173+
if (token == null) {
174+
token = {
175+
type: "Text",
176+
range: [start.range[0], end.range[1]],
177+
loc: {start: start.loc.start, end: end.loc.end},
178+
value,
179+
}
180+
}
181+
else if (token.type === "Text") {
182+
token.range[1] = end.range[1]
183+
token.loc.end = end.loc.end
184+
token.value += value
185+
}
186+
else {
187+
throw new Error("unreachable")
188+
}
189+
}
132190

133191
return token as IntermediateToken
134192
}
@@ -166,8 +224,19 @@ export class IntermediateTokenizer {
166224

167225
let result: IntermediateToken | null = null
168226

169-
if (this.currentToken != null && this.currentToken.type === "Text") {
170-
if (this.currentToken.range[1] === token.range[0]) {
227+
if (this.expressionStartToken != null) {
228+
// Defer this token until a VExpressionEnd token or a non-text token appear.
229+
const lastToken = lodash.last(this.expressionTokens) || this.expressionStartToken
230+
if (lastToken.range[1] === token.range[0]) {
231+
this.expressionTokens.push(token)
232+
return null
233+
}
234+
235+
result = this.commit()
236+
}
237+
else if (this.currentToken != null) {
238+
// Concatenate this token to the current text token.
239+
if (this.currentToken.type === "Text" && this.currentToken.range[1] === token.range[0]) {
171240
this.currentToken.value += token.value
172241
this.currentToken.range[1] = token.range[1]
173242
this.currentToken.loc.end = token.loc.end
@@ -195,9 +264,9 @@ export class IntermediateTokenizer {
195264
protected HTMLAssociation(token: Token): IntermediateToken | null {
196265
this.tokens.push(token)
197266

198-
if (this.currentAttribute != null) {
199-
this.currentAttribute.range[1] = token.range[1]
200-
this.currentAttribute.loc.end = token.loc.end
267+
if (this.attribute != null) {
268+
this.attribute.range[1] = token.range[1]
269+
this.attribute.loc.end = token.loc.end
201270

202271
if (this.currentToken == null || this.currentToken.type !== "StartTag") {
203272
throw new Error("unreachable")
@@ -242,7 +311,7 @@ export class IntermediateTokenizer {
242311

243312
let result: IntermediateToken | null = null
244313

245-
if (this.currentToken != null) {
314+
if (this.currentToken != null || this.expressionStartToken != null) {
246315
result = this.commit()
247316
}
248317

@@ -263,15 +332,15 @@ export class IntermediateTokenizer {
263332
protected HTMLIdentifier(token: Token): IntermediateToken | null {
264333
this.tokens.push(token)
265334

266-
if (this.currentToken == null || this.currentToken.type === "Text") {
335+
if (this.currentToken == null || this.currentToken.type === "Text" || this.currentToken.type === "Mustache") {
267336
throw new Error("unreachable")
268337
}
269338
if (this.currentToken.type === "EndTag") {
270339
this.reportParseError(token, "end-tag-with-attributes")
271340
return null
272341
}
273342

274-
this.currentAttribute = {
343+
this.attribute = {
275344
type: "VAttribute",
276345
range: [token.range[0], token.range[1]],
277346
loc: {start: token.loc.start, end: token.loc.end},
@@ -286,11 +355,11 @@ export class IntermediateTokenizer {
286355
},
287356
value: null,
288357
}
289-
this.currentAttribute.key.parent = this.currentAttribute
358+
this.attribute.key.parent = this.attribute
290359

291360
this.currentToken.range[1] = token.range[1]
292361
this.currentToken.loc.end = token.loc.end
293-
this.currentToken.attributes.push(this.currentAttribute)
362+
this.currentToken.attributes.push(this.attribute)
294363

295364
return null
296365
}
@@ -302,14 +371,14 @@ export class IntermediateTokenizer {
302371
protected HTMLLiteral(token: Token): IntermediateToken | null {
303372
this.tokens.push(token)
304373

305-
if (this.currentAttribute != null) {
306-
this.currentAttribute.range[1] = token.range[1]
307-
this.currentAttribute.loc.end = token.loc.end
308-
this.currentAttribute.value = {
374+
if (this.attribute != null) {
375+
this.attribute.range[1] = token.range[1]
376+
this.attribute.loc.end = token.loc.end
377+
this.attribute.value = {
309378
type: "VLiteral",
310379
range: [token.range[0], token.range[1]],
311380
loc: {start: token.loc.start, end: token.loc.end},
312-
parent: this.currentAttribute,
381+
parent: this.attribute,
313382
value: token.value,
314383
}
315384

@@ -389,7 +458,7 @@ export class IntermediateTokenizer {
389458

390459
let result: IntermediateToken | null = null
391460

392-
if (this.currentToken != null) {
461+
if (this.currentToken != null || this.expressionStartToken != null) {
393462
result = this.commit()
394463
}
395464

@@ -426,14 +495,46 @@ export class IntermediateTokenizer {
426495
* @param token The token to process.
427496
*/
428497
protected VExpressionStart(token: Token): IntermediateToken | null {
429-
return this.processText(token)
498+
if (this.expressionStartToken != null) {
499+
return this.processText(token)
500+
}
501+
502+
this.tokens.push(token)
503+
this.expressionStartToken = token
504+
505+
if (this.currentToken != null && this.currentToken.range[1] !== token.range[0]) {
506+
return this.commit()
507+
}
508+
return null
430509
}
431510

432511
/**
433512
* Process a VExpressionEnd token.
434513
* @param token The token to process.
435514
*/
436515
protected VExpressionEnd(token: Token): IntermediateToken | null {
437-
return this.processText(token)
516+
if (this.expressionStartToken == null) {
517+
return this.processText(token)
518+
}
519+
this.tokens.push(token)
520+
521+
// Clear state.
522+
const start = this.expressionStartToken
523+
const value = this.expressionTokens.reduce(concat, "")
524+
this.expressionStartToken = null
525+
this.expressionTokens = []
526+
527+
// Create token.
528+
const result = (this.currentToken != null) ? this.commit() : null
529+
this.currentToken = {
530+
type: "Mustache",
531+
range: [start.range[0], token.range[1]],
532+
loc: {start: start.loc.start, end: token.loc.end},
533+
value,
534+
startToken: start,
535+
endToken: token,
536+
}
537+
538+
return result || this.commit()
438539
}
439540
}

0 commit comments

Comments
 (0)