Skip to content

Commit 5af6621

Browse files
mysticateamysticatea
committed
Chore: add tests for eof-before-tag-name error
1 parent 979b1ed commit 5af6621

File tree

4 files changed

+182
-9
lines changed

4 files changed

+182
-9
lines changed

‎src/html/tokenizer.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ export class Tokenizer {
167167
* @param text The source code to tokenize.
168168
*/
169169
constructor(text: string) {
170+
debug("[html] the source code length: %d", text.length)
170171
this.text = text
171172
this.gaps = []
172173
this.lineTerminators = []
@@ -196,18 +197,15 @@ export class Tokenizer {
196197
* @returns The next token or null.
197198
*/
198199
public nextToken(): Token {
199-
let cp = NULL
200-
while (this.tokens.length === 0) {
200+
let cp = this.lastCodePoint
201+
while (this.tokens.length === 0 && (cp !== EOF || this.reconsuming)) {
201202
if (this.reconsuming) {
202203
this.reconsuming = false
203204
cp = this.lastCodePoint
204205
}
205206
else {
206207
cp = this.consumeNextCodePoint()
207208
}
208-
if (cp === EOF) {
209-
break
210-
}
211209

212210
debug("[html] parse", cp, this.state)
213211
this.state = this[this.state](cp)
@@ -396,15 +394,15 @@ export class Tokenizer {
396394
this.currentToken = null
397395
this.tokenStartOffset = -1
398396

397+
token.range[1] = offset
398+
token.loc.end.line = line
399+
token.loc.end.column = column
400+
399401
if (token.range[0] === offset && !provisional) {
400402
debug("[html] abandon token: %j %s %j", token.range, token.type, token.value)
401403
return null
402404
}
403405

404-
token.range[1] = offset
405-
token.loc.end.line = line
406-
token.loc.end.column = column
407-
408406
if (provisional) {
409407
this.provisionalTokens.push(token)
410408
debug("[html] provisional commit token: %j %s %j", token.range, token.type, token.value)
@@ -658,6 +656,7 @@ export class Tokenizer {
658656
return this.reconsumeAs("BOGUS_COMMENT")
659657
}
660658
if (cp === EOF) {
659+
this.clearStartTokenMark()
661660
this.reportParseError("eof-before-tag-name")
662661
this.appendTokenValue(LESS_THAN_SIGN, "HTMLText")
663662
return "DATA"
@@ -683,6 +682,7 @@ export class Tokenizer {
683682
return "DATA"
684683
}
685684
if (cp === EOF) {
685+
this.clearStartTokenMark()
686686
this.reportParseError("eof-before-tag-name")
687687
this.appendTokenValue(LESS_THAN_SIGN, "HTMLText")
688688
this.appendTokenValue(SOLIDUS, "HTMLText")

‎test/fixtures/ast/error-eof-before-tag-name/ast.json‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
{
2+
"type": "Program",
3+
"start": 0,
4+
"end": 0,
5+
"loc": {
6+
"start": {
7+
"line": 1,
8+
"column": 0
9+
},
10+
"end": {
11+
"line": 1,
12+
"column": 0
13+
}
14+
},
15+
"range": [
16+
0,
17+
0
18+
],
19+
"body": [],
20+
"sourceType": "script",
21+
"comments": [],
22+
"tokens": [],
23+
"templateBody": {
24+
"type": "VElement",
25+
"range": [
26+
0,
27+
16
28+
],
29+
"loc": {
30+
"start": {
31+
"line": 1,
32+
"column": 0
33+
},
34+
"end": {
35+
"line": 2,
36+
"column": 5
37+
}
38+
},
39+
"name": "template",
40+
"namespace": "http://www.w3.org/1999/xhtml",
41+
"startTag": {
42+
"type": "VStartTag",
43+
"range": [
44+
0,
45+
10
46+
],
47+
"loc": {
48+
"start": {
49+
"line": 1,
50+
"column": 0
51+
},
52+
"end": {
53+
"line": 1,
54+
"column": 10
55+
}
56+
},
57+
"attributes": []
58+
},
59+
"children": [
60+
{
61+
"type": "VText",
62+
"range": [
63+
10,
64+
16
65+
],
66+
"loc": {
67+
"start": {
68+
"line": 1,
69+
"column": 10
70+
},
71+
"end": {
72+
"line": 2,
73+
"column": 5
74+
}
75+
},
76+
"value": "\n <"
77+
}
78+
],
79+
"endTag": null,
80+
"variables": [],
81+
"tokens": [
82+
{
83+
"type": "HTMLTagOpen",
84+
"range": [
85+
0,
86+
9
87+
],
88+
"loc": {
89+
"start": {
90+
"line": 1,
91+
"column": 0
92+
},
93+
"end": {
94+
"line": 1,
95+
"column": 9
96+
}
97+
},
98+
"value": "template"
99+
},
100+
{
101+
"type": "HTMLTagClose",
102+
"range": [
103+
9,
104+
10
105+
],
106+
"loc": {
107+
"start": {
108+
"line": 1,
109+
"column": 9
110+
},
111+
"end": {
112+
"line": 1,
113+
"column": 10
114+
}
115+
},
116+
"value": ""
117+
},
118+
{
119+
"type": "HTMLWhitespace",
120+
"range": [
121+
10,
122+
15
123+
],
124+
"loc": {
125+
"start": {
126+
"line": 1,
127+
"column": 10
128+
},
129+
"end": {
130+
"line": 2,
131+
"column": 4
132+
}
133+
},
134+
"value": "\n "
135+
},
136+
{
137+
"type": "HTMLText",
138+
"range": [
139+
15,
140+
16
141+
],
142+
"loc": {
143+
"start": {
144+
"line": 2,
145+
"column": 4
146+
},
147+
"end": {
148+
"line": 2,
149+
"column": 5
150+
}
151+
},
152+
"value": "<"
153+
}
154+
],
155+
"comments": [],
156+
"errors": [
157+
{
158+
"message": "eof-before-tag-name (2:5)",
159+
"index": 16,
160+
"lineNumber": 2,
161+
"column": 5
162+
}
163+
]
164+
}
165+
}

‎test/fixtures/ast/error-eof-before-tag-name/source.vue‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
<template>
2+
<

‎test/fixtures/ast/error-eof-before-tag-name/token-ranges.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
"<template",
3+
">",
4+
"\n ",
5+
"<"
6+
]

0 commit comments

Comments
 (0)