Permalink
Browse files

Make the pure tokenizer a bit aggressive in recognizing regex literals.

At the same time, ensure that it can fall back if the regex literal
proved to be invalid.

Fix #1772
Fix #1873
Close gh-1889
  • Loading branch information...
ariya committed Nov 23, 2017
1 parent 512ce40 commit df749a7fd319a72beb46d28c612da6c8497e7488
View
@@ -1,5 +1,5 @@
import { ErrorHandler } from './error-handler';
import { Comment, Scanner, SourceLocation } from './scanner';
import { Comment, RawToken, Scanner, SourceLocation } from './scanner';
import { Token, TokenName } from './token';
type ReaderEntry = string | null;
@@ -58,7 +58,7 @@ class Reader {
case '}':
// Dividing a function by anything makes little sense,
// but we have to check for that.
regex = false;
regex = true;
if (this.values[this.curly - 3] === 'function') {
// Anonymous function, e.g. function(){} /42
const check = this.values[this.curly - 4];
@@ -160,10 +160,21 @@ export class Tokenizer {
};
}
const startRegex = (this.scanner.source[this.scanner.index] === '/') && this.reader.isRegexStart();
const token = startRegex ? this.scanner.scanRegExp() : this.scanner.lex();
this.reader.push(token);
const maybeRegex = (this.scanner.source[this.scanner.index] === '/') && this.reader.isRegexStart();
let token: RawToken;
if (maybeRegex) {
const state = this.scanner.saveState();
try {
token = this.scanner.scanRegExp();
} catch (e) {
this.scanner.restoreState(state);
token = this.scanner.lex();
}
} else {
token = this.scanner.lex();
}
this.reader.push(token);
const entry: BufferEntry = {
type: TokenName[token.type],
value: this.scanner.source.slice(token.start, token.end)
@@ -0,0 +1 @@
if(x){} /y/.test(z)
@@ -0,0 +1,222 @@
[
{
"type": "Keyword",
"value": "if",
"range": [
0,
2
],
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 2
}
}
},
{
"type": "Punctuator",
"value": "(",
"range": [
2,
3
],
"loc": {
"start": {
"line": 1,
"column": 2
},
"end": {
"line": 1,
"column": 3
}
}
},
{
"type": "Identifier",
"value": "x",
"range": [
3,
4
],
"loc": {
"start": {
"line": 1,
"column": 3
},
"end": {
"line": 1,
"column": 4
}
}
},
{
"type": "Punctuator",
"value": ")",
"range": [
4,
5
],
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 5
}
}
},
{
"type": "Punctuator",
"value": "{",
"range": [
5,
6
],
"loc": {
"start": {
"line": 1,
"column": 5
},
"end": {
"line": 1,
"column": 6
}
}
},
{
"type": "Punctuator",
"value": "}",
"range": [
6,
7
],
"loc": {
"start": {
"line": 1,
"column": 6
},
"end": {
"line": 1,
"column": 7
}
}
},
{
"type": "RegularExpression",
"value": "/y/",
"range": [
8,
11
],
"loc": {
"start": {
"line": 1,
"column": 8
},
"end": {
"line": 1,
"column": 11
}
},
"regex": {
"pattern": "y",
"flags": ""
}
},
{
"type": "Punctuator",
"value": ".",
"range": [
11,
12
],
"loc": {
"start": {
"line": 1,
"column": 11
},
"end": {
"line": 1,
"column": 12
}
}
},
{
"type": "Identifier",
"value": "test",
"range": [
12,
16
],
"loc": {
"start": {
"line": 1,
"column": 12
},
"end": {
"line": 1,
"column": 16
}
}
},
{
"type": "Punctuator",
"value": "(",
"range": [
16,
17
],
"loc": {
"start": {
"line": 1,
"column": 16
},
"end": {
"line": 1,
"column": 17
}
}
},
{
"type": "Identifier",
"value": "z",
"range": [
17,
18
],
"loc": {
"start": {
"line": 1,
"column": 17
},
"end": {
"line": 1,
"column": 18
}
}
},
{
"type": "Punctuator",
"value": ")",
"range": [
18,
19
],
"loc": {
"start": {
"line": 1,
"column": 18
},
"end": {
"line": 1,
"column": 19
}
}
}
]
@@ -0,0 +1,7 @@
{
"tokenize": true,
"index": 2,
"lineNumber": 1,
"column": 3,
"message": "Error: Line 1: Unexpected token ILLEGAL"
}
@@ -0,0 +1 @@
1e
@@ -34,5 +34,59 @@
"column": 4
}
}
},
{
"type": "Punctuator",
"value": "/",
"range": [
4,
5
],
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 5
}
}
},
{
"type": "Numeric",
"value": "42",
"range": [
5,
7
],
"loc": {
"start": {
"line": 1,
"column": 5
},
"end": {
"line": 1,
"column": 7
}
}
},
{
"type": "Punctuator",
"value": "]",
"range": [
7,
8
],
"loc": {
"start": {
"line": 1,
"column": 7
},
"end": {
"line": 1,
"column": 8
}
}
}
]

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,38 @@
[
{
"type": "Punctuator",
"value": "/",
"range": [
0,
1
],
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 1
}
}
},
{
"type": "Numeric",
"value": "42",
"range": [
1,
3
],
"loc": {
"start": {
"line": 1,
"column": 1
},
"end": {
"line": 1,
"column": 3
}
}
}
]

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit df749a7

Please sign in to comment.