Skip to content
This repository has been archived by the owner on May 19, 2018. It is now read-only.

Fix: major revision to valid and invalid numeric literal separator "sibling" characters #745

Merged
merged 3 commits into from
Sep 29, 2017
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
38 changes: 38 additions & 0 deletions src/tokenizer/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,31 @@ const forbiddenNumericSeparatorSiblings = {
],
};

const allowedNumericSeparatorSiblings = {};
allowedNumericSeparatorSiblings.bin = [
// 0 - 1
48, 49
];
allowedNumericSeparatorSiblings.oct = [
// 0 - 7
...allowedNumericSeparatorSiblings.bin,
50, 51, 52, 53, 54, 55,
];
allowedNumericSeparatorSiblings.dec = [
// 0 - 9
...allowedNumericSeparatorSiblings.oct,
56, 57,
];

allowedNumericSeparatorSiblings.hex = [
// 0 - 9, A - F, a - f,
...allowedNumericSeparatorSiblings.dec,
// A - F
65, 66, 67, 68, 69, 70,
// a - f
97, 98, 99, 100, 101, 102,
];

// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
Expand Down Expand Up @@ -718,6 +743,15 @@ export default class Tokenizer extends LocationParser {
radix === 16
? forbiddenNumericSeparatorSiblings.hex
: forbiddenNumericSeparatorSiblings.decBinOct;
const allowedSiblings =
radix === 16
? allowedNumericSeparatorSiblings.hex
: (radix === 10
? allowedNumericSeparatorSiblings.dec
: (radix === 8
? allowedNumericSeparatorSiblings.oct
: allowedNumericSeparatorSiblings.bin));

let total = 0;

for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
Expand All @@ -728,6 +762,10 @@ export default class Tokenizer extends LocationParser {
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) {
if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token");
}

if (
forbiddenSiblings.indexOf(prev) > -1 ||
forbiddenSiblings.indexOf(next) > -1 ||
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_
1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1_
1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x1_1_
0x1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1__)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_.1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1._1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_e1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_E1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1e_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1E_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xa_1_
0xa_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0xa_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_a_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x__1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_11)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_0_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_a_1
0x_a_1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1__}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_.1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1._1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x__1_1_
0x__1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_e1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_E1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1e_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1E_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0xa_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_a_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x__1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1__1
0x_1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_11}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_0_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1_1_
0x_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1
0o_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_11
0o_11
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_01_1_
0o_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_0_1_1
0b_0_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1__
1_1__
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_01_1_
0b_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b01_1_
0b01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o1_1_
0o1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1_
0o_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
._1_1
._1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token (1:0)" }
{ "throws": "Unexpected token (1:0)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o01_8
0o01_8
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token, expected ; (1:5)" }
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b2_1
0b2_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 2 (1:2)" }
{ "throws": "Expected number in radix 2 (1:2)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xZ_1
0xZ_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 16 (1:2)" }
{ "throws": "Expected number in radix 16 (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1__1
1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1__;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1__1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_.1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1._1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_e1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_E1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1e_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1E_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0x1_1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }