Skip to content
This repository has been archived by the owner on May 19, 2018. It is now read-only.

Commit

Permalink
Fix: major revision to valid and invalid numeric literal separator "s…
Browse files Browse the repository at this point in the history
…ibling" characters (#745)
  • Loading branch information
rwaldron authored and hzoo committed Sep 29, 2017
1 parent 17be936 commit 18c6b4e
Show file tree
Hide file tree
Showing 289 changed files with 589 additions and 84 deletions.
55 changes: 55 additions & 0 deletions src/tokenizer/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,48 @@ const forbiddenNumericSeparatorSiblings = {
],
};

const allowedNumericSeparatorSiblings = {};
allowedNumericSeparatorSiblings.bin = [
// 0 - 1
48,
49,
];
allowedNumericSeparatorSiblings.oct = [
// 0 - 7
...allowedNumericSeparatorSiblings.bin,
50,
51,
52,
53,
54,
55,
];
allowedNumericSeparatorSiblings.dec = [
// 0 - 9
...allowedNumericSeparatorSiblings.oct,
56,
57,
];

allowedNumericSeparatorSiblings.hex = [
// 0 - 9, A - F, a - f,
...allowedNumericSeparatorSiblings.dec,
// A - F
65,
66,
67,
68,
69,
70,
// a - f
97,
98,
99,
100,
101,
102,
];

// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
Expand Down Expand Up @@ -718,6 +760,15 @@ export default class Tokenizer extends LocationParser {
radix === 16
? forbiddenNumericSeparatorSiblings.hex
: forbiddenNumericSeparatorSiblings.decBinOct;
const allowedSiblings =
radix === 16
? allowedNumericSeparatorSiblings.hex
: radix === 10
? allowedNumericSeparatorSiblings.dec
: radix === 8
? allowedNumericSeparatorSiblings.oct
: allowedNumericSeparatorSiblings.bin;

let total = 0;

for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
Expand All @@ -728,6 +779,10 @@ export default class Tokenizer extends LocationParser {
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) {
if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token");
}

if (
forbiddenSiblings.indexOf(prev) > -1 ||
forbiddenSiblings.indexOf(next) > -1 ||
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_
1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1_
1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x1_1_
0x1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1__)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1_.1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1._1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_e1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1_E1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1e_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(1_1.1E_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xa_1_
0xa_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0xa_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_a_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x__1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1__1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0x_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_11)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_0_1_1)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_a_1
0x_a_1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b_01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0b01_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(0o_1_1_)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1__}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1_.1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1._1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x__1_1_
0x__1_1_
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_e1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1_E1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1e_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{1_1.1E_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0xa_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_a_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x__1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1__1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0x_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1__1
0x_1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_11}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_0_1_1}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b_01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0b01_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:7)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:6)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{0o_1_1_}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0x_1_1_
0x_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1
0o_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_11
0o_11
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_01_1_
0o_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_0_1_1
0b_0_1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_1__
1_1__
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b_01_1_
0b_01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b01_1_
0b01_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o1_1_
0o1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o_1_1_
0o_1_1_
Original file line number Diff line number Diff line change
@@ -1 +1 @@
._1_1
._1_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token (1:0)" }
{ "throws": "Unexpected token (1:0)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0o01_8
0o01_8
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Unexpected token, expected ; (1:5)" }
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0b2_1
0b2_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 2 (1:2)" }
{ "throws": "Expected number in radix 2 (1:2)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0xZ_1
0xZ_1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "throws": "Expected number in radix 16 (1:2)" }
{ "throws": "Expected number in radix 16 (1:2)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1__1
1__1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1__;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1__1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:1)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1_.1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:3)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1._1_1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:4)" }
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1_1.1_e1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "throws": "Invalid or unexpected token (1:5)" }

0 comments on commit 18c6b4e

Please sign in to comment.