add numeric separators

acorn-loose/src/expression.js

@@ -266,7 +266,7 @@ lp.parseExprAtom = function() {
     node = this.startNode()
     node.value = this.tok.value
     node.raw = this.input.slice(this.tok.start, this.tok.end)
-    if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1)
+    if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "")
     this.next()
     return this.finishNode(node, "Literal")
 

acorn/src/expression.js

@@ -517,7 +517,7 @@ pp.parseLiteral = function(value) {
   let node = this.startNode()
   node.value = value
   node.raw = this.input.slice(this.start, this.end)
-  if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1)
+  if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "")
   this.next()
   return this.finishNode(node, "Literal")
 }

acorn/src/tokenize.js

@@ -438,30 +438,67 @@ pp.readRegexp = function() {
 // were read, the integer value otherwise. When `len` is given, this
 // will return `null` unless the integer has exactly `len` digits.
 
-pp.readInt = function(radix, len) {
-  let start = this.pos, total = 0
-  for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
+pp.readInt = function(radix, len, maybeLegacyOctalNumericLiteral) {
+  // `len` is used for character escape sequences. In that case, disallow separators.
+  const allowSeparators = this.options.ecmaVersion >= 12 && len === undefined
+
+  // `maybeLegacyOctalNumericLiteral` is true if it doesn't have prefix (0x,0o,0b)
+  // and isn't fraction part nor exponent part. In that case, if the first digit
+  // is zero then disallow separators.
+  const isLegacyOctalNumericLiteral = maybeLegacyOctalNumericLiteral && this.input.charCodeAt(this.pos) === 48
+
+  let start = this.pos, total = 0, lastCode = 0
+  for (let i = 0, e = len == null ? Infinity : len; i < e; ++i, ++this.pos) {
     let code = this.input.charCodeAt(this.pos), val
+
+    if (allowSeparators && code === 95) {
+      if (isLegacyOctalNumericLiteral) this.raiseRecoverable(this.pos, "Numeric separator is not allowed in legacy octal numeric literals")
+      if (lastCode === 95) this.raiseRecoverable(this.pos, "Numeric separator must be exactly one underscore")
+      if (i === 0) this.raiseRecoverable(this.pos, "Numeric separator is not allowed at the first of digits")
+      lastCode = code
+      continue
+    }
+
     if (code >= 97) val = code - 97 + 10 // a
     else if (code >= 65) val = code - 65 + 10 // A
     else if (code >= 48 && code <= 57) val = code - 48 // 0-9
     else val = Infinity
     if (val >= radix) break
-    ++this.pos
+    lastCode = code
     total = total * radix + val
   }
+
+  if (allowSeparators && lastCode === 95) this.raiseRecoverable(this.pos - 1, "Numeric separator is not allowed at the last of digits")
   if (this.pos === start || len != null && this.pos - start !== len) return null
 
   return total
 }
 
+function stringToNumber(str, isLegacyOctalNumericLiteral) {
+  if (isLegacyOctalNumericLiteral) {
+    return parseInt(str, 8)
+  }
+
+  // `parseFloat(value)` stops parsing at the first numeric separator then returns a wrong value.
+  return parseFloat(str.replace(/_/g, ""))
+}
+
+function stringToBigInt(str) {
+  if (typeof BigInt !== "function") {
+    return null
+  }
+
+  // `BigInt(value)` throws syntax error if the string contains numeric separators.
+  return BigInt(str.replace(/_/g, ""))
+}
+
 pp.readRadixNumber = function(radix) {
   let start = this.pos
   this.pos += 2 // 0x
   let val = this.readInt(radix)
   if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix)
   if (this.options.ecmaVersion >= 11 && this.input.charCodeAt(this.pos) === 110) {
-    val = typeof BigInt !== "undefined" ? BigInt(this.input.slice(start, this.pos)) : null
+    val = stringToBigInt(this.input.slice(start, this.pos))
     ++this.pos
   } else if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
   return this.finishToken(tt.num, val)
@@ -471,13 +508,12 @@ pp.readRadixNumber = function(radix) {
 
 pp.readNumber = function(startsWithDot) {
   let start = this.pos
-  if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number")
+  if (!startsWithDot && this.readInt(10, undefined, true) === null) this.raise(start, "Invalid number")
   let octal = this.pos - start >= 2 && this.input.charCodeAt(start) === 48
   if (octal && this.strict) this.raise(start, "Invalid number")
   let next = this.input.charCodeAt(this.pos)
   if (!octal && !startsWithDot && this.options.ecmaVersion >= 11 && next === 110) {
-    let str = this.input.slice(start, this.pos)
-    let val = typeof BigInt !== "undefined" ? BigInt(str) : null
+    let val = stringToBigInt(this.input.slice(start, this.pos))
     ++this.pos
     if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
     return this.finishToken(tt.num, val)
@@ -495,8 +531,7 @@ pp.readNumber = function(startsWithDot) {
   }
   if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
 
-  let str = this.input.slice(start, this.pos)
-  let val = octal ? parseInt(str, 8) : parseFloat(str)
+  let val = stringToNumber(this.input.slice(start, this.pos), octal)
   return this.finishToken(tt.num, val)
 }
 

bin/run_test262.js

@@ -10,7 +10,6 @@ const unsupportedFeatures = [
   "class-static-fields-private",
   "class-static-fields-public",
   "class-static-methods-private",
-  "numeric-separator-literal",
   "logical-assignment-operators",
 ];
 

test/run.js

@@ -22,6 +22,7 @@
   require("./tests-nullish-coalescing.js");
   require("./tests-optional-chaining.js");
   require("./tests-logical-assignment-operators.js");
+  require("./tests-numeric-separators.js");
   var acorn = require("../acorn")
   var acorn_loose = require("../acorn-loose")
 

test/tests-numeric-separators.js

@@ -0,0 +1,199 @@
+// Tests for ECMAScript 2021 Numeric Separators
+
+if (typeof exports != 'undefined') {
+  var test = require('./driver.js').test;
+  var testFail = require('./driver.js').testFail;
+}
+
+function bigint(str) {
+  if (typeof BigInt !== "function") {
+    return null
+  }
+  return BigInt(str)
+}
+
+test(
+  "123_456",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 7,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 7,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 7,
+          "value": 123456,
+          "raw": "123_456"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  "123_456.123_456e+123_456",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 24,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 24,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 24,
+          "value": 123456.123456e+123456,
+          "raw": "123_456.123_456e+123_456"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  "0b1010_0001",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 11,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 11,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 11,
+          "value": 0b10100001,
+          "raw": "0b1010_0001"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  "0xDEAD_BEAF",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 11,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 11,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 11,
+          "value": 0xDEADBEAF,
+          "raw": "0xDEAD_BEAF"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  "0o755_666",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 9,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 9,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 9,
+          "value": 0o755666,
+          "raw": "0o755_666"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  "123_456n",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 8,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 8,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 8,
+          "value": bigint("123456"),
+          "raw": "123_456n",
+          "bigint": "123456"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+test(
+  ".012_345",
+  {
+    "type": "Program",
+    "start": 0,
+    "end": 8,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start": 0,
+        "end": 8,
+        "expression": {
+          "type": "Literal",
+          "start": 0,
+          "end": 8,
+          "value": 0.012345,
+          "raw": ".012_345"
+        }
+      }
+    ],
+    "sourceType": "script"
+  },
+  { ecmaVersion: 12 }
+);
+
+testFail("123_456", "Identifier directly after number (1:3)", { ecmaVersion: 11 });
+testFail("123__456", "Numeric separator must be exactly one underscore (1:4)", { ecmaVersion: 12 });
+testFail("0._123456", "Numeric separator is not allowed at the first of digits (1:2)", { ecmaVersion: 12 });
+testFail("123456_", "Numeric separator is not allowed at the last of digits (1:6)", { ecmaVersion: 12 });
+testFail("012_345", "Numeric separator is not allowed in legacy octal numeric literals (1:3)", { ecmaVersion: 12 });
+
+testFail("'\\x2_0'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 });
+testFail("'\\u00_20'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 });
+testFail("'\\u{2_0}'", "Bad character escape sequence (1:4)", { ecmaVersion: 12 });