support bigint and numeric seperators

src/character.ts

@@ -64,8 +64,15 @@ export const Character = {
             (cp >= 0x61 && cp <= 0x66);         // a..f
     },
 
+    isHexDigitChar(ch: string): ch is '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a'| 'b'| 'c'| 'd'| 'e'| 'f'| 'A'| 'B'| 'C'| 'D'| 'E'| 'F' {
+        return ch.length === 1 && Character.isHexDigit(ch.charCodeAt(0));
+    },
+
     isOctalDigit(cp: number): boolean {
         return (cp >= 0x30 && cp <= 0x37);      // 0..7
-    }
+    },
 
+    isOctalDigitChar(ch: string): ch is '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' {
+        return ch.length === 1 && Character.isOctalDigit(ch.charCodeAt(0));
+    }
 };

src/messages.ts

@@ -35,6 +35,8 @@ export const Messages = {
     NewlineAfterThrow: 'Illegal newline after throw',
     NoAsAfterImportNamespace: 'Unexpected token',
     NoCatchOrFinally: 'Missing catch or finally after try',
+    NumericSeperatorOneUnderscore: 'Numeric separator must be exactly one underscore',
+    NumericSeperatorNotAllowedHere: 'Numeric separator is not allowed here',
     ParameterAfterRestParameter: 'Rest parameter must be last formal parameter',
     PropertyAfterRestProperty: 'Unexpected token',
     Redeclaration: '%0 \'%1\' has already been declared',

src/nodes.ts

@@ -154,6 +154,19 @@ export class AwaitExpression {
     }
 }
 
+export class BigIntLiteral {
+    readonly type: string;
+    readonly value: null | string; //should be bigint
+    readonly raw: string;
+    readonly bigint: string;
+    constructor(value: null | string /*should be bigint*/, raw: string, bigint: string) {
+        this.type = Syntax.Literal;
+        this.value = value;
+        this.raw = raw;
+        this.bigint = bigint;
+    }
+}
+
 export class BinaryExpression {
     readonly type: string;
     readonly operator: string;

src/parser.ts

@@ -609,7 +609,7 @@ export class Parser {
         const node = this.createNode();
 
         let expr: Node.Expression;
-        let token, raw;
+        let token: RawToken, raw;
 
         switch (this.lookahead.type) {
             case Token.Identifier:
@@ -620,6 +620,7 @@ export class Parser {
                 break;
 
             case Token.NumericLiteral:
+            case Token.BigIntLiteral:
             case Token.StringLiteral:
                 if (this.context.strict && this.lookahead.octal) {
                     this.tolerateUnexpectedToken(this.lookahead, Messages.StrictOctalLiteral);
@@ -628,7 +629,10 @@ export class Parser {
                 this.context.isBindingElement = false;
                 token = this.nextToken();
                 raw = this.getTokenRaw(token);
-                expr = this.finalize(node, new Node.Literal(token.value, raw));
+                if (token.type == Token.BigIntLiteral)
+                    expr = this.finalize(node, new Node.BigIntLiteral(token.value as string, raw, token.value.toString()));
+                else
+                    expr = this.finalize(node, new Node.Literal(token.value as string, raw));
                 break;
 
             case Token.BooleanLiteral:
@@ -670,7 +674,7 @@ export class Parser {
                         this.scanner.index = this.startMarker.index;
                         token = this.nextRegexToken();
                         raw = this.getTokenRaw(token);
-                        expr = this.finalize(node, new Node.RegexLiteral(token.regex as RegExp, raw, token.pattern, token.flags));
+                        expr = this.finalize(node, new Node.RegexLiteral(token.regex as RegExp, raw, token.pattern as string, token.flags as string));
                         break;
                     default:
                         expr = this.throwUnexpectedToken(this.nextToken());
@@ -803,11 +807,16 @@ export class Parser {
         switch (token.type) {
             case Token.StringLiteral:
             case Token.NumericLiteral:
+            case Token.BigIntLiteral:
                 if (this.context.strict && token.octal) {
                     this.tolerateUnexpectedToken(token, Messages.StrictOctalLiteral);
                 }
                 const raw = this.getTokenRaw(token);
-                key = this.finalize(node, new Node.Literal(token.value as string, raw));
+                if (token.type === Token.BigIntLiteral)
+                    key = this.finalize(node, new Node.BigIntLiteral(token.value as string, raw, token.value.toString()));
+                else
+                    key = this.finalize(node, new Node.Literal(token.value as string, raw));
+
                 break;
 
             case Token.Identifier:

src/scanner.ts

@@ -35,7 +35,7 @@ type NotEscapeSequenceHead = '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8'
 
 export interface RawToken {
     type: Token;
-    value: string | number;
+    value: string | number; //add bigint
     pattern?: string;
     flags?: string;
     regex?: RegExp | null;
@@ -686,14 +686,7 @@ export class Scanner {
     // https://tc39.github.io/ecma262/#sec-literals-numeric-literals
 
     private scanHexLiteral(start: number): RawToken {
-        let num = '';
-
-        while (!this.eof()) {
-            if (!Character.isHexDigit(this.source.charCodeAt(this.index))) {
-                break;
-            }
-            num += this.source[this.index++];
-        }
+        let num = this.scanLiteralPart(Character.isHexDigitChar);
 
         if (num.length === 0) {
             this.throwUnexpectedToken();
@@ -714,16 +707,9 @@ export class Scanner {
     }
 
     private scanBinaryLiteral(start: number): RawToken {
-        let num = '';
         let ch;
 
-        while (!this.eof()) {
-            ch = this.source[this.index];
-            if (ch !== '0' && ch !== '1') {
-                break;
-            }
-            num += this.source[this.index++];
-        }
+        let num = this.scanLiteralPart(c => c === '0' || c === '1');
 
         if (num.length === 0) {
             // only 0b or 0B
@@ -759,12 +745,7 @@ export class Scanner {
             ++this.index;
         }
 
-        while (!this.eof()) {
-            if (!Character.isOctalDigit(this.source.charCodeAt(this.index))) {
-                break;
-            }
-            num += this.source[this.index++];
-        }
+        num += this.scanLiteralPart(Character.isOctalDigitChar);
 
         if (!octal && num.length === 0) {
             // only 0o or 0O
@@ -802,6 +783,26 @@ export class Scanner {
         return true;
     }
 
+    private scanLiteralPart(check: (str: string) => boolean) {
+        let num = '';
+
+        if (this.source[this.index] === '_')
+            this.throwUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
+
+        while (this.source[this.index] && (check(this.source[this.index]) || this.source[this.index] === '_')) {
+            if (this.source[this.index] !== '_')
+                num += this.source[this.index];
+            this.index++;
+            if (this.source[this.index - 1] === '_' && this.source[this.index] === '_')
+                this.throwUnexpectedToken(Messages.NumericSeperatorOneUnderscore);
+        }
+
+        if (this.source[this.index - 1] === '_')
+            this.throwUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
+
+        return num;
+    }
+
     private scanNumericLiteral(): RawToken {
         const start = this.index;
         let ch = this.source[start];
@@ -837,17 +838,14 @@ export class Scanner {
                 }
             }
 
-            while (Character.isDecimalDigit(this.source.charCodeAt(this.index))) {
-                num += this.source[this.index++];
-            }
+            this.index--;
+            num = this.scanLiteralPart(Character.isDecimalDigitChar);
             ch = this.source[this.index];
         }
 
         if (ch === '.') {
             num += this.source[this.index++];
-            while (Character.isDecimalDigit(this.source.charCodeAt(this.index))) {
-                num += this.source[this.index++];
-            }
+            num += this.scanLiteralPart(Character.isDecimalDigitChar);
             ch = this.source[this.index];
         }
 
@@ -859,12 +857,20 @@ export class Scanner {
                 num += this.source[this.index++];
             }
             if (Character.isDecimalDigit(this.source.charCodeAt(this.index))) {
-                while (Character.isDecimalDigit(this.source.charCodeAt(this.index))) {
-                    num += this.source[this.index++];
-                }
+                num += this.scanLiteralPart(Character.isDecimalDigitChar);
             } else {
                 this.throwUnexpectedToken();
             }
+        } else if (ch === 'n') {
+            this.index++;
+            return {
+                type: Token.BigIntLiteral,
+                value: num, // should be BigInt(num),
+                lineNumber: this.lineNumber,
+                lineStart: this.lineStart,
+                start: start,
+                end: this.index
+            };
         }
 
         if (Character.isIdentifierStart(this.source.charCodeAt(this.index))) {

src/token.ts

@@ -8,7 +8,8 @@ export const enum Token {
     Punctuator,
     StringLiteral,
     RegularExpression,
-    Template
+    Template,
+    BigIntLiteral
 }
 
 export const TokenName = {};
@@ -22,3 +23,4 @@ TokenName[Token.Punctuator] = 'Punctuator';
 TokenName[Token.StringLiteral] = 'String';
 TokenName[Token.RegularExpression] = 'RegularExpression';
 TokenName[Token.Template] = 'Template';
+TokenName[Token.BigIntLiteral] = 'BigIntLiteral';