JS: support parsing of numeric literal separators

tdewolff · Mar 31, 2022 · ff140b1 · ff140b1
1 parent 9d035a3
commit ff140b1
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 6 deletions.
diff --git a/js/lex.go b/js/lex.go
@@ -515,6 +515,18 @@ func (l *Lexer) consumeIdentifierToken() bool {
 	return true
 }
 
+func (l *Lexer) consumeNumericSeparator(f func() bool) bool {
+	if l.r.Peek(0) != '_' {
+		return false
+	}
+	l.r.Move(1)
+	if !f() {
+		l.r.Move(-1)
+		return false
+	}
+	return true
+}
+
 func (l *Lexer) consumeNumericToken() TokenType {
 	// assume to be on 0 1 2 3 4 5 6 7 8 9 .
 	first := l.r.Peek(0)
@@ -523,7 +535,7 @@ func (l *Lexer) consumeNumericToken() TokenType {
 		if l.r.Peek(0) == 'x' || l.r.Peek(0) == 'X' {
 			l.r.Move(1)
 			if l.consumeHexDigit() {
-				for l.consumeHexDigit() {
+				for l.consumeHexDigit() || l.consumeNumericSeparator(l.consumeHexDigit) {
 				}
 				return HexadecimalToken
 			}
@@ -532,7 +544,7 @@ func (l *Lexer) consumeNumericToken() TokenType {
 		} else if l.r.Peek(0) == 'b' || l.r.Peek(0) == 'B' {
 			l.r.Move(1)
 			if l.consumeBinaryDigit() {
-				for l.consumeBinaryDigit() {
+				for l.consumeBinaryDigit() || l.consumeNumericSeparator(l.consumeBinaryDigit) {
 				}
 				return BinaryToken
 			}
@@ -541,7 +553,7 @@ func (l *Lexer) consumeNumericToken() TokenType {
 		} else if l.r.Peek(0) == 'o' || l.r.Peek(0) == 'O' {
 			l.r.Move(1)
 			if l.consumeOctalDigit() {
-				for l.consumeOctalDigit() {
+				for l.consumeOctalDigit() || l.consumeNumericSeparator(l.consumeOctalDigit) {
 				}
 				return OctalToken
 			}
@@ -555,15 +567,15 @@ func (l *Lexer) consumeNumericToken() TokenType {
 			return ErrorToken
 		}
 	} else if first != '.' {
-		for l.consumeDigit() {
+		for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
 		}
 	}
 	// we have parsed a 0 or an integer number
 	c := l.r.Peek(0)
 	if c == '.' {
 		l.r.Move(1)
 		if l.consumeDigit() {
-			for l.consumeDigit() {
+			for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
 			}
 			c = l.r.Peek(0)
 		} else if first == '.' {
@@ -587,7 +599,7 @@ func (l *Lexer) consumeNumericToken() TokenType {
 			l.err = parse.NewErrorLexer(l.r, "invalid number")
 			return ErrorToken
 		}
-		for l.consumeDigit() {
+		for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
 		}
 	}
 	return DecimalToken

diff --git a/js/lex_test.go b/js/lex_test.go
@@ -19,6 +19,7 @@ func TestTokens(t *testing.T) {
 		{" \t\v\f\u00A0\uFEFF\u2000", TTs{}}, // WhitespaceToken
 		{"\n\r\r\n\u2028\u2029", TTs{LineTerminatorToken}},
 		{"5.2 .04 1. 2.e3 0x0F 5e99", TTs{DecimalToken, DecimalToken, DecimalToken, DecimalToken, HexadecimalToken, DecimalToken}},
+		{"2_3 5_4.1_2 1_1n 0o2_3 0b1_1 0xF_F", TTs{DecimalToken, DecimalToken, BigIntToken, OctalToken, BinaryToken, HexadecimalToken}},
 		{"0o22 0b11", TTs{OctalToken, BinaryToken}},
 		{"0n 2345n 435.333n", TTs{BigIntToken, BigIntToken, DecimalToken, ErrorToken}},
 		{"a = 'string'", TTs{IdentifierToken, EqToken, StringToken}},