Skip to content

Commit

Permalink
fix(lexer): optimized number scanning
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFlash
KFlash committed Jun 28, 2019
1 parent c3efc64 commit 0a09e9e
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 43 deletions.
65 changes: 32 additions & 33 deletions src/lexer/numeric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,24 @@ export const enum NumberKind {
DecimalWithLeadingZero = 1 << 5
}

/**
* Scans numeric literal
*
* @param parser Parser object
* @param context Context masks
* @param isFloat
*/
export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1): Token {
let kind: NumberKind = NumberKind.Decimal;
let char = parser.nextCP;
let value: any = 0;
let digit = 9;
let atStart = !isFloat;
let digits = 0;
let allowSeparator: 0 | 1 = 0;

if (isFloat) {
if (char === Chars.Underscore) report(parser, Errors.Unexpected);
value += '.' + scanDecimalDigitsOrSeparator(parser, char);
value = '.' + scanDecimalDigitsOrSeparator(parser, char);
char = parser.nextCP;
} else {
if (char === Chars.Zero) {
Expand All @@ -31,8 +39,6 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
// Hex
if ((char | 32) === Chars.LowerX) {
kind = NumberKind.Hex;
let digits = 0;
let allowSeparator: 0 | 1 = 0;
char = nextCP(parser);
while (CharTypes[char] & (CharFlags.Hex | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
Expand All @@ -49,15 +55,11 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
char = nextCP(parser);
}

if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);

char = parser.nextCP;
if (digits < 1) report(parser, Errors.MissingHexDigits);
if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);
// Octal
} else if ((char | 32) === Chars.LowerO) {
kind = NumberKind.Octal;
let digits = 0;
let allowSeparator: 0 | 1 = 0;
char = nextCP(parser);
while (CharTypes[char] & (CharFlags.Octal | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
Expand All @@ -73,13 +75,10 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
digits++;
char = nextCP(parser);
}

if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);
if (digits < 1) report(parser, Errors.ExpectedNumberInRadix, `${8}`);
if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);
} else if ((char | 32) === Chars.LowerB) {
kind = NumberKind.Binary;
let digits = 0;
let allowSeparator: 0 | 1 = 0;
char = nextCP(parser);
while (CharTypes[char] & (CharFlags.Binary | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
Expand All @@ -95,8 +94,8 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
digits++;
char = nextCP(parser);
}
if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);
if (digits < 1) report(parser, Errors.ExpectedNumberInRadix, `${2}`);
if (!allowSeparator) report(parser, Errors.TrailingNumericSeparator);
} else if (CharTypes[char] & CharFlags.Octal) {
// Octal integer literals are not permitted in strict mode code
if (context & Context.Strict) report(parser, Errors.StrictOctalEscape);
Expand All @@ -122,24 +121,21 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1

// Parse decimal digits and allow trailing fractional part
if (kind & (NumberKind.Decimal | NumberKind.DecimalWithLeadingZero)) {
let seenSeparator = 0;

if (atStart) {
while (digit >= 0 && CharTypes[char] & (CharFlags.Decimal | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
char = nextCP(parser);
if (char === Chars.Underscore) report(parser, Errors.ContinuousNumericSeparator);
seenSeparator = 1;
allowSeparator = 1;
continue;
}
seenSeparator = 0;
allowSeparator = 0;
value = 10 * value + (char - Chars.Zero);
char = nextCP(parser);
--digit;
}
if (seenSeparator) {
report(parser, Errors.TrailingNumericSeparator);
}

if (allowSeparator) report(parser, Errors.TrailingNumericSeparator);

if (digit >= 0 && !isIdentifierStart(char) && char !== Chars.Period && char !== Chars.Underscore) {
// Most numbers are pure decimal integers without fractional component
Expand All @@ -157,7 +153,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
// Consume any decimal dot and fractional component.
if (char === Chars.Period) {
char = nextCP(parser);
if ((char as number) === Chars.Underscore) report(parser, Errors.Unexpected);
if (char === Chars.Underscore) report(parser, Errors.Unexpected);
isFloat = 1;
value += '.' + scanDecimalDigitsOrSeparator(parser, char);
char = parser.nextCP;
Expand Down Expand Up @@ -195,50 +191,53 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1

// The source character immediately following a numeric literal must
// not be an identifier start or a decimal digit
if (CharTypes[char] & CharFlags.Decimal || isIdentifierStart(char)) {
if ((parser.index < parser.end && CharTypes[char] & CharFlags.Decimal) || isIdentifierStart(char)) {
report(parser, Errors.IDStartAfterNumber);
}

if (context & Context.OptionsRaw || isBigInt) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);

if (isBigInt) {
parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);
parser.tokenValue = parseInt(value, 0xa);
return Token.BigIntLiteral;
}

if (context & Context.OptionsRaw) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);

parser.tokenValue =
kind & (NumberKind.ImplicitOctal | NumberKind.Binary | NumberKind.Hex | NumberKind.Octal)
? value
: kind & NumberKind.DecimalWithLeadingZero
? parseFloat(parser.source.slice(parser.tokenIndex, parser.index))
: +value;

if (context & Context.OptionsRaw) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);

return Token.NumericLiteral;
}

/**
* Scans numeric literal and skip underscore '_' if it exist
*
* @param parser Parser object
* @param char Code point
*/
export function scanDecimalDigitsOrSeparator(parser: ParserState, char: number): string {
let allowSeparator = false;
let allowSeparator: 0 | 1 = 0;
let start = parser.index;
let ret = '';
while (CharTypes[char] & (CharFlags.Decimal | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
const preUnderscoreIndex = parser.index;
char = nextCP(parser);
if (char === Chars.Underscore) report(parser, Errors.ContinuousNumericSeparator);
allowSeparator = true;
allowSeparator = 1;
ret += parser.source.substring(start, preUnderscoreIndex);
start = parser.index;
continue;
}
allowSeparator = false;
allowSeparator = 0;
char = nextCP(parser);
}
if (allowSeparator) {
report(parser, Errors.TrailingNumericSeparator);
}

if (allowSeparator) report(parser, Errors.TrailingNumericSeparator);

return ret + parser.source.substring(start, parser.index);
}
36 changes: 26 additions & 10 deletions test/lexer/numbers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe('src/lexer/scan', () => {
[Context.None, Token.NumericLiteral, '98', 98],
[Context.None, Token.NumericLiteral, '7890', 7890],
[Context.None, Token.NumericLiteral, '123', 123],
[Context.None, Token.NumericLiteral, '.5', 0.5],
[Context.OptionsRaw, Token.NumericLiteral, '.5', 0.5],
[Context.None, Token.NumericLiteral, '.9', 0.9],
[Context.None, Token.NumericLiteral, '.123', 0.123],
[Context.None, Token.NumericLiteral, '.1234567890', 0.123456789],
Expand All @@ -31,7 +31,7 @@ describe('src/lexer/scan', () => {
[Context.None, Token.NumericLiteral, '0.0', 0],
[Context.None, Token.NumericLiteral, '4.0', 4],
[Context.None, Token.NumericLiteral, '0.0', 0],
[Context.None, Token.NumericLiteral, '456.345', 456.345],
[Context.OptionsRaw, Token.NumericLiteral, '456.345', 456.345],
[Context.None, Token.NumericLiteral, '1234567890.0987654321', 1234567890.0987654321],

// Numeric literals with exponent
Expand All @@ -51,15 +51,15 @@ describe('src/lexer/scan', () => {
[Context.None, Token.NumericLiteral, '.5e3', 500],
[Context.None, Token.NumericLiteral, '.5e-3', 0.0005],
[Context.None, Token.NumericLiteral, '0.5e3', 500],
[Context.None, Token.NumericLiteral, '55.55e10', 555500000000],
[Context.OptionsRaw, Token.NumericLiteral, '55.55e10', 555500000000],
[Context.None, Token.NumericLiteral, '0e-100', 0],
[Context.None, Token.NumericLiteral, '0E-100', 0],
[Context.None, Token.NumericLiteral, '0e+1', 0],
[Context.None, Token.NumericLiteral, '0e01', 0],
[Context.None, Token.NumericLiteral, '6e+1', 60],
[Context.None, Token.NumericLiteral, '9e+1', 90],
[Context.None, Token.NumericLiteral, '1E-1', 0.1],
[Context.None, Token.NumericLiteral, '0e-1', 0],
[Context.OptionsRaw, Token.NumericLiteral, '0e-1', 0],
[Context.None, Token.NumericLiteral, '7E1', 70],
[Context.None, Token.NumericLiteral, '0e0', 0],
[Context.None, Token.NumericLiteral, '0E0', 0],
Expand All @@ -68,7 +68,7 @@ describe('src/lexer/scan', () => {
[Context.None, Token.NumericLiteral, '.1e-100', 1e-101],
[Context.None, Token.NumericLiteral, '0e+100', 0],
[Context.None, Token.NumericLiteral, '1E+100', 1e100],
[Context.None, Token.NumericLiteral, '.1E+100', 1e99],
[Context.OptionsRaw, Token.NumericLiteral, '.1E+100', 1e99],

// Hex
[Context.None, Token.NumericLiteral, '0xcafe', 51966],
Expand Down Expand Up @@ -120,7 +120,7 @@ describe('src/lexer/scan', () => {
[Context.None, Token.NumericLiteral, '0B01001', 9],
[Context.None, Token.NumericLiteral, '0B011111111111111111111111111111', 536870911],
[Context.None, Token.NumericLiteral, '0B00000111111100000011', 32515],
[Context.None, Token.NumericLiteral, '0B0000000000000000000000000000000000000000000000001111111111', 1023],
[Context.OptionsRaw, Token.NumericLiteral, '0B0000000000000000000000000000000000000000000000001111111111', 1023],

// Octals
[Context.None, Token.NumericLiteral, '0O12345670', 2739128],
Expand Down Expand Up @@ -157,12 +157,12 @@ describe('src/lexer/scan', () => {
[Context.None, Token.BigIntLiteral, '0x9an', 154],
[Context.None, Token.BigIntLiteral, '9007199254740991n', 9007199254740991],
[Context.None, Token.BigIntLiteral, '100000000000000000n', 100000000000000000],
[Context.None, Token.BigIntLiteral, '123456789000000000000000n', 123456789000000000000000],
[Context.OptionsRaw, Token.BigIntLiteral, '123456789000000000000000n', 123456789000000000000000],
[Context.None, Token.BigIntLiteral, '0xfn', 15],
[Context.None, Token.BigIntLiteral, '0n', 0],

// Numeric separators
[Context.None, Token.NumericLiteral, '0', 0],
[Context.OptionsRaw, Token.NumericLiteral, '0', 0],
[Context.None, Token.NumericLiteral, '1.1', 1.1],
[Context.None, Token.NumericLiteral, '1_1', 11],
[Context.None, Token.NumericLiteral, '1.1_1', 1.11],
Expand All @@ -173,7 +173,7 @@ describe('src/lexer/scan', () => {
1.3333333333333334e26
],

[Context.None, Token.NumericLiteral, '0O01_1', 9],
[Context.OptionsRaw, Token.NumericLiteral, '0O01_1', 9],
[Context.None, Token.NumericLiteral, '0O0_7_7', 63],
[Context.None, Token.NumericLiteral, '0B0_1', 1],
[Context.None, Token.NumericLiteral, '0B010_01', 9],
Expand Down Expand Up @@ -241,7 +241,6 @@ describe('src/lexer/scan', () => {
fail('fails on 0b2n', '0b2n', Context.None);
fail('fails on 008.3', '008.3', Context.Strict);
fail('fails on 008.3n', '008.3n', Context.None);
//fail('fails on 0b001E-100', '0b001E-100', Context.None);
fail('fails on 0b2', '0b2', Context.None);
fail('fails on 00b0', '00b0', Context.None);
fail('fails on 0b', '0b', Context.None);
Expand Down Expand Up @@ -290,6 +289,23 @@ describe('src/lexer/scan', () => {
fail('fails on 0x1_', '0x1_', Context.None);
fail('fails on 0x_1', '0x_1', Context.None);
fail('fails on 0o_1', '0o_1', Context.None);
fail('fails on 0o_', '0o_', Context.None);
fail('fails on 0o_', '0o_', Context.None);
fail('fails on 0o2_', '0o_', Context.None);
fail('fails on 0o2_________', '0o_', Context.None);
fail('fails on 0b_1', '0b_1', Context.None);
fail('fails on 0b1_', '0b1_', Context.None);
fail('fails on 0b', '0b', Context.None);
fail('fails on 0o', '0o', Context.None);
fail('fails on 0x', '0x', Context.None);
fail('fails on 1_', '1_', Context.None);
fail('fails on 1__', '1__', Context.None);
fail('fails on 1_1_', '1_1_', Context.None);
fail('fails on 1__1_', '1__1_', Context.None);
fail('fails on 1_1__', '1_1__', Context.None);
fail('fails on 0O1_', '0O1_', Context.None);
fail('fails on 0O1__', '0O1__', Context.None);
fail('fails on 0O1_1_', '0O1_1_', Context.None);
fail('fails on 0O1__1_', '0O1__1_', Context.None);
fail('fails on 0O1_1__', '0O1_1__', Context.None);
});

0 comments on commit 0a09e9e

Please sign in to comment.