Skip to content

Commit

Permalink
fix(lexer): tweaked numeric separators implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFlash
KFlash committed Jun 28, 2019
1 parent 8ba7461 commit 4cfcb28
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 36 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "meriyah",
"version": "1.1.0",
"version": "1.2.0",
"description": "A 100% compliant, self-hosted javascript parser with high focus on both performance and stability",
"main": "dist/meriyah.umd.js",
"module": "dist/meriyah.esm.js",
Expand Down
82 changes: 47 additions & 35 deletions src/lexer/numeric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,30 +19,6 @@ export const enum SeparatorState {
Previous = 1 << 1
}

export function scanDigits(parser: ParserState, char: number): string {
let seenSeparator = false;
let start = parser.index;
let ret = '';
while (CharTypes[char] & (CharFlags.Decimal | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
const preUnderscoreIndex = parser.index;
char = nextCP(parser);
if (char === Chars.Underscore) report(parser, Errors.ContinuousNumericSeparator);
seenSeparator = true;
ret += parser.source.substring(start, preUnderscoreIndex);
start = parser.index;
continue;
}
seenSeparator = false;
char = nextCP(parser);
}
if (seenSeparator) {
report(parser, Errors.TrailingNumericSeparator);
}

return ret + parser.source.substring(start, parser.index);
}

export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1): Token {
let kind: NumberKind = NumberKind.Decimal;
let char = parser.nextCP;
Expand All @@ -53,7 +29,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1

if (isFloat) {
if (char === Chars.Underscore) report(parser, Errors.Unexpected);
value += '.' + scanDigits(parser, char);
value += '.' + scanDecimalDigitsOrSeparator(parser, char);
char = parser.nextCP;
} else {
if (char === Chars.Zero) {
Expand All @@ -67,13 +43,12 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
char = nextCP(parser);
while (CharTypes[char] & (CharFlags.Hex | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
// let seenSeparator = 1;
if (state & SeparatorState.Allowed) {
state = (state | SeparatorState.Allowed | SeparatorState.Previous) ^ SeparatorState.Allowed;
} else if (state & SeparatorState.Previous) {
report(parser, Errors.ContinuousNumericSeparator);
} else {
report(parser, Errors.ContinuousNumericSeparator);
report(parser, Errors.TrailingNumericSeparator);
}
char = nextCP(parser);
continue;
Expand Down Expand Up @@ -103,7 +78,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
} else if (state & SeparatorState.Previous) {
report(parser, Errors.ContinuousNumericSeparator);
} else {
report(parser, Errors.ContinuousNumericSeparator);
report(parser, Errors.TrailingNumericSeparator);
}
char = nextCP(parser);
continue;
Expand All @@ -129,7 +104,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
} else if (state & SeparatorState.Previous) {
report(parser, Errors.ContinuousNumericSeparator);
} else {
report(parser, Errors.ContinuousNumericSeparator);
report(parser, Errors.TrailingNumericSeparator);
}
char = nextCP(parser);
continue;
Expand Down Expand Up @@ -195,7 +170,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
}
}

value += scanDigits(parser, char);
value += scanDecimalDigitsOrSeparator(parser, char);

char = parser.nextCP;

Expand All @@ -204,7 +179,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
char = nextCP(parser);
if ((char as number) === Chars.Underscore) report(parser, Errors.Unexpected);
isFloat = 1;
value += '.' + scanDigits(parser, char);
value += '.' + scanDecimalDigitsOrSeparator(parser, char);
char = parser.nextCP;
}
}
Expand Down Expand Up @@ -232,7 +207,7 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1
if ((CharTypes[char] & CharFlags.Decimal) < 1) report(parser, Errors.MissingExponent);

// Consume exponential digits
value += parser.source.substring(end, preNumericPart) + scanDigits(parser, char);
value += parser.source.substring(end, preNumericPart) + scanDecimalDigitsOrSeparator(parser, char);

char = parser.nextCP;
}
Expand All @@ -246,11 +221,48 @@ export function scanNumber(parser: ParserState, context: Context, isFloat: 0 | 1

if (context & Context.OptionsRaw || isBigInt) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);

if (kind & NumberKind.DecimalWithLeadingZero)
parser.tokenValue = parseFloat(parser.source.slice(parser.tokenIndex, parser.index));
else parser.tokenValue = parseFloat(value);
if (isBigInt) {
parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);
parser.tokenValue = parseInt(value, 0xa);
return Token.BigIntLiteral;
}

if (context & Context.OptionsRaw) parser.tokenRaw = parser.source.slice(parser.tokenIndex, parser.index);

parser.tokenValue =
kind & (NumberKind.ImplicitOctal | NumberKind.Binary | NumberKind.Hex | NumberKind.Octal)
? value
: kind & NumberKind.DecimalWithLeadingZero
? parseFloat(parser.source.slice(parser.tokenIndex, parser.index))
: +value;

if (isBigInt) return Token.BigIntLiteral;

return Token.NumericLiteral;
}

export function scanDecimalDigitsOrSeparator(parser: ParserState, char: number): string {
let state = SeparatorState.None;
let start = parser.index;
let ret = '';
while (CharTypes[char] & (CharFlags.Decimal | CharFlags.Underscore)) {
if (char === Chars.Underscore) {
if (state & SeparatorState.Allowed) {
state = (state | SeparatorState.Allowed | SeparatorState.Previous) ^ SeparatorState.Allowed;
ret += parser.source.substring(start, parser.index);
} else if (state & SeparatorState.Previous) {
report(parser, Errors.ContinuousNumericSeparator);
} else {
report(parser, Errors.TrailingNumericSeparator);
}
char = nextCP(parser);
start = parser.index;
continue;
}
state = (state | SeparatorState.Allowed | SeparatorState.Previous) ^ SeparatorState.Previous;
char = nextCP(parser);
}
if (parser.source.charCodeAt(parser.index - 1) === Chars.Underscore) report(parser, Errors.TrailingNumericSeparator);

return ret + parser.source.substring(start, parser.index);
}

2 comments on commit 4cfcb28

@KFlash
Copy link
Contributor Author

@KFlash KFlash commented on 4cfcb28 Jun 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aladdin-add Do I miss any features, or do you think I can start to focus on JSX now?

@aladdin-add
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you mean stage-3 features, or something else?

I don't have a full view of the repo, but I noticed tokenizer is missing somehow. :)

Please sign in to comment.