Skip to content

Commit

Permalink
perf(lexer): improved lexer perf
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFlash
KFlash committed Oct 27, 2020
1 parent 9d74a49 commit bc5e647
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 68 deletions.
124 changes: 61 additions & 63 deletions src/lexer/scan.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,48 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
advanceChar(parser);
return token;

// Look for an identifier
case Token.Identifier:
return scanIdentifier(parser, context, /* isValidAsKeyword */ 0);

// Look for identifier or keyword
case Token.Keyword:
return scanIdentifier(parser, context, /* isValidAsKeyword */ 1);

// Look for a decimal number.
case Token.NumericLiteral:
return scanNumber(parser, context, NumberKind.Decimal | NumberKind.ValidBigIntKind);

// Look for a string literal
case Token.StringLiteral:
return scanString(parser, context, char);

// Look for a template string
case Token.Template:
return scanTemplate(parser, context);

// Look for a escaped identifier
case Token.EscapedIdentifier:
return scanUnicodeIdentifier(parser, context);

// `#` (private name)
case Token.PrivateField:
return scanPrivateName(parser);

case Token.WhiteSpace:
advanceChar(parser);
break;

case Token.CarriageReturn:
state |= LexerState.NewLine | LexerState.LastIsCR;
scanNewLine(parser);
break;

case Token.LineFeed:
consumeLineFeed(parser, state);
state = (state & ~LexerState.LastIsCR) | LexerState.NewLine;
break;

// `<`, `<=`, `<<`, `<<=`, `</`, `<!--`
case Token.LessThan:
let ch = advanceChar(parser);
Expand Down Expand Up @@ -293,7 +335,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
// `=`, `==`, `===`, `=>`
case Token.Assign: {
advanceChar(parser);
if (parser.index >= parser.end) return Token.Assign;

const ch = parser.currentChar;

if (ch === Chars.EqualSign) {
Expand Down Expand Up @@ -378,7 +420,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
// `-`, `--`, `-=`, `-->`
case Token.Subtract: {
advanceChar(parser);
if (parser.index >= parser.end) return Token.Subtract;

const ch = parser.currentChar;

if (ch === Chars.Hyphen) {
Expand Down Expand Up @@ -454,10 +496,25 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
return Token.Divide;
}

// `.`, `...`, `.123` (numeric literal)
case Token.Period:
const next = advanceChar(parser);
if (next >= Chars.Zero && next <= Chars.Nine)
return scanNumber(parser, context, NumberKind.Float | NumberKind.Decimal);
if (next === Chars.Period) {
const index = parser.index + 1;
if (index < parser.end && source.charCodeAt(index) === Chars.Period) {
parser.column += 2;
parser.currentChar = source.charCodeAt((parser.index += 2));
return Token.Ellipsis;
}
}
return Token.Period;

// `|`, `||`, `|=`
case Token.BitwiseOr: {
advanceChar(parser);
if (parser.index >= parser.end) return Token.BitwiseOr;

const ch = parser.currentChar;

if (ch === Chars.VerticalBar) {
Expand All @@ -476,8 +533,6 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
case Token.GreaterThan: {
advanceChar(parser);

if (parser.index >= parser.end) return Token.GreaterThan;

const ch = parser.currentChar;

if (ch === Chars.EqualSign) {
Expand Down Expand Up @@ -511,7 +566,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
// `&`, `&&`, `&=`
case Token.BitwiseAnd: {
advanceChar(parser);
if (parser.index >= parser.end) return Token.BitwiseAnd;

const ch = parser.currentChar;

if (ch === Chars.Ampersand) {
Expand All @@ -526,20 +581,6 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le

return Token.BitwiseAnd;
}
// `.`, `...`, `.123` (numeric literal)
case Token.Period:
const next = advanceChar(parser);
if (next >= Chars.Zero && next <= Chars.Nine)
return scanNumber(parser, context, NumberKind.Float | NumberKind.Decimal);
if (next === Chars.Period) {
const index = parser.index + 1;
if (index < parser.end && source.charCodeAt(index) === Chars.Period) {
parser.column += 2;
parser.currentChar = source.charCodeAt((parser.index += 2));
return Token.Ellipsis;
}
}
return Token.Period;

// `?`, `??`, `?.`
case Token.QuestionMark: {
Expand All @@ -564,48 +605,6 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
return Token.QuestionMark;
}

// Look for identifier or keyword
case Token.Keyword:
return scanIdentifier(parser, context, /* isValidAsKeyword */ 1);

// Look for an identifier
case Token.Identifier:
return scanIdentifier(parser, context, /* isValidAsKeyword */ 0);

// Look for a decimal number.
case Token.NumericLiteral:
return scanNumber(parser, context, NumberKind.Decimal | NumberKind.ValidBigIntKind);

// Look for a string literal
case Token.StringLiteral:
return scanString(parser, context, char);

// Look for a template string
case Token.Template:
return scanTemplate(parser, context);

// Look for a escaped identifier
case Token.EscapedIdentifier:
return scanUnicodeIdentifier(parser, context);

// `#` (private name)
case Token.PrivateField:
return scanPrivateName(parser);

case Token.WhiteSpace:
advanceChar(parser);
break;

case Token.CarriageReturn:
state |= LexerState.NewLine | LexerState.LastIsCR;
scanNewLine(parser);
break;

case Token.LineFeed:
consumeLineFeed(parser, state);
state = (state & ~LexerState.LastIsCR) | LexerState.NewLine;
break;

default:
// unreachable
}
Expand All @@ -627,7 +626,6 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
}

parser.column++;

parser.tokenValue = '';
return scanIdentifierSlowCase(parser, context, /* hasEscape */ 0, /* canBeKeyword */ 0);
}
Expand Down
4 changes: 3 additions & 1 deletion src/lexer/string.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export const enum Escape {
/**
* Scan a string token.
*/
export function scanString(parser: ParserState, context: Context, quote: number): any {
export function scanString(parser: ParserState, context: Context, quote: number): Token {
const { index: start } = parser;

let ret: string | void = '';
Expand Down Expand Up @@ -55,6 +55,8 @@ export function scanString(parser: ParserState, context: Context, quote: number)
report(parser, Errors.UnterminatedString);
}

// TODO! Use table lookup

export function parseEscape(parser: ParserState, context: Context, first: number): number {
switch (first) {
// https://tc39.github.io/ecma262/#prod-SingleEscapeCharacter
Expand Down
6 changes: 2 additions & 4 deletions src/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2487,9 +2487,7 @@ function parseImportDeclaration(
case Token.LeftParen:
return parseImportCallDeclaration(parser, context, start, line, column);
case Token.Period:
if (context & Context.OptionsNext) {
return parseImportMetaDeclaration(parser, context, start, line, column);
}
return parseImportMetaDeclaration(parser, context, start, line, column);
default:
report(parser, Errors.UnexpectedToken, KeywordDescTable[parser.token & Token.Type]);
}
Expand Down Expand Up @@ -4356,7 +4354,7 @@ function parseImportCallOrMetaExpression(

let expr: ESTree.Identifier | ESTree.ImportExpression = parseIdentifier(parser, context, 0);

if (context & Context.OptionsNext && parser.token === Token.Period) {
if (parser.token === Token.Period) {
return parseImportMetaExpression(parser, context, expr, start, line, column);
}

Expand Down

0 comments on commit bc5e647

Please sign in to comment.