fix(lexer): minor optimization tweaks

meriyah · Jun 25, 2019 · 20a118c · 20a118c
1 parent 2f85c58
commit 20a118c
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 38 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "meriyah",
-  "version": "0.6.02",
+  "version": "0.6.0",
   "description": "A 100% compliant, self-hosted javascript parser with high focus on both performance and stability",
   "main": "dist/meriyah.umd.js",
   "module": "dist/meriyah.esm.js",

diff --git a/src/lexer/comments.ts b/src/lexer/comments.ts
@@ -1,6 +1,6 @@
-import { nextCodePoint, CharTypes, CharFlags, ScannerState, advanceNewline } from './';
+import { nextCodePoint, CharTypes, CharFlags, LexerState, advanceNewline, consumeLineFeed } from './';
 import { Chars } from '../chars';
-import { ParserState, Flags } from '../common';
+import { ParserState } from '../common';
 import { report, Errors } from '../errors';
 
 /**
@@ -21,7 +21,7 @@ export function skipHashBang(parser: ParserState): void {
     if (index < parser.end && parser.source.charCodeAt(index) === Chars.Exclamation) {
       parser.index = index + 1;
       parser.nextCP = parser.source.charCodeAt(parser.index);
-      skipSingleLineComment(parser, ScannerState.None);
+      skipSingleLineComment(parser, LexerState.None);
     } else {
       report(parser, Errors.IllegalCaracter, '#');
     }
@@ -32,11 +32,12 @@ export function skipHashBang(parser: ParserState): void {
  * Skips single line comment
  *
  * @param parser  Parser object
+ * @param state  Lexer state
  */
-export function skipSingleLineComment(parser: ParserState, state: ScannerState): ScannerState {
+export function skipSingleLineComment(parser: ParserState, state: LexerState): LexerState {
   while (parser.index < parser.end) {
     if (CharTypes[parser.nextCP] & CharFlags.LineTerminator || (parser.nextCP ^ Chars.LineSeparator) <= 1) {
-      state = (state | ScannerState.LastIsCR | ScannerState.NewLine) ^ ScannerState.LastIsCR;
+      state = (state | LexerState.LastIsCR | LexerState.NewLine) ^ LexerState.LastIsCR;
       advanceNewline(parser);
       return state;
     }
@@ -48,33 +49,26 @@ export function skipSingleLineComment(parser: ParserState, state: ScannerState):
 /**
  * Skips multiline comment
  *
- * @param parser  Parser object
+ * @param parser Parser object
+ * @param state Lexer state
  */
-export function skipMultiLineComment(parser: ParserState, state: ScannerState): any {
+export function skipMultiLineComment(parser: ParserState, state: LexerState): LexerState | void {
   while (parser.index < parser.end) {
-    while (CharTypes[parser.nextCP] & CharFlags.Asterisk) {
+    while (parser.nextCP === Chars.Asterisk) {
       if (nextCodePoint(parser) === Chars.Slash) {
         nextCodePoint(parser);
         return state;
       }
     }
 
-    // ES 2020 11.3 Line Terminators
-    if (CharTypes[parser.nextCP] & CharFlags.LineTerminator) {
-      if (CharTypes[parser.nextCP] & CharFlags.CarriageReturn) {
-        state |= ScannerState.NewLine | ScannerState.LastIsCR;
-        advanceNewline(parser);
-      } else {
-        if (state & ScannerState.LastIsCR) {
-          parser.column = 0;
-          parser.line++;
-        }
-        state = (state | ScannerState.LastIsCR | ScannerState.NewLine) ^ ScannerState.LastIsCR;
-        parser.nextCP = parser.source.charCodeAt(++parser.index);
-        parser.flags |= Flags.NewLine;
-      }
+    if (parser.nextCP === Chars.CarriageReturn) {
+      state |= LexerState.NewLine | LexerState.LastIsCR;
+      advanceNewline(parser);
+    } else if (parser.nextCP === Chars.LineFeed) {
+      consumeLineFeed(parser, (state & LexerState.LastIsCR) !== 0);
+      state = (state | LexerState.LastIsCR | LexerState.NewLine) ^ LexerState.LastIsCR;
     } else if ((parser.nextCP ^ Chars.LineSeparator) <= 1) {
-      state = (state | ScannerState.LastIsCR | ScannerState.NewLine) ^ ScannerState.LastIsCR;
+      state = (state | LexerState.LastIsCR | LexerState.NewLine) ^ LexerState.LastIsCR;
       advanceNewline(parser);
     } else {
       nextCodePoint(parser);

diff --git a/src/lexer/common.ts b/src/lexer/common.ts
@@ -3,7 +3,7 @@ import { ParserState, Flags } from '../common';
 import { unicodeLookup } from '../unicode';
 import { report, Errors } from '../errors';
 
-export const enum ScannerState {
+export const enum LexerState {
   None = 0,
   NewLine = 1 << 0,
   SameLine = 1 << 1,

diff --git a/src/lexer/identifier.ts b/src/lexer/identifier.ts
@@ -53,7 +53,7 @@ export function scanIdentifierSlowCase(
 ): Token {
   let start = parser.index;
   while (parser.index < parser.end) {
-    if (CharTypes[parser.nextCP] & CharFlags.BackSlash) {
+    if (parser.nextCP === Chars.Backslash) {
       parser.tokenValue += parser.source.slice(start, parser.index);
       hasEscape = 1;
       const code = scanIdentifierUnicodeEscape(parser) as number;

diff --git a/src/lexer/index.ts b/src/lexer/index.ts
@@ -8,7 +8,7 @@ export {
   toHex,
   consumeLineFeed,
   advanceNewline,
-  ScannerState
+  LexerState
 } from './common';
 export { CharTypes, CharFlags, isIdentifierStart, isIdentifierPart } from './charClassifier';
 export {

diff --git a/src/lexer/scan.ts b/src/lexer/scan.ts
@@ -1,4 +1,4 @@
-import { skipSingleLineComment, skipMultiLineComment, ScannerState } from './';
+import { skipSingleLineComment, skipMultiLineComment, LexerState } from './';
 import { CharTypes, CharFlags } from './charClassifier';
 import { Chars } from '../chars';
 import { Token } from '../token';
@@ -178,10 +178,10 @@ export function nextToken(parser: ParserState, context: Context): void {
   parser.startIndex = parser.index;
   parser.startColumn = parser.column;
   parser.startLine = parser.line;
-  parser.token = scanSingleToken(parser, context, ScannerState.None);
+  parser.token = scanSingleToken(parser, context, LexerState.None);
 }
 
-export function scanSingleToken(parser: ParserState, context: Context, state: ScannerState): Token {
+export function scanSingleToken(parser: ParserState, context: Context, state: LexerState): Token {
   const isStartOfLine = parser.index === 0;
 
   while (parser.index < parser.end) {
@@ -218,13 +218,13 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Sc
           break;
 
         case Token.CarriageReturn:
-          state |= ScannerState.NewLine | ScannerState.LastIsCR;
+          state |= LexerState.NewLine | LexerState.LastIsCR;
           advanceNewline(parser);
           break;
 
         case Token.LineFeed:
-          consumeLineFeed(parser, (state & ScannerState.LastIsCR) !== 0);
-          state = (state | ScannerState.LastIsCR | ScannerState.NewLine) ^ ScannerState.LastIsCR;
+          consumeLineFeed(parser, (state & LexerState.LastIsCR) !== 0);
+          state = (state | LexerState.LastIsCR | LexerState.NewLine) ^ LexerState.LastIsCR;
           parser.flags |= Flags.NewLine;
           break;
 
@@ -319,7 +319,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Sc
             nextCodePoint(parser);
             if (
               (context & Context.Module) === 0 &&
-              (state & ScannerState.NewLine || isStartOfLine) &&
+              (state & LexerState.NewLine || isStartOfLine) &&
               parser.nextCP === Chars.GreaterThan
             ) {
               if ((context & Context.OptionsWebCompat) === 0) report(parser, Errors.HtmlCommentInWebCompat);
@@ -348,7 +348,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Sc
               continue;
             } else if (ch === Chars.Asterisk) {
               nextCodePoint(parser);
-              state = skipMultiLineComment(parser, state);
+              state = skipMultiLineComment(parser, state) as LexerState;
               continue;
             } else if (context & Context.AllowRegExp) {
               return scanRegularExpression(parser, context);
@@ -504,7 +504,7 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Sc
       }
     } else {
       if ((first ^ Chars.LineSeparator) <= 1) {
-        state = (state | ScannerState.LastIsCR | ScannerState.NewLine) ^ ScannerState.LastIsCR;
+        state = (state | LexerState.LastIsCR | LexerState.NewLine) ^ LexerState.LastIsCR;
         advanceNewline(parser);
         continue;
       }

diff --git a/test/lexer/whitespace.ts b/test/lexer/whitespace.ts
@@ -352,7 +352,7 @@ describe('Lexer - Whitespace', () => {
     value: '',
     index: 26,
     line: 2,
-    column: 25
+    column: 5
   });
 
   pass('skips multiline comments with Windows newlines', {
@@ -372,7 +372,7 @@ describe('Lexer - Whitespace', () => {
     value: '',
     index: 33,
     line: 3,
-    column: 5
+    column: 17
   });
 
   pass('skips no break space in a string', {