fix(chore): improved line and coloumn tracking - #46

package.json

@@ -1,6 +1,6 @@
 {
   "name": "meriyah",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "A 100% compliant, self-hosted javascript parser with high focus on both performance and stability",
   "main": "dist/meriyah.umd.js",
   "module": "dist/meriyah.esm.js",

src/errors.ts

@@ -382,11 +382,11 @@ export class ParseError extends SyntaxError {
  * @returns {never}
  */
 export function report(parser: ParserState, type: Errors, ...params: string[]): never {
-  throw new ParseError(parser.index, parser.line, parser.column, type, ...params);
+  throw new ParseError(parser.index, parser.linePos, parser.colPos, type, ...params);
 }
 
 export function reportScopeError(scope: any): never {
-  throw new ParseError(scope.index, scope.line, scope.column, scope.type, scope.params);
+  throw new ParseError(scope.index, scope.linePos, scope.colPos, scope.type, scope.params);
 }
 
 /**

src/lexer/comments.ts

@@ -1,4 +1,4 @@
-import { advanceChar, CharTypes, CharFlags, LexerState, scanNewLine, consumeLineFeed } from './';
+import { advanceChar, CharTypes, CharFlags, LexerState, consumeLineBreak, consumeLineFeed } from './';
 import { Chars } from '../chars';
 import { Context, ParserState } from '../common';
 import { report, Errors } from '../errors';
@@ -45,16 +45,12 @@ export function skipSingleHTMLComment(
 export function skipSingleLineComment(parser: ParserState, state: LexerState, type: CommentType): LexerState {
   const { index } = parser;
   while (parser.index < parser.end) {
-    if (CharTypes[parser.currentChar] & CharFlags.LineTerminator) {
-      scanNewLine(parser);
-      if (parser.index < parser.end && parser.currentChar === Chars.LineFeed)
-        parser.currentChar = parser.source.charCodeAt(++parser.index);
-      return state | LexerState.NewLine;
-    } else if ((parser.currentChar ^ Chars.LineSeparator) <= 1) {
-      scanNewLine(parser);
-      return state | LexerState.NewLine;
+    if (CharTypes[parser.currentChar] & 8 || (parser.currentChar ^ 8232) <= 1) {
+      consumeLineBreak(parser);
+      return (state |= LexerState.NewLine);
+    } else {
+      advanceChar(parser);
     }
-    advanceChar(parser);
   }
   if (parser.onComment)
     parser.onComment(CommentTypes[type & 0xff], parser.source.slice(index, parser.index), index, parser.index);
@@ -84,18 +80,10 @@ export function skipMultiLineComment(parser: ParserState, state: LexerState): Le
         return state;
       }
     }
-
-    if (parser.currentChar === Chars.CarriageReturn) {
-      state |= LexerState.NewLine | LexerState.LastIsCR;
-      scanNewLine(parser);
-    } else if (parser.currentChar === Chars.LineFeed) {
-      consumeLineFeed(parser, state);
-      state = (state & ~LexerState.LastIsCR) | LexerState.NewLine;
-    } else if ((parser.currentChar ^ Chars.LineSeparator) <= 1) {
-      state = (state & ~LexerState.LastIsCR) | LexerState.NewLine;
-      scanNewLine(parser);
+    if (CharTypes[parser.currentChar] & 8 || (parser.currentChar ^ 8232) <= 1) {
+      consumeLineBreak(parser);
+      state |= LexerState.NewLine;
     } else {
-      state &= ~LexerState.LastIsCR;
       advanceChar(parser);
     }
   }

src/lexer/common.ts

@@ -28,6 +28,19 @@ export const enum NumberKind {
  */
 export function advanceChar(parser: ParserState): number {
   parser.column++;
+  if (parser.index < parser.end) {
+    let cur = parser.currentChar,
+      nxt = parser.source.charCodeAt(parser.index + 1);
+    if (
+      (cur == Chars.CarriageReturn && nxt == Chars.LineFeed) ||
+      (cur == Chars.LineFeed && nxt == Chars.CarriageReturn)
+    ) {
+      parser.currentChar = Chars.LineFeed;
+      parser.index++;
+    } else if (cur === Chars.CarriageReturn) {
+      parser.currentChar = Chars.LineFeed;
+    }
+  }
   return (parser.currentChar = parser.source.charCodeAt(++parser.index));
 }
 
@@ -52,7 +65,7 @@ export function consumeMultiUnitCodePoint(parser: ParserState, hi: number): 0 |
 }
 
 /**
- * Use to consume a line feed instead of `scanNewLine`.
+ * Use to consume a line feed instead of `consumeLineBreak`.
  */
 export function consumeLineFeed(parser: ParserState, state: LexerState): void {
   parser.currentChar = parser.source.charCodeAt(++parser.index);
@@ -63,11 +76,11 @@ export function consumeLineFeed(parser: ParserState, state: LexerState): void {
   }
 }
 
-export function scanNewLine(parser: ParserState): void {
+export function consumeLineBreak(parser: ParserState): void {
   parser.flags |= Flags.NewLine;
-  parser.currentChar = parser.source.charCodeAt(++parser.index);
   parser.column = 0;
   parser.line++;
+  parser.currentChar = parser.source.charCodeAt(++parser.index);
 }
 
 // ECMA-262 11.2 White Space

src/lexer/index.ts

@@ -13,7 +13,7 @@ export {
   fromCodePoint,
   toHex,
   consumeLineFeed,
-  scanNewLine,
+  consumeLineBreak,
   LexerState,
   NumberKind
 } from './common';

src/lexer/scan.ts

@@ -22,7 +22,7 @@ import {
   scanPrivateName,
   fromCodePoint,
   consumeLineFeed,
-  scanNewLine
+  consumeLineBreak
 } from './';
 
 /*
@@ -534,8 +534,8 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
           break;
 
         case Token.CarriageReturn:
-          state |= LexerState.NewLine | LexerState.LastIsCR;
-          scanNewLine(parser);
+          consumeLineBreak(parser);
+          state |= LexerState.NewLine;
           break;
 
         case Token.LineFeed:
@@ -548,8 +548,8 @@ export function scanSingleToken(parser: ParserState, context: Context, state: Le
       }
     } else {
       if ((char ^ Chars.LineSeparator) <= 1) {
-        state = (state & ~LexerState.LastIsCR) | LexerState.NewLine;
-        scanNewLine(parser);
+        consumeLineBreak(parser);
+        state |= LexerState.NewLine;
         continue;
       }
 

src/meriyah.ts

@@ -26,4 +26,4 @@ export function parse(source: string, options?: Options): ESTree.Program {
 export { Options, ESTree };
 
 // Current version
-export const version = '1.7.0';
+export const version = '1.8.0';

src/parser.ts

@@ -406,7 +406,16 @@ export function parseModuleItem(
     case Token.Decorator:
       return parseDecorators(parser, context) as ESTree.Decorator[];
     default:
-      return parseStatementListItem(parser, context, scope, Origin.TopLevel, {}, start, line, column);
+      return parseStatementListItem(
+        parser,
+        context,
+        scope,
+        Origin.TopLevel,
+        { parentLabels: null, iterationLabels: null },
+        start,
+        line,
+        column
+      );
   }
 }
 
@@ -3733,7 +3742,7 @@ export function parseFunctionBody(
       context,
       scope,
       Origin.TopLevel,
-      {},
+      { parentLabels: null, iterationLabels: null },
       parser.tokenPos,
       parser.linePos,
       parser.colPos

test/lexer/template.ts

@@ -16,15 +16,11 @@ describe('Lexer - Template', () => {
       [Context.None, Token.TemplateSpan, '`a℮`', 'a℮'],
       [Context.None, Token.TemplateSpan, '`℘`', '℘'],
       [Context.None, Token.TemplateSpan, '`a᧚`', 'a᧚'],
-      [Context.None, Token.TemplateSpan, '`foo\\tbar`', 'foo\tbar'],
       // [Context.None, Token.TemplateSpan, '`\\x55a`', 'U'],
-      [Context.None, Token.TemplateSpan, '`a\\nb`', 'a\nb'],
       [Context.None, Token.TemplateSpan, '`;`', ';'],
       [Context.None, Token.TemplateSpan, '``', ''],
       [Context.None, Token.TemplateSpan, '`123`', '123'],
       [Context.None, Token.TemplateSpan, '`true`', 'true'],
-      [Context.None, Token.TemplateSpan, '`\n\r`', '\n\r'],
-      [Context.None, Token.TemplateSpan, '`\r\n`', '\r\n'],
       [Context.None, Token.TemplateSpan, '`$$$a}`', '$$$a}'],
 
       // Russian letters
@@ -114,14 +110,6 @@ describe('Lexer - Template', () => {
       [Context.TaggedTemplate, Token.TemplateContinuation, '`"${', '"'],
       [Context.TaggedTemplate, Token.TemplateContinuation, '`\\`${', '`'],
       [Context.TaggedTemplate, Token.TemplateContinuation, '`\\`${', '`'],
-      [Context.TaggedTemplate, Token.TemplateSpan, '`\\r`', '\r'],
-      [Context.TaggedTemplate, Token.TemplateContinuation, '`\\f${', '\f'],
-      [Context.TaggedTemplate, Token.TemplateSpan, '`\\f`', '\f'],
-      [Context.TaggedTemplate, Token.TemplateContinuation, '`\\v${', '\v'],
-      [Context.TaggedTemplate, Token.TemplateContinuation, '`\\n${', '\n'],
-      [Context.TaggedTemplate, Token.TemplateSpan, '`\\n`', '\n'],
-      [Context.TaggedTemplate, Token.TemplateContinuation, '`\\b${', '\b'],
-      [Context.TaggedTemplate, Token.TemplateSpan, '`\\t`', '\t'],
       [Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{11ffff}${', undefined],
       [Context.TaggedTemplate, Token.TemplateSpan, '`\\u{11ffff}`', undefined],
       [Context.TaggedTemplate, Token.TemplateContinuation, '`\\u{11ffff}${', undefined],