fix(lexer): fix line continuation with \r\n

closes #146
meriyah · Oct 29, 2020 · 1423e81 · 1423e81
1 parent 7264369
commit 1423e81
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 8 deletions.
diff --git a/src/lexer/string.ts b/src/lexer/string.ts
@@ -36,13 +36,12 @@ export function scanString(parser: ParserState, context: Context, quote: number)
       ret += parser.source.slice(marker, parser.index);
       char = advanceChar(parser);
 
-      if (char > 0x7e) {
-        ret += fromCodePoint(char);
-      } else {
+      if (char < 0x7f || char === Chars.LineSeparator || char === Chars.ParagraphSeparator) {
         const code = parseEscape(parser, context, char);
-
         if (code >= 0) ret += fromCodePoint(code);
         else handleStringError(parser, code as Escape, /* isTemplate */ 0);
+      } else {
+        ret += fromCodePoint(char);
       }
       marker = parser.index + 1;
     }
@@ -77,9 +76,10 @@ export function parseEscape(parser: ParserState, context: Context, first: number
     // Line continuations
     case Chars.CarriageReturn: {
       if (parser.index < parser.end) {
-        if (parser.currentChar === Chars.LineFeed) {
+        const nextChar = parser.source.charCodeAt(parser.index + 1);
+        if (nextChar === Chars.LineFeed) {
           parser.index = parser.index + 1;
-          parser.currentChar = parser.source.charCodeAt(parser.index);
+          parser.currentChar = nextChar;
         }
       }
     }

diff --git a/test/lexer/strings.ts b/test/lexer/strings.ts
@@ -136,7 +136,18 @@ describe('Lexer - String', () => {
     // \8 \9 are acceptable in web compatibility mode
     [Context.OptionsWebCompat, Token.StringLiteral, '"\\8"', '8'],
     [Context.OptionsWebCompat, Token.StringLiteral, '"\\9"', '9'],
-    [Context.OptionsWebCompat, Token.StringLiteral, '"\\9999"', '9999']
+    [Context.OptionsWebCompat, Token.StringLiteral, '"a\\9999"', 'a9999'],
+
+    // Line continuation
+    [Context.None, Token.StringLiteral, '"a\\\nb"', 'ab'],
+    [Context.None, Token.StringLiteral, '"a\\\rb"', 'ab'],
+    [Context.None, Token.StringLiteral, '"a\\\r\nb"', 'ab'],
+    [Context.None, Token.StringLiteral, '"a\\\u2028b"', 'ab'], // unicode LineSeparator
+    [Context.None, Token.StringLiteral, '"a\\\u2029b"', 'ab'], // unicode ParagraphSeparator
+    [Context.None, Token.StringLiteral, '"\\\n"', ''],
+    [Context.None, Token.StringLiteral, '"a\\\r"', 'a'],
+    [Context.None, Token.StringLiteral, '"\\\r\nb"', 'b'],
+    [Context.None, Token.StringLiteral, '"\\\r\n"', '']
   ];
 
   for (const [ctx, token, op, value] of tokens) {

diff --git a/test/test262-parser-tests/parser-tests.ts b/test/test262-parser-tests/parser-tests.ts
@@ -6,7 +6,6 @@ const Test262Dir = 'node_modules/test262-parser-tests';
 
 const expectations = {
   pass: [
-    '110fa1efdd0868b8.js',
     '1a1c717109ab67e1.js',
     '206ebb4e67a6daa9.js',
     '4ad6e3a59e27e9b1.js',