Skip to content

Commit

Permalink
Merge pull request #150 from meriyah/fix-string-line-conti
Browse files Browse the repository at this point in the history 
fix(lexer): fix line continuation with \r\n
  • Loading branch information
@3cp
3cp committed Oct 30, 2020
2 parents 7264369 + d3947af commit aaa91f5
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 61 deletions.
12 changes: 6 additions & 6 deletions src/lexer/string.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,12 @@ export function scanString(parser: ParserState, context: Context, quote: number)
ret += parser.source.slice(marker, parser.index);
char = advanceChar(parser);

if (char > 0x7e) {
ret += fromCodePoint(char);
} else {
if (char < 0x7f || char === Chars.LineSeparator || char === Chars.ParagraphSeparator) {
const code = parseEscape(parser, context, char);

if (code >= 0) ret += fromCodePoint(code);
else handleStringError(parser, code as Escape, /* isTemplate */ 0);
} else {
ret += fromCodePoint(char);
}
marker = parser.index + 1;
}
Expand Down Expand Up @@ -77,9 +76,10 @@ export function parseEscape(parser: ParserState, context: Context, first: number
// Line continuations
case Chars.CarriageReturn: {
if (parser.index < parser.end) {
if (parser.currentChar === Chars.LineFeed) {
const nextChar = parser.source.charCodeAt(parser.index + 1);
if (nextChar === Chars.LineFeed) {
parser.index = parser.index + 1;
parser.currentChar = parser.source.charCodeAt(parser.index);
parser.currentChar = nextChar;
}
}
}
Expand Down
13 changes: 12 additions & 1 deletion test/lexer/strings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,18 @@ describe('Lexer - String', () => {
// \8 \9 are acceptable in web compatibility mode
[Context.OptionsWebCompat, Token.StringLiteral, '"\\8"', '8'],
[Context.OptionsWebCompat, Token.StringLiteral, '"\\9"', '9'],
[Context.OptionsWebCompat, Token.StringLiteral, '"\\9999"', '9999']
[Context.OptionsWebCompat, Token.StringLiteral, '"a\\9999"', 'a9999'],

// Line continuation
[Context.None, Token.StringLiteral, '"a\\\nb"', 'ab'],
[Context.None, Token.StringLiteral, '"a\\\rb"', 'ab'],
[Context.None, Token.StringLiteral, '"a\\\r\nb"', 'ab'],
[Context.None, Token.StringLiteral, '"a\\\u2028b"', 'ab'], // unicode LineSeparator
[Context.None, Token.StringLiteral, '"a\\\u2029b"', 'ab'], // unicode ParagraphSeparator
[Context.None, Token.StringLiteral, '"\\\n"', ''],
[Context.None, Token.StringLiteral, '"a\\\r"', 'a'],
[Context.None, Token.StringLiteral, '"\\\r\nb"', 'b'],
[Context.None, Token.StringLiteral, '"\\\r\n"', '']
];

for (const [ctx, token, op, value] of tokens) {
Expand Down
90 changes: 36 additions & 54 deletions test/test262-parser-tests/parser-tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,75 +4,57 @@ import * as t from 'assert';

const Test262Dir = 'node_modules/test262-parser-tests';

const expectations = {
pass: [
'110fa1efdd0868b8.js',
'1a1c717109ab67e1.js',
'206ebb4e67a6daa9.js',
'4ad6e3a59e27e9b1.js',
'a62c6323a3696fa8.js',
'fc020c065098cbd5.js',
'77a541b0502d0bde.js',
'3b36d546985cd9cb.js',
'ac09566949f0db57.js',
'116cacc3c80a5a3e.js',
'10f0ef998c05c611.js',
'bf1db420b006027f.js'
],
explicit: [
'3b36d546985cd9cb.js',
'110fa1efdd0868b8.js',
'1a1c717109ab67e1.js',
'206ebb4e67a6daa9.js',
'fc020c065098cbd5.js',
'4ad6e3a59e27e9b1.js'
],
const skipped = {
fail: [
'e4a43066905a597b.js',
'f872cf801765a723.js',
'bf49ec8d96884562.js',
// https://github.com/tc39/test262-parser-tests/issues/26
'8af69d8f15295ed2.js',
'84633c379e4010bf.js',
'7fc173197c3cc75d.js',
'78c215fabdf13bae.js',
'f4467d719dcee086.js',
'66e383bfd18e66ab.js',
'647e21f8f157c338.js',
'324ab48c6d89125d.js',
'2687d6d9043bd5cb.js',
'7b876ca5139f1ca8.js',
'3bc2b27a7430f818.js',

// Seems nothing wrong with the code
'e3fbcf63d7e43ead.js',
'15a6123f6b825c38.js',
'147fa078a7436e0e.js',
'fd2a45941e114896.js',

// https://github.com/tc39/test262-parser-tests/issues/25
'0d5e450f1da8a92a.js',
'748656edbfb2d0bb.js',
'79f882da06f88c9f.js',
'92b6af54adef3624.js'
'92b6af54adef3624.js',

// FIXME: "use strict" should be preflighted
'147fa078a7436e0e.js',
'15a6123f6b825c38.js',
'3bc2b27a7430f818.js',

// FIXME
'2687d6d9043bd5cb.js',
'84633c379e4010bf.js',
'f4467d719dcee086.js',
'7b876ca5139f1ca8.js',
'fd2a45941e114896.js'
],
early: [
'eeb5ffaafa815bb2.js',
'ec31fa5e521c5df4.js',
'e262ea7682c36f92.js',
'cc0122b6d261427b.js',
// Seems nothing wrong with the code
'06593c3474233eca.js',

// FIXME: if possible
'be7329119eaa3d47.js',
'9d893d3b185cdca0.js',
'f3943346692438ba.js',
'de9d9a6cd61407f5.js',
'cc0122b6d261427b.js',
'e25b19185b3988dc.js',
'b4994b656f8c8e75.js',
'8f2d88f14f125b42.js',
'e262ea7682c36f92.js',
'ec31fa5e521c5df4.js',
'eeb5ffaafa815bb2.js',
'f3943346692438ba.js',
'587f8eba459c585c.js',
'811e434afe688eb4.js',
'4de83a7417cd30dd.js',
'8f2d88f14f125b42.js',
'3644a964c2a8522c.js',
'1aff49273f3e3a98.js',
'12a74c60f52a60de.js',
'0f5f47108da5c34e.js',
'06593c3474233eca.js',
// Need a fix
'de9d9a6cd61407f5.js'

// FIXME
'b4994b656f8c8e75.js'
]
};

Expand All @@ -84,7 +66,7 @@ const isModule = (val: string) => /\.module\.js/.test(val);
describe('Test262 Parser tests', () => {
describe('Pass', () => {
for (const f of readdirSync(`${Test262Dir}/pass`)) {
if (expectations.pass.indexOf(f) !== -1) continue;
// if (skipped.pass.indexOf(f) !== -1) continue;
it(`Should pass - [${f}]`, () => {
t.doesNotThrow(() => {
parse(readFileSync(`${Test262Dir}/pass/${f}`, 'utf8'), isModule(f));
Expand All @@ -95,7 +77,7 @@ describe('Test262 Parser tests', () => {

describe('Pass explicit', () => {
for (const f of readdirSync(`${Test262Dir}/pass-explicit`)) {
if (expectations.explicit.indexOf(f) !== -1) continue;
// if (skipped.explicit.indexOf(f) !== -1) continue;
it(`Should pass - [${f}]`, () => {
t.doesNotThrow(() => {
parse(readFileSync(`${Test262Dir}/pass-explicit/${f}`, 'utf8'), isModule(f));
Expand All @@ -106,7 +88,7 @@ describe('Test262 Parser tests', () => {

describe('Fail', () => {
for (const f of readdirSync(`${Test262Dir}/fail`)) {
if (expectations.fail.indexOf(f) !== -1) continue;
if (skipped.fail.indexOf(f) !== -1) continue;
it(`Should fail on - [${f}]`, () => {
t.throws(() => {
parse(readFileSync(`${Test262Dir}/fail/${f}`, 'utf8'), isModule(f));
Expand All @@ -117,7 +99,7 @@ describe('Test262 Parser tests', () => {

describe('Early errors', () => {
for (const f of readdirSync(`${Test262Dir}/early`)) {
if (expectations.early.indexOf(f) !== -1) continue;
if (skipped.early.indexOf(f) !== -1) continue;
it(`should fail on early error [${f}]`, () => {
t.throws(() => {
parse(readFileSync(`${Test262Dir}/early/${f}`, 'utf8'), isModule(f));
Expand Down

0 comments on commit aaa91f5

Please sign in to comment.