diff --git a/src/acorn.ts b/src/acorn.ts index 682c962..a1ea521 100644 --- a/src/acorn.ts +++ b/src/acorn.ts @@ -1,11 +1,12 @@ import { tokenizer } from 'acorn' +import type { Parser, Token } from 'acorn' /** * Strip literal using Acorn's tokenizer. * * Will throw error if the input is not valid JavaScript. */ -export function stripLiteralAcorn(code: string) { +export function _stripLiteralAcorn(code: string) { const FILL = ' ' let result = '' function fulfill(index: number) { @@ -13,31 +14,57 @@ export function stripLiteralAcorn(code: string) { result += code.slice(result.length, index).replace(/[^\n]/g, FILL) } - const tokens = tokenizer(code, { + const tokens: Token[] = [] + const pasers = tokenizer(code, { ecmaVersion: 'latest', sourceType: 'module', allowHashBang: true, allowAwaitOutsideFunction: true, allowImportExportEverywhere: true, - }) - const inter = tokens[Symbol.iterator]() + }) as Parser & ReturnType + const iter = pasers[Symbol.iterator]() - while (true) { - const { done, value: token } = inter.next() - if (done) - break - fulfill(token.start) - if (token.type.label === 'string') - result += code[token.start] + FILL.repeat(token.end - token.start - 2) + code[token.end - 1] - else if (token.type.label === 'template') - result += FILL.repeat(token.end - token.start) - else - result += code.slice(token.start, token.end) + let error: any + try { + while (true) { + const { done, value: token } = iter.next() + if (done) + break + tokens.push(token) + fulfill(token.start) + if (token.type.label === 'string') + result += code[token.start] + FILL.repeat(token.end - token.start - 2) + code[token.end - 1] + else if (token.type.label === 'template') + result += FILL.repeat(token.end - token.start) + else if (token.type.label === 'regexp') + result += code.slice(token.start, token.end).replace(/\/(.*)\/(\w?)$/g, (_, $1, $2) => `/${FILL.repeat($1.length)}/${$2}`) + else + result += code.slice(token.start, token.end) + } + + fulfill(code.length) + } + catch (e) { + error = e } - fulfill(code.length) + return { + error, + result, + tokens, + } +} - return result +/** + * Strip literal using Acorn's tokenizer. + * + * Will throw error if the input is not valid JavaScript. + */ +export function stripLiteralAcorn(code: string) { + const result = _stripLiteralAcorn(code) + if (result.error) + throw result.error + return result.result } /** diff --git a/src/index.ts b/src/index.ts index cbf7171..b358144 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { stripLiteralAcorn } from './acorn' +import { _stripLiteralAcorn } from './acorn' import { stripLiteralRegex } from './regex' export { stripLiteralAcorn, createIsLiteralPositionAcorn } from './acorn' @@ -10,10 +10,33 @@ export { stripLiteralRegex } from './regex' * Using Acorn's tokenizer first, and fallback to Regex if Acorn fails. */ export function stripLiteral(code: string) { - try { - return stripLiteralAcorn(code) + return stripLiteralDetailed(code).result +} + +/** + * Strip literal from code, return more detailed information. + * + * Using Acorn's tokenizer first, and fallback to Regex if Acorn fails. + */ +export function stripLiteralDetailed(code: string): { + mode: 'acorn' | 'regex' + result: string + acorn: { + tokens: any[] + error?: any + } +} { + const acorn = _stripLiteralAcorn(code) + if (!acorn.error) { + return { + mode: 'acorn', + result: acorn.result, + acorn, + } } - catch (e) { - return stripLiteralRegex(code) + return { + mode: 'regex', + result: stripLiteralRegex(acorn.result + code.slice(acorn.result.length)), + acorn, } } diff --git a/test/__snapshots__/index.test.ts.snap b/test/__snapshots__/index.test.ts.snap index e61b71d..0bb56e7 100644 --- a/test/__snapshots__/index.test.ts.snap +++ b/test/__snapshots__/index.test.ts.snap @@ -1,8 +1,8 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`escape character 1`] = ` -{ - "code": "' ' +"// mode: acorn +' ' \\" \\" \\" \\" \\" \\" @@ -14,88 +14,70 @@ exports[`escape character 1`] = ` ' ' \\" \\" \\" \\" -\\" \\"", - "mode": "acorn", -} +\\" \\"" `; exports[`regexp affect 1`] = ` -{ - "code": "[ - /'/, +"// mode: acorn +[ + / /, ' ', - /\\"/, + / /, \\" \\" -]", - "mode": "acorn", -} +]" `; exports[`strings comment nested 1`] = ` -{ - "code": " +"// mode: acorn + const a = \\" \\" - ", - "mode": "acorn", -} + " `; exports[`strings comment nested 2`] = ` -{ - "code": " +"// mode: acorn + const a = \\" \\" - ", - "mode": "acorn", -} + " `; exports[`strings comment nested 3`] = ` -{ - "code": " +"// mode: acorn + const a = \\" \\" - ", - "mode": "acorn", -} + " `; exports[`strings comment nested 4`] = ` -{ - "code": "const a = \\" \\" -console.log(\\" \\")", - "mode": "acorn", -} +"// mode: acorn +const a = \\" \\" +console.log(\\" \\")" `; exports[`strings comment nested 5`] = ` -{ - "code": "const a = \\" \\" +"// mode: acorn +const a = \\" \\" console.log(\\" \\") -const b = \\" \\"", - "mode": "acorn", -} +const b = \\" \\"" `; exports[`strings comment nested 6`] = ` -{ - "code": "const a = \\" \\" +"// mode: acorn +const a = \\" \\" console.log(\\" \\") -const b = \\" \\"", - "mode": "acorn", -} +const b = \\" \\"" `; exports[`strings comment nested 7`] = ` -{ - "code": "const a = \\" \\" +"// mode: acorn +const a = \\" \\" console.log(\\" \\") -const b = \\" \\"", - "mode": "acorn", -} +const b = \\" \\"" `; exports[`works 1`] = ` -{ - "code": " +"// mode: acorn + const a = ' ' const b = \\" \\" @@ -106,7 +88,5 @@ const b = \\" \\" const c = \` \${a}\` -let d = /re\\\\\\\\ge/g", - "mode": "acorn", -} +let d = / /g" `; diff --git a/test/fixtures.test.ts b/test/fixtures.test.ts index 8e70794..ec616f2 100644 --- a/test/fixtures.test.ts +++ b/test/fixtures.test.ts @@ -7,8 +7,8 @@ describe('fixtures', () => { if (path.includes('.output.')) continue test(path, async () => { - const result = executeWithVerify(await input(), !!path.match(/\.(ts|js)$/)) - const code = `// mode: ${result.mode}\n${result.code}` + const raw = await input() + const code = executeWithVerify(raw, !!path.match(/\.(ts|js)$/) && !raw.includes('skip-verify')) await expect(code) .toMatchFileSnapshot(path.replace(/\.(\w+)$/, '.output.$1')) }) diff --git a/test/fixtures/backtick-in-regex.js b/test/fixtures/backtick-in-regex.js new file mode 100644 index 0000000..8ad1d3d --- /dev/null +++ b/test/fixtures/backtick-in-regex.js @@ -0,0 +1,4 @@ +// skip-verify +var r = /`/; +foobar(`${foo({ class: "foo" })}`); +const a = 1; diff --git a/test/fixtures/backtick-in-regex.output.js b/test/fixtures/backtick-in-regex.output.js new file mode 100644 index 0000000..de731bb --- /dev/null +++ b/test/fixtures/backtick-in-regex.output.js @@ -0,0 +1,5 @@ +// mode: regex + +var r = / /; +foobar(`${foo({ class: " " })}`); +const a = 1; \ No newline at end of file diff --git a/test/index.test.ts b/test/index.test.ts index 44e2f96..b558fc4 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -96,68 +96,52 @@ test('acorn syntax error', () => { foo(\`fooo \${foo({ class: "foo" })} bar\`) `, false)) .toMatchInlineSnapshot(` - { - "code": "foo(\` \${foo({ class: \\" \\" })} \`)", - "mode": "regex", - } + "// mode: regex + foo(\` \${foo({ class: \\" \\" })} \`)" `) }) test('template string nested', () => { let str = '`aaaa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \`", - "mode": "acorn", - } + "// mode: acorn + \` \`" `) str = '`aaaa` `aaaa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \` \` \`", - "mode": "acorn", - } + "// mode: acorn + \` \` \` \`" `) str = '`aa${a}aa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \${a} \`", - "mode": "acorn", - } + "// mode: acorn + \` \${a} \`" `) str = '`aa${a + `a` + a}aa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \${a + \` \` + a} \`", - "mode": "acorn", - } + "// mode: acorn + \` \${a + \` \` + a} \`" `) str = '`aa${a + `a` + a}aa` `aa${a + `a` + a}aa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \${a + \` \` + a} \` \` \${a + \` \` + a} \`", - "mode": "acorn", - } + "// mode: acorn + \` \${a + \` \` + a} \` \` \${a + \` \` + a} \`" `) str = '`aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \`", - "mode": "acorn", - } + "// mode: acorn + \` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \`" `) str = '`aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa` `aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa`' expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "\` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \` \` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \`", - "mode": "acorn", - } + "// mode: acorn + \` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \` \` \${a + \` \${c + (a = {b: 1}) + d}\` + a} \`" `) }) @@ -168,11 +152,9 @@ test('backtick escape', () => { 'this.error(`\\``)', ].join('\n') expect(executeWithVerify(str)).toMatchInlineSnapshot(` - { - "code": "this.error(\` \`) + "// mode: acorn + this.error(\` \`) this.error(\` \`) - this.error(\` \`)", - "mode": "acorn", - } + this.error(\` \`)" `) }) diff --git a/test/utils.ts b/test/utils.ts index d892f9a..30db472 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -1,37 +1,26 @@ import { parse } from 'acorn' import { expect } from 'vitest' -import { stripLiteralAcorn, stripLiteralRegex } from '../src' +import { stripLiteralDetailed } from '../src' export function executeWithVerify(code: string, verifyAst = true) { code = code.trim() - let result: string - let mode = 'acorn' - // let parseError: any - try { - result = stripLiteralAcorn(code) - } - catch (e) { - result = stripLiteralRegex(code) - mode = 'regex' - // parseError = e - } + const result = stripLiteralDetailed(code) - // if (verifyAst && parseError) - // console.error(parseError) + // if (verifyAst && result.acorn.error) + // console.error(result.acorn.error) - for (let i = 0; i < result.length; i++) { - if (!result[i].match(/\s/)) - expect(result[i]).toBe(code[i]) + const stripped = result.result + + for (let i = 0; i < stripped.length; i++) { + if (!stripped[i].match(/\s/)) + expect(stripped[i]).toBe(code[i]) } - expect(result.length).toBe(code.length) + expect(stripped.length).toBe(code.length) // make sure no syntax errors if (verifyAst) - parse(result, { ecmaVersion: 'latest', sourceType: 'module' }) + parse(stripped, { ecmaVersion: 'latest', sourceType: 'module' }) - return { - code: result, - mode, - } + return `// mode: ${result.mode}\n${stripped}` }