fix: improve fallback handling

src/acorn.ts

@@ -1,43 +1,70 @@
 import { tokenizer } from 'acorn'
+import type { Parser, Token } from 'acorn'
 
 /**
  * Strip literal using Acorn's tokenizer.
  *
  * Will throw error if the input is not valid JavaScript.
  */
-export function stripLiteralAcorn(code: string) {
+export function _stripLiteralAcorn(code: string) {
   const FILL = ' '
   let result = ''
   function fulfill(index: number) {
     if (index > result.length)
       result += code.slice(result.length, index).replace(/[^\n]/g, FILL)
   }
 
-  const tokens = tokenizer(code, {
+  const tokens: Token[] = []
+  const pasers = tokenizer(code, {
     ecmaVersion: 'latest',
     sourceType: 'module',
     allowHashBang: true,
     allowAwaitOutsideFunction: true,
     allowImportExportEverywhere: true,
-  })
-  const inter = tokens[Symbol.iterator]()
+  }) as Parser & ReturnType<typeof tokenizer>
+  const iter = pasers[Symbol.iterator]()
 
-  while (true) {
-    const { done, value: token } = inter.next()
-    if (done)
-      break
-    fulfill(token.start)
-    if (token.type.label === 'string')
-      result += code[token.start] + FILL.repeat(token.end - token.start - 2) + code[token.end - 1]
-    else if (token.type.label === 'template')
-      result += FILL.repeat(token.end - token.start)
-    else
-      result += code.slice(token.start, token.end)
+  let error: any
+  try {
+    while (true) {
+      const { done, value: token } = iter.next()
+      if (done)
+        break
+      tokens.push(token)
+      fulfill(token.start)
+      if (token.type.label === 'string')
+        result += code[token.start] + FILL.repeat(token.end - token.start - 2) + code[token.end - 1]
+      else if (token.type.label === 'template')
+        result += FILL.repeat(token.end - token.start)
+      else if (token.type.label === 'regexp')
+        result += code.slice(token.start, token.end).replace(/\/(.*)\/(\w?)$/g, (_, $1, $2) => `/${FILL.repeat($1.length)}/${$2}`)
+      else
+        result += code.slice(token.start, token.end)
+    }
+
+    fulfill(code.length)
+  }
+  catch (e) {
+    error = e
   }
 
-  fulfill(code.length)
+  return {
+    error,
+    result,
+    tokens,
+  }
+}
 
-  return result
+/**
+ * Strip literal using Acorn's tokenizer.
+ *
+ * Will throw error if the input is not valid JavaScript.
+ */
+export function stripLiteralAcorn(code: string) {
+  const result = _stripLiteralAcorn(code)
+  if (result.error)
+    throw result.error
+  return result.result
 }
 
 /**

src/index.ts

@@ -1,4 +1,4 @@
-import { stripLiteralAcorn } from './acorn'
+import { _stripLiteralAcorn } from './acorn'
 import { stripLiteralRegex } from './regex'
 
 export { stripLiteralAcorn, createIsLiteralPositionAcorn } from './acorn'
@@ -10,10 +10,33 @@ export { stripLiteralRegex } from './regex'
  * Using Acorn's tokenizer first, and fallback to Regex if Acorn fails.
  */
 export function stripLiteral(code: string) {
-  try {
-    return stripLiteralAcorn(code)
+  return stripLiteralDetailed(code).result
+}
+
+/**
+ * Strip literal from code, return more detailed information.
+ *
+ * Using Acorn's tokenizer first, and fallback to Regex if Acorn fails.
+ */
+export function stripLiteralDetailed(code: string): {
+  mode: 'acorn' | 'regex'
+  result: string
+  acorn: {
+    tokens: any[]
+    error?: any
+  }
+} {
+  const acorn = _stripLiteralAcorn(code)
+  if (!acorn.error) {
+    return {
+      mode: 'acorn',
+      result: acorn.result,
+      acorn,
+    }
   }
-  catch (e) {
-    return stripLiteralRegex(code)
+  return {
+    mode: 'regex',
+    result: stripLiteralRegex(acorn.result + code.slice(acorn.result.length)),
+    acorn,
   }
 }

test/__snapshots__/index.test.ts.snap

@@ -1,8 +1,8 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`escape character 1`] = `
-{
-  "code": "'    '
+"// mode: acorn
+'    '
 \\"    \\"
 \\"       \\"
 \\"        \\"
@@ -14,88 +14,70 @@ exports[`escape character 1`] = `
 '         '
 \\"     \\"
 \\"       \\"
-\\"         \\"",
-  "mode": "acorn",
-}
+\\"         \\""
 `;
 
 exports[`regexp affect 1`] = `
-{
-  "code": "[
-  /'/,
+"// mode: acorn
+[
+  / /,
   ' ',
-  /\\"/,
+  / /,
   \\" \\"
-]",
-  "mode": "acorn",
-}
+]"
 `;
 
 exports[`strings comment nested 1`] = `
-{
-  "code": "                    
+"// mode: acorn
+                    
 const a = \\"    \\"
-                    ",
-  "mode": "acorn",
-}
+                    "
 `;
 
 exports[`strings comment nested 2`] = `
-{
-  "code": "                    
+"// mode: acorn
+                    
 const a = \\"    \\"
-                    ",
-  "mode": "acorn",
-}
+                    "
 `;
 
 exports[`strings comment nested 3`] = `
-{
-  "code": "                    
+"// mode: acorn
+                    
 const a = \\"    \\"
-                    ",
-  "mode": "acorn",
-}
+                    "
 `;
 
 exports[`strings comment nested 4`] = `
-{
-  "code": "const a = \\"    \\"
-console.log(\\"       \\")",
-  "mode": "acorn",
-}
+"// mode: acorn
+const a = \\"    \\"
+console.log(\\"       \\")"
 `;
 
 exports[`strings comment nested 5`] = `
-{
-  "code": "const a = \\"    \\"
+"// mode: acorn
+const a = \\"    \\"
 console.log(\\"       \\")
-const b = \\"    \\"",
-  "mode": "acorn",
-}
+const b = \\"    \\""
 `;
 
 exports[`strings comment nested 6`] = `
-{
-  "code": "const a = \\"    \\"
+"// mode: acorn
+const a = \\"    \\"
 console.log(\\"       \\")
-const b = \\"    \\"",
-  "mode": "acorn",
-}
+const b = \\"    \\""
 `;
 
 exports[`strings comment nested 7`] = `
-{
-  "code": "const a = \\"    \\"
+"// mode: acorn
+const a = \\"    \\"
 console.log(\\"       \\")
-const b = \\"    \\"",
-  "mode": "acorn",
-}
+const b = \\"    \\""
 `;
 
 exports[`works 1`] = `
-{
-  "code": "           
+"// mode: acorn
+           
 const a = '    '
               
 const b = \\"    \\"
@@ -106,7 +88,5 @@ const b = \\"    \\"
                             
 const c = \`   \${a}\`
 
-let d = /re\\\\\\\\ge/g",
-  "mode": "acorn",
-}
+let d = /      /g"
 `;

test/fixtures.test.ts

@@ -7,8 +7,8 @@ describe('fixtures', () => {
     if (path.includes('.output.'))
       continue
     test(path, async () => {
-      const result = executeWithVerify(await input(), !!path.match(/\.(ts|js)$/))
-      const code = `// mode: ${result.mode}\n${result.code}`
+      const raw = await input()
+      const code = executeWithVerify(raw, !!path.match(/\.(ts|js)$/) && !raw.includes('skip-verify'))
       await expect(code)
         .toMatchFileSnapshot(path.replace(/\.(\w+)$/, '.output.$1'))
     })

test/fixtures/backtick-in-regex.js

@@ -0,0 +1,4 @@
+// skip-verify
+var r = /`/;
+foobar(`${foo({ class: "foo" })}`);
+const a = 1;

test/fixtures/backtick-in-regex.output.js

@@ -0,0 +1,5 @@
+// mode: regex
+
+var r = / /;
+foobar(`${foo({ class: "   " })}`);
+const a = 1;

test/index.test.ts

@@ -96,68 +96,52 @@ test('acorn syntax error', () => {
 foo(\`fooo \${foo({ class: "foo" })} bar\`)
   `, false))
     .toMatchInlineSnapshot(`
-      {
-        "code": "foo(\`     \${foo({ class: \\"   \\" })}    \`)",
-        "mode": "regex",
-      }
+      "// mode: regex
+      foo(\`     \${foo({ class: \\"   \\" })}    \`)"
     `)
 })
 
 test('template string nested', () => {
   let str = '`aaaa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`    \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`    \`"
   `)
 
   str = '`aaaa` `aaaa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`    \` \`    \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`    \` \`    \`"
   `)
 
   str = '`aa${a}aa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`  \${a}  \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`  \${a}  \`"
   `)
 
   str = '`aa${a + `a` + a}aa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`  \${a + \` \` + a}  \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`  \${a + \` \` + a}  \`"
   `)
 
   str = '`aa${a + `a` + a}aa` `aa${a + `a` + a}aa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`  \${a + \` \` + a}  \` \`  \${a + \` \` + a}  \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`  \${a + \` \` + a}  \` \`  \${a + \` \` + a}  \`"
   `)
 
   str = '`aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \`"
   `)
 
   str = '`aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa` `aa${a + `aaaa${c + (a = {b: 1}) + d}` + a}aa`'
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "\`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \` \`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \`",
-      "mode": "acorn",
-    }
+    "// mode: acorn
+    \`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \` \`  \${a + \`    \${c + (a = {b: 1}) + d}\` + a}  \`"
   `)
 })
 
@@ -168,11 +152,9 @@ test('backtick escape', () => {
     'this.error(`\\``)',
   ].join('\n')
   expect(executeWithVerify(str)).toMatchInlineSnapshot(`
-    {
-      "code": "this.error(\`                                                          \`)
+    "// mode: acorn
+    this.error(\`                                                          \`)
     this.error(\`  \`)
-    this.error(\`  \`)",
-      "mode": "acorn",
-    }
+    this.error(\`  \`)"
   `)
 })