feat(transformers): add transformerRemoveComments (#1144)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: crazylogic03 <saithrishul2006@gmail.com>
Co-authored-by: Anthony Fu <github@antfu.me>

Author: 4 people

docs/components.d.ts

@@ -1,8 +1,11 @@
 /* eslint-disable */
 // @ts-nocheck
+// biome-ignore lint: disable
+// oxlint-disable
+// ------
 // Generated by unplugin-vue-components
 // Read more: https://github.com/vuejs/core/pull/3399
-// biome-ignore lint: disable
+
 export {}
 
 /* prettier-ignore */

docs/packages/markdown-it.md

@@ -52,6 +52,7 @@ md.use(await Shiki({
 You can pass [transformers](/guide/transformers) to the plugin options to customize the highlighted code.
 
 ```ts twoslash
+// @twoslash-cache: {"v":1,"hash":"7251d42df549db899085560c6ca05a99dc9f338642b7849507ef0ead24fa6dba","data":"N4Igdg9gJgpgziAXAbVAFwJ4AcZJACwgDcYAnEAGhDRgA808AKAQwBsBLZuASgAIAzAK5gAxmnYQwvAMr52Aa3aMIWcZLiJeAWWal5UCAHcwASTSyF7APKqJYHpoAKpCAFt2cGAB5Gr3fqMwdjRNHT0DYzM+AF4APl4iCHYoWIAdINcsCFI0GTlFShA4NF0GRAB2KlYYMABzNHwkAFYqEtJamDKQCwKq9jBcRAAGKhF8XWYxMiRygF8KdGxBgmJp1roulg4uPiFRNSk0UmZ7fmzXMgA5CBKDgBF2fn5lW3UAfk0AFWPT86ub5j3R78GwHBx5SzfE5wM6kC6kdLsTLZXJHaGw+HXW52B5PQoGEQIRAgACqnl4AANkABCETQGC8ADUjIAuhTeCcoJSaXTYLwALT8tm8NAQXh+PQcqCwLmc3ikGCuVZcjgDOAAOkKxVKzRGIGqdQaSAATABmVq6DpdNG/OH/bGSXH8QqqwYARlG42OU3IiAAnPNFjg8IQSOR1vQmGxODxeP0aKR+JMGWEApE0OktjG+HT7LlUxFTCFtP5C2YAMLqI6CMTZRHInIl8KBMz4iCEvA6fq8LC6TykAD0CrAsAVpF4IlYXA16XSAGIFwveGTmB1Z2AKZuAFbMIhcESkdiqdIDge8SCwdVbuAUXipECT6fsET33iGZgYRDpPfjgst3LRPKMAAI6COwCqMAA5BKaZgPywSQdwFDpLwqHilygEDIYTawWYjDcAA3N+uhAXAgisAB6HqsOo5QXO4qloE8G5KQ5EwAAXux1KIURYAnmeF4wFeN4ivgDJwMwFy3gARoIuSGME+C8GRtS6F+YA/uhvCAQqoHgTAUEwYWzGIfhvGaQqZEUdpVE0WQdEMc2xjMfKbGcdxhHrqevDSS4hj9m+ikQHJvAAIJaHct7MNKMBcqKd4gApI5GK+kjKQeR65KwEDRfxvBYjAt4NGQDK6AykC8FAXD4OqxHjq4GGBclhjqkZgTBGZdWkeRlENdRNS0ZB9Ftc5wSudU7k8ekm4Uuu0j9LU1S8K6QEjmQC23gpDTBbkvbHLUxxYEphiHep03bru+6HseGkkQ1Nm6WBEHQYxo1oKZnm3eOlk9TZfV2aQJhgK6UEAPqgyNcHBOD40cVxU0bpu66LkuACitCSVg1TrjNO57nAGU3d5dKuEqYAweK9JdfdOkgU9BkvU5UPvdwUEk2TMEI3lsBJr9SqwNTjWPfphmvczH28XlNTMNJS0wGGGANAtgsPXTIuM7BJms8AKG8PgaCuKwmjVoVuuqoo/CfiKrGm1IIrYBAB3MEdZDGzb6SzJ9M3IyjMgYGAJS0Hr7C1PgHCh+IdQ49ucBdWH16q3pz1yKH4f68JiG8N5+toFgGininYch/r17qtktQDuumk00BScM5DWuMDrduF2nxZ7GIdi8IwxSkLeU51HwzdoXG/DdwPtS8AAZFPeusKXVoADInLUgirgZE/cEPusj0cGC8MPI9oQqaCCKQUjxxqrfF2gjAT7evdFTb3DqnurCCDAvFH7wswToCYzd3BkPWYO8f7rhHifM+UhIKQQIlnM8ghyQbDIGANglUYC82svAEQztlZ2xAWAT2vFvZ8TAFYccW0lJCFYKwN8h0cDjlWKQQ8fJGCPF4BgYK54YCxQ5HAOAIcpCPn4SKMUFIvBYAVLECk3AzqIwpHjK6mU47zzgInemUFr4RwznwbOaBc75wHFokuGpy6V1Id5UKYg160J5swX6b8P6xy+lpWmddRZM0bofYx7dhCdzSj3I4/cV7bztqhdhd8V7T1npfdUS8V5rw6JEweoTv57wPqAiBnQoG8EghIhUE4pz8OiPeS+95YheF5DAWIkEmSZO/iPWJPjkm1AfkE62H8X6OIZIyepDTUJ5IHFUipA5JHVNgfU3+OC0AAMYEAg+BCR4ENAZA8+uT8kMmEXAEpBBVHlMqfSGpTIqJyXYKwDU2DnYwAABIG1YIE0gfBGTrKGYcrwoypETPwR7L2m4tRtDKG6N0xoqg1HqI0RAbo9RtCtJ2MWrY+gDBNJ6CYPoZiBmoEsEMTDCg0EjMSXMxR0KhHhQwKg2ochIDdAAFj9KCw0EKQXUEtJ0PADUXT9EGNSlF3oExIAAGwYswMGYkoY1jUA2FGbYsY/zpnwiSpmZhu6MmNLwJh2Voo8HrFkRssqixtg7MSLsUg9r9iHANEq44tm1VISjeiK41ykNxpdAm10MxgG8oJYSt57xbOfK+d8n4up6qVW4jRGtjIISQqAmuWEcJllvp9Cy8BfqAX+ha0gDkG5jVYhNeGn08peuvEVMSylJKFR8iFShylBCqVIOpauQs1bPWzSzTqLifrWTTVAfqa1M1DUcprHNbl82Sw9WeXyRgAqUJ2mFCKUUYpxTFPeJKERUpSFdZlZaOUoB5QKiWkqHICkVSqnAGqKtAKrqMK1MWHUk0kU7b1HtAMs1ixcrmuGHliFI1IfNOoS0VoAw2oFbaIU9qrkOsdU60cFEusJu6xt6j1atoll1R9f1n0ZqBiDSC4NW0ww/ZNX5s1bW+3RpjbGTqLr43g3ldmkgKb8xgBe2u4aUOs0gvR8m/gubjvQZg3ITGWPCxbW+qN7apaoNlgyeWZBFZyCji4muIn65ibbd4u5bsP7ITtubR4VsTY6dQsKx2kHXYdOY4Q4jPtFx+wDswIOPi8HnQUc4zSl8kPJ2Lm3HR8C9b6LzogAu3mb6lzMVXO6Tb3ERqYuJ7xIWI6aA7gcbuj9lohIyWE0e48okzzngvToy86iJI3iE1JR90mH2/qsi+qj1TNPvspdpJsulsA/l/I+Uz/5KTmaDYBoDllZZq7k2BfnEEyfoCgtBdjfqXKwHg1CBCiEufXOQkDVDyK0JOs7BhaqwwsIZGwsenDBDcN4U+WoQiilqPiuIsZ0jZEwcUZum67nVGeYZj43zeiDFBaMQlkxZd2jmLylY0+02MH2Ost0tzkWPseKHep3WPikt+JSw84Jg9MsjwiRPaJ+WNTxOK+vFpW9scVdIPvKrR9ht5LGYU6cOyykgAqcM2pvSsv9LQk0gHt9GtpZa6/NrPS+n9MGcM9592vnfy6zMnr8zgCLLQoNrJp81l04KVspneyWcHNgEc55fVTnnPVHNm5dyHlPJeRLj54yOtLeI/8nUkKmjcv1GCo0kLoUsq6CGsl+pOVIDd2MVFfLIVuiFVi0VOKIxdEJYJqACrcL+4pYC/lAAOel4LkXMvaKy4k7LEWDH5TyyYYehiR5FSsMMuLJXEl8J0QgUA+B+/VONxgWNa39CcO/Wo/R5qZGqLI+N/5lWqvVTungBqiQgAAFQh/6DLaos/1yLx3cpHAIhHjsF4Z3vvUhq195ICaiYrg1HxjFCIM+w5dp9jIHGPMJwRBCXSGYSCaityINyMwGtdbRF/xoUpD336F8CgFvHAzP24HZGrSv2YQWhtXnDIwxkH0sxc2e1oxcWCDIEBGyHhxizen5FhHgmBk5QRkQzDWQzUw+lFxHjb08DYQTBwL7lyTOAgFBgVCxmTEglvEgjxXelvGSy7kYFFHkBqBEmSFoHKy5zQhELEOQAkJZHVFzBoADhslkPsHkKgFoEUOUJqDQH6k4Of0YAHFYIHFaVyWkl0ARmkMW0d3JQBSpXKA9HdwZSQHNFz1hWJHGw5SRUQBLwfC9DL2mEQFNEr2WDFXDAlXxRACzB2AhEUBeDBCTwTR6GsFeHsGH2cDcA8G8F8FvWLD9xiHiESGSDSAyB1VyFSKd0pUhXT2cINGz0QBaA8Pz26HyHYB8OL1LzRUhVCIWExSrwiNr2iI7xcAYUwD4ErFgFBDsDgC0B6iPGqE+DEguDgC8AACFhAoBqgoBljFRql1RipVinBShOBWAvAAAlGAOkUgKALwXuYDPYi4K4vvXuQEOwUKMAfeAAHxkCOAWkXiwOOFYBJCCEkA2K2J2KeIrQeLqFiHhOnzwFCgYiwDVTHjpGynHFQVWP/yOPgBtWWI8A5BoSnQ4S4XijgE3303FAWKxgZDxLUVhFEgZA6AGGOBoC5CqQQPkTQGcR8TICUPpE+AgFuUNkYCqVvEPgnk0EgmvG4N1gZM0GpzbhlKICwP4RgH5DbnlKyyqj0FVPVM8H5D1PkB1MWx+RW1IQAHUzlaFWTsCaA5EZp9ZDZ0h7jewN1MBqgdkMTshEA5wgU3QCJBQz1LBjT/B/Snh+AiIWcql3lKSTgyiZpcVVwiRkBkAigeFCgc5AtTw+8GhBBpJBTXABxQzFBrxSz2i5w24TTjSbF+QGSQAWQWR7Dnc3Q/R6iPcIV/CYVWjGyi8Zhuiw9jQwjsUa9Y8mBJEVAyAJjlob5NB7w1SaANStSb57xqiyhjQhhmiGjPdmjeyug25OikBM8AjQ9gjqVRzo9xyojNgpzxiMA+ATSFyQAlz4AjSTT1zWyaizR3DdyIU3cDy8ATTjz/QhyLyrzq9xU+DJyxiZzHzeAoRbR4QZj1BDifgYQ/hSA4APh4j2AkLMK7RSBkAWReBfitiMFOUoBETiQCKMQyBGTcDio8KewjwYBXRNRvzNz+UmV/yqUmUgLiQbRCL4QEABzw9wLfQRz+jhVwiY9bypVsxrZ0QsKsR3jHRgREjZjcK6LVKAQgQnhUKMjNBUjdKiLtUURlLkL7R1KwAnQaLSRyQqRaR6QmRWR2Q5RnKqkBQhR2R4oKZooZQOQRwgIlQSAVROUNQNyTR09eKuyTR3DBLqAML6LSA1KDLnRxLgVJKg9ZgWyHx6Q8AkQKiWL+AXBXBckAABMs9gCslDCyxsYAKykSmyjKn+AQcqqqmqis4S1KuASCBq/MUlDqtwXJeq0hePVxEfOVbgdcI3eg5gd8MaVIpuBUlY+AJU0BFU3JN8lc7UozNCZ8naw0zUk0s0n+A63qrCjQXgZAUBK6oi9KnETS6NO2Fsn5bgQoC4EoJAUAZBewOwPAXkkAWYWYIAA=="}
 import Shiki from '@shikijs/markdown-it'
 import { transformerNotationDiff } from '@shikijs/transformers'
 import MarkdownIt from 'markdown-it'

docs/packages/transformers.md

@@ -544,3 +544,45 @@ CSS output:
   --shiki-light-bg: #ffffff;
 }
 ```
+
+### `transformerRemoveComments`
+
+Remove comments from the code. It works by checking the internal grammar token metadata to determine if the token is a comment.
+
+This transformer requires `includeExplanation: true` to work.
+
+```ts
+import { transformerRemoveComments } from '@shikijs/transformers'
+
+const html = await codeToHtml(code, {
+  lang: 'ts',
+  includeExplanation: true, // [!code highlight]
+  transformers: [
+    transformerRemoveComments(), // [!code highlight]
+  ],
+})
+```
+
+Options:
+
+- `removeEmptyLines`: Remove lines that become empty after removing comments. Default `true`.
+
+For example:
+
+````md
+```js
+// This is a comment
+const x = 1 // Inline comment
+/* Block comment */
+const y = 2
+
+// Another comment
+```
+````
+
+Will renders:
+
+```js
+const x = 1
+const y = 2
+```

packages/transformers/src/index.ts

@@ -9,6 +9,7 @@ export * from './transformers/notation-focus'
 export * from './transformers/notation-highlight'
 export * from './transformers/notation-highlight-word'
 export * from './transformers/notation-map'
+export * from './transformers/remove-comments'
 export * from './transformers/remove-line-breaks'
 export * from './transformers/remove-notation-escape'
 export * from './transformers/render-indent-guides'

packages/transformers/src/transformers/remove-comments.ts

@@ -0,0 +1,53 @@
+import type { ShikiTransformer } from '@shikijs/types'
+
+export interface TransformerRemoveCommentsOptions {
+  /**
+   * Remove lines that become empty after removing comments.
+   * @default true
+   */
+  removeEmptyLines?: boolean
+}
+
+/**
+ * Remove comments from the code.
+ */
+export function transformerRemoveComments(
+  options: TransformerRemoveCommentsOptions = {},
+): ShikiTransformer {
+  const { removeEmptyLines = true } = options
+
+  return {
+    name: '@shikijs/transformers:remove-comments',
+    preprocess(_code, options) {
+      if (options.includeExplanation !== true && options.includeExplanation !== 'scopeName')
+        throw new Error('`transformerRemoveComments` requires `includeExplanation` to be set to `true` or `\'scopeName\'`')
+    },
+    tokens(tokens) {
+      const result = []
+      for (const line of tokens) {
+        const filteredLine = []
+        let hasComment = false
+        for (const token of line) {
+          const isComment = token.explanation?.some(exp =>
+            exp.scopes.some(s => s.scopeName.startsWith('comment')),
+          )
+          if (isComment) {
+            hasComment = true
+          }
+          else {
+            filteredLine.push(token)
+          }
+        }
+
+        if (removeEmptyLines && hasComment) {
+          const isAllWhitespace = filteredLine.every(token => !token.content.trim())
+          if (isAllWhitespace)
+            continue
+        }
+
+        result.push(filteredLine)
+      }
+      return result
+    },
+  }
+}

packages/transformers/test/fixtures.test.ts

@@ -1,6 +1,6 @@
 /// <reference types="vite/client" />
 
-import type { ShikiTransformer } from 'shiki'
+import type { CodeToHastOptions, ShikiTransformer } from 'shiki'
 import { codeToHtml } from 'shiki'
 import { describe, expect, it } from 'vitest'
 import {
@@ -10,6 +10,7 @@ import {
   transformerNotationFocus,
   transformerNotationHighlight,
   transformerNotationWordHighlight,
+  transformerRemoveComments,
   transformerRemoveLineBreak,
   transformerRemoveNotationEscape,
   transformerRenderWhitespace,
@@ -25,6 +26,7 @@ function suite(
   transformers: ShikiTransformer[],
   replace?: (code: string) => string,
   outputSuffix = '',
+  options: Partial<CodeToHastOptions> = {},
 ) {
   describe(name, () => {
     for (const path of Object.keys(files)) {
@@ -36,6 +38,7 @@ function suite(
         const ext = path.split('.').pop()!
 
         let code = await codeToHtml(files[path], {
+          ...options,
           lang: ext,
           theme: 'github-dark',
           transformers,
@@ -265,3 +268,20 @@ body { margin: 0; }
 .line { display: block; width: 100%; height: 1.2em; }
 </style>`,
 )
+
+suite(
+  'remove-comments',
+  import.meta.glob('./fixtures/remove-comments/*.*', { query: '?raw', import: 'default', eager: true }),
+  [
+    transformerRemoveComments(),
+    transformerRemoveLineBreak(),
+  ],
+  code => `${code}
+<style>
+body { margin: 0; }
+.shiki { padding: 1em; }
+.line { display: block; width: 100%; height: 1.2em; }
+</style>`,
+  undefined,
+  { includeExplanation: true },
+)

packages/transformers/test/fixtures/remove-comments/basic.js

@@ -142,7 +142,8 @@ export interface CodeToHastOptionsCommon<Languages extends string = string>
   extends
   TransformerOptions,
   DecorationOptions,
-  Pick<TokenizeWithThemeOptions, 'colorReplacements' | 'tokenizeMaxLineLength' | 'tokenizeTimeLimit' | 'grammarState' | 'grammarContextCode'> {
+  Pick<TokenizeWithThemeOptions, 'colorReplacements' | 'tokenizeMaxLineLength' | 'tokenizeTimeLimit' | 'grammarState' | 'grammarContextCode' | 'includeExplanation'> {
+
   /**
    * The grammar name for the code.
    */

packages/transformers/test/fixtures/remove-comments/basic.js.output.html

@@ -12,6 +12,7 @@
   transformerNotationHighlight: function
   transformerNotationMap: function
   transformerNotationWordHighlight: function
+  transformerRemoveComments: function
   transformerRemoveLineBreak: function
   transformerRemoveNotationEscape: function
   transformerRenderIndentGuides: function