feat(core): introduce tokens hook to transformers

packages/shikiji-core/src/renderer-hast.ts

@@ -86,6 +86,9 @@ export function codeToHast(
     throw new Error('[shikiji] Invalid options, either `theme` or `themes` must be provided')
   }
 
+  for (const transformer of options.transformers || [])
+    tokens = transformer.tokens?.call(transformerContext, tokens) || tokens
+
   return tokensToHast(
     tokens,
     {

packages/shikiji-core/src/renderer-html-themes.ts

@@ -13,7 +13,7 @@ export function codeToTokensWithThemes(
   internal: ShikiInternal,
   code: string,
   options: CodeToTokensWithThemesOptions,
-) {
+): ThemedTokenWithVariants[][] {
   const themes = Object.entries(options.themes)
     .filter(i => i[1])
     .map(i => ({ color: i[0], theme: i[1]! }))

packages/shikiji-core/src/types.ts

@@ -471,6 +471,15 @@ export interface ShikijiTransformer {
    * Name of the transformer
    */
   name?: string
+  /**
+   * Transform the raw input code before passing to the highlighter.
+   */
+  preprocess?(this: ShikijiTransformerContextCommon, code: string, options: CodeToHastOptions): string | void
+  /**
+   * Transform the full tokens list before converting to HAST.
+   * Return a new tokens list will replace the original one.
+   */
+  tokens?(this: ShikijiTransformerContextCommon, tokens: ThemedToken[][]): ThemedToken[][] | void
   /**
    * Transform the entire generated HAST tree. Return a new Node will replace the original one.
    */
@@ -494,12 +503,6 @@ export interface ShikijiTransformer {
    * Transform each token `<span>` element.
    */
   token?(this: ShikijiTransformerContext, hast: Element, line: number, col: number, lineElement: Element): Element | void
-
-  /**
-   * Transform the raw input code before passing to the highlighter.
-   * This hook will only be called with `codeToHtml` or `codeToHast`.
-   */
-  preprocess?(this: ShikijiTransformerContextCommon, code: string, options: CodeToHastOptions): string | void
   /**
    * Transform the generated HTML string before returning.
    * This hook will only be called with `codeToHtml`.