feat(core): lighter token explanation with scopeName (#739)

Author: antfu

packages/core/src/code-to-tokens-base.ts

@@ -125,28 +125,6 @@ function _tokenizeWithTheme(
   let actual: ThemedToken[] = []
   const final: ThemedToken[][] = []
 
-  const themeSettingsSelectors: ThemeSettingsSelectors[] = []
-  if (options.includeExplanation) {
-    for (const setting of theme.settings) {
-      let selectors: string[]
-      switch (typeof setting.scope) {
-        case 'string':
-          selectors = setting.scope.split(/,/).map(scope => scope.trim())
-          break
-        case 'object':
-          selectors = setting.scope
-          break
-        default:
-          continue
-      }
-
-      themeSettingsSelectors.push({
-        settings: setting,
-        selectors: selectors.map(selector => selector.split(/ /)),
-      })
-    }
-  }
-
   for (let i = 0, len = lines.length; i < len; i++) {
     const [line, lineOffset] = lines[i]
     if (line === '') {
@@ -201,6 +179,29 @@ function _tokenizeWithTheme(
       }
 
       if (options.includeExplanation) {
+        const themeSettingsSelectors: ThemeSettingsSelectors[] = []
+
+        if (options.includeExplanation !== 'scopeName') {
+          for (const setting of theme.settings) {
+            let selectors: string[]
+            switch (typeof setting.scope) {
+              case 'string':
+                selectors = setting.scope.split(/,/).map(scope => scope.trim())
+                break
+              case 'object':
+                selectors = setting.scope
+                break
+              default:
+                continue
+            }
+
+            themeSettingsSelectors.push({
+              settings: setting,
+              selectors: selectors.map(selector => selector.split(/ /)),
+            })
+          }
+        }
+
         token.explanation = []
         let offset = 0
         while (startIndex + offset < nextStartIndex) {
@@ -213,7 +214,14 @@ function _tokenizeWithTheme(
           offset += tokenWithScopesText.length
           token.explanation.push({
             content: tokenWithScopesText,
-            scopes: explainThemeScopes(themeSettingsSelectors, tokenWithScopes.scopes),
+            scopes: options.includeExplanation === 'scopeName'
+              ? explainThemeScopesNameOnly(
+                tokenWithScopes.scopes,
+              )
+              : explainThemeScopesFull(
+                themeSettingsSelectors,
+                tokenWithScopes.scopes,
+              ),
           })
 
           tokensWithScopesIndex! += 1
@@ -233,17 +241,22 @@ function _tokenizeWithTheme(
   }
 }
 
-function explainThemeScopes(
+function explainThemeScopesNameOnly(
+  scopes: string[],
+): ThemedTokenScopeExplanation[] {
+  return scopes.map(scope => ({ scopeName: scope }))
+}
+
+function explainThemeScopesFull(
   themeSelectors: ThemeSettingsSelectors[],
   scopes: string[],
 ): ThemedTokenScopeExplanation[] {
   const result: ThemedTokenScopeExplanation[] = []
   for (let i = 0, len = scopes.length; i < len; i++) {
-    const parentScopes = scopes.slice(0, i)
     const scope = scopes[i]
     result[i] = {
       scopeName: scope,
-      themeMatches: explainThemeScope(themeSelectors, scope, parentScopes),
+      themeMatches: explainThemeScope(themeSelectors, scope, scopes.slice(0, i)),
     }
   }
   return result

packages/core/src/types/tokens.ts

@@ -1,4 +1,5 @@
 import type { GrammarState } from '../grammar-state'
+import type { IRawThemeSetting } from '../textmate'
 import type { SpecialLanguage } from './langs'
 import type { SpecialTheme, ThemeRegistrationAny } from './themes'
 import type { CodeOptionsThemes } from './options'
@@ -36,7 +37,7 @@ export interface CodeToTokensWithThemesOptions<Languages = string, Themes = stri
 
 export interface ThemedTokenScopeExplanation {
   scopeName: string
-  themeMatches: any[]
+  themeMatches?: IRawThemeSetting[]
 }
 
 export interface ThemedTokenExplanation {
@@ -149,9 +150,12 @@ export interface TokenizeWithThemeOptions {
   /**
    * Include explanation of why a token is given a color.
    *
+   * You can optionally pass `scopeName` to only include explanation for scopes,
+   * which is more performant than full explanation.
+   *
    * @default false
    */
-  includeExplanation?: boolean
+  includeExplanation?: boolean | 'scopeName'
 
   /**
    * A map of color names to new color values.