Add an option to skip generating the explanation text

packages/shiki/src/highlighter.ts

@@ -34,6 +34,14 @@ export async function getHighlighter(options: HighlighterOptions) {
   return await s.getHighlighter()
 }
 
+interface ThemedTokenizerOptions {
+  /**
+   * Whether to include explanation of each token's matching scopes
+   * and why it's given its color. Default to false.
+   */
+  includeExplanation: boolean
+}
+
 class Shiki {
   private _resolver: Resolver
   private _registry: Registry
@@ -70,14 +78,14 @@ class Shiki {
     )
 
     return {
-      codeToThemedTokens: (code, lang) => {
+      codeToThemedTokens: (code, lang, options = { includeExplanation: true }) => {
         if (isPlaintext(lang)) {
           throw Error('Cannot tokenize plaintext')
         }
         if (!ltog[lang]) {
           throw Error(`No language registration for ${lang}`)
         }
-        return tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang])
+        return tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang], options)
       },
       codeToHtml: (code, lang) => {
         if (isPlaintext(lang)) {
@@ -89,7 +97,9 @@ class Shiki {
         if (!ltog[lang]) {
           throw Error(`No language registration for ${lang}`)
         }
-        const tokens = tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang])
+        const tokens = tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang], {
+          includeExplanation: false
+        })
         return renderToHtml(tokens, {
           bg: this._theme.bg
         })
@@ -108,7 +118,11 @@ function isPlaintext(lang) {
  */
 type StringLiteralUnion<T extends U, U = string> = T | (U & {})
 export interface Highlighter {
-  codeToThemedTokens(code: string, lang: StringLiteralUnion<Lang>): IThemedToken[][]
+  codeToThemedTokens(
+    code: string,
+    lang: StringLiteralUnion<Lang>,
+    options?: ThemedTokenizerOptions
+  ): IThemedToken[][]
   codeToHtml?(code: string, lang: StringLiteralUnion<Lang>): string
 
   // codeToRawHtml?(code: string): string

packages/shiki/src/themedTokenizer.ts

@@ -16,17 +16,83 @@ export interface IThemedTokenExplanation {
   scopes: IThemedTokenScopeExplanation[]
 }
 
+/**
+ * A single token with color, and optionally with explanation.
+ *
+ * For example:
+ *
+ * {
+ *   "content": "shiki",
+ *   "color": "#D8DEE9",
+ *   "explanation": [
+ *     {
+ *       "content": "shiki",
+ *       "scopes": [
+ *         {
+ *           "scopeName": "source.js",
+ *           "themeMatches": []
+ *         },
+ *         {
+ *           "scopeName": "meta.objectliteral.js",
+ *           "themeMatches": []
+ *         },
+ *         {
+ *           "scopeName": "meta.object.member.js",
+ *           "themeMatches": []
+ *         },
+ *         {
+ *           "scopeName": "meta.array.literal.js",
+ *           "themeMatches": []
+ *         },
+ *         {
+ *           "scopeName": "variable.other.object.js",
+ *           "themeMatches": [
+ *             {
+ *               "name": "Variable",
+ *               "scope": "variable.other",
+ *               "settings": {
+ *                 "foreground": "#D8DEE9"
+ *               }
+ *             },
+ *             {
+ *               "name": "[JavaScript] Variable Other Object",
+ *               "scope": "source.js variable.other.object",
+ *               "settings": {
+ *                 "foreground": "#D8DEE9"
+ *               }
+ *             }
+ *           ]
+ *         }
+ *       ]
+ *     }
+ *   ]
+ * }
+ *
+ */
 export interface IThemedToken {
+  /**
+   * The content of the token
+   */
   content: string
+  /**
+   * 6 or 8 digit hex code representation of the token's color
+   */
   color?: string
+  /**
+   * Explanation of
+   *
+   * - token text's matching scopes
+   * - reason that token text is given a color (one matching scope matches a rule (scope -> color) in the theme)
+   */
   explanation?: IThemedTokenExplanation[]
 }
 
 export function tokenizeWithTheme(
   theme: IRawTheme,
   colorMap: string[],
   fileContents: string,
-  grammar: IGrammar
+  grammar: IGrammar,
+  options: { includeExplanation?: boolean }
 ): IThemedToken[][] {
   let lines = fileContents.split(/\r\n|\r|\n/)
 
@@ -60,22 +126,25 @@ export function tokenizeWithTheme(
       let foregroundColor = colorMap[foreground]
 
       let explanation: IThemedTokenExplanation[] = []
-      let offset = 0
-      while (startIndex + offset < nextStartIndex) {
-        let tokenWithScopes = tokensWithScopes[tokensWithScopesIndex]
-
-        let tokenWithScopesText = line.substring(
-          tokenWithScopes.startIndex,
-          tokenWithScopes.endIndex
-        )
-        offset += tokenWithScopesText.length
-        explanation.push({
-          content: tokenWithScopesText,
-          scopes: explainThemeScopes(theme, tokenWithScopes.scopes)
-        })
-
-        tokensWithScopesIndex++
+      if (options.includeExplanation) {
+        let offset = 0
+        while (startIndex + offset < nextStartIndex) {
+          let tokenWithScopes = tokensWithScopes[tokensWithScopesIndex]
+
+          let tokenWithScopesText = line.substring(
+            tokenWithScopes.startIndex,
+            tokenWithScopes.endIndex
+          )
+          offset += tokenWithScopesText.length
+          explanation.push({
+            content: tokenWithScopesText,
+            scopes: explainThemeScopes(theme, tokenWithScopes.scopes)
+          })
+
+          tokensWithScopesIndex++
+        }
       }
+
       actual.push({
         content: line.substring(startIndex, nextStartIndex),
         color: foregroundColor,