From b1fe16c67b4b48bb159fac64a4c19e7344897e12 Mon Sep 17 00:00:00 2001 From: Gerrit Birkeland Date: Sun, 2 Aug 2020 18:50:02 -0600 Subject: [PATCH 1/2] perf: Add option to disable generating the explanation for tokens --- packages/shiki/src/highlighter.ts | 14 ++++++++--- packages/shiki/src/themedTokenizer.ts | 36 +++++++++++++++------------ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/shiki/src/highlighter.ts b/packages/shiki/src/highlighter.ts index bf97b9b1c..472059f00 100644 --- a/packages/shiki/src/highlighter.ts +++ b/packages/shiki/src/highlighter.ts @@ -70,14 +70,14 @@ class Shiki { ) return { - codeToThemedTokens: (code, lang) => { + codeToThemedTokens: (code, lang, options = { includeExplanation: true }) => { if (isPlaintext(lang)) { throw Error('Cannot tokenize plaintext') } if (!ltog[lang]) { throw Error(`No language registration for ${lang}`) } - return tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang]) + return tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang], options) }, codeToHtml: (code, lang) => { if (isPlaintext(lang)) { @@ -89,7 +89,9 @@ class Shiki { if (!ltog[lang]) { throw Error(`No language registration for ${lang}`) } - const tokens = tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang]) + const tokens = tokenizeWithTheme(this._theme, this._colorMap, code, ltog[lang], { + includeExplanation: false + }) return renderToHtml(tokens, { bg: this._theme.bg }) @@ -108,7 +110,11 @@ function isPlaintext(lang) { */ type StringLiteralUnion = T | (U & {}) export interface Highlighter { - codeToThemedTokens(code: string, lang: StringLiteralUnion): IThemedToken[][] + codeToThemedTokens( + code: string, + lang: StringLiteralUnion, + options?: { includeExplanation?: boolean } + ): IThemedToken[][] codeToHtml?(code: string, lang: StringLiteralUnion): string // codeToRawHtml?(code: string): string diff --git a/packages/shiki/src/themedTokenizer.ts b/packages/shiki/src/themedTokenizer.ts index 4d03322fa..29d874950 100644 --- a/packages/shiki/src/themedTokenizer.ts +++ b/packages/shiki/src/themedTokenizer.ts @@ -26,7 +26,8 @@ export function tokenizeWithTheme( theme: IRawTheme, colorMap: string[], fileContents: string, - grammar: IGrammar + grammar: IGrammar, + options: { includeExplanation?: boolean } ): IThemedToken[][] { let lines = fileContents.split(/\r\n|\r|\n/) @@ -60,22 +61,25 @@ export function tokenizeWithTheme( let foregroundColor = colorMap[foreground] let explanation: IThemedTokenExplanation[] = [] - let offset = 0 - while (startIndex + offset < nextStartIndex) { - let tokenWithScopes = tokensWithScopes[tokensWithScopesIndex] - - let tokenWithScopesText = line.substring( - tokenWithScopes.startIndex, - tokenWithScopes.endIndex - ) - offset += tokenWithScopesText.length - explanation.push({ - content: tokenWithScopesText, - scopes: explainThemeScopes(theme, tokenWithScopes.scopes) - }) - - tokensWithScopesIndex++ + if (options.includeExplanation) { + let offset = 0 + while (startIndex + offset < nextStartIndex) { + let tokenWithScopes = tokensWithScopes[tokensWithScopesIndex] + + let tokenWithScopesText = line.substring( + tokenWithScopes.startIndex, + tokenWithScopes.endIndex + ) + offset += tokenWithScopesText.length + explanation.push({ + content: tokenWithScopesText, + scopes: explainThemeScopes(theme, tokenWithScopes.scopes) + }) + + tokensWithScopesIndex++ + } } + actual.push({ content: line.substring(startIndex, nextStartIndex), color: foregroundColor, From e82a04163674254fa0fd2fee51958dc58091fd2f Mon Sep 17 00:00:00 2001 From: Pine Wu Date: Thu, 20 Aug 2020 22:15:44 +0800 Subject: [PATCH 2/2] Add docs for `includeExplanation` API --- packages/shiki/src/highlighter.ts | 10 ++++- packages/shiki/src/themedTokenizer.ts | 65 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/packages/shiki/src/highlighter.ts b/packages/shiki/src/highlighter.ts index 472059f00..b55bdd956 100644 --- a/packages/shiki/src/highlighter.ts +++ b/packages/shiki/src/highlighter.ts @@ -34,6 +34,14 @@ export async function getHighlighter(options: HighlighterOptions) { return await s.getHighlighter() } +interface ThemedTokenizerOptions { + /** + * Whether to include explanation of each token's matching scopes + * and why it's given its color. Default to false. + */ + includeExplanation: boolean +} + class Shiki { private _resolver: Resolver private _registry: Registry @@ -113,7 +121,7 @@ export interface Highlighter { codeToThemedTokens( code: string, lang: StringLiteralUnion, - options?: { includeExplanation?: boolean } + options?: ThemedTokenizerOptions ): IThemedToken[][] codeToHtml?(code: string, lang: StringLiteralUnion): string diff --git a/packages/shiki/src/themedTokenizer.ts b/packages/shiki/src/themedTokenizer.ts index 29d874950..8a517168d 100644 --- a/packages/shiki/src/themedTokenizer.ts +++ b/packages/shiki/src/themedTokenizer.ts @@ -16,9 +16,74 @@ export interface IThemedTokenExplanation { scopes: IThemedTokenScopeExplanation[] } +/** + * A single token with color, and optionally with explanation. + * + * For example: + * + * { + * "content": "shiki", + * "color": "#D8DEE9", + * "explanation": [ + * { + * "content": "shiki", + * "scopes": [ + * { + * "scopeName": "source.js", + * "themeMatches": [] + * }, + * { + * "scopeName": "meta.objectliteral.js", + * "themeMatches": [] + * }, + * { + * "scopeName": "meta.object.member.js", + * "themeMatches": [] + * }, + * { + * "scopeName": "meta.array.literal.js", + * "themeMatches": [] + * }, + * { + * "scopeName": "variable.other.object.js", + * "themeMatches": [ + * { + * "name": "Variable", + * "scope": "variable.other", + * "settings": { + * "foreground": "#D8DEE9" + * } + * }, + * { + * "name": "[JavaScript] Variable Other Object", + * "scope": "source.js variable.other.object", + * "settings": { + * "foreground": "#D8DEE9" + * } + * } + * ] + * } + * ] + * } + * ] + * } + * + */ export interface IThemedToken { + /** + * The content of the token + */ content: string + /** + * 6 or 8 digit hex code representation of the token's color + */ color?: string + /** + * Explanation of + * + * - token text's matching scopes + * - reason that token text is given a color (one matching scope matches a rule (scope -> color) in the theme) + */ explanation?: IThemedTokenExplanation[] }