Skip to content

Toggle line comment doesn't work when tree sitter is enabled #229691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexr00 merged 1 commit into from
Sep 25, 2024
+23 −12
Merged

Toggle line comment doesn't work when tree sitter is enabled #229691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,14 @@ class Snapper {
return result;
}

private _themedTokenizeTreeSitter(tokens: IToken[]): IThemedToken[] {
private _themedTokenizeTreeSitter(tokens: IToken[], languageId: string): IThemedToken[] {
const colorMap = TokenizationRegistry.getColorMap();
const result: IThemedToken[] = Array(tokens.length);
const colorThemeData = this.themeService.getColorTheme() as ColorThemeData;
for (let i = 0, len = tokens.length; i < len; i++) {
const token = tokens[i];
const scopes = token.t.split(' ');
const metadata = findMetadata(colorThemeData, scopes[scopes.length - 1]);
const metadata = findMetadata(colorThemeData, scopes[scopes.length - 1], this.languageService.languageIdCodec.encodeLanguageId(languageId));
const color = TokenMetadata.getForeground(metadata);

result[i] = {
Expand Down Expand Up @@ -218,7 +218,7 @@ class Snapper {
return result;
}

private async _getTreeSitterThemesResult(tokens: IToken[]): Promise<IThemesResult> {
private async _getTreeSitterThemesResult(tokens: IToken[], languageId: string): Promise<IThemesResult> {
const currentTheme = this.themeService.getColorTheme();

const getThemeName = (id: string) => {
Expand All @@ -241,7 +241,7 @@ class Snapper {
const themeName = getThemeName(themeId);
result[themeName!] = {
document: new ThemeDocument(this.themeService.getColorTheme()),
tokens: this._themedTokenizeTreeSitter(tokens)
tokens: this._themedTokenizeTreeSitter(tokens, languageId)
};
}
}
Expand Down Expand Up @@ -333,7 +333,7 @@ class Snapper {
return [];
}
const result = (await this._treeSitterTokenize(tree, languageId)).filter(t => t.c.length > 0);
const themeTokens = await this._getTreeSitterThemesResult(result);
const themeTokens = await this._getTreeSitterThemesResult(result, languageId);
this._enrichResult(result, themeTokens);
return result;
}
Expand Down
12 changes: 9 additions & 3 deletions src/vs/workbench/services/themes/common/colorThemeData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { StorageScope, IStorageService, StorageTarget } from '../../../../platfo
import { ThemeConfiguration } from './themeConfiguration.js';
import { ColorScheme } from '../../../../platform/theme/common/theme.js';
import { FontStyle, MetadataConsts } from '../../../../editor/common/encodedTokenAttributes.js';
import { toStandardTokenType } from '../../../../editor/common/languages/supports/tokenization.js';

const colorRegistry = Registry.as<IColorRegistry>(ColorRegistryExtensions.ColorContribution);

Expand Down Expand Up @@ -899,13 +900,18 @@ function isSemanticTokenColorizationSetting(style: any): style is ISemanticToken
|| types.isBoolean(style.underline) || types.isBoolean(style.strikethrough) || types.isBoolean(style.bold));
}

export function findMetadata(colorThemeData: ColorThemeData, captureName: string): number {
export function findMetadata(colorThemeData: ColorThemeData, captureName: string, languageId: number): number {
let metadata = 0;

const standardToken = toStandardTokenType(captureName);
metadata |= (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET);
metadata |= (languageId << MetadataConsts.LANGUAGEID_OFFSET);

const tokenStyle: TokenStyle | undefined = colorThemeData.resolveScopes([[captureName]]);
if (!tokenStyle) {
return 0;
return metadata;
}

let metadata = 0;
if (typeof tokenStyle.italic !== 'undefined') {
const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0);
metadata |= italicBit | MetadataConsts.ITALIC_MASK;
Expand Down
13 changes: 9 additions & 4 deletions src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import type { Parser } from '@vscode/tree-sitter-wasm';
import { Emitter, Event } from '../../../../base/common/event.js';
import { Disposable, DisposableMap, DisposableStore, IDisposable } from '../../../../base/common/lifecycle.js';
import { AppResourcePath, FileAccess } from '../../../../base/common/network.js';
import { ITreeSitterTokenizationSupport, LazyTokenizationSupport, TreeSitterTokenizationRegistry } from '../../../../editor/common/languages.js';
import { ILanguageIdCodec, ITreeSitterTokenizationSupport, LazyTokenizationSupport, TreeSitterTokenizationRegistry } from '../../../../editor/common/languages.js';
import { ITextModel } from '../../../../editor/common/model.js';
import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult } from '../../../../editor/common/services/treeSitterParserService.js';
import { IModelTokensChangedEvent } from '../../../../editor/common/textModelEvents.js';
Expand All @@ -18,6 +18,7 @@ import { InstantiationType, registerSingleton } from '../../../../platform/insta
import { createDecorator, IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
import { IThemeService } from '../../../../platform/theme/common/themeService.js';
import { ColorThemeData, findMetadata } from '../../themes/common/colorThemeData.js';
import { ILanguageService } from '../../../../editor/common/languages/language.js';

const ALLOWED_SUPPORT = ['typescript'];
type TreeSitterQueries = string;
Expand All @@ -33,6 +34,7 @@ class TreeSitterTokenizationFeature extends Disposable implements ITreeSitterTok
private readonly _tokenizersRegistrations: DisposableMap<string, DisposableStore> = new DisposableMap();

constructor(
@ILanguageService private readonly _languageService: ILanguageService,
@IConfigurationService private readonly _configurationService: IConfigurationService,
@IInstantiationService private readonly _instantiationService: IInstantiationService,
@IFileService private readonly _fileService: IFileService
Expand Down Expand Up @@ -79,7 +81,7 @@ class TreeSitterTokenizationFeature extends Disposable implements ITreeSitterTok

private async _createTokenizationSupport(languageId: string): Promise<ITreeSitterTokenizationSupport & IDisposable | null> {
const queries = await this._fetchQueries(languageId);
return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, languageId);
return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, languageId, this._languageService.languageIdCodec);
}
}

Expand All @@ -93,6 +95,7 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok
constructor(
private readonly _queries: TreeSitterQueries,
private readonly _languageId: string,
private readonly _languageIdCodec: ILanguageIdCodec,
@ITreeSitterParserService private readonly _treeSitterService: ITreeSitterParserService,
@IThemeService private readonly _themeService: IThemeService,
) {
Expand Down Expand Up @@ -181,9 +184,11 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok
tokens = newTokens;
};

const encodedLanguageId = this._languageIdCodec.encodeLanguageId(this._languageId);

for (let captureIndex = 0; captureIndex < captures.length; captureIndex++) {
const capture = captures[captureIndex];
const metadata = findMetadata(this._colorThemeData, capture.name,);
const metadata = findMetadata(this._colorThemeData, capture.name, encodedLanguageId);
const tokenEndIndex = capture.node.endIndex < lineStartOffset + lineLength ? capture.node.endIndex : lineStartOffset + lineLength;
const tokenStartIndex = capture.node.startIndex < lineStartOffset ? lineStartOffset : capture.node.startIndex;

Expand All @@ -201,7 +206,7 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok
if ((previousTokenEnd >= 0) && (previousTokenEnd < intermediateTokenOffset)) {
// Add en empty token to cover the space where there were no captures
tokens[tokenIndex * 2] = intermediateTokenOffset;
tokens[tokenIndex * 2 + 1] = 0;
tokens[tokenIndex * 2 + 1] = findMetadata(this._colorThemeData, '', encodedLanguageId);
tokenIndex++;

increaseSizeOfTokensByOneToken();
Expand Down