Fixes https://github.com/microsoft/vscode-internalbacklog/issues/4101

src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

@@ -25,19 +25,21 @@ import { IInstantiationService } from 'vs/platform/instantiation/common/instanti
 import { ILogService } from 'vs/platform/log/common/log';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
+import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
 import { ExtensionMessageCollector, IExtensionPointUser } from 'vs/workbench/services/extensions/common/extensionsRegistry';
 import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
 import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
 import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
-import { missingTMGrammarErrorMessage, TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
-import { grammarsExtPoint, ITMSyntaxExtensionPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
+import { TMGrammarFactory, missingTMGrammarErrorMessage } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
+import { ITMSyntaxExtensionPoint, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
 import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, IValidTokenTypeMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
 import { ITextMateThemingRule, IWorkbenchColorTheme, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import type { IGrammar, IOnigLib, IRawTheme } from 'vscode-textmate';
 
 export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService {
+	private static reportTokenizationTimeCounter = 0;
 	public _serviceBrand: undefined;
 
 	private readonly _styleElement: HTMLStyleElement;
@@ -52,7 +54,10 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 	private readonly _tokenizersRegistrations = new DisposableStore();
 	private _currentTheme: IRawTheme | null = null;
 	private _currentTokenColorMap: string[] | null = null;
-	private readonly _workerHost = this._instantiationService.createInstance(TextMateWorkerHost);
+	private readonly _workerHost = this._instantiationService.createInstance(
+		TextMateWorkerHost,
+		(timeMs, languageId, sourceExtensionId, lineLength) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true)
+	);
 
 	constructor(
 		@ILanguageService private readonly _languageService: ILanguageService,
@@ -64,6 +69,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 		@IProgressService private readonly _progressService: IProgressService,
 		@IWorkbenchEnvironmentService private readonly _environmentService: IWorkbenchEnvironmentService,
 		@IInstantiationService private readonly _instantiationService: IInstantiationService,
+		@ITelemetryService private readonly _telemetryService: ITelemetryService,
 	) {
 		super();
 
@@ -179,6 +185,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			injectTo: grammar.injectTo,
 			balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
 			unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
+			sourceExtensionId: extension.description.id,
 		};
 	}
 
@@ -283,6 +290,9 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 				r.containsEmbeddedLanguages,
 				(textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
 				() => this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification'),
+				(timeMs, lineLength) => {
+					this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false);
+				}
 			);
 			tokenization.onDidEncounterLanguage((encodedLanguageId) => {
 				if (!this._encounteredLanguages[encodedLanguageId]) {
@@ -365,6 +375,44 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			return response;
 		}
 	}
+
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean): void {
+		// 50 events per hour (one event has a low probability)
+		if (TextMateTokenizationFeature.reportTokenizationTimeCounter > 50) {
+			// Don't flood telemetry with too many events
+			return;
+		}
+		if (TextMateTokenizationFeature.reportTokenizationTimeCounter === 0) {
+			setTimeout(() => {
+				TextMateTokenizationFeature.reportTokenizationTimeCounter = 0;
+			}, 1000 * 60 * 60);
+		}
+		TextMateTokenizationFeature.reportTokenizationTimeCounter++;
+
+		this._telemetryService.publicLog2<{
+			timeMs: number;
+			languageId: string;
+			lineLength: number;
+			fromWorker: boolean;
+			sourceExtensionId: string | undefined;
+		}, {
+			owner: 'hediet';
+
+			timeMs: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To understand how long it took to tokenize a random line' };
+			languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the language' };
+			lineLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the line length' };
+			fromWorker: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this line was tokenized sync or async' };
+			sourceExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out which extension contributed the grammar' };
+
+			comment: 'This event gives insight about the performance certain grammars.';
+		}>('editor.tokenizedLine', {
+			timeMs,
+			languageId,
+			lineLength,
+			fromWorker,
+			sourceExtensionId,
+		});
+	}
 }
 
 function toColorMap(colorMap: string[]): Color[] {

src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts

@@ -5,6 +5,7 @@
 
 import { Emitter, Event } from 'vs/base/common/event';
 import { Disposable } from 'vs/base/common/lifecycle';
+import { StopWatch } from 'vs/base/common/stopwatch';
 import { LanguageId, TokenMetadata } from 'vs/editor/common/encodedTokenAttributes';
 import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, IState, ITokenizationSupport, TokenizationResult } from 'vs/editor/common/languages';
 import { ITextModel } from 'vs/editor/common/model';
@@ -21,6 +22,7 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
 		private readonly _containsEmbeddedLanguages: boolean,
 		private readonly _createBackgroundTokenizer?: (textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined,
 		private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean = () => false,
+		private readonly _reportTokenizationTime?: (timeMs: number, lineLength: number) => void,
 	) {
 		super();
 	}
@@ -45,7 +47,13 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
 	}
 
 	public tokenizeEncoded(line: string, hasEOL: boolean, state: StateStack): EncodedTokenizationResult {
+		const shouldMeasure = this._reportTokenizationTime && (Math.random() * 10_000 < 1);
+		const sw = shouldMeasure ? new StopWatch(true) : undefined;
 		const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
+		if (shouldMeasure) {
+			const timeMS = sw!.elapsed();
+			this._reportTokenizationTime!(timeMS, line.length);
+		}
 
 		if (textMateResult.stoppedEarly) {
 			console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);

src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts

@@ -29,6 +29,7 @@ export interface IValidGrammarDefinitionDTO {
 	injectTo?: string[];
 	balancedBracketSelectors: string[];
 	unbalancedBracketSelectors: string[];
+	sourceExtensionId?: string;
 }
 
 export class TextMateTokenizationWorker {
@@ -50,6 +51,7 @@ export class TextMateTokenizationWorker {
 				injectTo: def.injectTo,
 				balancedBracketSelectors: def.balancedBracketSelectors,
 				unbalancedBracketSelectors: def.unbalancedBracketSelectors,
+				sourceExtensionId: def.sourceExtensionId,
 			};
 		});
 		this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
@@ -134,6 +136,10 @@ export class TextMateTokenizationWorker {
 		this._host.setTokensAndStates(resource, versionId, tokens, stateDeltas);
 	}
 
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
+		this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
+	}
+
 	// #endregion
 }
 

src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts

@@ -3,20 +3,20 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { RunOnceScheduler } from 'vs/base/common/async';
+import { observableValue } from 'vs/base/common/observable';
 import { URI } from 'vs/base/common/uri';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 import { LanguageId } from 'vs/editor/common/encodedTokenAttributes';
 import { IModelChangedEvent, MirrorTextModel } from 'vs/editor/common/model/mirrorTextModel';
 import { TokenizerWithStateStore } from 'vs/editor/common/model/textModelTokens';
-import { diffStateStacksRefEq, StateStack, StackDiff } from 'vscode-textmate';
 import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
 import { LineTokens } from 'vs/editor/common/tokens/lineTokens';
 import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
+import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
 import { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
-import { RunOnceScheduler } from 'vs/base/common/async';
+import { StackDiff, StateStack, diffStateStacksRefEq } from 'vscode-textmate';
 import { TextMateTokenizationWorker } from './textMate.worker';
-import { observableValue } from 'vs/base/common/observable';
-import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
-import { LineRange } from 'vs/editor/common/core/lineRange';
 
 export class TextMateWorkerModel extends MirrorTextModel {
 	private _tokenizationStateStore: TokenizerWithStateStore<StateStack> | null = null;
@@ -95,7 +95,8 @@ export class TextMateWorkerModel extends MirrorTextModel {
 			if (r.grammar) {
 				const tokenizationSupport = new TokenizationSupportWithLineLimit(
 					this._encodedLanguageId,
-					new TextMateTokenizationSupport(r.grammar, r.initialState, false),
+					new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, undefined,
+						(timeMs, lineLength) => { this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength); }),
 					this._maxTokenizationLineLength
 				);
 				this._tokenizationStateStore = new TokenizerWithStateStore(this._lines.length, tokenizationSupport);

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

@@ -9,7 +9,7 @@ import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } fro
 import { IObservable } from 'vs/base/common/observable';
 import { isWeb } from 'vs/base/common/platform';
 import { URI, UriComponents } from 'vs/base/common/uri';
-import { createWebWorker, MonacoWebWorker } from 'vs/editor/browser/services/webWorker';
+import { MonacoWebWorker, createWebWorker } from 'vs/editor/browser/services/webWorker';
 import { IBackgroundTokenizationStore, IBackgroundTokenizer } from 'vs/editor/common/languages';
 import { ILanguageService } from 'vs/editor/common/languages/language';
 import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry';
@@ -38,6 +38,7 @@ export class TextMateWorkerHost implements IDisposable {
 	private _grammarDefinitions: IValidGrammarDefinition[] = [];
 
 	constructor(
+		private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number) => void,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IModelService private readonly _modelService: IModelService,
 		@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
@@ -203,6 +204,10 @@ export class TextMateWorkerHost implements IDisposable {
 		}
 	}
 
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
+		this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
+	}
+
 	// #endregion
 }
 

src/vs/workbench/services/textMate/common/TMGrammarFactory.ts

@@ -3,10 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { URI } from 'vs/base/common/uri';
-import type { IGrammar, Registry, StateStack, IOnigLib, IRawTheme } from 'vscode-textmate';
 import { Disposable } from 'vs/base/common/lifecycle';
-import { TMScopeRegistry, IValidGrammarDefinition, IValidEmbeddedLanguagesMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
+import { URI } from 'vs/base/common/uri';
+import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, TMScopeRegistry } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
+import type { IGrammar, IOnigLib, IRawTheme, Registry, StateStack } from 'vscode-textmate';
 
 interface ITMGrammarFactoryHost {
 	logTrace(msg: string): void;
@@ -19,6 +19,7 @@ export interface ICreateGrammarResult {
 	grammar: IGrammar | null;
 	initialState: StateStack;
 	containsEmbeddedLanguages: boolean;
+	sourceExtensionId?: string;
 }
 
 export const missingTMGrammarErrorMessage = 'No TM Grammar registered for this language.';
@@ -160,7 +161,8 @@ export class TMGrammarFactory extends Disposable {
 			languageId: languageId,
 			grammar: grammar,
 			initialState: this._initialState,
-			containsEmbeddedLanguages: containsEmbeddedLanguages
+			containsEmbeddedLanguages: containsEmbeddedLanguages,
+			sourceExtensionId: grammarDefinition.sourceExtensionId,
 		};
 	}
 }

src/vs/workbench/services/textMate/common/TMScopeRegistry.ts

@@ -16,6 +16,7 @@ export interface IValidGrammarDefinition {
 	injectTo?: string[];
 	balancedBracketSelectors: string[];
 	unbalancedBracketSelectors: string[];
+	sourceExtensionId?: string;
 }
 
 export interface IValidTokenTypeMap {