Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sends telemetry when a line is slow to tokenize. #187835

Merged
merged 1 commit into from
Jul 13, 2023
Merged

Sends telemetry when a line is slow to tokenize. #187835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
private _currentTokenColorMap: string[] | null = null;
private readonly _workerHost = this._instantiationService.createInstance(
TextMateWorkerHost,
(timeMs, languageId, sourceExtensionId, lineLength) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true)
(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample)
);

constructor(
Expand Down Expand Up @@ -291,9 +291,10 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
r.containsEmbeddedLanguages,
(textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
() => this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification'),
(timeMs, lineLength) => {
this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false);
}
(timeMs, lineLength, isRandomSample) => {
this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
},
true,
);
tokenization.onDidEncounterLanguage((encodedLanguageId) => {
if (!this._encounteredLanguages[encodedLanguageId]) {
Expand Down Expand Up @@ -377,7 +378,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
}
}

public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean): void {
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean, isRandomSample: boolean): void {
// 50 events per hour (one event has a low probability)
if (TextMateTokenizationFeature.reportTokenizationTimeCounter > 50) {
// Don't flood telemetry with too many events
Expand All @@ -396,6 +397,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
lineLength: number;
fromWorker: boolean;
sourceExtensionId: string | undefined;
isRandomSample: boolean;
}, {
owner: 'hediet';

Expand All @@ -404,6 +406,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
lineLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the line length' };
fromWorker: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this line was tokenized sync or async' };
sourceExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out which extension contributed the grammar' };
isRandomSample: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this is a random sample or measured because of some other condition.' };

comment: 'This event gives insight about the performance certain grammars.';
}>('editor.tokenizedLine', {
Expand All @@ -412,6 +415,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
lineLength,
fromWorker,
sourceExtensionId,
isRandomSample,
});
}
}
Expand Down
14 changes: 9 additions & 5 deletions ...vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
private readonly _grammar: IGrammar,
private readonly _initialState: StateStack,
private readonly _containsEmbeddedLanguages: boolean,
private readonly _createBackgroundTokenizer?: (textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined,
private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean = () => false,
private readonly _reportTokenizationTime?: (timeMs: number, lineLength: number) => void,
private readonly _createBackgroundTokenizer: ((textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined) | undefined,
private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean,
private readonly _reportTokenizationTime: (timeMs: number, lineLength: number, isRandomSample: boolean) => void,
private readonly _reportSlowTokenization: boolean,
) {
super();
}
Expand All @@ -47,12 +48,15 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
}

public tokenizeEncoded(line: string, hasEOL: boolean, state: StateStack): EncodedTokenizationResult {
const shouldMeasure = this._reportTokenizationTime && (Math.random() * 10_000 < 1);
const isRandomSample = Math.random() * 10_000 < 1;
const shouldMeasure = this._reportSlowTokenization || isRandomSample;
const sw = shouldMeasure ? new StopWatch(true) : undefined;
const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
if (shouldMeasure) {
const timeMS = sw!.elapsed();
this._reportTokenizationTime!(timeMS, line.length);
if (isRandomSample || timeMS > 32) {
this._reportTokenizationTime!(timeMS, line.length, isRandomSample);
}
}

if (textMateResult.stoppedEarly) {
Expand Down
4 changes: 2 additions & 2 deletions src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,8 @@ export class TextMateTokenizationWorker {
this._host.setTokensAndStates(resource, versionId, tokens, stateDeltas);
}

public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void {
this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
}

// #endregion
Expand Down
8 changes: 6 additions & 2 deletions src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,12 @@ export class TextMateWorkerModel extends MirrorTextModel {
if (r.grammar) {
const tokenizationSupport = new TokenizationSupportWithLineLimit(
this._encodedLanguageId,
new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, undefined,
(timeMs, lineLength) => { this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength); }),
new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, () => false,
(timeMs, lineLength, isRandomSample) => {
this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, isRandomSample);
},
false
),
this._maxTokenizationLineLength
);
this._tokenizationStateStore = new TokenizerWithStateStore(this._lines.length, tokenizationSupport);
Expand Down
6 changes: 3 additions & 3 deletions src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export class TextMateWorkerHost implements IDisposable {
private _grammarDefinitions: IValidGrammarDefinition[] = [];

constructor(
private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number) => void,
private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean) => void,
@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
@IModelService private readonly _modelService: IModelService,
@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
Expand Down Expand Up @@ -204,8 +204,8 @@ export class TextMateWorkerHost implements IDisposable {
}
}

public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void {
this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
}

// #endregion
Expand Down