Add telemetry for interactive session provider result/perf

src/vs/workbench/api/common/extHost.protocol.ts

@@ -1103,6 +1103,10 @@ export interface IInteractiveResponseDto {
 	followups?: string[];
 	commandFollowups?: IInteractiveSessionResponseCommandFollowup[];
 	errorDetails?: IInteractiveResponseErrorDetails;
+	timings: {
+		firstProgress: number;
+		totalElapsed: number;
+	};
 }
 
 export interface IInteractiveResponseProgressDto {

src/vs/workbench/api/common/extHostInteractiveSession.ts

@@ -5,6 +5,7 @@
 
 import { CancellationToken } from 'vs/base/common/cancellation';
 import { toDisposable } from 'vs/base/common/lifecycle';
+import { StopWatch } from 'vs/base/common/stopwatch';
 import { withNullAsUndefined } from 'vs/base/common/types';
 import { localize } from 'vs/nls';
 import { IRelaxedExtensionDescription } from 'vs/platform/extensions/common/extensions';
@@ -142,11 +143,20 @@ export class ExtHostInteractiveSession implements ExtHostInteractiveSessionShape
 				return;
 			}
 
+			// TODO clean up this API
 			this._proxy.$acceptInteractiveResponseProgress(handle, sessionId, { responsePart: res.content });
-			return { followups: res.followups };
+			return { followups: res.followups, timings: { firstProgress: 0, totalElapsed: 0 } };
 		} else if (entry.provider.provideResponseWithProgress) {
+			const stopWatch = StopWatch.create(false);
+			let firstProgress: number | undefined;
 			const progressObj: vscode.Progress<vscode.InteractiveProgress> = {
-				report: (progress: vscode.InteractiveProgress) => this._proxy.$acceptInteractiveResponseProgress(handle, sessionId, { responsePart: progress.content })
+				report: (progress: vscode.InteractiveProgress) => {
+					if (typeof firstProgress === 'undefined') {
+						firstProgress = stopWatch.elapsed();
+					}
+
+					this._proxy.$acceptInteractiveResponseProgress(handle, sessionId, { responsePart: progress.content });
+				}
 			};
 			let result: vscode.InteractiveResponseForProgress | undefined | null;
 			try {
@@ -168,7 +178,8 @@ export class ExtHostInteractiveSession implements ExtHostInteractiveSessionShape
 				this.logService.warn(err);
 			}
 
-			return { followups: result.followups, commandFollowups: result.commands, errorDetails: result.errorDetails };
+			const timings = { firstProgress: firstProgress ?? 0, totalElapsed: stopWatch.elapsed() };
+			return { followups: result.followups, commandFollowups: result.commands, errorDetails: result.errorDetails, timings };
 		}
 
 		throw new Error('Provider must implement either provideResponse or provideResponseWithProgress');

src/vs/workbench/contrib/interactiveSession/common/interactiveSessionService.ts

@@ -29,6 +29,10 @@ export interface IInteractiveResponse {
 	followups?: string[];
 	commandFollowups?: IInteractiveSessionResponseCommandFollowup[];
 	errorDetails?: IInteractiveResponseErrorDetails;
+	timings?: {
+		firstProgress: number;
+		totalElapsed: number;
+	};
 }
 
 export interface IInteractiveProgress {

src/vs/workbench/contrib/interactiveSession/common/interactiveSessionServiceImpl.ts

@@ -12,12 +12,29 @@ import { localize } from 'vs/nls';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
 import { ILogService } from 'vs/platform/log/common/log';
 import { IStorageService, StorageScope, StorageTarget } from 'vs/platform/storage/common/storage';
+import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { ISerializableInteractiveSessionsData, InteractiveSessionModel } from 'vs/workbench/contrib/interactiveSession/common/interactiveSessionModel';
 import { IInteractiveProgress, IInteractiveProvider, IInteractiveSessionService } from 'vs/workbench/contrib/interactiveSession/common/interactiveSessionService';
 import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
 
 const serializedInteractiveSessionKey = 'interactive.sessions';
 
+type InteractiveSessionProviderInvokedEvent = {
+	providerId: string;
+	timeToFirstProgress: number;
+	totalTime: number;
+	result: 'success' | 'error' | 'errorWithOutput';
+};
+
+type InteractiveSessionProviderInvokedClassification = {
+	providerId: { classification: 'PublicNonPersonalData'; purpose: 'FeatureInsight'; comment: 'The identifier of the provider that was invoked.' };
+	timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'The time in milliseconds from invoking the provider to getting the first data.' };
+	totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'The total time it took to run the provider\'s `provideResponseWithProgress`.' };
+	result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the InteractiveSessionProvider resulted in an error.' };
+	owner: 'roblourens';
+	comment: 'Provides insight into the performance of InteractiveSession providers.';
+};
+
 export class InteractiveSessionService extends Disposable implements IInteractiveSessionService {
 	declare _serviceBrand: undefined;
 
@@ -31,6 +48,7 @@ export class InteractiveSessionService extends Disposable implements IInteractiv
 		@ILogService private readonly logService: ILogService,
 		@IExtensionService private readonly extensionService: IExtensionService,
 		@IInstantiationService private readonly instantiationService: IInstantiationService,
+		@ITelemetryService private readonly telemetryService: ITelemetryService,
 	) {
 		super();
 		const sessionData = storageService.get(serializedInteractiveSessionKey, StorageScope.WORKSPACE, '');
@@ -126,32 +144,40 @@ export class InteractiveSessionService extends Disposable implements IInteractiv
 			return false;
 		}
 
-		const _sendRequest = async (): Promise<void> => {
-			try {
-				this._pendingRequestSessions.add(sessionId);
-				const request = model.addRequest(message);
-				const progressCallback = (progress: IInteractiveProgress) => {
-					this.trace('sendRequest', `Provider returned progress for session ${sessionId}, ${progress.responsePart.length} chars`);
-					model.mergeResponseContent(request, progress.responsePart);
-				};
-				let rawResponse = await provider.provideReply({ session: model.session, message }, progressCallback, token);
-				if (!rawResponse) {
-					this.trace('sendRequest', `Provider returned no response for session ${sessionId}`);
-					rawResponse = { session: model.session, errorDetails: { message: localize('emptyResponse', "Provider returned null response") } };
-				}
-
-				model.completeResponse(request, rawResponse);
-				this.trace('sendRequest', `Provider returned response for session ${sessionId} with ${rawResponse.followups} followups`);
-			} finally {
-				this._pendingRequestSessions.delete(sessionId);
-			}
-		};
-
 		// Return immediately that the request was accepted, don't wait
-		_sendRequest();
+		this._sendRequestAsync(model, provider, message, token);
 		return true;
 	}
 
+	private async _sendRequestAsync(model: InteractiveSessionModel, provider: IInteractiveProvider, message: string, token: CancellationToken): Promise<void> {
+		try {
+			this._pendingRequestSessions.add(model.sessionId);
+			const request = model.addRequest(message);
+			let gotProgress = false;
+			const progressCallback = (progress: IInteractiveProgress) => {
+				gotProgress = true;
+				this.trace('sendRequest', `Provider returned progress for session ${model.sessionId}, ${progress.responsePart.length} chars`);
+				model.mergeResponseContent(request, progress.responsePart);
+			};
+			let rawResponse = await provider.provideReply({ session: model.session, message }, progressCallback, token);
+			if (!rawResponse) {
+				this.trace('sendRequest', `Provider returned no response for session ${model.sessionId}`);
+				rawResponse = { session: model.session, errorDetails: { message: localize('emptyResponse', "Provider returned null response") } };
+			}
+
+			this.telemetryService.publicLog2<InteractiveSessionProviderInvokedEvent, InteractiveSessionProviderInvokedClassification>('interactiveSessionProviderInvoked', {
+				providerId: provider.id,
+				timeToFirstProgress: rawResponse.timings?.firstProgress ?? 0,
+				totalTime: rawResponse.timings?.totalElapsed ?? 0,
+				result: rawResponse.errorDetails && gotProgress ? 'errorWithOutput' : rawResponse.errorDetails ? 'error' : 'success'
+			});
+			model.completeResponse(request, rawResponse);
+			this.trace('sendRequest', `Provider returned response for session ${model.sessionId} with ${rawResponse.followups} followups`);
+		} finally {
+			this._pendingRequestSessions.delete(model.sessionId);
+		}
+	}
+
 	acceptNewSessionState(sessionId: number, state: any): void {
 		this.trace('acceptNewSessionState', `sessionId: ${sessionId}`);
 		const model = this._sessionModels.get(sessionId);