From b2f01e0fb0e99853b49d31383d319a9d912aca30 Mon Sep 17 00:00:00 2001 From: eleanorjboyd <26030610+eleanorjboyd@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:30:07 -0700 Subject: [PATCH 1/2] Add telemetry properties for environment resolution and package installation Co-authored-by: Copilot --- src/client/chat/baseTool.ts | 7 ++++- src/client/chat/configurePythonEnvTool.ts | 10 ++++++- src/client/chat/getExecutableTool.ts | 7 +++++ src/client/chat/getPythonEnvTool.ts | 14 ++++++++- src/client/chat/installPackagesTool.ts | 36 ++++++++++++++++++----- src/client/chat/utils.ts | 4 +++ src/client/telemetry/index.ts | 27 ++++++++++++++++- 7 files changed, 94 insertions(+), 11 deletions(-) diff --git a/src/client/chat/baseTool.ts b/src/client/chat/baseTool.ts index 2eedbbe226e3..d8e2e1d60d42 100644 --- a/src/client/chat/baseTool.ts +++ b/src/client/chat/baseTool.ts @@ -16,8 +16,10 @@ import { IResourceReference, isCancellationError, resolveFilePath } from './util import { ErrorWithTelemetrySafeReason } from '../common/errors/errorUtils'; import { sendTelemetryEvent } from '../telemetry'; import { EventName } from '../telemetry/constants'; +import { StopWatch } from '../common/utils/stopWatch'; export abstract class BaseTool implements LanguageModelTool { + protected extraTelemetryProperties: Record = {}; constructor(private readonly toolName: string) {} async invoke( @@ -29,8 +31,10 @@ export abstract class BaseTool implements Language new LanguageModelTextPart('Cannot use this tool in an untrusted workspace.'), ]); } + this.extraTelemetryProperties = {}; let error: Error | undefined; const resource = resolveFilePath(options.input.resourcePath); + const stopWatch = new StopWatch(); try { return await this.invokeImpl(options, resource, token); } catch (ex) { @@ -46,10 +50,11 @@ export abstract class BaseTool implements Language ? error.telemetrySafeReason : 'error' : undefined; - sendTelemetryEvent(EventName.INVOKE_TOOL, undefined, { + sendTelemetryEvent(EventName.INVOKE_TOOL, stopWatch.elapsedTime, { toolName: this.toolName, failed, failureCategory, + ...this.extraTelemetryProperties, }); } } diff --git a/src/client/chat/configurePythonEnvTool.ts b/src/client/chat/configurePythonEnvTool.ts index 0634b9c9ac34..914a92f81c52 100644 --- a/src/client/chat/configurePythonEnvTool.ts +++ b/src/client/chat/configurePythonEnvTool.ts @@ -18,6 +18,7 @@ import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { getEnvDetailsForResponse, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -58,6 +59,7 @@ export class ConfigurePythonEnvTool extends BaseTool ): Promise { const notebookResponse = getToolResponseIfNotebook(resource); if (notebookResponse) { + this.extraTelemetryProperties.resolveOutcome = 'notebook'; return notebookResponse; } @@ -67,6 +69,8 @@ export class ConfigurePythonEnvTool extends BaseTool ); if (workspaceSpecificEnv) { + this.extraTelemetryProperties.resolveOutcome = 'existingWorkspaceEnv'; + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(workspaceSpecificEnv); return getEnvDetailsForResponse( workspaceSpecificEnv, this.api, @@ -79,7 +83,9 @@ export class ConfigurePythonEnvTool extends BaseTool if (await this.createEnvTool.shouldCreateNewVirtualEnv(resource, token)) { try { - return await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + const result = await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + this.extraTelemetryProperties.resolveOutcome = 'createdVirtualEnv'; + return result; } catch (ex) { if (isCancellationError(ex)) { const input: ISelectPythonEnvToolArguments = { @@ -87,6 +93,7 @@ export class ConfigurePythonEnvTool extends BaseTool reason: 'cancelled', }; // If the user cancelled the tool, then we should invoke the select env tool. + this.extraTelemetryProperties.resolveOutcome = 'selectedEnvAfterCancelledCreate'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } throw ex; @@ -95,6 +102,7 @@ export class ConfigurePythonEnvTool extends BaseTool const input: ISelectPythonEnvToolArguments = { ...options.input, }; + this.extraTelemetryProperties.resolveOutcome = 'selectedEnv'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } } diff --git a/src/client/chat/getExecutableTool.ts b/src/client/chat/getExecutableTool.ts index 746a540d14f8..38dabce644a7 100644 --- a/src/client/chat/getExecutableTool.ts +++ b/src/client/chat/getExecutableTool.ts @@ -19,6 +19,7 @@ import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/termin import { getEnvDisplayName, getEnvironmentDetails, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, raceCancellationError, @@ -53,6 +54,12 @@ export class GetExecutableTool extends BaseTool implements L return notebookResponse; } + const envPath = this.api.getActiveEnvironmentPath(resourcePath); + const environment = await raceCancellationError(this.api.resolveEnvironment(envPath), token); + if (environment) { + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); + } + const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/getPythonEnvTool.ts b/src/client/chat/getPythonEnvTool.ts index ed1dd0374424..d25d72baeba8 100644 --- a/src/client/chat/getPythonEnvTool.ts +++ b/src/client/chat/getPythonEnvTool.ts @@ -17,7 +17,13 @@ import { IServiceContainer } from '../ioc/types'; import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { IProcessServiceFactory, IPythonExecutionFactory } from '../common/process/types'; -import { getEnvironmentDetails, getToolResponseIfNotebook, IResourceReference, raceCancellationError } from './utils'; +import { + getEnvironmentDetails, + getEnvTypeForTelemetry, + getToolResponseIfNotebook, + IResourceReference, + raceCancellationError, +} from './utils'; import { getPythonPackagesResponse } from './listPackagesTool'; import { ITerminalHelper } from '../common/terminal/types'; import { getEnvExtApi, useEnvExtension } from '../envExt/api.internal'; @@ -64,13 +70,16 @@ export class GetEnvironmentInfoTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); let packages = ''; + let responsePackageCount = 0; if (useEnvExtension()) { const api = await getEnvExtApi(); const env = await api.getEnvironment(resourcePath); const pkgs = env ? await api.getPackages(env) : []; if (pkgs && pkgs.length > 0) { + responsePackageCount = pkgs.length; // Installed Python packages, each in the format or (). The version may be omitted if unknown. Returns an empty array if no packages are installed. const response = [ 'Below is a list of the Python packages, each in the format or (). The version may be omitted if unknown: ', @@ -90,7 +99,10 @@ export class GetEnvironmentInfoTool extends BaseTool resourcePath, token, ); + // Count lines starting with '- ' to get the number of packages + responsePackageCount = (packages.match(/^- /gm) || []).length; } + this.extraTelemetryProperties.responsePackageCount = String(responsePackageCount); const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/installPackagesTool.ts b/src/client/chat/installPackagesTool.ts index f7795620cf13..2caa3819554a 100644 --- a/src/client/chat/installPackagesTool.ts +++ b/src/client/chat/installPackagesTool.ts @@ -16,6 +16,7 @@ import { PythonExtension } from '../api/types'; import { IServiceContainer } from '../ioc/types'; import { getEnvDisplayName, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -51,6 +52,7 @@ export class InstallPackagesTool extends BaseTool ): Promise { const packageCount = options.input.packageList.length; const packagePlurality = packageCount === 1 ? 'package' : 'packages'; + this.extraTelemetryProperties.packageCount = String(packageCount); const notebookResponse = getToolResponseIfNotebook(resourcePath); if (notebookResponse) { return notebookResponse; @@ -84,9 +86,11 @@ export class InstallPackagesTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); const isConda = isCondaEnv(environment); const installers = this.serviceContainer.getAll(IModuleInstaller); const installerType = isConda ? ModuleInstallerType.Conda : ModuleInstallerType.Pip; + this.extraTelemetryProperties.installerType = isConda ? 'conda' : 'pip'; const installer = installers.find((i) => i.type === installerType); if (!installer) { throw new ErrorWithTelemetrySafeReason( @@ -100,15 +104,33 @@ export class InstallPackagesTool extends BaseTool 'installerNotSupported', ); } + const succeeded: string[] = []; + const failed: { pkg: string; error: string }[] = []; for (const packageName of options.input.packageList) { - await installer.installModule(packageName, resourcePath, token, undefined, { - installAsProcess: true, - hideProgress: true, - }); + try { + await installer.installModule(packageName, resourcePath, token, undefined, { + installAsProcess: true, + hideProgress: true, + }); + succeeded.push(packageName); + } catch (error) { + if (isCancellationError(error)) { + throw error; + } + failed.push({ pkg: packageName, error: `${error}` }); + } } - // format and return - const resultMessage = `Successfully installed ${packagePlurality}: ${options.input.packageList.join(', ')}`; - return new LanguageModelToolResult([new LanguageModelTextPart(resultMessage)]); + const parts: string[] = []; + if (succeeded.length > 0) { + parts.push(`Successfully installed: ${succeeded.join(', ')}`); + } + if (failed.length > 0) { + parts.push(`Failed to install: ${failed.map((f) => `${f.pkg} (${f.error})`).join(', ')}`); + } + if (succeeded.length === 0 && failed.length > 0) { + throw new ErrorWithTelemetrySafeReason(parts.join('. '), 'installFailed'); + } + return new LanguageModelToolResult([new LanguageModelTextPart(parts.join('. '))]); } catch (error) { if (isCancellationError(error)) { throw error; diff --git a/src/client/chat/utils.ts b/src/client/chat/utils.ts index 84df2901341b..2309316bcbdd 100644 --- a/src/client/chat/utils.ts +++ b/src/client/chat/utils.ts @@ -76,6 +76,10 @@ export function isCondaEnv(env: ResolvedEnvironment) { return (env.environment?.type || '').toLowerCase() === 'conda'; } +export function getEnvTypeForTelemetry(env: ResolvedEnvironment): string { + return (env.environment?.type || 'unknown').toLowerCase(); +} + export async function getEnvironmentDetails( resourcePath: Uri | undefined, api: PythonExtension['environments'], diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 738c5f8a2776..c5faec51d01d 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1992,7 +1992,12 @@ export interface IEventNamePropertyMapping { "duration" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "owner": "donjayamanne" }, "toolName" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "donjayamanne" }, "failed": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Whether there was a failure. Common to most of the events.", "owner": "donjayamanne" }, - "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" } + "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" }, + "resolveOutcome": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which code path resolved the environment in configure_python_environment.", "owner": "donjayamanne" }, + "envType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"The type of Python environment (e.g. venv, conda, system).", "owner": "donjayamanne" }, + "packageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages requested for installation (install_python_packages only).", "owner": "donjayamanne" }, + "installerType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which installer was used: pip or conda (install_python_packages only).", "owner": "donjayamanne" }, + "responsePackageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages in the environment response (get_python_environment_details only).", "owner": "donjayamanne" } } */ [EventName.INVOKE_TOOL]: { @@ -2009,6 +2014,26 @@ export interface IEventNamePropertyMapping { * A reason the error was thrown. */ failureCategory?: string; + /** + * Which code path resolved the environment (configure_python_environment only). + */ + resolveOutcome?: string; + /** + * The type of Python environment (e.g. venv, conda, system). + */ + envType?: string; + /** + * Number of packages requested for installation (install_python_packages only). + */ + packageCount?: string; + /** + * Which installer was used: pip or conda (install_python_packages only). + */ + installerType?: string; + /** + * Number of packages in the environment response (get_python_environment_details only). + */ + responsePackageCount?: string; }; /** * Telemetry event sent if and when user configure tests command. This command can be trigerred from multiple places in the extension. (Command palette, prompt etc.) From 0309d4a1721833fce3383626fd87b8759aa5f908 Mon Sep 17 00:00:00 2001 From: eleanorjboyd <26030610+eleanorjboyd@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:39:21 -0700 Subject: [PATCH 2/2] remove unrelated fix --- src/client/chat/installPackagesTool.ts | 32 ++++++-------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/src/client/chat/installPackagesTool.ts b/src/client/chat/installPackagesTool.ts index 2caa3819554a..5d3d456361f9 100644 --- a/src/client/chat/installPackagesTool.ts +++ b/src/client/chat/installPackagesTool.ts @@ -104,33 +104,15 @@ export class InstallPackagesTool extends BaseTool 'installerNotSupported', ); } - const succeeded: string[] = []; - const failed: { pkg: string; error: string }[] = []; for (const packageName of options.input.packageList) { - try { - await installer.installModule(packageName, resourcePath, token, undefined, { - installAsProcess: true, - hideProgress: true, - }); - succeeded.push(packageName); - } catch (error) { - if (isCancellationError(error)) { - throw error; - } - failed.push({ pkg: packageName, error: `${error}` }); - } + await installer.installModule(packageName, resourcePath, token, undefined, { + installAsProcess: true, + hideProgress: true, + }); } - const parts: string[] = []; - if (succeeded.length > 0) { - parts.push(`Successfully installed: ${succeeded.join(', ')}`); - } - if (failed.length > 0) { - parts.push(`Failed to install: ${failed.map((f) => `${f.pkg} (${f.error})`).join(', ')}`); - } - if (succeeded.length === 0 && failed.length > 0) { - throw new ErrorWithTelemetrySafeReason(parts.join('. '), 'installFailed'); - } - return new LanguageModelToolResult([new LanguageModelTextPart(parts.join('. '))]); + // format and return + const resultMessage = `Successfully installed ${packagePlurality}: ${options.input.packageList.join(', ')}`; + return new LanguageModelToolResult([new LanguageModelTextPart(resultMessage)]); } catch (error) { if (isCancellationError(error)) { throw error;