feat(testing): support alternative semantic index endpoints

extensions/copilot/src/extension/extension/vscode-node/services.ts

@@ -86,6 +86,7 @@ import { ITestDepsResolver, TestDepsResolver } from '../../../platform/testing/n
 import { ITokenizerProvider, TokenizerProvider } from '../../../platform/tokenizer/node/tokenizer';
 import { GithubAvailableEmbeddingTypesService, IGithubAvailableEmbeddingTypesService } from '../../../platform/workspaceChunkSearch/common/githubAvailableEmbeddingTypes';
 import { IRerankerService, RerankerService } from '../../../platform/workspaceChunkSearch/common/rerankerService';
+import { ScenarioAutomationWorkspaceChunkSearchService } from '../../../platform/workspaceChunkSearch/node/scenarioAutomationWorkspaceChunkSearchService';
 import { IWorkspaceChunkSearchService, WorkspaceChunkSearchService } from '../../../platform/workspaceChunkSearch/node/workspaceChunkSearchService';
 import { IWorkspaceFileIndex, WorkspaceFileIndex } from '../../../platform/workspaceChunkSearch/node/workspaceFileIndex';
 import { IInstantiationServiceBuilder } from '../../../util/common/services';
@@ -198,12 +199,17 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
 		builder.define(IAuthenticationService, new SyncDescriptor(StaticGitHubAuthenticationService, [createStaticGitHubTokenProvider()]));
 		builder.define(IEndpointProvider, new SyncDescriptor(ScenarioAutomationEndpointProviderImpl));
 		builder.define(IIgnoreService, new SyncDescriptor(NullIgnoreService));
+		builder.define(IWorkspaceChunkSearchService, new SyncDescriptor(ScenarioAutomationWorkspaceChunkSearchService));
 	} else {
 		builder.define(IAuthenticationService, new SyncDescriptor(AuthenticationService));
 		builder.define(IEndpointProvider, new SyncDescriptor(ProductionEndpointProvider));
 		builder.define(IIgnoreService, new SyncDescriptor(VsCodeIgnoreService));
+		builder.define(IWorkspaceChunkSearchService, new SyncDescriptor(WorkspaceChunkSearchService));
 	}
 
+	builder.define(IGithubCodeSearchService, new SyncDescriptor(GithubCodeSearchService));
+	builder.define(IGithubAvailableEmbeddingTypesService, new SyncDescriptor(GithubAvailableEmbeddingTypesService));
+
 	builder.define(ITestGenInfoStorage, new SyncDescriptor(TestGenInfoStorage)); // Used for test generation (/tests intent)
 	builder.define(IParserService, new SyncDescriptor(ParserServiceImpl, [/*useWorker*/ true]));
 	builder.define(IIntentService, new SyncDescriptor(IntentService));
@@ -235,9 +241,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
 	builder.define(IFeedbackReporter, new SyncDescriptor(FeedbackReporter));
 	builder.define(IApiEmbeddingsIndex, new SyncDescriptor(ApiEmbeddingsIndex, [/*useRemoteCache*/ true]));
 	builder.define(IGithubApiFetcherService, new SyncDescriptor(GithubApiFetcherService));
-	builder.define(IGithubCodeSearchService, new SyncDescriptor(GithubCodeSearchService));
 	builder.define(IAdoCodeSearchService, new SyncDescriptor(AdoCodeSearchService));
-	builder.define(IWorkspaceChunkSearchService, new SyncDescriptor(WorkspaceChunkSearchService));
 	builder.define(ISettingsEditorSearchService, new SyncDescriptor(SettingsEditorSearchServiceImpl));
 	builder.define(INewWorkspacePreviewContentManager, new SyncDescriptor(NewWorkspacePreviewContentManagerImpl));
 	builder.define(IPromptVariablesService, new SyncDescriptor(PromptVariablesServiceImpl));
@@ -254,7 +258,6 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
 	builder.define(IWorkspaceListenerService, new SyncDescriptor(WorkspacListenerService));
 	builder.define(ICodeSearchAuthenticationService, new SyncDescriptor(VsCodeCodeSearchAuthenticationService));
 	builder.define(ITodoListContextProvider, new SyncDescriptor(TodoListContextProvider));
-	builder.define(IGithubAvailableEmbeddingTypesService, new SyncDescriptor(GithubAvailableEmbeddingTypesService));
 	builder.define(IRerankerService, new SyncDescriptor(RerankerService));
 	builder.define(IProxyModelsService, new SyncDescriptor(ProxyModelsService));
 	builder.define(IPowerService, new SyncDescriptor(PowerService));

extensions/copilot/src/extension/prompt/vscode-node/scenarioAutomationEndpointProviderImpl.ts

@@ -38,6 +38,16 @@ export class ScenarioAutomationEndpointProviderImpl extends ProductionEndpointPr
 			}
 		}
 
-		return super.getChatEndpoint(requestOrFamilyOrModel);
+		try {
+			return await super.getChatEndpoint(requestOrFamilyOrModel);
+		} catch (error) {
+			// In scenario automation, some model families (e.g. copilot-fast → gpt-4o-mini) may
+			// not be available via the capi proxy. Fall back to copilot-base.
+			if (typeof requestOrFamilyOrModel === 'string') {
+				this._logService.trace(`ScenarioAutomation: failed to resolve model family '${requestOrFamilyOrModel}', falling back to copilot-base`);
+				return super.getChatEndpoint('copilot-base');
+			}
+			throw error;
+		}
 	}
 }

extensions/copilot/src/platform/workspaceChunkSearch/node/scenarioAutomationWorkspaceChunkSearchService.ts

@@ -0,0 +1,207 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type * as vscode from 'vscode';
+import { shouldInclude } from '../../../util/common/glob';
+import { Result } from '../../../util/common/result';
+import { TelemetryCorrelationId } from '../../../util/common/telemetryCorrelationId';
+import type { CancellationToken } from '../../../util/vs/base/common/cancellation';
+import { Event } from '../../../util/vs/base/common/event';
+import { URI } from '../../../util/vs/base/common/uri';
+import { Range } from '../../../util/vs/editor/common/core/range';
+import { FileChunkAndScore } from '../../chunking/common/chunk';
+import { stripChunkTextMetadata, truncateToMaxUtf8Length } from '../../chunking/common/chunkingStringUtils';
+import { EmbeddingType } from '../../embeddings/common/embeddingsComputer';
+import { getGitHubRepoInfoFromContext, IGitService, toGithubNwo } from '../../git/common/gitService';
+import { ILogService } from '../../log/common/logService';
+import { IFetcherService } from '../../networking/common/fetcherService';
+import { WorkspaceChunkQuery, WorkspaceChunkSearchOptions } from '../common/workspaceChunkSearch';
+import { BuildIndexTriggerReason, TriggerIndexingError } from './codeSearch/codeSearchRepo';
+import {
+	IWorkspaceChunkSearchService,
+	WorkspaceChunkSearchResult,
+	WorkspaceChunkSearchSizing,
+	WorkspaceIndexState,
+} from './workspaceChunkSearchService';
+
+/**
+ * The Blackbird local server endpoint for embeddings code search.
+ * In scenario automation (msbench), Blackbird always runs at this address.
+ */
+const BLACKBIRD_EMBEDDINGS_URL = 'http://localhost:4443/api/embeddings/code/search';
+
+interface BlackbirdSearchResponse {
+	readonly embedding_model?: string;
+	readonly results: ReadonlyArray<{
+		readonly location: {
+			readonly path: string;
+		};
+		readonly chunk: {
+			readonly text: string;
+			readonly line_range: {
+				readonly start: number;
+				readonly end: number;
+			};
+		};
+		readonly distance: number;
+	}>;
+}
+
+/**
+ * Scenario automation implementation of {@link IWorkspaceChunkSearchService}.
+ *
+ * This is a minimal implementation that directly calls the Blackbird local
+ * embeddings endpoint without depending on the production
+ * {@link WorkspaceChunkSearchService} or any of its strategies.
+ * All methods except {@link searchFileChunks} and {@link getIndexState}
+ * are no-ops.
+ */
+export class ScenarioAutomationWorkspaceChunkSearchService implements IWorkspaceChunkSearchService {
+	declare readonly _serviceBrand: undefined;
+
+	readonly onDidChangeIndexState: Event<void> = Event.None;
+
+	constructor(
+		@IFetcherService private readonly _fetcherService: IFetcherService,
+		@IGitService private readonly _gitService: IGitService,
+		@ILogService private readonly _logService: ILogService,
+	) { }
+
+	async getIndexState(): Promise<WorkspaceIndexState> {
+		return {
+			remoteIndexState: { status: 'loaded', repos: [] },
+		};
+	}
+
+	async isAvailable(): Promise<boolean> {
+		return true;
+	}
+
+	async searchFileChunks(
+		sizing: WorkspaceChunkSearchSizing,
+		query: WorkspaceChunkQuery,
+		options: WorkspaceChunkSearchOptions,
+		_telemetryInfo: TelemetryCorrelationId,
+		_progress: vscode.Progress<vscode.ChatResponsePart> | undefined,
+		token: CancellationToken,
+	): Promise<WorkspaceChunkSearchResult> {
+		if (token.isCancellationRequested) {
+			return { chunks: [] };
+		}
+
+		const repo = this._gitService.repositories[0];
+		const repoInfo = repo ? getGitHubRepoInfoFromContext(repo) : undefined;
+		const nwo = repoInfo ? toGithubNwo(repoInfo.id) : (process.env.SWEBENCH_REPO ?? '');
+
+		const queryText = query.queryText;
+		const maxResults = sizing.maxResults ?? 20;
+
+		this._logService.trace(`ScenarioAutomationWorkspaceChunkSearchService: searching for "${queryText}" in repo ${nwo}`);
+
+		if (!nwo) {
+			this._logService.error('ScenarioAutomationWorkspaceChunkSearchService: no repo NWO available (git has no remotes and SWEBENCH_REPO is unset)');
+			return { chunks: [] };
+		}
+
+		const requestBody = {
+			scoping_query: `repo:${nwo}`,
+			prompt: truncateToMaxUtf8Length(queryText, 7800),
+			include_embeddings: false,
+			limit: maxResults,
+			embedding_model: EmbeddingType.metis_1024_I16_Binary.id,
+		};
+
+		let response;
+		const abortController = this._fetcherService.makeAbortController();
+		const tokenListener = token.onCancellationRequested(() => abortController.abort());
+		try {
+			response = await this._fetcherService.fetch(BLACKBIRD_EMBEDDINGS_URL, {
+				callSite: 'ScenarioAutomationWorkspaceChunkSearchService.searchFileChunks',
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify(requestBody),
+				signal: abortController.signal,
+			});
+		} catch (e) {
+			if (token.isCancellationRequested || this._fetcherService.isAbortError(e)) {
+				this._logService.trace('ScenarioAutomationWorkspaceChunkSearchService: search cancelled');
+				return { chunks: [] };
+			}
+			this._logService.error(`ScenarioAutomationWorkspaceChunkSearchService: fetch failed: ${e instanceof Error ? e.message : e}`);
+			return { chunks: [] };
+		} finally {
+			tokenListener.dispose();
+		}
+
+		if (!response.ok) {
+			const errorBody = await response.text().catch(() => '<unable to read body>');
+			this._logService.error(`ScenarioAutomationWorkspaceChunkSearchService: search failed with status ${response.status}, body: ${errorBody}`);
+			return { chunks: [] };
+		}
+
+		if (token.isCancellationRequested) {
+			return { chunks: [] };
+		}
+
+		let body: unknown;
+		try {
+			body = await response.json();
+		} catch (e) {
+			if (token.isCancellationRequested || this._fetcherService.isAbortError(e)) {
+				this._logService.trace('ScenarioAutomationWorkspaceChunkSearchService: search cancelled');
+				return { chunks: [] };
+			}
+			this._logService.error(`ScenarioAutomationWorkspaceChunkSearchService: failed to parse response JSON: ${e instanceof Error ? e.message : e}`);
+			return { chunks: [] };
+		}
+
+		const parsedBody = body as Partial<BlackbirdSearchResponse>;
+		if (!Array.isArray(parsedBody.results)) {
+			this._logService.error('ScenarioAutomationWorkspaceChunkSearchService: unexpected response shape');
+			return { chunks: [] };
+		}
+
+		const embeddingType = new EmbeddingType(parsedBody.embedding_model ?? EmbeddingType.metis_1024_I16_Binary.id);
+		const chunks: FileChunkAndScore[] = [];
+		for (const result of parsedBody.results) {
+			const fileUri = repo?.rootUri
+				? URI.joinPath(repo.rootUri, result.location.path)
+				: URI.from({ scheme: 'githubRepoResult', path: '/' + result.location.path });
+
+			if (!shouldInclude(fileUri, options.globPatterns)) {
+				continue;
+			}
+
+			chunks.push({
+				chunk: {
+					file: fileUri,
+					text: stripChunkTextMetadata(result.chunk.text),
+					rawText: undefined,
+					range: new Range(result.chunk.line_range.start, 0, result.chunk.line_range.end, 0),
+					isFullFile: false,
+				},
+				distance: {
+					embeddingType,
+					value: result.distance,
+				},
+			});
+		}
+
+		this._logService.trace(`ScenarioAutomationWorkspaceChunkSearchService: got ${chunks.length} chunks`);
+		return { chunks };
+	}
+
+	async triggerIndexing(_trigger: BuildIndexTriggerReason, _onProgress: (message: string) => void, _telemetryInfo: TelemetryCorrelationId, _token: CancellationToken): Promise<Result<true, TriggerIndexingError>> {
+		return Result.ok(true);
+	}
+
+	async deleteExternalIngestWorkspaceIndex(): Promise<void> {
+		// noop
+	}
+
+	dispose(): void {
+		// noop
+	}
+}