Introduce a SemanticSimilarityService in the Command Palette

src/vs/editor/standalone/browser/quickAccess/standaloneCommandsQuickAccess.ts

@@ -37,9 +37,13 @@ export class StandaloneCommandsQuickAccessProvider extends AbstractEditorCommand
 		super({ showAlias: false }, instantiationService, keybindingService, commandService, telemetryService, dialogService);
 	}
 
-	protected async getCommandPicks(): Promise<Array<ICommandQuickPick>> {
+	protected getCommandPicks(): Array<ICommandQuickPick> {
 		return this.getCodeEditorCommandPicks();
 	}
+
+	protected async getAdditionalCommandPicks(): Promise<ICommandQuickPick[]> {
+		return [];
+	}
 }
 
 export class GotoLineAction extends EditorAction {

src/vs/platform/quickinput/browser/commandsQuickAccess.ts

@@ -17,7 +17,7 @@ import { IConfigurationChangeEvent, IConfigurationService } from 'vs/platform/co
 import { IDialogService } from 'vs/platform/dialogs/common/dialogs';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
 import { IKeybindingService } from 'vs/platform/keybinding/common/keybinding';
-import { IPickerQuickAccessItem, IPickerQuickAccessProviderOptions, PickerQuickAccessProvider } from 'vs/platform/quickinput/browser/pickerQuickAccess';
+import { FastAndSlowPicks, IPickerQuickAccessItem, IPickerQuickAccessProviderOptions, PickerQuickAccessProvider, Picks } from 'vs/platform/quickinput/browser/pickerQuickAccess';
 import { IQuickAccessProviderRunOptions } from 'vs/platform/quickinput/common/quickAccess';
 import { IQuickPickSeparator } from 'vs/platform/quickinput/common/quickInput';
 import { IStorageService, StorageScope, StorageTarget } from 'vs/platform/storage/common/storage';
@@ -56,10 +56,10 @@ export abstract class AbstractCommandsQuickAccessProvider extends PickerQuickAcc
 		this.options = options;
 	}
 
-	protected async _getPicks(filter: string, _disposables: DisposableStore, token: CancellationToken, runOptions?: IQuickAccessProviderRunOptions): Promise<Array<ICommandQuickPick | IQuickPickSeparator>> {
+	protected _getPicks(filter: string, _disposables: DisposableStore, token: CancellationToken, runOptions?: IQuickAccessProviderRunOptions): Picks<ICommandQuickPick> | FastAndSlowPicks<ICommandQuickPick> {
 
 		// Ask subclass for all command picks
-		const allCommandPicks = await this.getCommandPicks(token);
+		const allCommandPicks = this.getCommandPicks(token);
 
 		if (token.isCancellationRequested) {
 			return [];
@@ -195,13 +195,21 @@ export abstract class AbstractCommandsQuickAccessProvider extends PickerQuickAcc
 			});
 		}
 
-		return commandPicks;
+		return {
+			picks: commandPicks,
+			additionalPicks: this.getAdditionalCommandPicks(allCommandPicks, filteredCommandPicks, filter, token)
+		};
 	}
 
 	/**
 	 * Subclasses to provide the actual command entries.
 	 */
-	protected abstract getCommandPicks(token: CancellationToken): Promise<Array<ICommandQuickPick>>;
+	protected abstract getCommandPicks(token: CancellationToken): Array<ICommandQuickPick>;
+
+	/**
+	 * Subclasses to provide the actual command entries.
+	 */
+	protected abstract getAdditionalCommandPicks(allPicks: ICommandQuickPick[], picksSoFar: ICommandQuickPick[], filter: string, token: CancellationToken): Promise<Array<ICommandQuickPick>>;
 }
 
 interface ISerializedCommandHistory {

src/vs/workbench/browser/quickaccess.ts

@@ -23,6 +23,7 @@ export interface IWorkbenchQuickAccessConfiguration {
 			preserveInput: boolean;
 			experimental: {
 				suggestCommands: boolean;
+				useSemanticSimilarity: boolean;
 			};
 		};
 		quickOpen: {

src/vs/workbench/contrib/quickaccess/browser/commandsQuickAccess.ts

@@ -7,9 +7,9 @@ import { localize } from 'vs/nls';
 import { ICommandQuickPick, CommandsHistory } from 'vs/platform/quickinput/browser/commandsQuickAccess';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { IMenuService, MenuId, MenuItemAction, SubmenuItemAction, Action2 } from 'vs/platform/actions/common/actions';
-import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
+// import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
 import { CancellationToken } from 'vs/base/common/cancellation';
-import { timeout } from 'vs/base/common/async';
+// import { timeout } from 'vs/base/common/async';
 import { AbstractEditorCommandsQuickAccessProvider } from 'vs/editor/contrib/quickAccess/browser/commandsQuickAccess';
 import { IEditor } from 'vs/editor/common/editorCommon';
 import { Language } from 'vs/base/common/platform';
@@ -33,17 +33,22 @@ import { IPreferencesService } from 'vs/workbench/services/preferences/common/pr
 import { stripIcons } from 'vs/base/common/iconLabels';
 import { isFirefox } from 'vs/base/browser/browser';
 import { IProductService } from 'vs/platform/product/common/productService';
+import { ISemanticSimilarityService } from 'vs/workbench/contrib/quickaccess/browser/semanticSimilarityService';
+import { timeout } from 'vs/base/common/async';
 
 export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAccessProvider {
 
+	// TODO: bring this back once we have a chosen strategy for FastAndSlowPicks where Fast is also Promise based
 	// If extensions are not yet registered, we wait for a little moment to give them
 	// a chance to register so that the complete set of commands shows up as result
 	// We do not want to delay functionality beyond that time though to keep the commands
 	// functional.
-	private readonly extensionRegistrationRace = Promise.race([
-		timeout(800),
-		this.extensionService.whenInstalledExtensionsRegistered()
-	]);
+	// private readonly extensionRegistrationRace = Promise.race([
+	// 	timeout(800),
+	// 	this.extensionService.whenInstalledExtensionsRegistered()
+	// ]);
+
+	private useSemanticSimilarity = false;
 
 	protected get activeTextEditorControl(): IEditor | undefined { return this.editorService.activeTextEditorControl; }
 
@@ -58,7 +63,7 @@ export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAcce
 	constructor(
 		@IEditorService private readonly editorService: IEditorService,
 		@IMenuService private readonly menuService: IMenuService,
-		@IExtensionService private readonly extensionService: IExtensionService,
+		// @IExtensionService private readonly extensionService: IExtensionService,
 		@IInstantiationService instantiationService: IInstantiationService,
 		@IKeybindingService keybindingService: IKeybindingService,
 		@ICommandService commandService: ICommandService,
@@ -67,18 +72,19 @@ export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAcce
 		@IConfigurationService private readonly configurationService: IConfigurationService,
 		@IEditorGroupsService private readonly editorGroupService: IEditorGroupsService,
 		@IPreferencesService private readonly preferencesService: IPreferencesService,
-		@IProductService private readonly productService: IProductService
+		@IProductService private readonly productService: IProductService,
+		@ISemanticSimilarityService private readonly semanticSimilarityService: ISemanticSimilarityService,
 	) {
 		super({
 			showAlias: !Language.isDefaultVariant(),
 			noResultsPick: {
 				label: localize('noCommandResults', "No matching commands"),
 				commandId: ''
-			}
+			},
 		}, instantiationService, keybindingService, commandService, telemetryService, dialogService);
 
-		this._register(configurationService.onDidChangeConfiguration((e) => this.updateSuggestedCommandIds(e)));
-		this.updateSuggestedCommandIds();
+		this._register(configurationService.onDidChangeConfiguration((e) => this.updateOptions(e)));
+		this.updateOptions();
 	}
 
 	private get configuration() {
@@ -90,8 +96,8 @@ export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAcce
 		};
 	}
 
-	private updateSuggestedCommandIds(e?: IConfigurationChangeEvent): void {
-		if (e && !e.affectsConfiguration('workbench.commandPalette.experimental.suggestCommands')) {
+	private updateOptions(e?: IConfigurationChangeEvent): void {
+		if (e && !e.affectsConfiguration('workbench.commandPalette.experimental')) {
 			return;
 		}
 
@@ -100,12 +106,14 @@ export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAcce
 			? new Set(this.productService.commandPaletteSuggestedCommandIds)
 			: undefined;
 		this.options.suggestedCommandIds = suggestedCommandIds;
+		this.useSemanticSimilarity = config.experimental.useSemanticSimilarity;
 	}
 
-	protected async getCommandPicks(token: CancellationToken): Promise<Array<ICommandQuickPick>> {
+	protected getCommandPicks(token: CancellationToken): Array<ICommandQuickPick> {
 
+		// TODO: bring this back once we have a chosen strategy for FastAndSlowPicks where Fast is also Promise based
 		// wait for extensions registration or 800ms once
-		await this.extensionRegistrationRace;
+		// await this.extensionRegistrationRace;
 
 		if (token.isCancellationRequested) {
 			return [];
@@ -127,6 +135,34 @@ export class CommandsQuickAccessProvider extends AbstractEditorCommandsQuickAcce
 		}));
 	}
 
+	protected async getAdditionalCommandPicks(allPicks: ICommandQuickPick[], picksSoFar: ICommandQuickPick[], filter: string, token: CancellationToken): Promise<ICommandQuickPick[]> {
+		if (!this.useSemanticSimilarity || filter === '' || token.isCancellationRequested || !this.semanticSimilarityService.isEnabled()) {
+			return [];
+		}
+		const format = allPicks.map(p => p.commandId);
+		let scores: number[];
+		try {
+			await timeout(800, token);
+			scores = await this.semanticSimilarityService.getSimilarityScore(filter, format, token);
+		} catch (e) {
+			return [];
+		}
+		const sortedIndices = scores.map((_, i) => i).sort((a, b) => scores[b] - scores[a]);
+		const setOfPicksSoFar = new Set(picksSoFar.map(p => p.commandId));
+		const additionalPicks: ICommandQuickPick[] = [];
+		for (const i of sortedIndices) {
+			const score = scores[i];
+			if (score < 0.8) {
+				break;
+			}
+			const pick = allPicks[i];
+			if (!setOfPicksSoFar.has(pick.commandId)) {
+				additionalPicks.push(pick);
+			}
+		}
+		return additionalPicks;
+	}
+
 	private getGlobalCommandPicks(): ICommandQuickPick[] {
 		const globalCommandPicks: ICommandQuickPick[] = [];
 		const scopedContextKeyService = this.editorService.activeEditorPane?.scopedContextKeyService || this.editorGroupService.activeGroup.scopedContextKeyService;

src/vs/workbench/contrib/quickaccess/browser/semanticSimilarityService.ts

@@ -0,0 +1,103 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { CommandsRegistry, ICommandService } from 'vs/platform/commands/common/commands';
+import { IFileService } from 'vs/platform/files/common/files';
+import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions';
+import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
+import { URI } from 'vs/base/common/uri';
+import { CancellationToken } from 'vs/base/common/cancellation';
+import { raceCancellation } from 'vs/base/common/async';
+
+export const ISemanticSimilarityService = createDecorator<ISemanticSimilarityService>('IEmbeddingsService');
+
+export interface ISemanticSimilarityService {
+	isEnabled(): boolean;
+	getSimilarityScore(string1: string, comparisons: string[], token: CancellationToken): Promise<number[]>;
+}
+
+interface ICommandsEmbeddingsCache {
+	[commandId: string]: { embedding: number[] };
+}
+
+// TODO: use proper API for this instead of commands
+export class SemanticSimilarityService implements ISemanticSimilarityService {
+	declare _serviceBrand: undefined;
+
+	static readonly CALCULATE_EMBEDDING_COMMAND_ID = '_vscode.ai.calculateEmbedding';
+	static readonly COMMAND_EMBEDDING_CACHE_COMMAND_ID = '_vscode.ai.commandEmbeddingsCache';
+
+	private cache: Promise<ICommandsEmbeddingsCache>;
+
+	constructor(
+		@ICommandService private readonly commandService: ICommandService,
+		@IFileService private readonly fileService: IFileService
+	) {
+		this.cache = this.loadCache();
+	}
+
+	private async loadCache(): Promise<ICommandsEmbeddingsCache> {
+		const path = await this.commandService.executeCommand<string>(SemanticSimilarityService.COMMAND_EMBEDDING_CACHE_COMMAND_ID);
+		if (!path) {
+			return {};
+		}
+		const content = await this.fileService.readFile(URI.parse(path));
+		return JSON.parse(content.value.toString());
+	}
+
+	isEnabled(): boolean {
+		return !!CommandsRegistry.getCommand(SemanticSimilarityService.CALCULATE_EMBEDDING_COMMAND_ID);
+	}
+
+	async getSimilarityScore(str: string, comparisons: string[], token: CancellationToken): Promise<number[]> {
+
+		const embedding1 = await this.computeEmbedding(str, token);
+		const scores: number[] = [];
+		for (const comparison of comparisons) {
+			if (token.isCancellationRequested) {
+				scores.push(0);
+				continue;
+			}
+			const embedding2 = await this.getCommandEmbeddingFromCache(comparison, token);
+			if (embedding2) {
+				scores.push(this.getEmbeddingSimilarityScore(embedding1, embedding2));
+				continue;
+			}
+			scores.push(0);
+		}
+		return scores;
+	}
+
+	private async computeEmbedding(text: string, token: CancellationToken): Promise<number[]> {
+		if (!this.isEnabled()) {
+			throw new Error('Embeddings are not enabled');
+		}
+		const result = await raceCancellation(this.commandService.executeCommand<number[][]>(SemanticSimilarityService.CALCULATE_EMBEDDING_COMMAND_ID, text), token);
+		if (!result) {
+			throw new Error('No result');
+		}
+		return result[0];
+	}
+
+	private async getCommandEmbeddingFromCache(commandId: string, token: CancellationToken): Promise<number[] | undefined> {
+		const cache = await raceCancellation(this.cache, token);
+		return cache?.[commandId]?.embedding;
+	}
+
+	/**
+	 * Performs cosine similarity on two vectors to determine their similarity.
+	 * @param embedding1 The first embedding
+	 * @param embedding2 The second embedding
+	 * @returns A number between 0 and 1 for how similar the two embeddings are
+	 */
+	private getEmbeddingSimilarityScore(embedding1: number[], embedding2: number[]): number {
+		const dotProduct = embedding1.reduce((sum, value, index) => sum + value * embedding2[index], 0);
+		const magnitude1 = Math.sqrt(embedding1.reduce((sum, value) => sum + value * value, 0));
+		const magnitude2 = Math.sqrt(embedding2.reduce((sum, value) => sum + value * value, 0));
+		return dotProduct / (magnitude1 * magnitude2);
+	}
+}
+
+registerSingleton(ISemanticSimilarityService, SemanticSimilarityService, InstantiationType.Delayed);