Skip to content

chat: add multiple 'initiator' types to provideLanguageModelResponse #250018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@ const _allApiProposals = {
},
chatProvider: {
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatProvider.d.ts',
version: 2
},
chatReferenceBinaryData: {
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatReferenceBinaryData.d.ts',
28 changes: 14 additions & 14 deletions src/vs/workbench/api/browser/mainThreadLanguageModels.ts
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import { ILogService } from '../../../platform/log/common/log.js';
import { resizeImage } from '../../contrib/chat/browser/imageUtils.js';
import { ILanguageModelIgnoredFilesService } from '../../contrib/chat/common/ignoredFiles.js';
import { ILanguageModelStatsService } from '../../contrib/chat/common/languageModelStats.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatResponse, ILanguageModelChatSelector, ILanguageModelsService } from '../../contrib/chat/common/languageModels.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatResponse, ILanguageModelChatSelector, ILanguageModelsService, LanguageModelInitiatorKind, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { IAuthenticationAccessService } from '../../services/authentication/browser/authenticationAccessService.js';
import { AuthenticationSession, AuthenticationSessionsChangeEvent, IAuthenticationProvider, IAuthenticationService, INTERNAL_AUTH_PROVIDER_PREFIX } from '../../services/authentication/common/authentication.js';
import { IExtHostContext, extHostNamedCustomer } from '../../services/extensions/common/extHostCustomers.js';
@@ -60,7 +60,7 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
const dipsosables = new DisposableStore();
dipsosables.add(this._chatProviderService.registerLanguageModelChat(identifier, {
metadata,
sendChatRequest: async (messages, from, options, token) => {
sendChatRequest: async (messages, initiator, options, token) => {
const requestId = (Math.random() * 1e6) | 0;
const defer = new DeferredPromise<any>();
const stream = new AsyncIterableSource<IChatResponseFragment | IChatResponseFragment[]>();
@@ -74,7 +74,7 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
part.value.data = VSBuffer.wrap(await resizeImage(part.value.data.buffer));
})
);
await this._proxy.$startChatRequest(handle, requestId, from, new SerializableObjectWithBuffers(messages), options, token);
await this._proxy.$startChatRequest(handle, requestId, initiator, new SerializableObjectWithBuffers(messages), options, token);
} catch (err) {
this._pendingProgress.delete(requestId);
throw err;
@@ -127,18 +127,18 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
return this._chatProviderService.selectLanguageModels(selector);
}

$whenLanguageModelChatRequestMade(identifier: string, extensionId: ExtensionIdentifier, participant?: string | undefined, tokenCount?: number | undefined): void {
this._languageModelStatsService.update(identifier, extensionId, participant, tokenCount);
$whenLanguageModelChatRequestMade(identifier: string, initiator: LanguageModelRequestInitiator, participant?: string | undefined, tokenCount?: number | undefined): void {
this._languageModelStatsService.update(identifier, initiator, participant, tokenCount);
}

async $tryStartChatRequest(extension: ExtensionIdentifier, providerId: string, requestId: number, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: {}, token: CancellationToken): Promise<any> {
this._logService.trace('[CHAT] request STARTED', extension.value, requestId);
async $tryStartChatRequest(extensionId: ExtensionIdentifier, providerId: string, requestId: number, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: {}, token: CancellationToken): Promise<any> {
this._logService.trace('[CHAT] request STARTED', extensionId.value, requestId);

let response: ILanguageModelChatResponse;
try {
response = await this._chatProviderService.sendChatRequest(providerId, extension, messages.value, options, token);
response = await this._chatProviderService.sendChatRequest(providerId, { kind: LanguageModelInitiatorKind.Extension, extensionId }, messages.value, options, token);
} catch (err) {
this._logService.error('[CHAT] request FAILED', extension.value, requestId, err);
this._logService.error('[CHAT] request FAILED', extensionId.value, requestId, err);
throw err;
}

@@ -149,22 +149,22 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
const streaming = (async () => {
try {
for await (const part of response.stream) {
this._logService.trace('[CHAT] request PART', extension.value, requestId, part);
this._logService.trace('[CHAT] request PART', extensionId.value, requestId, part);
await this._proxy.$acceptResponsePart(requestId, part);
}
this._logService.trace('[CHAT] request DONE', extension.value, requestId);
this._logService.trace('[CHAT] request DONE', extensionId.value, requestId);
} catch (err) {
this._logService.error('[CHAT] extension request ERRORED in STREAM', toErrorMessage(err, true), extension.value, requestId);
this._logService.error('[CHAT] extension request ERRORED in STREAM', toErrorMessage(err, true), extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, transformErrorForSerialization(err));
}
})();

// When the response is done (signaled via its result) we tell the EH
Promise.allSettled([response.result, streaming]).then(() => {
this._logService.debug('[CHAT] extension request DONE', extension.value, requestId);
this._logService.debug('[CHAT] extension request DONE', extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, undefined);
}, err => {
this._logService.error('[CHAT] extension request ERRORED', toErrorMessage(err, true), extension.value, requestId);
this._logService.error('[CHAT] extension request ERRORED', toErrorMessage(err, true), extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, transformErrorForSerialization(err));
});
}
3 changes: 2 additions & 1 deletion src/vs/workbench/api/common/extHost.api.impl.ts
Original file line number Diff line number Diff line change
@@ -1846,7 +1846,8 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
ChatErrorLevel: extHostTypes.ChatErrorLevel,
McpHttpServerDefinition: extHostTypes.McpHttpServerDefinition,
McpStdioServerDefinition: extHostTypes.McpStdioServerDefinition,
SettingsSearchResultKind: extHostTypes.SettingsSearchResultKind
SettingsSearchResultKind: extHostTypes.SettingsSearchResultKind,
LanguageModelRequestInitiatorKind: extHostTypes.LanguageModelRequestInitiatorKind,
};
};
}
6 changes: 3 additions & 3 deletions src/vs/workbench/api/common/extHost.protocol.ts
Original file line number Diff line number Diff line change
@@ -60,7 +60,7 @@ import { IChatProgressHistoryResponseContent } from '../../contrib/chat/common/c
import { IChatContentInlineReference, IChatFollowup, IChatNotebookEdit, IChatProgress, IChatResponseErrorDetails, IChatTask, IChatTaskDto, IChatUserActionEvent, IChatVoteAction } from '../../contrib/chat/common/chatService.js';
import { IChatRequestVariableValue } from '../../contrib/chat/common/chatVariables.js';
import { ChatAgentLocation } from '../../contrib/chat/common/constants.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatSelector, ILanguageModelsChangeEvent } from '../../contrib/chat/common/languageModels.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatSelector, ILanguageModelsChangeEvent, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { IPreparedToolInvocation, IToolInvocation, IToolProgressStep, IToolResult } from '../../contrib/chat/common/languageModelToolsService.js';
import { DebugConfigurationProviderTriggerKind, IAdapterDescriptor, IConfig, IDebugSessionReplMode, IDebugTestRunReference, IDebugVisualization, IDebugVisualizationContext, IDebugVisualizationTreeItem, MainThreadDebugVisualization } from '../../contrib/debug/common/debug.js';
import { McpCollectionDefinition, McpConnectionState, McpServerDefinition, McpServerLaunch } from '../../contrib/mcp/common/mcpTypes.js';
@@ -1264,7 +1264,7 @@ export interface MainThreadLanguageModelsShape extends IDisposable {
$reportResponsePart(requestId: number, chunk: IChatResponseFragment | IChatResponseFragment[]): Promise<void>;
$reportResponseDone(requestId: number, error: SerializedError | undefined): Promise<void>;
$selectChatModels(selector: ILanguageModelChatSelector): Promise<string[]>;
$whenLanguageModelChatRequestMade(identifier: string, extension: ExtensionIdentifier, participant?: string, tokenCount?: number): void;
$whenLanguageModelChatRequestMade(identifier: string, initiator: LanguageModelRequestInitiator, participant?: string, tokenCount?: number): void;
$countTokens(provider: string, value: string | IChatMessage, token: CancellationToken): Promise<number>;
$fileIsIgnored(uri: UriComponents, token: CancellationToken): Promise<boolean>;
$registerFileIgnoreProvider(handle: number): void;
@@ -1274,7 +1274,7 @@ export interface MainThreadLanguageModelsShape extends IDisposable {
export interface ExtHostLanguageModelsShape {
$acceptChatModelMetadata(data: ILanguageModelsChangeEvent): void;
$updateModelAccesslist(data: { from: ExtensionIdentifier; to: ExtensionIdentifier; enabled: boolean }[]): void;
$startChatRequest(handle: number, requestId: number, from: ExtensionIdentifier, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: { [name: string]: any }, token: CancellationToken): Promise<void>;
$startChatRequest(handle: number, requestId: number, initiator: LanguageModelRequestInitiator, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: { [name: string]: any }, token: CancellationToken): Promise<void>;
$acceptResponsePart(requestId: number, chunk: IChatResponseFragment | IChatResponseFragment[]): Promise<void>;
$acceptResponseDone(requestId: number, error: SerializedError | undefined): Promise<void>;
$provideTokenLength(handle: number, value: string | IChatMessage, token: CancellationToken): Promise<number>;
10 changes: 5 additions & 5 deletions src/vs/workbench/api/common/extHostLanguageModels.ts
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ import { ExtensionIdentifier, ExtensionIdentifierMap, ExtensionIdentifierSet, IE
import { createDecorator } from '../../../platform/instantiation/common/instantiation.js';
import { ILogService } from '../../../platform/log/common/log.js';
import { Progress } from '../../../platform/progress/common/progress.js';
import { ChatImageMimeType, IChatMessage, IChatResponseFragment, IChatResponsePart, ILanguageModelChatMetadata } from '../../contrib/chat/common/languageModels.js';
import { ChatImageMimeType, IChatMessage, IChatResponseFragment, IChatResponsePart, ILanguageModelChatMetadata, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { INTERNAL_AUTH_PROVIDER_PREFIX } from '../../services/authentication/common/authentication.js';
import { checkProposedApiEnabled } from '../../services/extensions/common/extensions.js';
import { ExtHostLanguageModelsShape, MainContext, MainThreadLanguageModelsShape } from './extHost.protocol.js';
@@ -204,8 +204,8 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
capabilities: metadata.capabilities,
});

const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ extensionId, participant, tokenCount }) => {
this._proxy.$whenLanguageModelChatRequestMade(identifier, new ExtensionIdentifier(extensionId), participant, tokenCount);
const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ initiator, participant, tokenCount }) => {
this._proxy.$whenLanguageModelChatRequestMade(identifier, typeConvert.LanguageModelRequestInitiator.from(initiator), participant, tokenCount);
});

return toDisposable(() => {
@@ -215,7 +215,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
});
}

async $startChatRequest(handle: number, requestId: number, from: ExtensionIdentifier, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: vscode.LanguageModelChatRequestOptions, token: CancellationToken): Promise<void> {
async $startChatRequest(handle: number, requestId: number, initiator: LanguageModelRequestInitiator, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: vscode.LanguageModelChatRequestOptions, token: CancellationToken): Promise<void> {
const data = this._languageModels.get(handle);
if (!data) {
throw new Error('Provider not found');
@@ -269,7 +269,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
value = data.provider.provideLanguageModelResponse(
messages.value.map(typeConvert.LanguageModelChatMessage2.to),
options,
ExtensionIdentifier.toKey(from),
typeConvert.LanguageModelRequestInitiator.to(initiator),
progress,
token
);
24 changes: 23 additions & 1 deletion src/vs/workbench/api/common/extHostTypeConverters.ts
Original file line number Diff line number Diff line change
@@ -34,7 +34,7 @@ import * as languageSelector from '../../../editor/common/languageSelector.js';
import * as languages from '../../../editor/common/languages.js';
import { EndOfLineSequence, TrackedRangeStickiness } from '../../../editor/common/model.js';
import { ITextEditorOptions } from '../../../platform/editor/common/editor.js';
import { IExtensionDescription, IRelaxedExtensionDescription } from '../../../platform/extensions/common/extensions.js';
import { ExtensionIdentifier, IExtensionDescription, IRelaxedExtensionDescription } from '../../../platform/extensions/common/extensions.js';
import { IMarkerData, IRelatedInformation, MarkerSeverity, MarkerTag } from '../../../platform/markers/common/markers.js';
import { ProgressLocation as MainProgressLocation } from '../../../platform/progress/common/progress.js';
import { DEFAULT_EDITOR_ASSOCIATION, SaveReason } from '../../common/editor.js';
@@ -2516,6 +2516,28 @@ export namespace LanguageModelChatMessage2 {
}
}

export namespace LanguageModelRequestInitiator {
export function to(init: chatProvider.LanguageModelRequestInitiator): vscode.LanguageModelRequestInitiator {
if (init.kind === chatProvider.LanguageModelInitiatorKind.Extension) {
return { kind: types.LanguageModelRequestInitiatorKind.Extension, extensionId: ExtensionIdentifier.toKey(init.extensionId) };
} else if (init.kind === chatProvider.LanguageModelInitiatorKind.Editor) {
return { kind: types.LanguageModelRequestInitiatorKind.Editor, reason: init.reason };
} else {
return { kind: types.LanguageModelRequestInitiatorKind.McpServer, id: init.id, label: init.label };
}
}

export function from(init: vscode.LanguageModelRequestInitiator): chatProvider.LanguageModelRequestInitiator {
if (init.kind === types.LanguageModelRequestInitiatorKind.Extension) {
return { kind: chatProvider.LanguageModelInitiatorKind.Extension, extensionId: new ExtensionIdentifier(init.extensionId) };
} else if (init.kind === types.LanguageModelRequestInitiatorKind.Editor) {
return { kind: chatProvider.LanguageModelInitiatorKind.Editor, reason: init.reason };
} else {
return { kind: chatProvider.LanguageModelInitiatorKind.McpServer, id: init.id, label: init.label };
}
}
}

function isImageDataPart(part: types.LanguageModelDataPart): boolean {
switch (part.mimeType) {
case types.ChatImageMimeType.PNG:
8 changes: 7 additions & 1 deletion src/vs/workbench/api/common/extHostTypes.ts
Original file line number Diff line number Diff line change
@@ -5123,6 +5123,12 @@ export enum LanguageModelChatToolMode {
Required = 2
}

export enum LanguageModelRequestInitiatorKind {
Extension = 1,
McpServer = 2,
Editor = 3,
}

//#endregion

//#region ai
@@ -5181,7 +5187,7 @@ export enum InlineEditTriggerKind {

//#endregion

//#region MC
//#region MCP
export class McpStdioServerDefinition implements vscode.McpStdioServerDefinition {
cwd?: URI;

16 changes: 9 additions & 7 deletions src/vs/workbench/contrib/chat/common/languageModelStats.ts
Original file line number Diff line number Diff line change
@@ -3,21 +3,21 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { Codicon } from '../../../../base/common/codicons.js';
import { Disposable } from '../../../../base/common/lifecycle.js';
import { localize } from '../../../../nls.js';
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
import { Registry } from '../../../../platform/registry/common/platform.js';
import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
import { ExtensionIdentifier } from '../../../../platform/extensions/common/extensions.js';
import { Extensions, IExtensionFeaturesManagementService, IExtensionFeaturesRegistry } from '../../../services/extensionManagement/common/extensionFeatures.js';
import { Registry } from '../../../../platform/registry/common/platform.js';
import { localize } from '../../../../nls.js';
import { Codicon } from '../../../../base/common/codicons.js';
import { LanguageModelInitiatorKind, LanguageModelRequestInitiator } from './languageModels.js';

export const ILanguageModelStatsService = createDecorator<ILanguageModelStatsService>('ILanguageModelStatsService');

export interface ILanguageModelStatsService {
readonly _serviceBrand: undefined;

update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void>;
update(model: string, initator: LanguageModelRequestInitiator, agent: string | undefined, tokenCount: number | undefined): Promise<void>;
}

export class LanguageModelStatsService extends Disposable implements ILanguageModelStatsService {
@@ -37,8 +37,10 @@ export class LanguageModelStatsService extends Disposable implements ILanguageMo
}
}

async update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void> {
await this.extensionFeaturesManagementService.getAccess(extensionId, CopilotUsageExtensionFeatureId);
async update(model: string, initator: LanguageModelRequestInitiator, agent: string | undefined, tokenCount: number | undefined): Promise<void> {
if (initator.kind === LanguageModelInitiatorKind.Extension) {
await this.extensionFeaturesManagementService.getAccess(initator.extensionId, CopilotUsageExtensionFeatureId);
}
}

}
Loading
Oops, something went wrong.
Loading
Oops, something went wrong.