Skip to content

chat: add multiple 'initiator' types to provideLanguageModelResponse #250018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ const _allApiProposals = {
},
chatProvider: {
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatProvider.d.ts',
version: 2
},
chatReferenceBinaryData: {
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatReferenceBinaryData.d.ts',
Expand Down
28 changes: 14 additions & 14 deletions src/vs/workbench/api/browser/mainThreadLanguageModels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { ILogService } from '../../../platform/log/common/log.js';
import { resizeImage } from '../../contrib/chat/browser/imageUtils.js';
import { ILanguageModelIgnoredFilesService } from '../../contrib/chat/common/ignoredFiles.js';
import { ILanguageModelStatsService } from '../../contrib/chat/common/languageModelStats.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatResponse, ILanguageModelChatSelector, ILanguageModelsService } from '../../contrib/chat/common/languageModels.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatResponse, ILanguageModelChatSelector, ILanguageModelsService, LanguageModelInitiatorKind, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { IAuthenticationAccessService } from '../../services/authentication/browser/authenticationAccessService.js';
import { AuthenticationSession, AuthenticationSessionsChangeEvent, IAuthenticationProvider, IAuthenticationService, INTERNAL_AUTH_PROVIDER_PREFIX } from '../../services/authentication/common/authentication.js';
import { IExtHostContext, extHostNamedCustomer } from '../../services/extensions/common/extHostCustomers.js';
Expand Down Expand Up @@ -60,7 +60,7 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
const dipsosables = new DisposableStore();
dipsosables.add(this._chatProviderService.registerLanguageModelChat(identifier, {
metadata,
sendChatRequest: async (messages, from, options, token) => {
sendChatRequest: async (messages, initiator, options, token) => {
const requestId = (Math.random() * 1e6) | 0;
const defer = new DeferredPromise<any>();
const stream = new AsyncIterableSource<IChatResponseFragment | IChatResponseFragment[]>();
Expand All @@ -74,7 +74,7 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
part.value.data = VSBuffer.wrap(await resizeImage(part.value.data.buffer));
})
);
await this._proxy.$startChatRequest(handle, requestId, from, new SerializableObjectWithBuffers(messages), options, token);
await this._proxy.$startChatRequest(handle, requestId, initiator, new SerializableObjectWithBuffers(messages), options, token);
} catch (err) {
this._pendingProgress.delete(requestId);
throw err;
Expand Down Expand Up @@ -127,18 +127,18 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
return this._chatProviderService.selectLanguageModels(selector);
}

$whenLanguageModelChatRequestMade(identifier: string, extensionId: ExtensionIdentifier, participant?: string | undefined, tokenCount?: number | undefined): void {
this._languageModelStatsService.update(identifier, extensionId, participant, tokenCount);
$whenLanguageModelChatRequestMade(identifier: string, initiator: LanguageModelRequestInitiator, participant?: string | undefined, tokenCount?: number | undefined): void {
this._languageModelStatsService.update(identifier, initiator, participant, tokenCount);
}

async $tryStartChatRequest(extension: ExtensionIdentifier, providerId: string, requestId: number, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: {}, token: CancellationToken): Promise<any> {
this._logService.trace('[CHAT] request STARTED', extension.value, requestId);
async $tryStartChatRequest(extensionId: ExtensionIdentifier, providerId: string, requestId: number, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: {}, token: CancellationToken): Promise<any> {
this._logService.trace('[CHAT] request STARTED', extensionId.value, requestId);

let response: ILanguageModelChatResponse;
try {
response = await this._chatProviderService.sendChatRequest(providerId, extension, messages.value, options, token);
response = await this._chatProviderService.sendChatRequest(providerId, { kind: LanguageModelInitiatorKind.Extension, extensionId }, messages.value, options, token);
} catch (err) {
this._logService.error('[CHAT] request FAILED', extension.value, requestId, err);
this._logService.error('[CHAT] request FAILED', extensionId.value, requestId, err);
throw err;
}

Expand All @@ -149,22 +149,22 @@ export class MainThreadLanguageModels implements MainThreadLanguageModelsShape {
const streaming = (async () => {
try {
for await (const part of response.stream) {
this._logService.trace('[CHAT] request PART', extension.value, requestId, part);
this._logService.trace('[CHAT] request PART', extensionId.value, requestId, part);
await this._proxy.$acceptResponsePart(requestId, part);
}
this._logService.trace('[CHAT] request DONE', extension.value, requestId);
this._logService.trace('[CHAT] request DONE', extensionId.value, requestId);
} catch (err) {
this._logService.error('[CHAT] extension request ERRORED in STREAM', toErrorMessage(err, true), extension.value, requestId);
this._logService.error('[CHAT] extension request ERRORED in STREAM', toErrorMessage(err, true), extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, transformErrorForSerialization(err));
}
})();

// When the response is done (signaled via its result) we tell the EH
Promise.allSettled([response.result, streaming]).then(() => {
this._logService.debug('[CHAT] extension request DONE', extension.value, requestId);
this._logService.debug('[CHAT] extension request DONE', extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, undefined);
}, err => {
this._logService.error('[CHAT] extension request ERRORED', toErrorMessage(err, true), extension.value, requestId);
this._logService.error('[CHAT] extension request ERRORED', toErrorMessage(err, true), extensionId.value, requestId);
this._proxy.$acceptResponseDone(requestId, transformErrorForSerialization(err));
});
}
Expand Down
3 changes: 2 additions & 1 deletion src/vs/workbench/api/common/extHost.api.impl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1846,7 +1846,8 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
ChatErrorLevel: extHostTypes.ChatErrorLevel,
McpHttpServerDefinition: extHostTypes.McpHttpServerDefinition,
McpStdioServerDefinition: extHostTypes.McpStdioServerDefinition,
SettingsSearchResultKind: extHostTypes.SettingsSearchResultKind
SettingsSearchResultKind: extHostTypes.SettingsSearchResultKind,
LanguageModelRequestInitiatorKind: extHostTypes.LanguageModelRequestInitiatorKind,
};
};
}
6 changes: 3 additions & 3 deletions src/vs/workbench/api/common/extHost.protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ import { IChatProgressHistoryResponseContent } from '../../contrib/chat/common/c
import { IChatContentInlineReference, IChatFollowup, IChatNotebookEdit, IChatProgress, IChatResponseErrorDetails, IChatTask, IChatTaskDto, IChatUserActionEvent, IChatVoteAction } from '../../contrib/chat/common/chatService.js';
import { IChatRequestVariableValue } from '../../contrib/chat/common/chatVariables.js';
import { ChatAgentLocation } from '../../contrib/chat/common/constants.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatSelector, ILanguageModelsChangeEvent } from '../../contrib/chat/common/languageModels.js';
import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatSelector, ILanguageModelsChangeEvent, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { IPreparedToolInvocation, IToolInvocation, IToolProgressStep, IToolResult } from '../../contrib/chat/common/languageModelToolsService.js';
import { DebugConfigurationProviderTriggerKind, IAdapterDescriptor, IConfig, IDebugSessionReplMode, IDebugTestRunReference, IDebugVisualization, IDebugVisualizationContext, IDebugVisualizationTreeItem, MainThreadDebugVisualization } from '../../contrib/debug/common/debug.js';
import { McpCollectionDefinition, McpConnectionState, McpServerDefinition, McpServerLaunch } from '../../contrib/mcp/common/mcpTypes.js';
Expand Down Expand Up @@ -1264,7 +1264,7 @@ export interface MainThreadLanguageModelsShape extends IDisposable {
$reportResponsePart(requestId: number, chunk: IChatResponseFragment | IChatResponseFragment[]): Promise<void>;
$reportResponseDone(requestId: number, error: SerializedError | undefined): Promise<void>;
$selectChatModels(selector: ILanguageModelChatSelector): Promise<string[]>;
$whenLanguageModelChatRequestMade(identifier: string, extension: ExtensionIdentifier, participant?: string, tokenCount?: number): void;
$whenLanguageModelChatRequestMade(identifier: string, initiator: LanguageModelRequestInitiator, participant?: string, tokenCount?: number): void;
$countTokens(provider: string, value: string | IChatMessage, token: CancellationToken): Promise<number>;
$fileIsIgnored(uri: UriComponents, token: CancellationToken): Promise<boolean>;
$registerFileIgnoreProvider(handle: number): void;
Expand All @@ -1274,7 +1274,7 @@ export interface MainThreadLanguageModelsShape extends IDisposable {
export interface ExtHostLanguageModelsShape {
$acceptChatModelMetadata(data: ILanguageModelsChangeEvent): void;
$updateModelAccesslist(data: { from: ExtensionIdentifier; to: ExtensionIdentifier; enabled: boolean }[]): void;
$startChatRequest(handle: number, requestId: number, from: ExtensionIdentifier, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: { [name: string]: any }, token: CancellationToken): Promise<void>;
$startChatRequest(handle: number, requestId: number, initiator: LanguageModelRequestInitiator, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: { [name: string]: any }, token: CancellationToken): Promise<void>;
$acceptResponsePart(requestId: number, chunk: IChatResponseFragment | IChatResponseFragment[]): Promise<void>;
$acceptResponseDone(requestId: number, error: SerializedError | undefined): Promise<void>;
$provideTokenLength(handle: number, value: string | IChatMessage, token: CancellationToken): Promise<number>;
Expand Down
10 changes: 5 additions & 5 deletions src/vs/workbench/api/common/extHostLanguageModels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { ExtensionIdentifier, ExtensionIdentifierMap, ExtensionIdentifierSet, IE
import { createDecorator } from '../../../platform/instantiation/common/instantiation.js';
import { ILogService } from '../../../platform/log/common/log.js';
import { Progress } from '../../../platform/progress/common/progress.js';
import { ChatImageMimeType, IChatMessage, IChatResponseFragment, IChatResponsePart, ILanguageModelChatMetadata } from '../../contrib/chat/common/languageModels.js';
import { ChatImageMimeType, IChatMessage, IChatResponseFragment, IChatResponsePart, ILanguageModelChatMetadata, LanguageModelRequestInitiator } from '../../contrib/chat/common/languageModels.js';
import { INTERNAL_AUTH_PROVIDER_PREFIX } from '../../services/authentication/common/authentication.js';
import { checkProposedApiEnabled } from '../../services/extensions/common/extensions.js';
import { ExtHostLanguageModelsShape, MainContext, MainThreadLanguageModelsShape } from './extHost.protocol.js';
Expand Down Expand Up @@ -204,8 +204,8 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
capabilities: metadata.capabilities,
});

const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ extensionId, participant, tokenCount }) => {
this._proxy.$whenLanguageModelChatRequestMade(identifier, new ExtensionIdentifier(extensionId), participant, tokenCount);
const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ initiator, participant, tokenCount }) => {
this._proxy.$whenLanguageModelChatRequestMade(identifier, typeConvert.LanguageModelRequestInitiator.from(initiator), participant, tokenCount);
});

return toDisposable(() => {
Expand All @@ -215,7 +215,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
});
}

async $startChatRequest(handle: number, requestId: number, from: ExtensionIdentifier, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: vscode.LanguageModelChatRequestOptions, token: CancellationToken): Promise<void> {
async $startChatRequest(handle: number, requestId: number, initiator: LanguageModelRequestInitiator, messages: SerializableObjectWithBuffers<IChatMessage[]>, options: vscode.LanguageModelChatRequestOptions, token: CancellationToken): Promise<void> {
const data = this._languageModels.get(handle);
if (!data) {
throw new Error('Provider not found');
Expand Down Expand Up @@ -269,7 +269,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
value = data.provider.provideLanguageModelResponse(
messages.value.map(typeConvert.LanguageModelChatMessage2.to),
options,
ExtensionIdentifier.toKey(from),
typeConvert.LanguageModelRequestInitiator.to(initiator),
progress,
token
);
Expand Down
24 changes: 23 additions & 1 deletion src/vs/workbench/api/common/extHostTypeConverters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import * as languageSelector from '../../../editor/common/languageSelector.js';
import * as languages from '../../../editor/common/languages.js';
import { EndOfLineSequence, TrackedRangeStickiness } from '../../../editor/common/model.js';
import { ITextEditorOptions } from '../../../platform/editor/common/editor.js';
import { IExtensionDescription, IRelaxedExtensionDescription } from '../../../platform/extensions/common/extensions.js';
import { ExtensionIdentifier, IExtensionDescription, IRelaxedExtensionDescription } from '../../../platform/extensions/common/extensions.js';
import { IMarkerData, IRelatedInformation, MarkerSeverity, MarkerTag } from '../../../platform/markers/common/markers.js';
import { ProgressLocation as MainProgressLocation } from '../../../platform/progress/common/progress.js';
import { DEFAULT_EDITOR_ASSOCIATION, SaveReason } from '../../common/editor.js';
Expand Down Expand Up @@ -2516,6 +2516,28 @@ export namespace LanguageModelChatMessage2 {
}
}

export namespace LanguageModelRequestInitiator {
export function to(init: chatProvider.LanguageModelRequestInitiator): vscode.LanguageModelRequestInitiator {
if (init.kind === chatProvider.LanguageModelInitiatorKind.Extension) {
return { kind: types.LanguageModelRequestInitiatorKind.Extension, extensionId: ExtensionIdentifier.toKey(init.extensionId) };
} else if (init.kind === chatProvider.LanguageModelInitiatorKind.Editor) {
return { kind: types.LanguageModelRequestInitiatorKind.Editor, reason: init.reason };
} else {
return { kind: types.LanguageModelRequestInitiatorKind.McpServer, id: init.id, label: init.label };
}
}

export function from(init: vscode.LanguageModelRequestInitiator): chatProvider.LanguageModelRequestInitiator {
if (init.kind === types.LanguageModelRequestInitiatorKind.Extension) {
return { kind: chatProvider.LanguageModelInitiatorKind.Extension, extensionId: new ExtensionIdentifier(init.extensionId) };
} else if (init.kind === types.LanguageModelRequestInitiatorKind.Editor) {
return { kind: chatProvider.LanguageModelInitiatorKind.Editor, reason: init.reason };
} else {
return { kind: chatProvider.LanguageModelInitiatorKind.McpServer, id: init.id, label: init.label };
}
}
}

function isImageDataPart(part: types.LanguageModelDataPart): boolean {
switch (part.mimeType) {
case types.ChatImageMimeType.PNG:
Expand Down
8 changes: 7 additions & 1 deletion src/vs/workbench/api/common/extHostTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5123,6 +5123,12 @@ export enum LanguageModelChatToolMode {
Required = 2
}

export enum LanguageModelRequestInitiatorKind {
Extension = 1,
McpServer = 2,
Editor = 3,
}

//#endregion

//#region ai
Expand Down Expand Up @@ -5181,7 +5187,7 @@ export enum InlineEditTriggerKind {

//#endregion

//#region MC
//#region MCP
export class McpStdioServerDefinition implements vscode.McpStdioServerDefinition {
cwd?: URI;

Expand Down
16 changes: 9 additions & 7 deletions src/vs/workbench/contrib/chat/common/languageModelStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { Codicon } from '../../../../base/common/codicons.js';
import { Disposable } from '../../../../base/common/lifecycle.js';
import { localize } from '../../../../nls.js';
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
import { Registry } from '../../../../platform/registry/common/platform.js';
import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
import { ExtensionIdentifier } from '../../../../platform/extensions/common/extensions.js';
import { Extensions, IExtensionFeaturesManagementService, IExtensionFeaturesRegistry } from '../../../services/extensionManagement/common/extensionFeatures.js';
import { Registry } from '../../../../platform/registry/common/platform.js';
import { localize } from '../../../../nls.js';
import { Codicon } from '../../../../base/common/codicons.js';
import { LanguageModelInitiatorKind, LanguageModelRequestInitiator } from './languageModels.js';

export const ILanguageModelStatsService = createDecorator<ILanguageModelStatsService>('ILanguageModelStatsService');

export interface ILanguageModelStatsService {
readonly _serviceBrand: undefined;

update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void>;
update(model: string, initator: LanguageModelRequestInitiator, agent: string | undefined, tokenCount: number | undefined): Promise<void>;
}

export class LanguageModelStatsService extends Disposable implements ILanguageModelStatsService {
Expand All @@ -37,8 +37,10 @@ export class LanguageModelStatsService extends Disposable implements ILanguageMo
}
}

async update(model: string, extensionId: ExtensionIdentifier, agent: string | undefined, tokenCount: number | undefined): Promise<void> {
await this.extensionFeaturesManagementService.getAccess(extensionId, CopilotUsageExtensionFeatureId);
async update(model: string, initator: LanguageModelRequestInitiator, agent: string | undefined, tokenCount: number | undefined): Promise<void> {
if (initator.kind === LanguageModelInitiatorKind.Extension) {
await this.extensionFeaturesManagementService.getAccess(initator.extensionId, CopilotUsageExtensionFeatureId);
}
}

}
Expand Down
Loading
Loading