github image uploader service

src/vs/platform/webContentExtractor/common/webContentExtractor.ts

@@ -23,6 +23,7 @@ export interface IWebContentExtractorService {
 export interface ISharedWebContentExtractorService {
 	_serviceBrand: undefined;
 	readImage(uri: URI, token: CancellationToken): Promise<VSBuffer | undefined>;
+	chatImageUploader(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<string>;
 }
 
 /**
@@ -39,6 +40,9 @@ export class NullWebContentExtractorService implements IWebContentExtractorServi
 }
 
 export class NullSharedWebContentExtractorService implements ISharedWebContentExtractorService {
+	chatImageUploader(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<string> {
+		throw new Error('Method not implemented.');
+	}
 	_serviceBrand: undefined;
 	readImage(_uri: URI, _token: CancellationToken): Promise<VSBuffer | undefined> {
 		throw new Error('Not implemented');

src/vs/platform/webContentExtractor/node/sharedWebContentExtractorService.ts

@@ -35,4 +35,34 @@ export class SharedWebContentExtractorService implements ISharedWebContentExtrac
 			return undefined;
 		}
 	}
+
+	async chatImageUploader(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<string> {
+		if (mimeType && token) {
+			const url = `https://uploads.github.com/copilot/chat/attachments?name=${name}&content_type=${mimeType}`;
+			const init: RequestInit = {
+				method: 'POST',
+				body: binaryData,
+				// Using default 'cors' mode but with credentials
+				credentials: 'include',
+				headers: new Headers({
+					'Content-Type': 'application/octet-stream',
+					'Authorization': `Bearer ${token}`
+				})
+			};
+			try {
+				const response = await fetch(url, init);
+				if (!response.ok) {
+					console.error(`Invalid GitHub URL provided: ${response.status} ${response.statusText}`);
+					return '';
+				}
+				const result = await response.json();
+				return result.url;
+			} catch (error) {
+				console.error('Error uploading image:', error);
+				return '';
+			}
+		}
+		return '';
+	}
+
 }

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -3019,6 +3019,7 @@ export namespace ChatPromptReference {
 			value = new types.ChatReferenceBinaryData(
 				variable.mimeType ?? 'image/png',
 				() => Promise.resolve(new Uint8Array(Object.values(variable.value as number[]))),
+				variable.url ?? '',
 				ref && URI.isUri(ref) ? ref : undefined
 			);
 		} else if (variable.kind === 'diagnostic') {

src/vs/workbench/api/common/extHostTypes.ts

@@ -4784,12 +4784,14 @@ export class ChatRequestNotebookData implements vscode.ChatRequestNotebookData {
 
 export class ChatReferenceBinaryData implements vscode.ChatReferenceBinaryData {
 	mimeType: string;
+	url: string;
 	data: () => Thenable<Uint8Array>;
 	reference?: vscode.Uri;
-	constructor(mimeType: string, data: () => Thenable<Uint8Array>, reference?: vscode.Uri) {
+	constructor(mimeType: string, data: () => Thenable<Uint8Array>, url: string, reference?: vscode.Uri) {
 		this.mimeType = mimeType;
 		this.data = data;
 		this.reference = reference;
+		this.url = url;
 	}
 }
 

src/vs/workbench/contrib/chat/browser/chatAttachmentModel.ts

@@ -160,7 +160,7 @@ export class ChatAttachmentModel extends Disposable {
 		} else if (uri.scheme === Schemas.http || uri.scheme === Schemas.https) {
 			const extractedImages = await this.webContentExtractorService.readImage(uri, CancellationToken.None);
 			if (extractedImages) {
-				return await resolveImageEditorAttachContext(this.fileService, this.dialogService, uri, extractedImages);
+				return await resolveImageEditorAttachContext(this.fileService, this.dialogService, uri, undefined, extractedImages);
 			}
 		}
 

src/vs/workbench/contrib/chat/browser/chatAttachmentResolve.ts

@@ -27,10 +27,22 @@ import { getNotebookEditorFromEditorPane } from '../../notebook/browser/notebook
 import { IChatRequestVariableEntry, IDiagnosticVariableEntry, IDiagnosticVariableEntryFilterData, ISymbolVariableEntry, OmittedState } from '../common/chatModel.js';
 import { imageToHash } from './chatPasteProviders.js';
 import { resizeImage } from './imageUtils.js';
+import { IProductService } from '../../../../platform/product/common/productService.js';
+import { IAuthenticationExtensionsService, IAuthenticationService } from '../../../services/authentication/common/authentication.js';
+import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
+import product from '../../../../platform/product/common/product.js';
+import { ISharedWebContentExtractorService } from '../../../../platform/webContentExtractor/common/webContentExtractor.js';
+
+const defaultChat = {
+	chatExtensionId: product.defaultChatAgent?.chatExtensionId ?? '',
+	providerId: product.defaultChatAgent?.providerId ?? '',
+	enterpriseProviderId: product.defaultChatAgent?.enterpriseProviderId ?? '',
+	completionsAdvancedSetting: product.defaultChatAgent?.completionsAdvancedSetting ?? '',
+};
 
 // --- EDITORS ---
 
-export async function resolveEditorAttachContext(editor: EditorInput | IDraggedResourceEditorInput, fileService: IFileService, editorService: IEditorService, textModelService: ITextModelService, extensionService: IExtensionService, dialogService: IDialogService): Promise<IChatRequestVariableEntry | undefined> {
+export async function resolveEditorAttachContext(editor: EditorInput | IDraggedResourceEditorInput, fileService: IFileService, editorService: IEditorService, textModelService: ITextModelService, extensionService: IExtensionService, dialogService: IDialogService, authenticationService: IAuthenticationService, configurationService: IConfigurationService, authenticationExtensionsService: IAuthenticationExtensionsService, productService: IProductService, sharedWebContentExtractorService: ISharedWebContentExtractorService): Promise<IChatRequestVariableEntry | undefined> {
 	// untitled editor
 	if (isUntitledResourceEditorInput(editor)) {
 		return await resolveUntitledEditorAttachContext(editor, editorService, textModelService);
@@ -51,7 +63,18 @@ export async function resolveEditorAttachContext(editor: EditorInput | IDraggedR
 		return undefined;
 	}
 
-	const imageContext = await resolveImageEditorAttachContext(fileService, dialogService, editor.resource);
+	let providerId: string | undefined = undefined;
+	if (configurationService.getValue<string | undefined>(`${defaultChat?.completionsAdvancedSetting}.authProvider`) === defaultChat?.enterpriseProviderId) {
+		providerId = defaultChat?.enterpriseProviderId;
+	} else {
+		providerId = defaultChat?.providerId;
+	}
+
+	const accountName = authenticationExtensionsService.getAccountPreference(defaultChat.chatExtensionId, providerId);
+	const session = await authenticationService.getSessions(providerId);
+	const token = session?.find(s => s.account.label === accountName)?.accessToken;
+
+	const imageContext = await resolveImageEditorAttachContext(fileService, dialogService, editor.resource, token, undefined, undefined, sharedWebContentExtractorService);
 	if (imageContext) {
 		return extensionService.extensions.some(ext => isProposedApiEnabled(ext, 'chatReferenceBinaryData')) ? imageContext : undefined;
 	}
@@ -113,10 +136,11 @@ export type ImageTransferData = {
 	id?: string;
 	mimeType?: string;
 	omittedState?: OmittedState;
+	token?: string;
 };
 const SUPPORTED_IMAGE_EXTENSIONS_REGEX = /\.(png|jpg|jpeg|gif|webp)$/i;
 
-export async function resolveImageEditorAttachContext(fileService: IFileService, dialogService: IDialogService, resource: URI, data?: VSBuffer, mimeType?: string): Promise<IChatRequestVariableEntry | undefined> {
+export async function resolveImageEditorAttachContext(fileService: IFileService, dialogService: IDialogService, resource: URI, token?: string, data?: VSBuffer, mimeType?: string, sharedWebContentExtractorService?: ISharedWebContentExtractorService): Promise<IChatRequestVariableEntry | undefined> {
 	if (!resource) {
 		return undefined;
 	}
@@ -165,25 +189,30 @@ export async function resolveImageEditorAttachContext(fileService: IFileService,
 		icon: Codicon.fileMedia,
 		resource: resource,
 		mimeType: mimeType,
-		omittedState: isPartiallyOmitted ? OmittedState.Partial : OmittedState.NotOmitted
-	}]);
+		omittedState: isPartiallyOmitted ? OmittedState.Partial : OmittedState.NotOmitted,
+		token
+	}], sharedWebContentExtractorService);
 
 	return imageFileContext[0];
 }
 
-export async function resolveImageAttachContext(images: ImageTransferData[]): Promise<IChatRequestVariableEntry[]> {
-	return Promise.all(images.map(async image => ({
-		id: image.id || await imageToHash(image.data),
-		name: image.name,
-		fullName: image.resource ? image.resource.path : undefined,
-		value: await resizeImage(image.data, image.mimeType),
-		icon: image.icon,
-		kind: 'image',
-		isFile: false,
-		isDirectory: false,
-		omittedState: image.omittedState || OmittedState.NotOmitted,
-		references: image.resource ? [{ reference: image.resource, kind: 'reference' }] : []
-	})));
+export async function resolveImageAttachContext(images: ImageTransferData[], sharedWebContentExtractorService?: ISharedWebContentExtractorService): Promise<IChatRequestVariableEntry[]> {
+	return Promise.all(images.map(async image => {
+		const binaryData = await resizeImage(image.data, image.mimeType);
+		return {
+			id: image.id || await imageToHash(image.data),
+			name: image.name,
+			fullName: image.resource ? image.resource.path : undefined,
+			value: binaryData,
+			url: sharedWebContentExtractorService ? await sharedWebContentExtractorService.chatImageUploader(binaryData, image.name, image.mimeType, image.token) : undefined,
+			icon: image.icon,
+			kind: 'image',
+			isFile: false,
+			isDirectory: false,
+			omittedState: image.omittedState || OmittedState.NotOmitted,
+			references: image.resource ? [{ reference: image.resource, kind: 'reference' }] : []
+		};
+	}));
 }
 
 const MIME_TYPES: Record<string, string> = {

src/vs/workbench/contrib/chat/browser/chatDragAndDrop.ts

@@ -29,6 +29,9 @@ import { ImageTransferData, resolveEditorAttachContext, resolveImageAttachContex
 import { ChatAttachmentModel } from './chatAttachmentModel.js';
 import { IChatInputStyles } from './chatInputPart.js';
 import { convertStringToUInt8Array } from './imageUtils.js';
+import { IProductService } from '../../../../platform/product/common/productService.js';
+import { IAuthenticationExtensionsService, IAuthenticationService } from '../../../services/authentication/common/authentication.js';
+import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
 
 enum ChatDragAndDropType {
 	FILE_INTERNAL,
@@ -62,6 +65,10 @@ export class ChatDragAndDrop extends Themable {
 		@ISharedWebContentExtractorService private readonly webContentExtractorService: ISharedWebContentExtractorService,
 		@IChatWidgetService private readonly chatWidgetService: IChatWidgetService,
 		@ILogService private readonly logService: ILogService,
+		@IAuthenticationService private readonly authenticationService: IAuthenticationService,
+		@IAuthenticationExtensionsService private readonly authenticationExtensionsService: IAuthenticationExtensionsService,
+		@IConfigurationService private readonly configurationService: IConfigurationService,
+		@IProductService private readonly productService: IProductService,
 	) {
 		super(themeService);
 
@@ -235,7 +242,7 @@ export class ChatDragAndDrop extends Themable {
 		const editorDragData = extractEditorsDropData(e);
 		if (editorDragData.length > 0) {
 			return coalesce(await Promise.all(editorDragData.map(editorInput => {
-				return resolveEditorAttachContext(editorInput, this.fileService, this.editorService, this.textModelService, this.extensionService, this.dialogService);
+				return resolveEditorAttachContext(editorInput, this.fileService, this.editorService, this.textModelService, this.extensionService, this.dialogService, this.authenticationService, this.configurationService, this.authenticationExtensionsService, this.productService, this.webContentExtractorService);
 			})));
 		}
 
@@ -244,8 +251,7 @@ export class ChatDragAndDrop extends Themable {
 			const uriList = UriList.parse(internal);
 			if (uriList.length) {
 				return coalesce(await Promise.all(
-					uriList.map(uri => resolveEditorAttachContext({ resource: URI.parse(uri) }, this.fileService, this.editorService, this.textModelService, this.extensionService, this.dialogService))
-				));
+					uriList.map(uri => resolveEditorAttachContext({ resource: URI.parse(uri) }, this.fileService, this.editorService, this.textModelService, this.extensionService, this.dialogService, this.authenticationService, this.configurationService, this.authenticationExtensionsService, this.productService, this.webContentExtractorService))));
 			}
 		}
 

src/vs/workbench/contrib/chat/common/chatModel.ts

@@ -117,6 +117,7 @@ export interface IImageVariableEntry extends IBaseChatRequestVariableEntry {
 	readonly isPasted?: boolean;
 	readonly isURL?: boolean;
 	readonly mimeType?: string;
+	readonly url?: string;
 }
 
 export interface INotebookOutputVariableEntry extends Omit<IBaseChatRequestVariableEntry, 'kind'> {

src/vs/workbench/contrib/inlineChat/browser/inlineChatController.ts

@@ -1192,7 +1192,7 @@ export class InlineChatController1 implements IEditorContribution {
 		} else if (attachment.scheme === Schemas.http || attachment.scheme === Schemas.https) {
 			const extractedImages = await this._webContentExtractorService.readImage(attachment, CancellationToken.None);
 			if (extractedImages) {
-				return await resolveImageEditorAttachContext(this._fileService, this._dialogService, attachment, extractedImages);
+				return await resolveImageEditorAttachContext(this._fileService, this._dialogService, attachment, undefined, extractedImages);
 			}
 		}
 
@@ -1487,7 +1487,7 @@ export class InlineChatController2 implements IEditorContribution {
 		} else if (attachment.scheme === Schemas.http || attachment.scheme === Schemas.https) {
 			const extractedImages = await this._webContentExtractorService.readImage(attachment, CancellationToken.None);
 			if (extractedImages) {
-				return await resolveImageEditorAttachContext(this._fileService, this._dialogService, attachment, extractedImages);
+				return await resolveImageEditorAttachContext(this._fileService, this._dialogService, attachment, undefined, extractedImages);
 			}
 		}
 		return undefined;

src/vs/workbench/contrib/mcp/browser/mcpResourceQuickAccess.ts

@@ -90,7 +90,7 @@ export class McpResourcePickHelper {
 	}
 
 	private async _resourceToAttachment(resource: { uri: URI; name: string; mimeType?: string }): Promise<IChatRequestVariableEntry | undefined> {
-		const asImage = await resolveImageEditorAttachContext(this._fileService, this._dialogService, resource.uri, undefined, resource.mimeType);
+		const asImage = await resolveImageEditorAttachContext(this._fileService, this._dialogService, resource.uri, undefined, undefined, resource.mimeType);
 		if (asImage) {
 			return asImage;
 		}

src/vscode-dts/vscode.proposed.chatReferenceBinaryData.d.ts

@@ -20,10 +20,16 @@ declare module 'vscode' {
 
 		/**
 		 * Retrieves the binary data of the reference. This is primarily used to receive image attachments from the chat.
+		 * Note: base64 will only be used as a fallback if upload fails.
 		 * @returns A promise that resolves to the binary data as a Uint8Array.
 		 */
 		data(): Thenable<Uint8Array>;
 
+		/**
+		 * URL of image that was uploaded to GitHub
+		 */
+		url: string;
+
 		/**
 		 * Retrieves a URI reference to the binary data, if available.
 		 */
@@ -33,6 +39,6 @@ declare module 'vscode' {
 		 * @param mimeType The MIME type of the binary data.
 		 * @param data The binary data of the reference.
 		 */
-		constructor(mimeType: string, data: () => Thenable<Uint8Array>);
+		constructor(mimeType: string, data: () => Thenable<Uint8Array>, url: string, reference?: Uri);
 	}
 }