Browser view native "add to chat" features

src/vs/platform/browserView/common/browserView.ts

@@ -6,6 +6,7 @@
 import { Event } from '../../../base/common/event.js';
 import { VSBuffer } from '../../../base/common/buffer.js';
 import { UriComponents } from '../../../base/common/uri.js';
+import { IElementData } from '../../browserElements/common/browserElements.js';
 import { localize } from '../../../nls.js';
 
 const commandPrefix = 'workbench.action.browser';
@@ -59,7 +60,8 @@ export interface IBrowserViewBounds {
 
 export interface IBrowserViewCaptureScreenshotOptions {
 	quality?: number;
-	rect?: { x: number; y: number; width: number; height: number };
+	screenRect?: { x: number; y: number; width: number; height: number };
+	pageRect?: { x: number; y: number; width: number; height: number };
 }
 
 export interface IBrowserViewState {
@@ -371,6 +373,36 @@ export interface IBrowserViewService {
 	 */
 	untrustCertificate(id: string, host: string, fingerprint: string): Promise<void>;
 
+	/**
+	 * Get captured console logs for a browser view.
+	 * Console messages are automatically captured from the moment the view is created.
+	 * @param id The browser view identifier
+	 * @returns The captured console logs as a single string
+	 */
+	getConsoleLogs(id: string): Promise<string>;
+
+	/**
+	 * Start element inspection mode in a browser view. Sets up a CDP overlay that
+	 * highlights elements on hover. When the user clicks an element, its data is
+	 * returned and the overlay is removed.
+	 * @param id The browser view identifier
+	 * @param cancellationId An identifier that can be passed to {@link cancel} to abort
+	 * @returns The inspected element data, or undefined if cancelled
+	 */
+	getElementData(id: string, cancellationId: number): Promise<IElementData | undefined>;
+
+	/**
+	 * Get element data for the currently focused element in the browser view.
+	 * @param id The browser view identifier
+	 * @returns The focused element's data, or undefined if no element is focused
+	 */
+	getFocusedElementData(id: string): Promise<IElementData | undefined>;
+
+	/**
+	 * Cancel an in-progress request.
+	 */
+	cancel(cancellationId: number): Promise<void>;
+
 	/**
 	 * Update the keybinding accelerators used in browser view context menus.
 	 * @param keybindings A map of command ID to accelerator label

src/vs/platform/browserView/electron-main/browserView.ts

@@ -7,7 +7,10 @@ import { WebContentsView, webContents } from 'electron';
 import { Disposable } from '../../../base/common/lifecycle.js';
 import { Emitter, Event } from '../../../base/common/event.js';
 import { VSBuffer } from '../../../base/common/buffer.js';
+import { CancellationToken } from '../../../base/common/cancellation.js';
 import { IBrowserViewBounds, IBrowserViewDevToolsStateEvent, IBrowserViewFocusEvent, IBrowserViewKeyDownEvent, IBrowserViewState, IBrowserViewNavigationEvent, IBrowserViewLoadingEvent, IBrowserViewLoadError, IBrowserViewTitleChangeEvent, IBrowserViewFaviconChangeEvent, IBrowserViewNewPageRequest, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, IBrowserViewFindInPageResult, IBrowserViewVisibilityEvent, BrowserNewPageLocation, browserViewIsolatedWorldId, browserZoomFactors, browserZoomDefaultIndex } from '../common/browserView.js';
+import { IElementData } from '../../browserElements/common/browserElements.js';
+import { getElementData, getFocusedElementData } from './browserViewElementInspector.js';
 import { IWindowsMainService } from '../../windows/electron-main/windows.js';
 import { ICodeWindow } from '../../window/electron-main/window.js';
 import { IAuxiliaryWindowsMainService } from '../../auxiliaryWindow/electron-main/auxiliaryWindows.js';
@@ -38,6 +41,9 @@ export class BrowserView extends Disposable implements ICDPTarget {
 	private _window: ICodeWindow | IAuxiliaryWindow | undefined;
 	private _isDisposed = false;
 
+	private static readonly MAX_CONSOLE_LOG_ENTRIES = 1000;
+	private readonly _consoleLogs: string[] = [];
+
 	private readonly _onDidNavigate = this._register(new Emitter<IBrowserViewNavigationEvent>());
 	readonly onDidNavigate: Event<IBrowserViewNavigationEvent> = this._onDidNavigate.event;
 
@@ -278,6 +284,7 @@ export class BrowserView extends Disposable implements ICDPTarget {
 		// Chromium resets the zoom factor to its per-origin default (100%) when
 		// navigating to a new document. Re-apply our stored zoom to override it.
 		webContents.on('did-navigate', () => {
+			this._consoleLogs.length = 0; // Clear console logs on navigation since they are per-page
 			this._view.webContents.setZoomFactor(browserZoomFactors[this._browserZoomIndex]);
 		});
 
@@ -357,6 +364,14 @@ export class BrowserView extends Disposable implements ICDPTarget {
 				finalUpdate: result.finalUpdate
 			});
 		});
+
+		// Capture console messages for sharing with chat
+		this._view.webContents.on('console-message', (event) => {
+			this._consoleLogs.push(`[${event.level}] ${event.message}`);
+			if (this._consoleLogs.length > BrowserView.MAX_CONSOLE_LOG_ENTRIES) {
+				this._consoleLogs.splice(0, this._consoleLogs.length - BrowserView.MAX_CONSOLE_LOG_ENTRIES);
+			}
+		});
 	}
 
 	private consumePopupPermission(location: BrowserNewPageLocation): boolean {
@@ -456,6 +471,29 @@ export class BrowserView extends Disposable implements ICDPTarget {
 		this._onDidChangeVisibility.fire({ visible });
 	}
 
+	/**
+	 * Get captured console logs.
+	 */
+	getConsoleLogs(): string {
+		return this._consoleLogs.join('\n');
+	}
+
+	/**
+	 * Start element inspection mode. Sets up a CDP overlay that highlights elements
+	 * on hover. When the user clicks, the element data is returned and the overlay is removed.
+	 * @param token Cancellation token to abort the inspection.
+	 */
+	async getElementData(token: CancellationToken): Promise<IElementData | undefined> {
+		return getElementData(this, token);
+	}
+
+	/**
+	 * Get element data for the currently focused element.
+	 */
+	async getFocusedElementData(): Promise<IElementData | undefined> {
+		return getFocusedElementData(this);
+	}
+
 	/**
 	 * Load a URL in this view
 	 */
@@ -518,13 +556,22 @@ export class BrowserView extends Disposable implements ICDPTarget {
 	 */
 	async captureScreenshot(options?: IBrowserViewCaptureScreenshotOptions): Promise<VSBuffer> {
 		const quality = options?.quality ?? 80;
-		const image = await this._view.webContents.capturePage(options?.rect, {
+		if (options?.pageRect) {
+			const zoomFactor = this._view.webContents.getZoomFactor();
+			options.screenRect = {
+				x: options.pageRect.x * zoomFactor,
+				y: options.pageRect.y * zoomFactor,
+				width: options.pageRect.width * zoomFactor,
+				height: options.pageRect.height * zoomFactor
+			};
+		}
+		const image = await this._view.webContents.capturePage(options?.screenRect, {
 			stayHidden: true
 		});
 		const buffer = image.toJPEG(quality);
 		const screenshot = VSBuffer.wrap(buffer);
 		// Only update _lastScreenshot if capturing the full view
-		if (!options?.rect) {
+		if (!options?.screenRect) {
 			this._lastScreenshot = screenshot;
 		}
 		return screenshot;