diff --git a/src/vs/platform/browserView/common/browserView.ts b/src/vs/platform/browserView/common/browserView.ts index c2f2161f6e000..ca9ad6d51f526 100644 --- a/src/vs/platform/browserView/common/browserView.ts +++ b/src/vs/platform/browserView/common/browserView.ts @@ -6,6 +6,7 @@ import { Event } from '../../../base/common/event.js'; import { VSBuffer } from '../../../base/common/buffer.js'; import { UriComponents } from '../../../base/common/uri.js'; +import { IElementData } from '../../browserElements/common/browserElements.js'; import { localize } from '../../../nls.js'; const commandPrefix = 'workbench.action.browser'; @@ -59,7 +60,8 @@ export interface IBrowserViewBounds { export interface IBrowserViewCaptureScreenshotOptions { quality?: number; - rect?: { x: number; y: number; width: number; height: number }; + screenRect?: { x: number; y: number; width: number; height: number }; + pageRect?: { x: number; y: number; width: number; height: number }; } export interface IBrowserViewState { @@ -371,6 +373,36 @@ export interface IBrowserViewService { */ untrustCertificate(id: string, host: string, fingerprint: string): Promise; + /** + * Get captured console logs for a browser view. + * Console messages are automatically captured from the moment the view is created. + * @param id The browser view identifier + * @returns The captured console logs as a single string + */ + getConsoleLogs(id: string): Promise; + + /** + * Start element inspection mode in a browser view. Sets up a CDP overlay that + * highlights elements on hover. When the user clicks an element, its data is + * returned and the overlay is removed. + * @param id The browser view identifier + * @param cancellationId An identifier that can be passed to {@link cancel} to abort + * @returns The inspected element data, or undefined if cancelled + */ + getElementData(id: string, cancellationId: number): Promise; + + /** + * Get element data for the currently focused element in the browser view. + * @param id The browser view identifier + * @returns The focused element's data, or undefined if no element is focused + */ + getFocusedElementData(id: string): Promise; + + /** + * Cancel an in-progress request. + */ + cancel(cancellationId: number): Promise; + /** * Update the keybinding accelerators used in browser view context menus. * @param keybindings A map of command ID to accelerator label diff --git a/src/vs/platform/browserView/electron-main/browserView.ts b/src/vs/platform/browserView/electron-main/browserView.ts index 8f61ab5cf8004..7e2dbd709df9f 100644 --- a/src/vs/platform/browserView/electron-main/browserView.ts +++ b/src/vs/platform/browserView/electron-main/browserView.ts @@ -7,7 +7,10 @@ import { WebContentsView, webContents } from 'electron'; import { Disposable } from '../../../base/common/lifecycle.js'; import { Emitter, Event } from '../../../base/common/event.js'; import { VSBuffer } from '../../../base/common/buffer.js'; +import { CancellationToken } from '../../../base/common/cancellation.js'; import { IBrowserViewBounds, IBrowserViewDevToolsStateEvent, IBrowserViewFocusEvent, IBrowserViewKeyDownEvent, IBrowserViewState, IBrowserViewNavigationEvent, IBrowserViewLoadingEvent, IBrowserViewLoadError, IBrowserViewTitleChangeEvent, IBrowserViewFaviconChangeEvent, IBrowserViewNewPageRequest, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, IBrowserViewFindInPageResult, IBrowserViewVisibilityEvent, BrowserNewPageLocation, browserViewIsolatedWorldId, browserZoomFactors, browserZoomDefaultIndex } from '../common/browserView.js'; +import { IElementData } from '../../browserElements/common/browserElements.js'; +import { getElementData, getFocusedElementData } from './browserViewElementInspector.js'; import { IWindowsMainService } from '../../windows/electron-main/windows.js'; import { ICodeWindow } from '../../window/electron-main/window.js'; import { IAuxiliaryWindowsMainService } from '../../auxiliaryWindow/electron-main/auxiliaryWindows.js'; @@ -38,6 +41,9 @@ export class BrowserView extends Disposable implements ICDPTarget { private _window: ICodeWindow | IAuxiliaryWindow | undefined; private _isDisposed = false; + private static readonly MAX_CONSOLE_LOG_ENTRIES = 1000; + private readonly _consoleLogs: string[] = []; + private readonly _onDidNavigate = this._register(new Emitter()); readonly onDidNavigate: Event = this._onDidNavigate.event; @@ -278,6 +284,7 @@ export class BrowserView extends Disposable implements ICDPTarget { // Chromium resets the zoom factor to its per-origin default (100%) when // navigating to a new document. Re-apply our stored zoom to override it. webContents.on('did-navigate', () => { + this._consoleLogs.length = 0; // Clear console logs on navigation since they are per-page this._view.webContents.setZoomFactor(browserZoomFactors[this._browserZoomIndex]); }); @@ -357,6 +364,14 @@ export class BrowserView extends Disposable implements ICDPTarget { finalUpdate: result.finalUpdate }); }); + + // Capture console messages for sharing with chat + this._view.webContents.on('console-message', (event) => { + this._consoleLogs.push(`[${event.level}] ${event.message}`); + if (this._consoleLogs.length > BrowserView.MAX_CONSOLE_LOG_ENTRIES) { + this._consoleLogs.splice(0, this._consoleLogs.length - BrowserView.MAX_CONSOLE_LOG_ENTRIES); + } + }); } private consumePopupPermission(location: BrowserNewPageLocation): boolean { @@ -456,6 +471,29 @@ export class BrowserView extends Disposable implements ICDPTarget { this._onDidChangeVisibility.fire({ visible }); } + /** + * Get captured console logs. + */ + getConsoleLogs(): string { + return this._consoleLogs.join('\n'); + } + + /** + * Start element inspection mode. Sets up a CDP overlay that highlights elements + * on hover. When the user clicks, the element data is returned and the overlay is removed. + * @param token Cancellation token to abort the inspection. + */ + async getElementData(token: CancellationToken): Promise { + return getElementData(this, token); + } + + /** + * Get element data for the currently focused element. + */ + async getFocusedElementData(): Promise { + return getFocusedElementData(this); + } + /** * Load a URL in this view */ @@ -518,13 +556,22 @@ export class BrowserView extends Disposable implements ICDPTarget { */ async captureScreenshot(options?: IBrowserViewCaptureScreenshotOptions): Promise { const quality = options?.quality ?? 80; - const image = await this._view.webContents.capturePage(options?.rect, { + if (options?.pageRect) { + const zoomFactor = this._view.webContents.getZoomFactor(); + options.screenRect = { + x: options.pageRect.x * zoomFactor, + y: options.pageRect.y * zoomFactor, + width: options.pageRect.width * zoomFactor, + height: options.pageRect.height * zoomFactor + }; + } + const image = await this._view.webContents.capturePage(options?.screenRect, { stayHidden: true }); const buffer = image.toJPEG(quality); const screenshot = VSBuffer.wrap(buffer); // Only update _lastScreenshot if capturing the full view - if (!options?.rect) { + if (!options?.screenRect) { this._lastScreenshot = screenshot; } return screenshot; diff --git a/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts b/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts new file mode 100644 index 0000000000000..0e11946daa8c3 --- /dev/null +++ b/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts @@ -0,0 +1,380 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken } from '../../../base/common/cancellation.js'; +import { DisposableStore } from '../../../base/common/lifecycle.js'; +import { IElementData, IElementAncestor } from '../../browserElements/common/browserElements.js'; +import { ICDPConnection } from '../common/cdp/types.js'; +import type { BrowserView } from './browserView.js'; + +type Quad = [number, number, number, number, number, number, number, number]; + +interface IBoxModel { + content: Quad; + padding: Quad; + border: Quad; + margin: Quad; + width: number; + height: number; +} + +interface ICSSStyle { + cssText?: string; + cssProperties: Array<{ name: string; value: string }>; +} + +interface ISelectorList { + selectors: Array<{ text: string }>; +} + +interface ICSSRule { + selectorList: ISelectorList; + origin: string; + style: ICSSStyle; +} + +interface IRuleMatch { + rule: ICSSRule; +} + +interface IInheritedStyleEntry { + inlineStyle?: ICSSStyle; + matchedCSSRules: IRuleMatch[]; +} + +interface IMatchedStyles { + inlineStyle?: ICSSStyle; + matchedCSSRules?: IRuleMatch[]; + inherited?: IInheritedStyleEntry[]; +} + +interface INode { + nodeId: number; + backendNodeId: number; + parentId?: number; + localName: string; + attributes: string[]; + children?: INode[]; + pseudoElements?: INode[]; +} + +function useScopedDisposal() { + const store = new DisposableStore() as DisposableStore & { [Symbol.dispose](): void }; + store[Symbol.dispose] = () => store.dispose(); + return store; +} + +/** + * Start element inspection mode on a browser view. Sets up an + * overlay that highlights elements on hover. When the user clicks, the + * element data is returned and the overlay is removed. + * + * @param browser The browser view to inspect. + * @param token Cancellation token to abort the inspection. + */ +export async function getElementData(browser: BrowserView, token: CancellationToken): Promise { + using store = useScopedDisposal(); + + const connection = store.add(await browser.attach()); + + // Important: don't use `Runtime.*` commands in this flow so we can support element selection during debugging + await connection.sendCommand('DOM.enable'); + await connection.sendCommand('Overlay.enable'); + + store.add({ + dispose: async () => { + try { + await connection.sendCommand('Overlay.setInspectMode', { + mode: 'none', + highlightConfig: { + showInfo: false, + showStyles: false + } + }); + await connection.sendCommand('Overlay.hideHighlight'); + await connection.sendCommand('Overlay.disable'); + } catch { + // Best effort cleanup + } + } + }); + + const result = new Promise((resolve, reject) => { + store.add(token.onCancellationRequested(() => { + resolve(undefined); + })); + + store.add(connection.onEvent(async (event) => { + if (event.method !== 'Overlay.inspectNodeRequested') { + return; + } + + const params = event.params as { backendNodeId: number }; + if (!params?.backendNodeId) { + reject(new Error('Missing backendNodeId in inspectNodeRequested event')); + return; + } + + try { + const nodeData = await extractNodeData(connection, { backendNodeId: params.backendNodeId }); + resolve(nodeData); + } catch (err) { + reject(err); + } + })); + }); + + await connection.sendCommand('Overlay.setInspectMode', { + mode: 'searchForNode', + highlightConfig: inspectHighlightConfig, + }); + + return await result; +} + +/** + * Get element data for the currently focused element in a browser view. + * + * @param browser The browser view to inspect. + */ +export async function getFocusedElementData(browser: BrowserView): Promise { + using store = useScopedDisposal(); + + const connection = store.add(await browser.attach()); + await connection.sendCommand('Runtime.enable'); + + const { result } = await connection.sendCommand('Runtime.evaluate', { + expression: `document.activeElement`, + returnByValue: false, + }) as { result: { objectId?: string } }; + + if (!result?.objectId) { + return undefined; + } + + return extractNodeData(connection, { objectId: result.objectId }); +} + +// ---- private helpers -------------------------------------------------- + +async function extractNodeData(connection: ICDPConnection, id: { backendNodeId?: number; objectId?: string }): Promise { + using store = useScopedDisposal(); + + const discoveredNodesByNodeId: Record = {}; + store.add(connection.onEvent(event => { + if (event.method === 'DOM.setChildNodes') { + const { nodes } = event.params as { nodes: INode[] }; + for (const node of nodes) { + discoveredNodesByNodeId[node.nodeId] = node; + if (node.children) { + for (const child of node.children) { + discoveredNodesByNodeId[child.nodeId] = { + ...child, + parentId: node.nodeId + }; + } + } + if (node.pseudoElements) { + for (const pseudo of node.pseudoElements) { + discoveredNodesByNodeId[pseudo.nodeId] = { + ...pseudo, + parentId: node.nodeId + }; + } + } + } + } + })); + + await connection.sendCommand('DOM.enable'); + await connection.sendCommand('DOM.getDocument'); + await connection.sendCommand('CSS.enable'); + + const { node } = await connection.sendCommand('DOM.describeNode', id) as { node: INode }; + if (!node) { + throw new Error('Failed to describe node.'); + } + let nodeId = node.nodeId; + if (!nodeId) { + const { nodeIds } = await connection.sendCommand('DOM.pushNodesByBackendIdsToFrontend', { backendNodeIds: [node.backendNodeId] }) as { nodeIds: number[] }; + if (!nodeIds?.length) { + throw new Error('Failed to get node ID.'); + } + nodeId = nodeIds[0]; + } + + const { model } = await connection.sendCommand('DOM.getBoxModel', { nodeId }) as { model: IBoxModel }; + if (!model) { + throw new Error('Failed to get box model.'); + } + + const content = model.content; + const margin = model.margin; + const x = Math.min(margin[0], content[0]); + const y = Math.min(margin[1], content[1]); + const width = Math.max(margin[2] - margin[0], content[2] - content[0]); + const height = Math.max(margin[5] - margin[1], content[5] - content[1]); + + const matched = await connection.sendCommand('CSS.getMatchedStylesForNode', { nodeId }); + if (!matched) { + throw new Error('Failed to get matched css.'); + } + + const computedStyle = formatMatchedStyles(matched as IMatchedStyles); + const { outerHTML } = await connection.sendCommand('DOM.getOuterHTML', { nodeId }) as { outerHTML: string }; + if (!outerHTML) { + throw new Error('Failed to get outerHTML.'); + } + + const attributes = attributeArrayToRecord(node.attributes); + + let ancestors: IElementAncestor[] | undefined; + try { + ancestors = []; + let currentNode: INode | undefined = discoveredNodesByNodeId[nodeId]; + while (currentNode) { + const attributes = attributeArrayToRecord(currentNode.attributes); + ancestors.unshift({ + tagName: currentNode.localName, + id: attributes.id, + classNames: attributes.class?.trim().split(/\s+/).filter(Boolean) + }); + currentNode = currentNode.parentId ? discoveredNodesByNodeId[currentNode.parentId] : undefined; + } + } catch { } + + let computedStyles: Record | undefined; + try { + const { computedStyle: computedStyleArray } = await connection.sendCommand('CSS.getComputedStyleForNode', { nodeId }) as { computedStyle?: Array<{ name: string; value: string }> }; + if (computedStyleArray) { + computedStyles = {}; + for (const prop of computedStyleArray) { + if (prop.name && typeof prop.value === 'string') { + computedStyles[prop.name] = prop.value; + } + } + } + } catch { } + + return { + outerHTML, + computedStyle, + bounds: { x, y, width, height }, + ancestors, + attributes, + computedStyles, + dimensions: { top: y, left: x, width, height } + }; +} + +function formatMatchedStyles(matched: IMatchedStyles): string { + const lines: string[] = []; + + if (matched.inlineStyle?.cssProperties?.length) { + lines.push('/* Inline style */'); + lines.push('element {'); + for (const prop of matched.inlineStyle.cssProperties) { + if (prop.name && prop.value) { + lines.push(` ${prop.name}: ${prop.value};`); + } + } + lines.push('}\n'); + } + + if (matched.matchedCSSRules?.length) { + for (const ruleEntry of matched.matchedCSSRules) { + const rule = ruleEntry.rule; + const selectors = rule.selectorList.selectors.map((s: { text: string }) => s.text).join(', '); + lines.push(`/* Matched Rule from ${rule.origin} */`); + lines.push(`${selectors} {`); + for (const prop of rule.style.cssProperties) { + if (prop.name && prop.value) { + lines.push(` ${prop.name}: ${prop.value};`); + } + } + lines.push('}\n'); + } + } + + if (matched.inherited?.length) { + let level = 1; + for (const inherited of matched.inherited) { + if (inherited.inlineStyle) { + lines.push(`/* Inherited from ancestor level ${level} (inline) */`); + lines.push('element {'); + lines.push(inherited.inlineStyle.cssText || ''); + lines.push('}\n'); + } + + const rules = inherited.matchedCSSRules || []; + for (const ruleEntry of rules) { + const rule = ruleEntry.rule; + const selectors = rule.selectorList.selectors.map((s: { text: string }) => s.text).join(', '); + lines.push(`/* Inherited from ancestor level ${level} (${rule.origin}) */`); + lines.push(`${selectors} {`); + for (const prop of rule.style.cssProperties) { + if (prop.name && prop.value) { + lines.push(` ${prop.name}: ${prop.value};`); + } + } + lines.push('}\n'); + } + level++; + } + } + + return '\n' + lines.join('\n'); +} + +function attributeArrayToRecord(attributes: string[]): Record { + const record: Record = {}; + for (let i = 0; i < attributes.length; i += 2) { + const name = attributes[i]; + const value = attributes[i + 1]; + record[name] = value; + } + return record; +} + +/** Slightly customised CDP debugger inspect highlight colours. */ +const inspectHighlightConfig = { + showInfo: true, + showRulers: false, + showStyles: true, + showAccessibilityInfo: true, + showExtensionLines: false, + contrastAlgorithm: 'aa', + contentColor: { r: 173, g: 216, b: 255, a: 0.8 }, + paddingColor: { r: 150, g: 200, b: 255, a: 0.5 }, + borderColor: { r: 120, g: 180, b: 255, a: 0.7 }, + marginColor: { r: 200, g: 220, b: 255, a: 0.4 }, + eventTargetColor: { r: 130, g: 160, b: 255, a: 0.8 }, + shapeColor: { r: 130, g: 160, b: 255, a: 0.8 }, + shapeMarginColor: { r: 130, g: 160, b: 255, a: 0.5 }, + gridHighlightConfig: { + rowGapColor: { r: 140, g: 190, b: 255, a: 0.3 }, + rowHatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, + columnGapColor: { r: 140, g: 190, b: 255, a: 0.3 }, + columnHatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, + rowLineColor: { r: 120, g: 180, b: 255 }, + columnLineColor: { r: 120, g: 180, b: 255 }, + rowLineDash: true, + columnLineDash: true + }, + flexContainerHighlightConfig: { + containerBorder: { color: { r: 120, g: 180, b: 255 }, pattern: 'solid' }, + itemSeparator: { color: { r: 140, g: 190, b: 255 }, pattern: 'solid' }, + lineSeparator: { color: { r: 140, g: 190, b: 255 }, pattern: 'solid' }, + mainDistributedSpace: { hatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, fillColor: { r: 140, g: 190, b: 255, a: 0.4 } }, + crossDistributedSpace: { hatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, fillColor: { r: 140, g: 190, b: 255, a: 0.4 } }, + rowGapSpace: { hatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, fillColor: { r: 140, g: 190, b: 255, a: 0.4 } }, + columnGapSpace: { hatchColor: { r: 140, g: 190, b: 255, a: 0.7 }, fillColor: { r: 140, g: 190, b: 255, a: 0.4 } }, + }, + flexItemHighlightConfig: { + baseSizeBox: { hatchColor: { r: 130, g: 170, b: 255, a: 0.6 } }, + baseSizeBorder: { color: { r: 120, g: 180, b: 255 }, pattern: 'solid' }, + flexibilityArrow: { color: { r: 130, g: 190, b: 255 } } + }, +}; diff --git a/src/vs/platform/browserView/electron-main/browserViewMainService.ts b/src/vs/platform/browserView/electron-main/browserViewMainService.ts index d7e9ac1737f57..150bfd6b6b47e 100644 --- a/src/vs/platform/browserView/electron-main/browserViewMainService.ts +++ b/src/vs/platform/browserView/electron-main/browserViewMainService.ts @@ -6,9 +6,10 @@ import { Emitter, Event } from '../../../base/common/event.js'; import { Disposable, DisposableMap } from '../../../base/common/lifecycle.js'; import { VSBuffer } from '../../../base/common/buffer.js'; +import { CancellationToken, CancellationTokenSource } from '../../../base/common/cancellation.js'; import { IBrowserViewBounds, IBrowserViewState, IBrowserViewService, BrowserViewStorageScope, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, BrowserViewCommandId } from '../common/browserView.js'; +import { IElementData } from '../../browserElements/common/browserElements.js'; import { clipboard, Menu, MenuItem } from 'electron'; -import { ICDPTarget, CDPBrowserVersion, CDPWindowBounds, CDPTargetInfo, ICDPConnection, ICDPBrowserTarget } from '../common/cdp/types.js'; import { IEnvironmentMainService } from '../../environment/electron-main/environmentMainService.js'; import { createDecorator, IInstantiationService } from '../../instantiation/common/instantiation.js'; import { BrowserView } from './browserView.js'; @@ -21,11 +22,11 @@ import { IApplicationStorageMainService } from '../../storage/electron-main/stor import { CDPBrowserProxy } from '../common/cdp/proxy.js'; import { IntegratedBrowserOpenSource, logBrowserOpen } from '../common/browserViewTelemetry.js'; import { ITelemetryService } from '../../telemetry/common/telemetry.js'; -import { CancellationToken } from '../../../base/common/cancellation.js'; import { localize } from '../../../nls.js'; import { INativeHostMainService } from '../../native/electron-main/nativeHostMainService.js'; import { ITextEditorOptions } from '../../editor/common/editor.js'; import { htmlAttributeEncodeValue } from '../../../base/common/strings.js'; +import { ICDPTarget, CDPBrowserVersion, CDPWindowBounds, CDPTargetInfo, ICDPConnection, ICDPBrowserTarget } from '../common/cdp/types.js'; export const IBrowserViewMainService = createDecorator('browserViewMainService'); @@ -47,6 +48,7 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa } private readonly browserViews = this._register(new DisposableMap()); + private readonly _activeTokens = new Map(); private _keybindings: { [commandId: string]: string } = Object.create(null); // ICDPBrowserTarget events @@ -351,10 +353,37 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa await browserSession.clearData(); } + async getConsoleLogs(id: string): Promise { + return this._getBrowserView(id).getConsoleLogs(); + } + + async getElementData(id: string, cancellationId: number): Promise { + return this._makeCancellable(cancellationId, (token) => this._getBrowserView(id).getElementData(token)); + } + + async getFocusedElementData(id: string): Promise { + return this._getBrowserView(id).getFocusedElementData(); + } + + async cancel(cancellationId: number): Promise { + this._activeTokens.get(cancellationId)?.cancel(); + } + async updateKeybindings(keybindings: { [commandId: string]: string }): Promise { this._keybindings = keybindings; } + private async _makeCancellable(cancellationId: number, callback: (token: CancellationToken) => T | Promise): Promise { + const cts: CancellationTokenSource = new CancellationTokenSource(); + this._activeTokens.set(cancellationId, cts); + try { + return await callback(cts.token); + } finally { + this._activeTokens.delete(cancellationId); + cts.dispose(); + } + } + /** * Create a browser view backed by the given {@link BrowserSession}. */ diff --git a/src/vs/workbench/contrib/browserView/common/browserView.ts b/src/vs/workbench/contrib/browserView/common/browserView.ts index 8f9cfa3f96b69..5508cfc68ebab 100644 --- a/src/vs/workbench/contrib/browserView/common/browserView.ts +++ b/src/vs/workbench/contrib/browserView/common/browserView.ts @@ -7,11 +7,13 @@ import { createDecorator } from '../../../../platform/instantiation/common/insta import { Emitter, Event } from '../../../../base/common/event.js'; import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js'; import { VSBuffer } from '../../../../base/common/buffer.js'; +import { CancellationToken } from '../../../../base/common/cancellation.js'; import { CDPEvent, CDPRequest, CDPResponse } from '../../../../platform/browserView/common/cdp/types.js'; -import { IPlaywrightService } from '../../../../platform/browserView/common/playwrightService.js'; import { IDialogService } from '../../../../platform/dialogs/common/dialogs.js'; import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js'; import { localize } from '../../../../nls.js'; +import { IElementData } from '../../../../platform/browserElements/common/browserElements.js'; +import { IPlaywrightService } from '../../../../platform/browserView/common/playwrightService.js'; import { IBrowserViewBounds, IBrowserViewNavigationEvent, @@ -209,6 +211,9 @@ export interface IBrowserViewModel extends IDisposable { zoomIn(): Promise; zoomOut(): Promise; resetZoom(): Promise; + getConsoleLogs(): Promise; + getElementData(token: CancellationToken): Promise; + getFocusedElementData(): Promise; } export class BrowserViewModel extends Disposable implements IBrowserViewModel { @@ -464,7 +469,7 @@ export class BrowserViewModel extends Disposable implements IBrowserViewModel { async captureScreenshot(options?: IBrowserViewCaptureScreenshotOptions): Promise { const result = await this.browserViewService.captureScreenshot(this.id, options); // Store full-page screenshots for display in UI as placeholders - if (!options?.rect) { + if (!options?.screenRect && !options?.pageRect) { this._screenshot = result; } return result; @@ -541,6 +546,18 @@ export class BrowserViewModel extends Disposable implements IBrowserViewModel { } } + async getConsoleLogs(): Promise { + return this.browserViewService.getConsoleLogs(this.id); + } + + async getElementData(token: CancellationToken): Promise { + return this._wrapCancellable(token, (cid) => this.browserViewService.getElementData(this.id, cid)); + } + + async getFocusedElementData(): Promise { + return this.browserViewService.getFocusedElementData(this.id); + } + private static readonly SHARE_DONT_ASK_KEY = 'browserView.shareWithAgent.dontAskAgain'; async setSharedWithAgent(shared: boolean): Promise { @@ -601,6 +618,19 @@ export class BrowserViewModel extends Disposable implements IBrowserViewModel { } } + private static _cancellationIdPool = 0; + private async _wrapCancellable(token: CancellationToken, callback: (cancellationId: number) => Promise): Promise { + const cancellationId = BrowserViewModel._cancellationIdPool++; + const disposable = token.onCancellationRequested(() => { + this.browserViewService.cancel(cancellationId); + }); + try { + return await callback(cancellationId); + } finally { + disposable.dispose(); + } + } + /** * Log navigation telemetry event */ diff --git a/src/vs/workbench/contrib/browserView/electron-browser/features/browserEditorChatFeatures.ts b/src/vs/workbench/contrib/browserView/electron-browser/features/browserEditorChatFeatures.ts index b44e966f2db58..08eba39df4c29 100644 --- a/src/vs/workbench/contrib/browserView/electron-browser/features/browserEditorChatFeatures.ts +++ b/src/vs/workbench/contrib/browserView/electron-browser/features/browserEditorChatFeatures.ts @@ -14,17 +14,16 @@ import { KeyMod, KeyCode } from '../../../../../base/common/keyCodes.js'; import { IEditorService } from '../../../../services/editor/common/editorService.js'; import { Codicon } from '../../../../../base/common/codicons.js'; import { ThemeIcon } from '../../../../../base/common/themables.js'; -import { DisposableStore, IDisposable, toDisposable } from '../../../../../base/common/lifecycle.js'; +import { DisposableStore } from '../../../../../base/common/lifecycle.js'; import { Event } from '../../../../../base/common/event.js'; import { ILogService } from '../../../../../platform/log/common/log.js'; import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; -import { IBrowserElementsService } from '../../../../services/browserElements/browser/browserElementsService.js'; import { IChatWidgetService } from '../../../chat/browser/chat.js'; import { IChatRequestVariableEntry } from '../../../chat/common/attachments/chatVariableEntries.js'; import { ChatContextKeys } from '../../../chat/common/actions/chatContextKeys.js'; import { ChatConfiguration } from '../../../chat/common/constants.js'; -import { IElementData, IBrowserTargetLocator, getDisplayNameFromOuterHTML, createElementContextValue } from '../../../../../platform/browserElements/common/browserElements.js'; +import { IElementData, createElementContextValue } from '../../../../../platform/browserElements/common/browserElements.js'; import { BrowserViewCommandId } from '../../../../../platform/browserView/common/browserView.js'; import { IBrowserViewModel } from '../../../browserView/common/browserView.js'; import { BrowserEditorInput } from '../../common/browserEditorInput.js'; @@ -36,10 +35,11 @@ import { IConfigurationRegistry, Extensions as ConfigurationExtensions } from '. import { Registry } from '../../../../../platform/registry/common/platform.js'; import { PolicyCategory } from '../../../../../base/common/policy.js'; import { workbenchConfigurationNodeBase } from '../../../../common/configuration.js'; +import { safeSetInnerHtml } from '../../../../../base/browser/domSanitize.js'; +import { BrowserActionCategory } from '../browserViewActions.js'; // Register tools import '../tools/browserTools.contribution.js'; -import { BrowserActionCategory } from '../browserViewActions.js'; // Context key expression to check if browser editor is active const BROWSER_EDITOR_ACTIVE = ContextKeyExpr.equals('activeEditor', BrowserEditorInput.EDITOR_ID); @@ -56,7 +56,7 @@ const canShareBrowserWithAgentContext = ContextKeyExpr.and( /** * Contribution that manages element selection, element attachment to chat, - * console session lifecycle, console log attachment to chat, and agent sharing. + * console log attachment to chat, and agent sharing. */ export class BrowserEditorChatIntegration extends BrowserEditorContribution { private _elementSelectionCts: CancellationTokenSource | undefined; @@ -72,7 +72,6 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { @IInstantiationService instantiationService: IInstantiationService, @ITelemetryService private readonly telemetryService: ITelemetryService, @ILogService private readonly logService: ILogService, - @IBrowserElementsService private readonly browserElementsService: IBrowserElementsService, @IChatWidgetService private readonly chatWidgetService: IChatWidgetService, @IConfigurationService private readonly configurationService: IConfigurationService, ) { @@ -117,15 +116,6 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { } protected override subscribeToModel(model: IBrowserViewModel, store: DisposableStore): void { - // Start console session when a page URL is loaded - if (model.url) { - store.add(this._startConsoleSession(model.id)); - } else { - store.add(Event.once(Event.filter(model.onDidNavigate, e => !!e.url))(() => { - store.add(this._startConsoleSession(model.id)); - })); - } - // Manage sharing state this._updateSharingState(true); store.add(model.onDidChangeSharedWithAgent(() => { @@ -200,24 +190,16 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { this.telemetryService.publicLog2('integratedBrowser.addElementToChat.start', {}); try { - const browserViewId = this.editor.model?.id; - if (!browserViewId) { - throw new Error('No browser view ID found'); + const model = this.editor.model; + if (!model) { + throw new Error('No browser view model found'); } // Make the browser the focused view this.editor.ensureBrowserFocus(); - const locator: IBrowserTargetLocator = { browserViewId }; - - // Start debug session for integrated browser - await this.browserElementsService.startDebugSession(cts.token, locator); - - // Get the browser container bounds - const { width, height } = this.editor.browserContainer.getBoundingClientRect(); - // Get element data from user selection - const elementData = await this.browserElementsService.getElementData({ x: 0, y: 0, width, height }, cts.token, locator); + const elementData = await model.getElementData(cts.token); if (!elementData) { throw new Error('Element data not found'); } @@ -246,7 +228,7 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { this.logService.error('BrowserEditor.addElementToChat: Failed to select element', error); } } finally { - cts.dispose(); + cts.dispose(true); if (this._elementSelectionCts === cts) { this._elementSelectionCts = undefined; this._elementSelectionActiveContext.set(false); @@ -263,20 +245,18 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { } const cts = this._elementSelectionCts; - const browserViewId = this.editor.model?.id; - if (!browserViewId) { + const model = this.editor.model; + if (!model) { return; } - const locator: IBrowserTargetLocator = { browserViewId }; - const { width, height } = this.editor.browserContainer.getBoundingClientRect(); - const elementData = await this.browserElementsService.getFocusedElementData({ x: 0, y: 0, width, height }, cts.token, locator); + const elementData = await model.getFocusedElementData(); if (!elementData) { return; } await this._attachElementDataToChat(elementData); - cts.dispose(); + cts.dispose(true); if (this._elementSelectionCts === cts) { this._elementSelectionCts = undefined; this._elementSelectionActiveContext.set(false); @@ -287,14 +267,31 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { const bounds = elementData.bounds; const toAttach: IChatRequestVariableEntry[] = []; - const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML); + const container = document.createElement('div'); + safeSetInnerHtml(container, elementData.outerHTML); + const element = container.firstElementChild; + const innerText = container.textContent; + + let displayNameShort = element ? `${element.tagName.toLowerCase()}${element.id ? `#${element.id}` : ''}` : ''; + let displayNameFull = element ? `${displayNameShort}${element.classList.length ? `.${[...element.classList].join('.')}` : ''}` : ''; + if (elementData.ancestors && elementData.ancestors.length > 0) { + let last = elementData.ancestors[elementData.ancestors.length - 1]; + let pseudo = ''; + if (last.tagName.startsWith('::') && elementData.ancestors.length > 1) { + pseudo = last.tagName; + last = elementData.ancestors[elementData.ancestors.length - 2]; + } + displayNameShort = `${last.tagName.toLowerCase()}${last.id ? `#${last.id}` : ''}${pseudo}`; + displayNameFull = `${last.tagName.toLowerCase()}${last.id ? `#${last.id}` : ''}${last.classNames && last.classNames.length ? `.${last.classNames.join('.')}` : ''}${pseudo}`; + } + const attachCss = this.configurationService.getValue('chat.sendElementsToChat.attachCSS'); - const value = createElementContextValue(elementData, displayName, attachCss); + const value = createElementContextValue(elementData, displayNameFull, attachCss); toAttach.push({ id: 'element-' + Date.now(), - name: displayName, - fullName: displayName, + name: displayNameShort, + fullName: displayNameFull, value: value, modelDescription: attachCss ? 'Structured browser element context with HTML path, attributes, and computed styles.' @@ -305,7 +302,7 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { attributes: elementData.attributes, computedStyles: attachCss ? elementData.computedStyles : undefined, dimensions: elementData.dimensions, - innerText: elementData.innerText, + innerText, }); const attachImages = this.configurationService.getValue('chat.sendElementsToChat.attachImages'); @@ -313,7 +310,7 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { if (attachImages && model) { const screenshotBuffer = await model.captureScreenshot({ quality: 90, - rect: bounds + pageRect: bounds }); toAttach.push({ @@ -337,15 +334,13 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { * Grab the current console logs from the active console session and attach them to chat. */ async addConsoleLogsToChat(): Promise { - const browserViewId = this.editor.model?.id; - if (!browserViewId) { + const model = this.editor.model; + if (!model) { return; } - const locator: IBrowserTargetLocator = { browserViewId }; - try { - const logs = await this.browserElementsService.getConsoleLogs(locator); + const logs = await model.getConsoleLogs(); if (!logs) { return; } @@ -367,21 +362,6 @@ export class BrowserEditorChatIntegration extends BrowserEditorContribution { this.logService.error('BrowserEditor.addConsoleLogsToChat: Failed to get console logs', error); } } - - private _startConsoleSession(browserViewId: string): IDisposable { - const cts = new CancellationTokenSource(); - const locator: IBrowserTargetLocator = { browserViewId }; - - this.browserElementsService.startConsoleSession(cts.token, locator).catch(error => { - if (!cts.token.isCancellationRequested) { - this.logService.error('BrowserEditor: Failed to start console session', error); - } - }); - - return toDisposable(() => { - cts.dispose(true); - }); - } } // Register the contribution diff --git a/src/vs/workbench/contrib/browserView/electron-browser/tools/screenshotBrowserTool.ts b/src/vs/workbench/contrib/browserView/electron-browser/tools/screenshotBrowserTool.ts index b2bb1f9a3fed7..d663b66cc30ec 100644 --- a/src/vs/workbench/contrib/browserView/electron-browser/tools/screenshotBrowserTool.ts +++ b/src/vs/workbench/contrib/browserView/electron-browser/tools/screenshotBrowserTool.ts @@ -87,14 +87,7 @@ export class ScreenshotBrowserTool implements IToolImpl { } return locator.boundingBox(); }, selector, params.scrollIntoViewIfNeeded) || undefined; - const zoomFactor = browserViewModel.zoomFactor; - if (bounds) { - bounds.x *= zoomFactor; - bounds.y *= zoomFactor; - bounds.width *= zoomFactor; - bounds.height *= zoomFactor; - } - const screenshot = await browserViewModel.captureScreenshot({ rect: bounds }); + const screenshot = await browserViewModel.captureScreenshot({ pageRect: bounds }); return { content: [