Skip to content

Refactor browser element selection to be event-based #315362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kycutler merged 2 commits into from
May 8, 2026
Merged

Refactor browser element selection to be event-based #315362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af8775d
Refactor browser element selection to be event-based
kycutler May 8, 2026
a120fa2
comment
kycutler May 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 9 additions & 18 deletions src/vs/platform/browserView/common/browserView.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ export interface IBrowserViewState {
certificateError: IBrowserViewCertificateError | undefined;
storageScope: BrowserViewStorageScope;
browserZoomIndex: number;
isElementSelectionActive: boolean;
}

export interface IBrowserViewNavigationEvent {
Expand Down Expand Up @@ -266,6 +267,8 @@ export interface IBrowserViewService {
onDynamicDidChangeFavicon(id: string): Event<IBrowserViewFaviconChangeEvent>;
onDynamicDidFindInPage(id: string): Event<IBrowserViewFindInPageResult>;
onDynamicDidClose(id: string): Event<void>;
onDynamicDidSelectElement(id: string): Event<IElementData>;
onDynamicDidChangeElementSelectionActive(id: string): Event<boolean>;

/**
* Get all known browser views with their ownership and state information.
Expand Down Expand Up @@ -443,26 +446,14 @@ export interface IBrowserViewService {
getConsoleLogs(id: string): Promise<string>;

/**
* Start element inspection mode in a browser view. Sets up a CDP overlay that
* highlights elements on hover. When the user clicks an element, its data is
* returned and the overlay is removed.
* @param id The browser view identifier
* @param cancellationId An identifier that can be passed to {@link cancel} to abort
* @returns The inspected element data, or undefined if cancelled
*/
getElementData(id: string, cancellationId: number): Promise<IElementData | undefined>;

/**
* Get element data for the currently focused element in the browser view.
* Toggle element selection mode in a browser view.
* Element selections are delivered via {@link onDynamicDidSelectElement}.
* State changes are delivered via {@link onDynamicDidChangeElementSelectionActive}.
*
* @param id The browser view identifier
* @returns The focused element's data, or undefined if no element is focused
*/
getFocusedElementData(id: string): Promise<IElementData | undefined>;

/**
* Cancel an in-progress request.
* @param enabled Whether to enable or disable. Omit to toggle.
*/
cancel(cancellationId: number): Promise<void>;
toggleElementSelection(id: string, enabled?: boolean): Promise<void>;

/**
* Update the keybinding accelerators used in browser view context menus.
Expand Down
33 changes: 11 additions & 22 deletions src/vs/platform/browserView/electron-main/browserView.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ import { WebContentsView, webContents } from 'electron';
import { Disposable } from '../../../base/common/lifecycle.js';
import { Emitter, Event } from '../../../base/common/event.js';
import { VSBuffer } from '../../../base/common/buffer.js';
import { CancellationToken } from '../../../base/common/cancellation.js';
import { IBrowserViewBounds, IBrowserViewDevToolsStateEvent, IBrowserViewFocusEvent, IBrowserViewKeyDownEvent, IBrowserViewState, IBrowserViewNavigationEvent, IBrowserViewLoadingEvent, IBrowserViewLoadError, IBrowserViewTitleChangeEvent, IBrowserViewFaviconChangeEvent, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, IBrowserViewFindInPageResult, IBrowserViewVisibilityEvent, browserViewIsolatedWorldId, browserZoomFactors, browserZoomDefaultIndex, IElementData, IBrowserViewOwner, IBrowserViewOpenOptions } from '../common/browserView.js';
import { IBrowserViewBounds, IBrowserViewDevToolsStateEvent, IBrowserViewFocusEvent, IBrowserViewKeyDownEvent, IBrowserViewState, IBrowserViewNavigationEvent, IBrowserViewLoadingEvent, IBrowserViewLoadError, IBrowserViewTitleChangeEvent, IBrowserViewFaviconChangeEvent, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, IBrowserViewFindInPageResult, IBrowserViewVisibilityEvent, browserViewIsolatedWorldId, browserZoomFactors, browserZoomDefaultIndex, IBrowserViewOwner, IBrowserViewOpenOptions } from '../common/browserView.js';
import { BrowserViewElementInspector } from './browserViewElementInspector.js';
import { IWindowsMainService } from '../../windows/electron-main/windows.js';
import { ICodeWindow, LoadReason } from '../../window/electron-main/window.js';
Expand Down Expand Up @@ -42,7 +41,7 @@ export class BrowserView extends Disposable {
private _browserZoomIndex: number = browserZoomDefaultIndex;

readonly debugger: BrowserViewDebugger;
private readonly _inspector: BrowserViewElementInspector;
readonly inspector: BrowserViewElementInspector;

private _ownerWindow: ICodeWindow;
private _currentWindow: ICodeWindow | IAuxiliaryWindow | undefined;
Expand Down Expand Up @@ -187,7 +186,7 @@ export class BrowserView extends Disposable {
});

this.debugger = new BrowserViewDebugger(this, this.logService);
this._inspector = this._register(new BrowserViewElementInspector(this));
this.inspector = this._register(new BrowserViewElementInspector(this));

this.setupEventListeners();
}
Expand Down Expand Up @@ -343,6 +342,11 @@ export class BrowserView extends Disposable {

// Forward key down events that weren't handled by the page to the workbench for shortcut handling.
webContents.ipc.on('vscode:browserView:keydown', (_event, keyEvent: IBrowserViewKeyDownEvent) => {
// Intercept Ctrl/Cmd+Enter during element selection to pick the focused element.
if (this.inspector.isElementSelectionActive && keyEvent.key === 'Enter' && (keyEvent.ctrlKey || keyEvent.metaKey)) {
void this.inspector.pickFocusedElement();
Comment thread
kycutler marked this conversation as resolved.
return;
}
this._onDidKeyCommand.fire(keyEvent);
});
// If the page won't be able to handle events, forward key down events directly.
Expand Down Expand Up @@ -459,7 +463,8 @@ export class BrowserView extends Disposable {
lastError: this._lastError,
certificateError: this.session.trust.getCertificateError(url),
storageScope: this.session.storageScope,
browserZoomIndex: this._browserZoomIndex
browserZoomIndex: this._browserZoomIndex,
isElementSelectionActive: this.inspector.isElementSelectionActive
};
}

Expand Down Expand Up @@ -522,22 +527,6 @@ export class BrowserView extends Disposable {
return this._consoleLogs.join('\n');
}

/**
* Start element inspection mode. Sets up a CDP overlay that highlights elements
* on hover. When the user clicks, the element data is returned and the overlay is removed.
* @param token Cancellation token to abort the inspection.
*/
async getElementData(token: CancellationToken): Promise<IElementData | undefined> {
return this._inspector.getElementData(token);
}

/**
* Get element data for the currently focused element.
*/
async getFocusedElementData(): Promise<IElementData | undefined> {
return this._inspector.getFocusedElementData();
}

/**
* Load a URL in this view
*/
Expand Down Expand Up @@ -609,7 +598,7 @@ export class BrowserView extends Disposable {
if (options?.pageRect) {
const zoomFactor = this._view.webContents.getZoomFactor();
// The visual viewport scale accounts for pinch-to-zoom magnification, which is separate from the regular zoom factor.
const visualViewportScale = await this._inspector.getVisualViewportScale();
const visualViewportScale = await this.inspector.getVisualViewportScale();
options.screenRect = {
x: options.pageRect.x * visualViewportScale * zoomFactor,
y: options.pageRect.y * visualViewportScale * zoomFactor,
Expand Down
137 changes: 87 additions & 50 deletions src/vs/platform/browserView/electron-main/browserViewElementInspector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { CancellationToken } from '../../../base/common/cancellation.js';
import { Emitter, Event } from '../../../base/common/event.js';
import { Disposable, DisposableStore } from '../../../base/common/lifecycle.js';
import { IElementData, IElementAncestor } from '../common/browserView.js';
import { collapseToShorthands, formatMatchedStyles, keyComputedProperties, type IMatchedStyles } from '../common/cssHelpers.js';
Expand Down Expand Up @@ -53,6 +53,17 @@ export class BrowserViewElementInspector extends Disposable {

private readonly _connectionPromise: Promise<ICDPConnection>;

private readonly _onDidSelectElement = this._register(new Emitter<IElementData>());
readonly onDidSelectElement: Event<IElementData> = this._onDidSelectElement.event;

private readonly _onDidChangeElementSelectionActive = this._register(new Emitter<boolean>());
readonly onDidChangeElementSelectionActive: Event<boolean> = this._onDidChangeElementSelectionActive.event;

private _elementSelectionActive = false;
get isElementSelectionActive(): boolean { return this._elementSelectionActive; }

private _selectionStore: DisposableStore | undefined;

constructor(private readonly browser: BrowserView) {
super();

Expand Down Expand Up @@ -80,67 +91,93 @@ export class BrowserViewElementInspector extends Disposable {
}

/**
* Start element inspection mode on the browser view. Sets up an
* overlay that highlights elements on hover. When the user clicks, the
* element data is returned and the overlay is removed.
* Toggle element selection mode on the browser view.
*
* When enabled, sets up a CDP overlay that highlights elements on hover.
* When the user picks an element, its data is fired via {@link onDidSelectElement}.
*
* @param token Cancellation token to abort the inspection.
* @param enabled Whether to enable or disable selection. Omit to toggle.
*/
async getElementData(token: CancellationToken): Promise<IElementData | undefined> {
async toggleElementSelection(enabled?: boolean): Promise<void> {
const newEnabled = enabled ?? !this._elementSelectionActive;
if (newEnabled === this._elementSelectionActive) {
return;
}

if (!newEnabled) {
await this._stopElementSelection();
return;
}

// Start selection
const connection = await this._connectionPromise;
const store = new DisposableStore();
const result = new Promise<IElementData | undefined>((resolve, reject) => {
store.add(token.onCancellationRequested(() => {
resolve(undefined);
}));

store.add(connection.onEvent(async (event) => {
if (event.method !== 'Overlay.inspectNodeRequested') {
return;
}

const params = event.params as { backendNodeId: number };
if (!params?.backendNodeId) {
reject(new Error('Missing backendNodeId in inspectNodeRequested event'));
return;
}
// Clean up any prior selection state
this._selectionStore?.dispose();
const store = this._selectionStore = new DisposableStore();

try {
const nodeData = await extractNodeData(connection, { backendNodeId: params.backendNodeId });
resolve({
...nodeData,
url: this.browser.getURL()
});
} catch (err) {
reject(err);
}
}));
});
store.add(connection.onEvent(async (event) => {
if (event.method !== 'Overlay.inspectNodeRequested') {
return;
}

const params = event.params as { backendNodeId: number };
if (!params?.backendNodeId) {
return;
}

try {
await connection.sendCommand('Overlay.setInspectMode', {
mode: 'searchForNode',
highlightConfig: inspectHighlightConfig,
});
return await result;
} finally {
try {
await connection.sendCommand('Overlay.setInspectMode', {
mode: 'none',
highlightConfig: { showInfo: false, showStyles: false }
const nodeData = await extractNodeData(connection, { backendNodeId: params.backendNodeId });
this._onDidSelectElement.fire({
...nodeData,
url: this.browser.getURL()
});
await connection.sendCommand('Overlay.hideHighlight');
await this._stopElementSelection();
} catch {
// Best effort cleanup
// Best effort - selection continues
}
store.dispose();
}));

await connection.sendCommand('Overlay.setInspectMode', {
mode: 'searchForNode',
highlightConfig: inspectHighlightConfig,
});

this._elementSelectionActive = true;
this._onDidChangeElementSelectionActive.fire(true);
}

private async _stopElementSelection(): Promise<void> {
if (!this._elementSelectionActive) {
return;
}

this._elementSelectionActive = false;
this._selectionStore?.dispose();
this._selectionStore = undefined;
this._onDidChangeElementSelectionActive.fire(false);

try {
const connection = await this._connectionPromise;
await connection.sendCommand('Overlay.setInspectMode', {
mode: 'none',
highlightConfig: { showInfo: false, showStyles: false }
});
await connection.sendCommand('Overlay.hideHighlight');
} catch {
// Best effort cleanup
}
}

/**
* Get element data for the currently focused element.
* Fire a selection event for the currently focused element.
* Only effective when element selection is active.
*/
async getFocusedElementData(): Promise<IElementData | undefined> {
async pickFocusedElement(): Promise<void> {
if (!this._elementSelectionActive) {
return;
}

const connection = await this._connectionPromise;

await connection.sendCommand('Runtime.enable');
Expand All @@ -150,14 +187,14 @@ export class BrowserViewElementInspector extends Disposable {
}) as { result: { objectId?: string } };

if (!result?.objectId) {
return undefined;
return;
}

const nodeData = await extractNodeData(connection, { objectId: result.objectId });
return {
this._onDidSelectElement.fire({
...nodeData,
url: this.browser.getURL()
};
});
}

async getVisualViewportScale(): Promise<number> {
Expand Down
35 changes: 11 additions & 24 deletions src/vs/platform/browserView/electron-main/browserViewMainService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
import { Emitter, Event } from '../../../base/common/event.js';
import { Disposable, DisposableMap } from '../../../base/common/lifecycle.js';
import { VSBuffer } from '../../../base/common/buffer.js';
import { CancellationToken, CancellationTokenSource } from '../../../base/common/cancellation.js';
import { IBrowserViewBounds, IBrowserViewState, IBrowserViewService, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, BrowserViewCommandId, IElementData, IBrowserViewOwner, IBrowserViewInfo, IBrowserViewCreatedEvent, IBrowserViewOpenOptions, IBrowserViewCreateOptions } from '../common/browserView.js';
import { IBrowserViewBounds, IBrowserViewState, IBrowserViewService, IBrowserViewCaptureScreenshotOptions, IBrowserViewFindInPageOptions, BrowserViewCommandId, IBrowserViewOwner, IBrowserViewInfo, IBrowserViewCreatedEvent, IBrowserViewOpenOptions, IBrowserViewCreateOptions } from '../common/browserView.js';
import { clipboard, Menu, MenuItem } from 'electron';
import { IEnvironmentMainService } from '../../environment/electron-main/environmentMainService.js';
import { createDecorator, IInstantiationService } from '../../instantiation/common/instantiation.js';
Expand Down Expand Up @@ -45,7 +44,6 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
}

private readonly browserViews = this._register(new DisposableMap<string, BrowserView>());
private readonly _activeTokens = new Map<number, CancellationTokenSource>();
private _keybindings: { [commandId: string]: string } = Object.create(null);

private readonly _onDidCreateBrowserView = this._register(new Emitter<IBrowserViewCreatedEvent>());
Expand Down Expand Up @@ -178,6 +176,14 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
return this._getBrowserView(id).onDidClose;
}

onDynamicDidSelectElement(id: string) {
return this._getBrowserView(id).inspector.onDidSelectElement;
}

onDynamicDidChangeElementSelectionActive(id: string) {
return this._getBrowserView(id).inspector.onDidChangeElementSelectionActive;
}

async getState(id: string): Promise<IBrowserViewState> {
return this._getBrowserView(id).getState();
}
Expand Down Expand Up @@ -281,33 +287,14 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
return this._getBrowserView(id).getConsoleLogs();
}

async getElementData(id: string, cancellationId: number): Promise<IElementData | undefined> {
return this._makeCancellable(cancellationId, (token) => this._getBrowserView(id).getElementData(token));
}

async getFocusedElementData(id: string): Promise<IElementData | undefined> {
return this._getBrowserView(id).getFocusedElementData();
}

async cancel(cancellationId: number): Promise<void> {
this._activeTokens.get(cancellationId)?.cancel();
async toggleElementSelection(id: string, enabled?: boolean): Promise<void> {
return this._getBrowserView(id).inspector.toggleElementSelection(enabled);
}

async updateKeybindings(keybindings: { [commandId: string]: string }): Promise<void> {
this._keybindings = keybindings;
}

private async _makeCancellable<T>(cancellationId: number, callback: (token: CancellationToken) => T | Promise<T>): Promise<T> {
const cts: CancellationTokenSource = new CancellationTokenSource();
this._activeTokens.set(cancellationId, cts);
try {
return await callback(cts.token);
} finally {
this._activeTokens.delete(cancellationId);
cts.dispose();
}
}

/**
* Create a browser view backed by the given {@link BrowserSession}.
*/
Expand Down
Loading
Loading