attach logs to chat from simple browser

src/vs/platform/browserElements/common/browserElements.ts

@@ -23,4 +23,8 @@ export interface INativeBrowserElementsService {
 	readonly windowId: number;
 
 	getElementData(rect: IRectangle, token: CancellationToken, cancellationId?: number): Promise<IElementData | undefined>;
+
+	startConsoleSession(token: CancellationToken, cancelAndDetachId?: number): Promise<void>;
+
+	getConsoleLogs(): Promise<string | undefined>;
 }

src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts

@@ -15,7 +15,6 @@ import { createDecorator } from '../../instantiation/common/instantiation.js';
 import { Disposable } from '../../../base/common/lifecycle.js';
 import { AddFirstParameterToFunctions } from '../../../base/common/types.js';
 
-
 export const INativeBrowserElementsMainService = createDecorator<INativeBrowserElementsMainService>('browserElementsMainService');
 export interface INativeBrowserElementsMainService extends AddFirstParameterToFunctions<INativeBrowserElementsService, Promise<unknown> /* only methods, not events */, number | undefined /* window ID */> { }
 
@@ -25,6 +24,8 @@ interface NodeDataResponse {
 	bounds: IRectangle;
 }
 
+const allConsole = new Map<number, string[]>();
+
 export class NativeBrowserElementsMainService extends Disposable implements INativeBrowserElementsMainService {
 	_serviceBrand: undefined;
 
@@ -38,6 +39,138 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 
 	get windowId(): never { throw new Error('Not implemented in electron-main'); }
 
+	async getConsoleLogs(windowId: number | undefined): Promise<string | undefined> {
+		const window = this.windowById(windowId);
+		if (!window?.win) {
+			return undefined;
+		}
+
+		const readable = (allConsole.get(window.id) ?? []).join('\n');
+		return readable;
+	}
+
+	// wait for potential webviews to be ready. on startup, the iframe with the simple browser content is not yet available
+	async waitForWebviewTargets(debuggers: any, windowId: number): Promise<any> {
+		const start = Date.now();
+		const timeout = 10000;
+
+		while (Date.now() - start < timeout) {
+			const { targetInfos } = await debuggers.sendCommand('Target.getTargets');
+
+			let resultId: string | undefined = undefined;
+			let target: typeof targetInfos[number] | undefined = undefined;
+
+			const matchingTarget = targetInfos.find((targetInfo: { url: string }) => {
+				const url = new URL(targetInfo.url);
+				return url.searchParams.get('parentId') === windowId.toString() && url.searchParams.get('extensionId') === 'vscode.simple-browser';
+			});
+
+			if (matchingTarget) {
+				const url = new URL(matchingTarget.url);
+				resultId = url.searchParams.get('id')!;
+			}
+
+			if (resultId) {
+				target = targetInfos.find((targetInfo: { url: string }) => {
+					const url = new URL(targetInfo.url);
+					return url.searchParams.get('id') === resultId && url.searchParams.get('vscodeBrowserReqId')!;
+				});
+			}
+
+			if (target) {
+				return target.targetId;
+			}
+
+			await new Promise(resolve => setTimeout(resolve, 250));
+		}
+
+		return undefined;
+	}
+
+	async startConsoleSession(windowId: number | undefined, token: CancellationToken, cancelAndDetachId?: number): Promise<void> {
+		const window = this.windowById(windowId);
+		if (!window?.win) {
+			return undefined;
+		}
+
+		// Find the simple browser webview
+		const allWebContents = webContents.getAllWebContents();
+		const simpleBrowserWebview = allWebContents.find(webContent => webContent.id === window.id);
+
+		if (!simpleBrowserWebview) {
+			return undefined;
+		}
+
+		const debuggers = simpleBrowserWebview.debugger;
+		if (!debuggers.isAttached()) {
+			debuggers.attach();
+		}
+
+		const onMessage = (event: any, method: string, params: any) => {
+			if (method === 'Runtime.consoleAPICalled' || method === 'Runtime.exceptionThrown' || method === 'Log.entryAdded') {
+				console.log('listener was hit here');
+				const current = allConsole.get(windowId!) ?? [];
+				const serialized = JSON.stringify(params);
+				if (!current.includes(serialized)) {
+					current.push(serialized);
+					allConsole.set(windowId!, current);
+				}
+			}
+		};
+
+		try {
+			const matchingTargetId = await this.waitForWebviewTargets(debuggers, windowId!);
+			if (!matchingTargetId) {
+				return undefined;
+			}
+
+			const { sessionId } = await debuggers.sendCommand('Target.attachToTarget', {
+				targetId: matchingTargetId,
+				flatten: true,
+			});
+
+			await debuggers.sendCommand('Debugger.enable', {}, sessionId);
+			await debuggers.sendCommand('Runtime.enable', {}, sessionId);
+			await debuggers.sendCommand('Log.enable', {}, sessionId);
+			console.log('turning on debugger');
+			debuggers.on('message', onMessage);
+		} catch (e) {
+			// debuggers.off('message', onMessage);
+			if (debuggers.isAttached()) {
+				debuggers.detach();
+			}
+			throw new Error('No target found', e);
+		}
+
+		window.win.webContents.on('ipc-message', async (event, channel, closedCancelAndDetachId) => {
+			if (channel === `vscode:changeElementSelection${cancelAndDetachId}`) {
+				if (cancelAndDetachId !== closedCancelAndDetachId) {
+					console.log('cancelAndDetachId does not match');
+					return;
+				}
+				console.log('turning off debugger');
+				debuggers.off('message', onMessage);
+				if (window.win) {
+					window.win.webContents.removeAllListeners('ipc-message');
+				}
+			}
+		});
+	}
+
+	async finishOverlay(debuggers: any, sessionId: string | undefined): Promise<void> {
+		if (debuggers.isAttached() && sessionId) {
+			await debuggers.sendCommand('Overlay.setInspectMode', {
+				mode: 'none',
+				highlightConfig: {
+					showInfo: false,
+					showStyles: false
+				}
+			}, sessionId);
+			await debuggers.sendCommand('Overlay.hideHighlight', {}, sessionId);
+			await debuggers.sendCommand('Overlay.disable', {}, sessionId);
+		}
+	}
+
 	async getElementData(windowId: number | undefined, rect: IRectangle, token: CancellationToken, cancellationId?: number): Promise<IElementData | undefined> {
 		const window = this.windowById(windowId);
 		if (!window?.win) {
@@ -53,12 +186,14 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		}
 
 		const debuggers = simpleBrowserWebview.debugger;
-		debuggers.attach();
+		if (!debuggers.isAttached()) {
+			debuggers.attach();
+		}
 
 		const { targetInfos } = await debuggers.sendCommand('Target.getTargets');
 		let resultId: string | undefined = undefined;
 		let target: typeof targetInfos[number] | undefined = undefined;
-		let targetSessionId: number | undefined = undefined;
+		let targetSessionId: string | undefined = undefined;
 		try {
 			// find parent id and extract id
 			const matchingTarget = targetInfos.find((targetInfo: { url: string }) => {
@@ -173,17 +308,19 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 				},
 			}, sessionId);
 		} catch (e) {
-			debuggers.detach();
+			await this.finishOverlay(debuggers, targetSessionId);
 			throw new Error('No target found', e);
 		}
 
 		if (!targetSessionId) {
-			debuggers.detach();
+			await this.finishOverlay(debuggers, targetSessionId);
 			throw new Error('No target session id found');
 		}
 
+
+		console.log('if this is logged before, then it means the onmessage triggers the other listener for some reason');
 		const nodeData = await this.getNodeData(targetSessionId, debuggers, window.win, cancellationId);
-		debuggers.detach();
+		await this.finishOverlay(debuggers, targetSessionId);
 
 		const zoomFactor = simpleBrowserWebview.getZoomFactor();
 		const absoluteBounds = {
@@ -210,7 +347,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		return { outerHTML: nodeData.outerHTML, computedStyle: nodeData.computedStyle, bounds: scaledBounds };
 	}
 
-	async getNodeData(sessionId: number, debuggers: any, window: BrowserWindow, cancellationId?: number): Promise<NodeDataResponse> {
+	async getNodeData(sessionId: string, debuggers: any, window: BrowserWindow, cancellationId?: number): Promise<NodeDataResponse> {
 		return new Promise((resolve, reject) => {
 			const onMessage = async (event: any, method: string, params: { backendNodeId: number }) => {
 				if (method === 'Overlay.inspectNodeRequested') {
@@ -265,7 +402,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 						});
 					} catch (err) {
 						debuggers.off('message', onMessage);
-						debuggers.detach();
+						await this.finishOverlay(debuggers, sessionId);
 						reject(err);
 					}
 				}
@@ -277,9 +414,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 						return;
 					}
 					debuggers.off('message', onMessage);
-					if (debuggers.isAttached()) {
-						debuggers.detach();
-					}
+					await this.finishOverlay(debuggers, sessionId);
 					window.webContents.removeAllListeners('ipc-message');
 				}
 			});

src/vs/workbench/contrib/chat/browser/chatEditing/simpleBrowserEditorOverlay.ts

@@ -35,6 +35,7 @@ import { IChatRequestVariableEntry } from '../../common/chatModel.js';
 import { IPreferencesService } from '../../../../services/preferences/common/preferences.js';
 import { IBrowserElementsService } from '../../../../services/browserElements/browser/browserElementsService.js';
 
+
 class SimpleBrowserOverlayWidget {
 
 	private readonly _domNode: HTMLElement;
@@ -89,6 +90,9 @@ class SimpleBrowserOverlayWidget {
 		cancelButton.element.className = 'element-selection-cancel hidden';
 		cancelButton.label = localize('cancel', 'Cancel');
 
+		const attachLogs = this._showStore.add(new Button(this._domNode, { supportIcons: true, title: localize('chat.attachLogs', "Attach Logs") }));
+		attachLogs.icon = Codicon.terminal;
+
 		const configure = this._showStore.add(new Button(this._domNode, { supportIcons: true, title: localize('chat.configureElements', "Configure Attachments Sent") }));
 		configure.icon = Codicon.gear;
 
@@ -99,6 +103,9 @@ class SimpleBrowserOverlayWidget {
 		nextSelection.icon = Codicon.close;
 		nextSelection.element.classList.add('hidden');
 
+
+		// attachLogs.element.classList.add('hidden');
+
 		// shown if the overlay is collapsed
 		const expandOverlay = this._showStore.add(new Button(this._domNode, { supportIcons: true, title: localize('chat.expandOverlay', "Expand Overlay") }));
 		expandOverlay.icon = Codicon.layout;
@@ -166,6 +173,10 @@ class SimpleBrowserOverlayWidget {
 		this._showStore.add(addDisposableListener(configure.element, 'click', () => {
 			this._preferencesService.openSettings({ jsonEditor: false, query: '@id:chat.sendElementsToChat.enabled,chat.sendElementsToChat.attachCSS,chat.sendElementsToChat.attachImages' });
 		}));
+
+		this._showStore.add(addDisposableListener(attachLogs.element, 'click', async () => {
+			await this.addConsolesToChat();
+		}));
 	}
 
 	hideElement(element: HTMLElement) {
@@ -234,6 +245,23 @@ class SimpleBrowserOverlayWidget {
 		widget?.attachmentModel?.addContext(...toAttach);
 	}
 
+	async addConsolesToChat() {
+		const logs = await this._browserElementsService.getConsoleLogs();
+		const toAttach: IChatRequestVariableEntry[] = [];
+
+		toAttach.push({
+			id: 'element-' + Date.now(),
+			name: localize('consoleLogs', 'Console Logs'),
+			fullName: localize('consoleLogs', 'Console Logs'),
+			value: logs,
+			kind: 'element',
+			icon: ThemeIcon.fromId(Codicon.terminal.id),
+		});
+
+		const widget = this._chatWidgetService.lastFocusedWidget ?? await showChatView(this._viewService);
+		widget?.attachmentModel?.addContext(...toAttach);
+	}
+
 
 	getDisplayNameFromOuterHTML(outerHTML: string): string {
 		const firstElementMatch = outerHTML.match(/^<(\w+)([^>]*?)>/);
@@ -269,6 +297,7 @@ class SimpleBrowserOverlayController {
 		group: IEditorGroup,
 		@IInstantiationService instaService: IInstantiationService,
 		@IConfigurationService private readonly configurationService: IConfigurationService,
+		@IBrowserElementsService private readonly _browserElementsService: IBrowserElementsService,
 	) {
 
 		if (!this.configurationService.getValue('chat.sendElementsToChat.enabled')) {
@@ -286,14 +315,19 @@ class SimpleBrowserOverlayController {
 		this._store.add(toDisposable(() => this._domNode.remove()));
 		this._store.add(widget);
 
-		const show = () => {
+		let cts = new CancellationTokenSource();
+		const show = async () => {
+			cts = new CancellationTokenSource();
+			await this._browserElementsService.startConsoleSession(cts.token);
 			if (!container.contains(this._domNode)) {
 				container.appendChild(this._domNode);
 			}
 		};
 
 		const hide = () => {
 			if (container.contains(this._domNode)) {
+				console.log('Hiding overlay');
+				cts.cancel();
 				this._domNode.remove();
 			}
 		};

src/vs/workbench/contrib/chat/browser/media/simpleBrowserOverlay.css

@@ -41,7 +41,8 @@
 .element-selection-message .monaco-button.codicon.codicon-close,
 .element-expand-container .monaco-button.codicon.codicon-layout,
 .element-selection-message .monaco-button.codicon.codicon-chevron-right,
-.element-selection-message .monaco-button.codicon.codicon-gear {
+.element-selection-message .monaco-button.codicon.codicon-gear,
+.element-selection-message .monaco-button.codicon.codicon-terminal {
 	width: 17px;
 	height: 17px;
 	padding: 2px 2px;

src/vs/workbench/services/browserElements/browser/browserElementsService.ts

@@ -15,4 +15,8 @@ export interface IBrowserElementsService {
 
 	// no browser implementation yet
 	getElementData(rect: IRectangle, token: CancellationToken, cancellationId?: number): Promise<IElementData | undefined>;
+
+	startConsoleSession(token: CancellationToken): Promise<void>;
+
+	getConsoleLogs(): Promise<string | undefined>;
 }

src/vs/workbench/services/browserElements/electron-sandbox/browserElementsService.ts

@@ -24,6 +24,7 @@ class WorkbenchNativeBrowserElementsService extends NativeBrowserElementsService
 }
 
 let cancelSelectionIdPool = 0;
+let cancelAndDetachIdPool = 0;
 
 class WorkbenchBrowserElementsService implements IBrowserElementsService {
 	_serviceBrand: undefined;
@@ -32,6 +33,27 @@ class WorkbenchBrowserElementsService implements IBrowserElementsService {
 		@INativeBrowserElementsService private readonly simpleBrowser: INativeBrowserElementsService
 	) { }
 
+	async getConsoleLogs(): Promise<string | undefined> {
+		return await this.simpleBrowser.getConsoleLogs();
+	}
+
+	async startConsoleSession(token: CancellationToken): Promise<void> {
+		const cancelAndDetachId = cancelAndDetachIdPool++;
+		const onCancelChannel = `vscode:changeElementSelection${cancelAndDetachId}`;
+
+		const disposable = token.onCancellationRequested(() => {
+			console.log('cancellation requested', cancelAndDetachId);
+			ipcRenderer.send(onCancelChannel, cancelAndDetachId);
+			disposable.dispose();
+		});
+		try {
+			await this.simpleBrowser.startConsoleSession(token, cancelAndDetachId);
+		} catch (error) {
+			disposable.dispose();
+			throw new Error('No target found error coming from browser', error);
+		}
+	}
+
 	async getElementData(rect: IRectangle, token: CancellationToken): Promise<IElementData | undefined> {
 		const cancelSelectionId = cancelSelectionIdPool++;
 		const onCancelChannel = `vscode:cancelElementSelection${cancelSelectionId}`;