diff --git a/.claude/skills/playwright-dev/dashboard.md b/.claude/skills/playwright-dev/dashboard.md index 60d255468ea80..606ed221e22b3 100644 --- a/.claude/skills/playwright-dev/dashboard.md +++ b/.claude/skills/playwright-dev/dashboard.md @@ -32,6 +32,4 @@ npx playwright cli video-stop # afterwards, use ffmpeg to turn the video into mp4 for sharing. ``` -For more about using Playwright CLI, look at `npx playwright cli --help` and the referenced Skill. -While developing in in this repo, it's important to use `npx playwright cli` instead of `playwright-cli`. - +Full CLI reference: `packages/playwright-core/src/tools/cli-client/skill/SKILL.md`. In this repo, invoke as `npx playwright cli` instead of `playwright-cli`. diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsx index 6b507049afe00..c3b5a73c0d36e 100644 --- a/packages/dashboard/src/annotations.tsx +++ b/packages/dashboard/src/annotations.tsx @@ -112,13 +112,43 @@ function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw: }; } +export async function saveAnnotationAsDownload(blob: Blob): Promise { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const suggestedName = `annotations-${stamp}.png`; + const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise); + if (picker) { + try { + const handle = await picker({ + suggestedName, + startIn: 'downloads', + types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }], + }); + const writable = await handle.createWritable(); + await writable.write(blob); + await writable.close(); + } catch (e: any) { + if (e?.name !== 'AbortError') + throw e; + } + return; + } + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = suggestedName; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); +} + export const Annotations: React.FC<{ active: boolean; displayRef: React.RefObject; screenRef: React.RefObject; viewportWidth: number; viewportHeight: number; - onSubmit?: (blob: Blob, annotations: Annotation[]) => Promise | void; + onSubmit: (blob: Blob, annotations: Annotation[]) => Promise | void; }> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight, onSubmit }) => { const [annotations, setAnnotations] = React.useState([]); const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null); @@ -327,37 +357,10 @@ export const Annotations: React.FC<{ const blob = await new Promise(resolve => canvas.toBlob(resolve, 'image/png')); if (!blob) return; - if (onSubmit) { - await onSubmit(blob, annotations); - return; - } - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const suggestedName = `annotations-${stamp}.png`; - const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise); - if (picker) { - try { - const handle = await picker({ - suggestedName, - startIn: 'downloads', - types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }], - }); - const writable = await handle.createWritable(); - await writable.write(blob); - await writable.close(); - } catch (e: any) { - if (e?.name !== 'AbortError') - throw e; - } - return; - } - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = suggestedName; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - URL.revokeObjectURL(url); + await onSubmit(blob, annotations); + setAnnotations([]); + setSelection(null); + setDraft(null); } if (!active) diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx index f72c6484fbce2..4761f56c1f9b9 100644 --- a/packages/dashboard/src/dashboard.tsx +++ b/packages/dashboard/src/dashboard.tsx @@ -19,7 +19,7 @@ import './dashboard.css'; import { DashboardClientContext } from './dashboardContext'; import { asLocator } from '@isomorphic/locatorGenerators'; import { ChevronLeftIcon, ChevronRightIcon, ReloadIcon } from './icons'; -import { Annotations, getImageLayout, clientToViewport } from './annotations'; +import { Annotations, getImageLayout, clientToViewport, saveAnnotationAsDownload } from './annotations'; import type { Annotation } from './annotations'; import { ToolbarButton } from '@web/components/toolbarButton'; @@ -74,7 +74,6 @@ export const Dashboard: React.FC = () => { const [recording, setRecording] = React.useState(false); const [screenshotIcon, setScreenshotIcon] = React.useState<'device-camera' | 'clippy'>('device-camera'); const [flashTick, setFlashTick] = React.useState(0); - const [pendingAnnotate, setPendingAnnotate] = React.useState(false); const [cliAnnotate, setCliAnnotate] = React.useState(false); const displayRef = React.useRef(null); @@ -85,6 +84,8 @@ export const Dashboard: React.FC = () => { const interactiveBtnRef = React.useRef(null); const moveThrottleRef = React.useRef(0); const modeRef = React.useRef('readonly'); + const cliAnnotateRef = React.useRef(false); + const cliAnnotateEnteredRef = React.useRef(false); const aspect = frame && frame.viewportWidth && frame.viewportHeight ? frame.viewportWidth / frame.viewportHeight @@ -115,6 +116,22 @@ export const Dashboard: React.FC = () => { modeRef.current = mode; }, [mode]); + React.useEffect(() => { + if (!cliAnnotate) { + cliAnnotateEnteredRef.current = false; + return; + } + if (cliAnnotateEnteredRef.current || !frame) + return; + cliAnnotateEnteredRef.current = true; + setMode('annotate'); + }, [cliAnnotate, frame]); + + const updateCliAnnotate = React.useCallback((value: boolean) => { + cliAnnotateRef.current = value; + setCliAnnotate(value); + }, []); + const interactive = mode === 'interactive'; const annotating = mode === 'annotate'; @@ -135,23 +152,11 @@ export const Dashboard: React.FC = () => { }; }, [flashTick, interactive]); - const hasFrame = !!frame; - React.useEffect(() => { - if (!pendingAnnotate || !hasFrame) - return; - setMode('annotate'); - setCliAnnotate(true); - setPendingAnnotate(false); - }, [pendingAnnotate, hasFrame]); - - React.useEffect(() => { - if (!annotating) - setCliAnnotate(false); - }, [annotating]); - - const submitAnnotationToCli = React.useCallback(async (blob: Blob, annotations: Annotation[]) => { - if (!client) + const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => { + if (!client || !cliAnnotate) { + await saveAnnotationAsDownload(blob); return; + } const dataUrl = await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(reader.result as string); @@ -163,8 +168,9 @@ export const Dashboard: React.FC = () => { data, annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })), }); + updateCliAnnotate(false); setMode('readonly'); - }, [client]); + }, [client, cliAnnotate, updateCliAnnotate]); function flashInteractiveHint() { setFlashTick(tick => tick + 1); @@ -179,7 +185,7 @@ export const Dashboard: React.FC = () => { const onTabs = (params: DashboardChannelEvents['tabs']) => { const prev = prevTabsRef.current; const selected = params.tabs.find(t => t.selected); - if (prev && selected && !prev.some(t => t.page === selected.page)) + if (prev && selected && !prev.some(t => t.page === selected.page) && !cliAnnotateRef.current) setMode('interactive'); prevTabsRef.current = params.tabs; setTabs(params.tabs); @@ -210,27 +216,44 @@ export const Dashboard: React.FC = () => { setMode('interactive'); setPicking(true); }; - const onAnnotate = () => setPendingAnnotate(true); + const onAnnotate = () => updateCliAnnotate(true); + const onCancelAnnotate = () => { + if (modeRef.current === 'annotate') + setMode('readonly'); + updateCliAnnotate(false); + }; client.on('tabs', onTabs); client.on('frame', onFrame); client.on('elementPicked', onElementPicked); client.on('pickLocator', onPickLocator); client.on('annotate', onAnnotate); + client.on('cancelAnnotate', onCancelAnnotate); return () => { client.off('tabs', onTabs); client.off('frame', onFrame); client.off('elementPicked', onElementPicked); client.off('pickLocator', onPickLocator); client.off('annotate', onAnnotate); + client.off('cancelAnnotate', onCancelAnnotate); }; }, [client]); const selectedTab = tabs?.find(t => t.selected); const ready = !!client && !!selectedTab; + const prevSelectedPageRef = React.useRef(undefined); React.useEffect(() => { + const prev = prevSelectedPageRef.current; + const current = selectedTab?.page; + prevSelectedPageRef.current = current; setRecording(false); setPicking(false); + if (!prev || !current || prev === current) + return; + setFrame(undefined); + cliAnnotateEnteredRef.current = false; + if (modeRef.current === 'annotate') + setMode('readonly'); }, [selectedTab?.page]); function imgCoords(e: React.MouseEvent): { x: number; y: number } { @@ -498,7 +521,7 @@ export const Dashboard: React.FC = () => { screenRef={screenRef} viewportWidth={frame?.viewportWidth ?? 0} viewportHeight={frame?.viewportHeight ?? 0} - onSubmit={cliAnnotate ? submitAnnotationToCli : undefined} + onSubmit={onSubmitAnnotations} /> {overlayText &&
{overlayText}
} diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts index a6b44506752c4..d424c0911fc0b 100644 --- a/packages/dashboard/src/dashboardChannel.ts +++ b/packages/dashboard/src/dashboardChannel.ts @@ -37,6 +37,7 @@ export type DashboardChannelEvents = { elementPicked: { selector: string; ariaSnapshot?: string }; pickLocator: {}; annotate: {}; + cancelAnnotate: {}; }; export type MouseButton = 'left' | 'middle' | 'right'; diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md index bd4fdc308a55a..10a0ca04c46c1 100644 --- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md +++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md @@ -356,22 +356,13 @@ playwright-cli tracing-stop playwright-cli close ``` -## Example: Interactive element inspection +## Example: Interactive session -Ask the user to point at an element in the browser, then keep it visible while you work on it: +Ask the user to annotate the UI. User can provide contextual tasks or ask contextual questions using annotations: ```bash playwright-cli open https://example.com -# blocks until the user clicks an element; prints `ref: eN` and the locator -playwright-cli pick -# keep the picked element highlighted while iterating; style is optional -playwright-cli highlight e5 --style="outline: 3px dashed red" -playwright-cli highlight e7 -# ... inspect, generate code, etc. ... -# hide a single highlight, or drop them all in one shot -playwright-cli highlight e5 --hide -playwright-cli highlight --hide -playwright-cli close +playwright-cli show --annotate ``` ## Specific tasks diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index 9267dc725dc7d..f2b9d5091b8a7 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -345,15 +345,6 @@ const snapshot = declareCommand({ toolParams: ({ filename, target, depth }) => ({ filename, target, depth }), }); -const pick = declareCommand({ - name: 'pick', - description: 'Wait for the user to pick an element in the browser and print its ref and locator', - category: 'devtools', - args: z.object({}), - toolName: 'browser_pick_locator', - toolParams: () => ({}), -}); - const generateLocator = declareCommand({ name: 'generate-locator', description: 'Generate a Playwright locator for the given element', @@ -1085,7 +1076,6 @@ const commandsArray: AnyCommandSchema[] = [ pauseAt, resume, stepOver, - pick, generateLocator, highlight, diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts index f044243c98788..43f486126a215 100644 --- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts +++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts @@ -106,9 +106,20 @@ async function startDashboardServer(options: DashboardOptions): Promise { + pendingAnnotate = false; + for (const connection of connections) + connection.emitCancelAnnotate(); + }; + const registerAnnotateWaiter = (socket: net.Socket) => { waitingSockets.add(socket); - const cleanup = () => waitingSockets.delete(socket); + const cleanup = () => { + if (!waitingSockets.delete(socket)) + return; + if (waitingSockets.size === 0) + notifyAnnotateEnded(); + }; socket.on('close', cleanup); socket.on('error', cleanup); }; @@ -403,7 +414,7 @@ async function runAnnotateClient(options: DashboardOptions): Promise { console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`); } // eslint-disable-next-line no-console - console.log(`image available at: ${path.relative(process.cwd(), filePath)}`); + console.log(`image: ${path.relative(process.cwd(), filePath)}`); } function selfDestructOnParentGone() { diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts index 3460386245a5d..0bfebe4e67c14 100644 --- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts +++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts @@ -73,38 +73,29 @@ class BrowserTracker { private _wireContext(context: api.BrowserContext) { if (this._contextListeners.has(context)) return; - const listeners: Disposable[] = []; - this._contextListeners.set(context, listeners); - const watchPage = (page: api.Page) => { - listeners.push( - eventsHelper.addEventListener(page, 'load', () => this._callbacks.onTabsChanged()), - eventsHelper.addEventListener(page, 'framenavigated', (frame: api.Frame) => { - if (frame === page.mainFrame()) - this._callbacks.onTabsChanged(); - }), - eventsHelper.addEventListener(page, 'close', () => this._callbacks.onTabsChanged()), - ); - }; - listeners.push( - eventsHelper.addEventListener(context, 'page', (page: api.Page) => { - watchPage(page); - this._callbacks.onTabsChanged(); - }), - eventsHelper.addEventListener(context, 'picklocator', (page: api.Page) => { - this._callbacks.onPickLocator(page); - }), - eventsHelper.addEventListener(context, 'close', () => { - const ls = this._contextListeners.get(context); - if (ls) { - ls.forEach(d => d.dispose()); - this._contextListeners.delete(context); - } - this._callbacks.onContextClosed(context); + const onTabsChanged = () => this._callbacks.onTabsChanged(); + const listeners: Disposable[] = [ + eventsHelper.addEventListener(context, 'page', onTabsChanged), + eventsHelper.addEventListener(context, 'pageload', onTabsChanged), + eventsHelper.addEventListener(context, 'pageclose', onTabsChanged), + eventsHelper.addEventListener(context, 'framenavigated', (frame: api.Frame) => { + if (frame === frame.page().mainFrame()) this._callbacks.onTabsChanged(); - }), - ); - for (const page of context.pages()) - watchPage(page); + }), + eventsHelper.addEventListener(context, 'picklocator', (page: api.Page) => { + this._callbacks.onPickLocator(page); + }), + eventsHelper.addEventListener(context, 'close', () => { + const ls = this._contextListeners.get(context); + if (ls) { + ls.forEach(d => d.dispose()); + this._contextListeners.delete(context); + } + this._callbacks.onContextClosed(context); + this._callbacks.onTabsChanged(); + }), + ]; + this._contextListeners.set(context, listeners); this._callbacks.onTabsChanged(); } } @@ -304,6 +295,10 @@ export class DashboardConnection implements Transport { this.sendEvent?.('annotate', {}); } + emitCancelAnnotate() { + this.sendEvent?.('cancelAnnotate', {}); + } + _pushTabs() { if (this._pushTabsScheduled) return; @@ -344,14 +339,22 @@ export class DashboardConnection implements Transport { if (this._attachedPage?.page === page) return; this._attachedPage?.dispose(); - this._attachedPage = undefined; const browser = page.context().browser(); const slot = browser ? [...this._browsers.values()].find(s => s.browser === browser) : undefined; - if (!slot) + if (!slot) { + this._attachedPage = undefined; return; + } const attached = new AttachedPage(this, slot, page); - await attached.init(); this._attachedPage = attached; + try { + await attached.init(); + } catch (e) { + if (this._attachedPage === attached) + this._attachedPage = undefined; + attached.dispose(); + throw e; + } } _handleAttachedPageClose(context: api.BrowserContext) { @@ -455,6 +458,7 @@ class AttachedPage { private _listeners: Disposable[] = []; private _screencastRunning = false; private _recordingPath: string | null = null; + private _disposed = false; constructor(owner: DashboardConnection, slot: BrowserTracker, page: api.Page) { this._owner = owner; @@ -483,6 +487,7 @@ class AttachedPage { } dispose() { + this._disposed = true; this._listeners.forEach(d => d.dispose()); this._listeners = []; if (this._screencastRunning) @@ -581,7 +586,12 @@ class AttachedPage { private async _startScreencast(page: api.Page) { await page.screencast.start({ - onFrame: ({ data }: { data: Buffer }) => this._owner.emitFrame(data.toString('base64'), page.viewportSize()?.width ?? 0, page.viewportSize()?.height ?? 0), + onFrame: ({ data }: { data: Buffer }) => { + if (this._disposed) + return; + const vp = page.viewportSize(); + this._owner.emitFrame(data.toString('base64'), vp?.width ?? 0, vp?.height ?? 0); + }, size: { width: 1280, height: 800 }, ...(this._recordingPath ? { path: this._recordingPath } : {}), }); diff --git a/tests/mcp/cli-devtools.spec.ts b/tests/mcp/cli-devtools.spec.ts index 69a58ba607faf..fdd40636d7bc5 100644 --- a/tests/mcp/cli-devtools.spec.ts +++ b/tests/mcp/cli-devtools.spec.ts @@ -160,49 +160,6 @@ test('video-chapter', async ({ cli, server }) => { await cli('video-stop'); }); -test('pick', async ({ boundBrowser, cli }) => { - const page = await boundBrowser.newPage(); - await page.setContent(``); - - await cli('attach', 'default'); - await cli('snapshot'); - - const scriptReady = page.waitForEvent('console', msg => msg.text() === 'Recorder script ready for test'); - const pickPromise = cli('pick'); - await scriptReady; - - const box = await page.getByRole('button', { name: 'Submit' }).boundingBox(); - await page.mouse.click(box!.x + box!.width / 2, box!.y + box!.height / 2); - - const { output } = await pickPromise; - expect(output).toContain(`ref: e2`); - expect(output).toContain(`locator: getByRole('button', { name: 'Submit' })`); -}); - -test('pick activates dashboard session', async ({ boundBrowser, cli, startDashboardServer }) => { - const page = await boundBrowser.newPage(); - await page.setContent(``); - - await cli('attach', 'default'); - await cli('snapshot'); - - const dashboard = await startDashboardServer(); - await expect(dashboard.locator('div.dashboard-view')).toBeVisible(); - - const scriptReady = page.waitForEvent('console', msg => msg.text() === 'Recorder script ready for test'); - const pickPromise = cli('pick'); - await scriptReady; - - await expect(dashboard.locator('div.dashboard-view.interactive')).toBeVisible(); - - const box = await page.getByRole('button', { name: 'Submit' }).boundingBox(); - await page.mouse.click(box!.x + box!.width / 2, box!.y + box!.height / 2); - - const { output } = await pickPromise; - expect(output).toContain(`ref: e2`); - expect(output).toContain(`locator: getByRole('button', { name: 'Submit' })`); -}); - test('generate-locator', async ({ cli, server }) => { server.setContent('/', ``, 'text/html'); await cli('open', server.PREFIX); diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts index 15c57d21b712a..42490d8fd88c6 100644 --- a/tests/mcp/core.spec.ts +++ b/tests/mcp/core.spec.ts @@ -81,7 +81,7 @@ test('browser_navigate can navigate to file:// URLs allowUnrestrictedFileAccess name: 'browser_navigate', arguments: { url }, })).toHaveResponse({ - page: `- Page URL: ${url}`, + page: expect.stringContaining(`- Page URL: ${url}`), snapshot: `- generic [ref=e2]: Test file content`, }); }); diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts index a8a5c7f39b92d..a6c5fdbac8d4a 100644 --- a/tests/mcp/dashboard.spec.ts +++ b/tests/mcp/dashboard.spec.ts @@ -19,6 +19,7 @@ import os from 'os'; import path from 'path'; import { test, expect } from './cli-fixtures'; +import { inheritAndCleanEnv } from '../config/utils'; function displayPath(p: string): string { const home = os.homedir(); @@ -149,8 +150,8 @@ async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) { const lines = output.trim().split('\n'); expect(lines[0]).toMatch(new RegExp(`^\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}$`)); - expect(lines[lines.length - 1]).toMatch(/^image available at: \.playwright-cli[\\/]annotations-.*\.png$/); - const pngRel = lines[lines.length - 1].replace(/^image available at: /, ''); + expect(lines[lines.length - 1]).toMatch(/^image: \.playwright-cli[\\/]annotations-.*\.png$/); + const pngRel = lines[lines.length - 1].replace(/^image: /, ''); const pngPath = path.resolve(outputDir, pngRel); expect(fs.existsSync(pngPath)).toBe(true); expect(fs.statSync(pngPath).size).toBeGreaterThan(0); @@ -199,28 +200,138 @@ test('should start dashboard and annotate when no dashboard is running', async ( verifyAnnotateOutput(output, 'hi', test.info().outputDir); }); -test('should pick locator from browser', async ({ cli, server, startDashboardServer }) => { - server.setContent('/', '', 'text/html'); - - await cli('open', server.PREFIX); +test('should keep CLI annotate engaged across mode switches', async ({ connectToDashboard, cli, server }) => { + await cli('open', server.EMPTY_PAGE); + const bindTitle = `--playwright-internal--${crypto.randomUUID()}`; + await cli('show', { bindTitle }); + const browser = await connectToDashboard(bindTitle); - const dashboard = await startDashboardServer(); + const dashboard = browser.contexts()[0].pages()[0]; await dashboard.locator('.sidebar-tab').first().click(); - const pickPromise = cli('pick'); + const annotatePromise = cli('show', '--annotate'); + let done = false; + void annotatePromise.finally(() => { done = true; }); + + await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible(); + + await dashboard.locator('.mode-toggle.mode-interactive').click(); + await expect(dashboard.locator('div.dashboard-view')).toHaveClass(/interactive/); + await expect(dashboard.locator('div.dashboard-view')).not.toHaveClass(/annotate/); + + const box = await dashboard.locator('img#display').boundingBox(); + await dashboard.mouse.click(box!.x + box!.width / 2, box!.y + box!.height / 2); + + await dashboard.locator('.mode-toggle.mode-annotate').click(); + await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible(); + + await drawAndSubmitAnnotation(dashboard, 'round-trip'); + + const { output, exitCode } = await annotatePromise; + expect(done).toBe(true); + expect(exitCode).toBe(0); + verifyAnnotateOutput(output, 'round-trip', test.info().outputDir); +}); + +test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => { + await cli('-s=first', 'open', server.EMPTY_PAGE); + await cli('-s=second', 'open', server.EMPTY_PAGE); + + const bindTitle = `--playwright-internal--${crypto.randomUUID()}`; + const annotatePromise = cli('-s=second', 'show', '--annotate', { bindTitle }); + let done = false; + void annotatePromise.finally(() => { done = true; }); + + const browser = await connectToDashboard(bindTitle); + try { + const dashboard = browser.contexts()[0].pages()[0]; + await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible(); + const activeSession = dashboard.locator('.sidebar-session:has(.sidebar-tab.active)'); + await expect(activeSession.locator('.session-chip-name')).toHaveText('second'); + await drawAndSubmitAnnotation(dashboard, 'fresh'); + } finally { + await browser.close().catch(() => {}); + } + + const { exitCode } = await annotatePromise; + expect(done).toBe(true); + expect(exitCode).toBe(0); +}); + +test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => { + server.setContent('/red', '', 'text/html'); + server.setContent('/green', '', 'text/html'); + + await cli('-s=first', 'open', server.PREFIX + '/red'); + await cli('-s=second', 'open', server.PREFIX + '/green'); + + const bindTitle = `--playwright-internal--${crypto.randomUUID()}`; + await cli('-s=first', 'show', { bindTitle }); + const browser = await connectToDashboard(bindTitle); + const dashboard = browser.contexts()[0].pages()[0]; + await expect(dashboard.locator('#display')).toBeVisible(); + + const sampleCenter = () => dashboard.evaluate(() => { + const img = document.querySelector('#display') as HTMLImageElement | null; + if (!img || !img.naturalWidth) + return null; + const canvas = document.createElement('canvas'); + canvas.width = 1; + canvas.height = 1; + const ctx = canvas.getContext('2d')!; + ctx.drawImage(img, img.naturalWidth / 2, img.naturalHeight / 2, 1, 1, 0, 0, 1, 1); + const [r, g] = ctx.getImageData(0, 0, 1, 1).data; + return { r, g }; + }); + + await expect.poll(async () => { + const c = await sampleCenter(); + return !!(c && c.r > 200 && c.g < 50); + }, { timeout: 15000 }).toBe(true); + + const annotatePromise = cli('-s=second', 'show', '--annotate'); let done = false; - void pickPromise.finally(() => { done = true; }); + void annotatePromise.finally(() => { done = true; }); + + await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible(); + const activeSession = dashboard.locator('.sidebar-session:has(.sidebar-tab.active)'); + await expect(activeSession.locator('.session-chip-name')).toHaveText('second'); + + await expect.poll(async () => { + const c = await sampleCenter(); + return !!(c && c.g > 200 && c.r < 50); + }, { timeout: 15000 }).toBe(true); - await expect(dashboard.locator('div.dashboard-view.interactive')).toBeVisible(); + await drawAndSubmitAnnotation(dashboard, 'session switch'); + const { exitCode } = await annotatePromise; + expect(done).toBe(true); + expect(exitCode).toBe(0); +}); + +test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => { + await cli('open', server.EMPTY_PAGE); + const bindTitle = `--playwright-internal--${crypto.randomUUID()}`; + await cli('show', { bindTitle }); + const browser = await connectToDashboard(bindTitle); + + const dashboard = browser.contexts()[0].pages()[0]; + await dashboard.locator('.sidebar-tab').first().click(); + + const annotateClient = childProcess({ + command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'], + cwd: test.info().outputPath(), + env: inheritAndCleanEnv({ + ...cliEnv, + PLAYWRIGHT_MCP_BROWSER: mcpBrowser, + PLAYWRIGHT_MCP_HEADLESS: String(mcpHeadless), + }), + }); + + await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible(); - await expect(async () => { - const box = await dashboard.locator('img#display').boundingBox(); - await dashboard.mouse.click(box!.x + box!.width / 2, box!.y + box!.height / 2); - expect(done).toBe(true); - }).toPass(); + await annotateClient.kill(); - const { output } = await pickPromise; - expect(output).toContain(`getByRole('button', { name: 'Submit' })`); + await expect(dashboard.locator('div.dashboard-view')).not.toHaveClass(/annotate/); }); async function installSaveFilePickerMock(page: import('playwright-core').Page): Promise<() => Promise> {