feat(dashboard): cancel CLI annotate on client disconnect

.claude/skills/playwright-dev/dashboard.md

@@ -32,6 +32,4 @@ npx playwright cli video-stop
 # afterwards, use ffmpeg to turn the video into mp4 for sharing.
 ```
 
-For more about using Playwright CLI, look at `npx playwright cli --help` and the referenced Skill.
-While developing in in this repo, it's important to use `npx playwright cli` instead of `playwright-cli`.
-
+Full CLI reference: `packages/playwright-core/src/tools/cli-client/skill/SKILL.md`. In this repo, invoke as `npx playwright cli` instead of `playwright-cli`.

packages/dashboard/src/annotations.tsx

@@ -112,13 +112,43 @@ function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw:
   };
 }
 
+export async function saveAnnotationAsDownload(blob: Blob): Promise<void> {
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const suggestedName = `annotations-${stamp}.png`;
+  const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);
+  if (picker) {
+    try {
+      const handle = await picker({
+        suggestedName,
+        startIn: 'downloads',
+        types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }],
+      });
+      const writable = await handle.createWritable();
+      await writable.write(blob);
+      await writable.close();
+    } catch (e: any) {
+      if (e?.name !== 'AbortError')
+        throw e;
+    }
+    return;
+  }
+  const url = URL.createObjectURL(blob);
+  const link = document.createElement('a');
+  link.href = url;
+  link.download = suggestedName;
+  document.body.appendChild(link);
+  link.click();
+  document.body.removeChild(link);
+  URL.revokeObjectURL(url);
+}
+
 export const Annotations: React.FC<{
   active: boolean;
   displayRef: React.RefObject<HTMLImageElement | null>;
   screenRef: React.RefObject<HTMLDivElement | null>;
   viewportWidth: number;
   viewportHeight: number;
-  onSubmit?: (blob: Blob, annotations: Annotation[]) => Promise<void> | void;
+  onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void> | void;
 }> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight, onSubmit }) => {
   const [annotations, setAnnotations] = React.useState<Annotation[]>([]);
   const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);
@@ -327,37 +357,10 @@ export const Annotations: React.FC<{
     const blob = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));
     if (!blob)
       return;
-    if (onSubmit) {
-      await onSubmit(blob, annotations);
-      return;
-    }
-    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
-    const suggestedName = `annotations-${stamp}.png`;
-    const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);
-    if (picker) {
-      try {
-        const handle = await picker({
-          suggestedName,
-          startIn: 'downloads',
-          types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }],
-        });
-        const writable = await handle.createWritable();
-        await writable.write(blob);
-        await writable.close();
-      } catch (e: any) {
-        if (e?.name !== 'AbortError')
-          throw e;
-      }
-      return;
-    }
-    const url = URL.createObjectURL(blob);
-    const link = document.createElement('a');
-    link.href = url;
-    link.download = suggestedName;
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-    URL.revokeObjectURL(url);
+    await onSubmit(blob, annotations);
+    setAnnotations([]);
+    setSelection(null);
+    setDraft(null);
   }
 
   if (!active)

packages/dashboard/src/dashboard.tsx

@@ -19,7 +19,7 @@
 import { DashboardClientContext } from './dashboardContext';
 import { asLocator } from '@isomorphic/locatorGenerators';
 import { ChevronLeftIcon, ChevronRightIcon, ReloadIcon } from './icons';
-import { Annotations, getImageLayout, clientToViewport } from './annotations';
+import { Annotations, getImageLayout, clientToViewport, saveAnnotationAsDownload } from './annotations';
 
 import type { Annotation } from './annotations';
 import { ToolbarButton } from '@web/components/toolbarButton';
@@ -74,7 +74,6 @@
   const [recording, setRecording] = React.useState(false);
   const [screenshotIcon, setScreenshotIcon] = React.useState<'device-camera' | 'clippy'>('device-camera');
   const [flashTick, setFlashTick] = React.useState(0);
-  const [pendingAnnotate, setPendingAnnotate] = React.useState(false);
   const [cliAnnotate, setCliAnnotate] = React.useState(false);
 
   const displayRef = React.useRef<HTMLImageElement>(null);
@@ -85,6 +84,8 @@
   const interactiveBtnRef = React.useRef<HTMLButtonElement>(null);
   const moveThrottleRef = React.useRef(0);
   const modeRef = React.useRef<Mode>('readonly');
+  const cliAnnotateRef = React.useRef(false);
+  const cliAnnotateEnteredRef = React.useRef(false);
 
   const aspect = frame && frame.viewportWidth && frame.viewportHeight
     ? frame.viewportWidth / frame.viewportHeight
@@ -115,6 +116,22 @@
     modeRef.current = mode;
   }, [mode]);
 
+  React.useEffect(() => {
+    if (!cliAnnotate) {
+      cliAnnotateEnteredRef.current = false;
+      return;
+    }
+    if (cliAnnotateEnteredRef.current || !frame)
+      return;
+    cliAnnotateEnteredRef.current = true;
+    setMode('annotate');
+  }, [cliAnnotate, frame]);
+
+  const updateCliAnnotate = React.useCallback((value: boolean) => {
+    cliAnnotateRef.current = value;
+    setCliAnnotate(value);
+  }, []);
+
   const interactive = mode === 'interactive';
   const annotating = mode === 'annotate';
 
@@ -135,23 +152,11 @@
     };
   }, [flashTick, interactive]);
 
-  const hasFrame = !!frame;
-  React.useEffect(() => {
-    if (!pendingAnnotate || !hasFrame)
-      return;
-    setMode('annotate');
-    setCliAnnotate(true);
-    setPendingAnnotate(false);
-  }, [pendingAnnotate, hasFrame]);
-
-  React.useEffect(() => {
-    if (!annotating)
-      setCliAnnotate(false);
-  }, [annotating]);
-
-  const submitAnnotationToCli = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {
-    if (!client)
+  const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {
+    if (!client || !cliAnnotate) {
+      await saveAnnotationAsDownload(blob);
       return;
+    }
     const dataUrl = await new Promise<string>((resolve, reject) => {
       const reader = new FileReader();
       reader.onload = () => resolve(reader.result as string);
@@ -163,8 +168,9 @@
       data,
       annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
     });
+    updateCliAnnotate(false);
     setMode('readonly');
-  }, [client]);
+  }, [client, cliAnnotate, updateCliAnnotate]);
 
   function flashInteractiveHint() {
     setFlashTick(tick => tick + 1);
@@ -179,7 +185,7 @@
     const onTabs = (params: DashboardChannelEvents['tabs']) => {
       const prev = prevTabsRef.current;
       const selected = params.tabs.find(t => t.selected);
-      if (prev && selected && !prev.some(t => t.page === selected.page))
+      if (prev && selected && !prev.some(t => t.page === selected.page) && !cliAnnotateRef.current)
         setMode('interactive');
       prevTabsRef.current = params.tabs;
       setTabs(params.tabs);
@@ -210,27 +216,44 @@
       setMode('interactive');
       setPicking(true);
     };
-    const onAnnotate = () => setPendingAnnotate(true);
+    const onAnnotate = () => updateCliAnnotate(true);
+    const onCancelAnnotate = () => {
+      if (modeRef.current === 'annotate')
+        setMode('readonly');
+      updateCliAnnotate(false);
+    };
     client.on('tabs', onTabs);
     client.on('frame', onFrame);
     client.on('elementPicked', onElementPicked);
     client.on('pickLocator', onPickLocator);
     client.on('annotate', onAnnotate);
+    client.on('cancelAnnotate', onCancelAnnotate);
     return () => {
       client.off('tabs', onTabs);
       client.off('frame', onFrame);
       client.off('elementPicked', onElementPicked);
       client.off('pickLocator', onPickLocator);
       client.off('annotate', onAnnotate);
+      client.off('cancelAnnotate', onCancelAnnotate);
     };
   }, [client]);
 
   const selectedTab = tabs?.find(t => t.selected);
   const ready = !!client && !!selectedTab;
 
+  const prevSelectedPageRef = React.useRef<string | undefined>(undefined);
   React.useEffect(() => {
+    const prev = prevSelectedPageRef.current;
+    const current = selectedTab?.page;
+    prevSelectedPageRef.current = current;
     setRecording(false);
     setPicking(false);
+    if (!prev || !current || prev === current)
+      return;
+    setFrame(undefined);
+    cliAnnotateEnteredRef.current = false;
+    if (modeRef.current === 'annotate')
+      setMode('readonly');
   }, [selectedTab?.page]);
 
   function imgCoords(e: React.MouseEvent): { x: number; y: number } {
@@ -498,7 +521,7 @@
                 screenRef={screenRef}
                 viewportWidth={frame?.viewportWidth ?? 0}
                 viewportHeight={frame?.viewportHeight ?? 0}
-                onSubmit={cliAnnotate ? submitAnnotationToCli : undefined}
+                onSubmit={onSubmitAnnotations}
               />
             </div>
             {overlayText && <div className={'screen-overlay' + (frame ? ' has-frame' : '')}><span>{overlayText}</span></div>}

packages/dashboard/src/dashboardChannel.ts

@@ -37,6 +37,7 @@ export type DashboardChannelEvents = {
   elementPicked: { selector: string; ariaSnapshot?: string };
   pickLocator: {};
   annotate: {};
+  cancelAnnotate: {};
 };
 
 export type MouseButton = 'left' | 'middle' | 'right';

packages/playwright-core/src/tools/cli-client/skill/SKILL.md

@@ -356,22 +356,13 @@ playwright-cli tracing-stop
 playwright-cli close
 ```
 
-## Example: Interactive element inspection
+## Example: Interactive session
 
-Ask the user to point at an element in the browser, then keep it visible while you work on it:
+Ask the user to annotate the UI. User can provide contextual tasks or ask contextual questions using annotations:
 
 ```bash
 playwright-cli open https://example.com
-# blocks until the user clicks an element; prints `ref: eN` and the locator
-playwright-cli pick
-# keep the picked element highlighted while iterating; style is optional
-playwright-cli highlight e5 --style="outline: 3px dashed red"
-playwright-cli highlight e7
-# ... inspect, generate code, etc. ...
-# hide a single highlight, or drop them all in one shot
-playwright-cli highlight e5 --hide
-playwright-cli highlight --hide
-playwright-cli close
+playwright-cli show --annotate
 ```
 
 ## Specific tasks

packages/playwright-core/src/tools/cli-daemon/commands.ts

@@ -345,15 +345,6 @@ const snapshot = declareCommand({
   toolParams: ({ filename, target, depth }) => ({ filename, target, depth }),
 });
 
-const pick = declareCommand({
-  name: 'pick',
-  description: 'Wait for the user to pick an element in the browser and print its ref and locator',
-  category: 'devtools',
-  args: z.object({}),
-  toolName: 'browser_pick_locator',
-  toolParams: () => ({}),
-});
-
 const generateLocator = declareCommand({
   name: 'generate-locator',
   description: 'Generate a Playwright locator for the given element',
@@ -1085,7 +1076,6 @@ const commandsArray: AnyCommandSchema[] = [
   pauseAt,
   resume,
   stepOver,
-  pick,
   generateLocator,
   highlight,
 

packages/playwright-core/src/tools/dashboard/dashboardApp.ts

@@ -106,9 +106,20 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboar
       connection.emitAnnotate();
   };
 
+  const notifyAnnotateEnded = () => {
+    pendingAnnotate = false;
+    for (const connection of connections)
+      connection.emitCancelAnnotate();
+  };
+
   const registerAnnotateWaiter = (socket: net.Socket) => {
     waitingSockets.add(socket);
-    const cleanup = () => waitingSockets.delete(socket);
+    const cleanup = () => {
+      if (!waitingSockets.delete(socket))
+        return;
+      if (waitingSockets.size === 0)
+        notifyAnnotateEnded();
+    };
     socket.on('close', cleanup);
     socket.on('error', cleanup);
   };
@@ -403,7 +414,7 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {
     console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
   }
   // eslint-disable-next-line no-console
-  console.log(`image available at: ${path.relative(process.cwd(), filePath)}`);
+  console.log(`image: ${path.relative(process.cwd(), filePath)}`);
 }
 
 function selfDestructOnParentGone() {