Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions packages/playwright-core/src/tools/cli-client/program.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,13 @@ export async function program(options?: { embedderVersion?: string}) {
await new Promise<void>(resolve => child.on('exit', () => resolve()));
return;
}
if (args.annotate) {
const dashboard = spawn(process.execPath, daemonArgs, { detached: true, stdio: 'ignore' });
dashboard.unref();
const annotate = spawn(process.execPath, [...daemonArgs, '--annotate'], { stdio: 'inherit' });
await new Promise<void>(resolve => annotate.on('exit', () => resolve()));
return;
}
const foreground = args.port !== undefined;
const child = spawn(process.execPath, daemonArgs, {
detached: !foreground,
Expand Down
8 changes: 4 additions & 4 deletions packages/playwright-core/src/tools/cli-client/skill/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ playwright-cli video-start video.webm
playwright-cli video-chapter "Chapter Title" --description="Details" --duration=2000
playwright-cli video-stop

# launch the dashboard for UI review / design feedback — user annotates the page, you receive the annotated screenshot, snapshot, and notes
playwright-cli annotate
# launch the dashboard with annotation prompt to ask the user for input
playwright-cli show --annotate

# generate a Playwright locator for an element from its ref or selector
playwright-cli generate-locator e5 --raw
Expand Down Expand Up @@ -367,11 +367,11 @@ playwright-cli close

## Example: Interactive session

Ask the user for UI review or design feedback. The user draws boxes on the live page and types comments; you receive the annotated screenshot, the snapshot of the marked region, and the user's notes. Use this whenever the user asks for "UI review", "design feedback", or to "ask the user what they think / want / mean":
Ask the user to annotate the UI. User can provide contextual tasks or ask contextual questions using annotations:

```bash
playwright-cli open https://example.com
playwright-cli annotate
playwright-cli show --annotate
```

## Specific tasks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ playwright-cli resume # resume so that seed test runs fully
playwright-cli snapshot # inventory of interactive elements
playwright-cli click e5 # follow a flow
playwright-cli eval "location.href" # read URL / state
playwright-cli annotate # ask the user to point at something
playwright-cli show --annotate # ask the user to point at something
```

Map out:
Expand Down Expand Up @@ -262,7 +262,7 @@ The test is paused at the start. Step forward or run to until just before the fa
playwright-cli snapshot # did the element change / move / rename?
playwright-cli console # app-side errors?
playwright-cli network # failed request? wrong payload?
playwright-cli annotate # ask the user to point somewhere
playwright-cli show --annotate # ask the user to point somewhere
```

Common causes: selector drift, new wrapper element, label/ARIA rename, timing (transition, async load), assertion text updated in the app, test data leaking between runs.
Expand Down
12 changes: 1 addition & 11 deletions packages/playwright-core/src/tools/cli-daemon/commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -977,22 +977,13 @@ const dashboardShow = declareCommand({
options: z.object({
port: numberArg.optional().describe('Start as a blocking HTTP server on this port (use 0 for a random port)'),
host: z.string().optional().describe('Host to bind to when using --port (defaults to localhost)'),
annotate: z.boolean().optional().describe('Switch the dashboard into annotation mode.'),
kill: z.boolean().optional().describe('Kill the dashboard daemon.'),
}),
toolName: '',
toolParams: () => ({}),
});

const annotate = declareCommand({
name: 'annotate',
description: 'Ask the user to annotate the current page.',
category: 'devtools',
raw: true,
args: z.object({}),
toolName: 'browser_annotate',
toolParams: () => ({}),
});

const resume = declareCommand({
name: 'resume',
description: 'Resume the test execution',
Expand Down Expand Up @@ -1207,7 +1198,6 @@ const commandsArray: AnyCommandSchema[] = [
videoStop,
videoChapter,
dashboardShow,
annotate,
pauseAt,
resume,
stepOver,
Expand Down
4 changes: 1 addition & 3 deletions packages/playwright-core/src/tools/cli-daemon/daemon.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,6 @@ export async function startCliDaemonServer(

const server = net.createServer(socket => {
const connection = new SocketConnection(socket);
const abortController = new AbortController();
connection.onclose = () => abortController.abort();
connection.onmessage = async message => {
const { id, method, params } = message;
try {
Expand All @@ -93,7 +91,7 @@ export async function startCliDaemonServer(
} else if (method === 'run') {
const { toolName, toolParams } = parseCliCommand(params.args);
toolParams._meta = { cwd: params.cwd, raw: params.raw || params.json, json: !!params.json };
const response = await backend.callTool(toolName, toolParams, abortController.signal);
const response = await backend.callTool(toolName, toolParams);
await connection.send({ id, result: formatResult(response) });
} else {
throw new Error(`Unknown method: ${method}`);
Expand Down
29 changes: 16 additions & 13 deletions tests/mcp/dashboard.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,23 +152,24 @@ async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page
}

function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {
expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));
const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);
expect(imageMatch).not.toBeNull();
const pngPath = path.resolve(outputDir, imageMatch![1]);
const lines = output.trim().split('\n');
expect(lines[0]).toMatch(new RegExp(`^\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}$`));
expect(lines[lines.length - 1]).toMatch(/^image: \.playwright-cli[\\/]annotations-.*\.png$/);
const pngRel = lines[lines.length - 1].replace(/^image: /, '');
const pngPath = path.resolve(outputDir, pngRel);
expect(fs.existsSync(pngPath)).toBe(true);
expect(fs.statSync(pngPath).size).toBeGreaterThan(0);
}

test('should capture annotations via annotate', async ({ connectToDashboard, cli, server }) => {
test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {
await cli('open', server.EMPTY_PAGE);
await cli('show');
const browser = await connectToDashboard();

const dashboard = browser.contexts()[0].pages()[0];
await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();

const annotatePromise = cli('annotate');
const annotatePromise = cli('show', '--annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });

Expand All @@ -183,7 +184,8 @@ test('should capture annotations via annotate', async ({ connectToDashboard, cli
test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {
await cli('open', server.EMPTY_PAGE);

const annotatePromise = cli('annotate');
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
const annotatePromise = cli('show', '--annotate', { bindTitle });
let done = false;
void annotatePromise.finally(() => { done = true; });

Expand All @@ -201,11 +203,12 @@ test('should start dashboard and annotate when no dashboard is running', async (
verifyAnnotateOutput(output, 'hi', test.info().outputDir);
});

test('should enter annotate mode on fresh dashboard.tsx mount with -s annotate', async ({ connectToDashboard, cli, server }) => {
test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {
await cli('-s=first', 'open', server.EMPTY_PAGE);
await cli('-s=second', 'open', server.EMPTY_PAGE);

const annotatePromise = cli('-s=second', 'annotate');
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
const annotatePromise = cli('-s=second', 'show', '--annotate', { bindTitle });
let done = false;
void annotatePromise.finally(() => { done = true; });

Expand Down Expand Up @@ -305,7 +308,7 @@ test('should cancel browser_annotate when the MCP client disconnects', async ({
});


test('should switch screencast to -s session on annotate', async ({ connectToDashboard, cli, server }) => {
test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {
server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');
server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');

Expand Down Expand Up @@ -335,7 +338,7 @@ test('should switch screencast to -s session on annotate', async ({ connectToDas
return !!(c && c.r > 200 && c.g < 50);
}, { timeout: 15000 }).toBe(true);

const annotatePromise = cli('-s=second', 'annotate');
const annotatePromise = cli('-s=second', 'show', '--annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });

Expand All @@ -353,7 +356,7 @@ test('should switch screencast to -s session on annotate', async ({ connectToDas
expect(exitCode).toBe(0);
});

test('should disengage annotate mode when annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
await cli('open', server.EMPTY_PAGE);
await cli('show');
const browser = await connectToDashboard();
Expand All @@ -362,7 +365,7 @@ test('should disengage annotate mode when annotate client disconnects', async ({
await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();

const annotateClient = childProcess({
command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'annotate'],
command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],
cwd: test.info().outputPath(),
env: inheritAndCleanEnv({
...cliEnv,
Expand Down
Loading