From e57914049d3055a344483075e6f6e40e1c145b16 Mon Sep 17 00:00:00 2001 From: Shriya Lolabattu Date: Tue, 4 Aug 2026 20:37:34 -0700 Subject: [PATCH] evals: add the stagehand_code code-mode exposure --- packages/evals/core/tools/registry.ts | 4 + packages/evals/core/tools/stagehand_code.ts | 516 ++++++++++++++++++ .../evals/tests/core/tool-registry.test.ts | 10 + 3 files changed, 530 insertions(+) create mode 100644 packages/evals/core/tools/stagehand_code.ts diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index 65384f1371..7a14bce997 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -4,11 +4,13 @@ import { CdpCodeTool } from "./cdp_code.js"; import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js"; import { PlaywrightCodeTool } from "./playwright_code.js"; import { PlaywrightMcpTool } from "./playwright_mcp.js"; +import { StagehandCodeTool } from "./stagehand_code.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; export function listCoreTools(): ToolSurface[] { return [ "understudy_code", + "stagehand_code", "playwright_code", "cdp_code", "playwright_mcp", @@ -21,6 +23,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { switch (toolSurface) { case "understudy_code": return new UnderstudyCodeTool(); + case "stagehand_code": + return new StagehandCodeTool(); case "playwright_code": return new PlaywrightCodeTool(); case "cdp_code": diff --git a/packages/evals/core/tools/stagehand_code.ts b/packages/evals/core/tools/stagehand_code.ts new file mode 100644 index 0000000000..87ef2b5801 --- /dev/null +++ b/packages/evals/core/tools/stagehand_code.ts @@ -0,0 +1,516 @@ +import type { Locator, Page, Stagehand } from "@browserbasehq/stagehand"; +import type { ProbeEvidence } from "stagehand-v3"; +import { z } from "zod/v4"; +import { initStagehand, type StagehandInitResult } from "../../initStagehand.js"; +import type { PageRepresentation } from "../contracts/representation.js"; +import type { Artifact, ConnectionMode } from "../contracts/results.js"; +import type { ActionTarget, TargetKind, WaitSpec } from "../contracts/targets.js"; +import { + AGENT_RUN_TOOL_NAME, + type CoreCapability, + type CoreLocatorHandle, + type CorePageHandle, + type CoreSession, + type CoreTool, + type StartupProfile, + type ToolStartInput, + type ToolStartResult, +} from "../contracts/tool.js"; + +const SURFACE_MODEL = "openai/gpt-4.1-mini"; + +const SUPPORTED_CAPABILITIES: CoreCapability[] = [ + "session", + "navigation", + "evaluation", + "screenshot", + "viewport", + "wait", + "click", + "hover", + "scroll", + "type", + "press", + "tabs", + "representation", +]; + +class StagehandLocatorHandle implements CoreLocatorHandle { + constructor(private readonly locatorHandle: Locator) {} + + async count(): Promise { + return this.locatorHandle.count(); + } + + async click(): Promise { + await this.locatorHandle.click(); + } + + async hover(): Promise { + await this.locatorHandle.hover(); + } + + async fill(value: string): Promise { + await this.locatorHandle.fill(value); + } + + async type(text: string, opts?: { delay?: number }): Promise { + await this.locatorHandle.type(text, opts); + } + + async isVisible(): Promise { + return this.locatorHandle.isVisible(); + } + + async textContent(): Promise { + return this.locatorHandle.textContent(); + } + + async inputValue(): Promise { + return this.locatorHandle.inputValue(); + } +} + +class StagehandPageHandle implements CorePageHandle { + readonly id: string; + private lastUrl: string; + + constructor(private readonly page: Page) { + this.id = page.pageId; + this.lastUrl = page.ref.url ?? "about:blank"; + } + + private async refreshUrl(): Promise { + this.lastUrl = await this.page.url(); + } + + async goto( + url: string, + opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }, + ): Promise { + await this.page.goto(url, { + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + } + + async reload(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + await this.page.reload({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + } + + async back(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + const response = await this.page.goBack({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + return response !== null; + } + + async goBack(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + return this.back(opts); + } + + async forward(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + const response = await this.page.goForward({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + return response !== null; + } + + async goForward(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + return this.forward(opts); + } + + url(): string { + return this.lastUrl; + } + + async title(): Promise { + return this.page.title(); + } + + async evaluate( + pageFunctionOrExpression: string | ((arg: Arg) => R | Promise), + arg?: Arg, + ): Promise { + return this.page.evaluate(pageFunctionOrExpression, arg); + } + + async screenshot(opts?: { + fullPage?: boolean; + type?: "png" | "jpeg"; + quality?: number; + }): Promise { + return this.page.screenshot(opts); + } + + async setViewport(size: { width: number; height: number }): Promise { + await this.page.setViewportSize(size.width, size.height); + } + + async setViewportSize(width: number, height: number): Promise { + await this.page.setViewportSize(width, height); + } + + async wait(spec: WaitSpec): Promise { + switch (spec.kind) { + case "selector": + await this.page.waitForSelector(spec.selector, { + timeout: spec.timeoutMs, + state: spec.state, + }); + return; + case "timeout": + await this.page.waitForTimeout(spec.timeoutMs); + return; + case "load_state": + await this.page.waitForLoadState(spec.state, spec.timeoutMs); + return; + default: { + const exhaustive: never = spec; + throw new Error(`Unsupported wait spec: ${JSON.stringify(exhaustive)}`); + } + } + } + + async waitForSelector( + selector: string, + opts?: { + timeout?: number; + state?: "attached" | "detached" | "visible" | "hidden"; + }, + ): Promise { + return this.page.waitForSelector(selector, opts); + } + + async waitForTimeout(ms: number): Promise { + await this.page.waitForTimeout(ms); + } + + locator(selector: string): CoreLocatorHandle { + return new StagehandLocatorHandle(this.page.locator(selector)); + } + + async click(targetOrX: string | ActionTarget | number, y?: number): Promise { + if (typeof targetOrX === "number") { + if (typeof y !== "number") throw new Error("click(x, y) requires both numeric coordinates"); + await this.page.click(targetOrX, y); + return; + } + + const target = + typeof targetOrX === "string" ? ({ kind: "selector", value: targetOrX } as const) : targetOrX; + switch (target.kind) { + case "selector": + await this.page.locator(target.value).click(); + return; + case "coords": + await this.page.click(target.x, target.y); + return; + default: + throw new Error(`stagehand_code does not support click target kind "${target.kind}" yet`); + } + } + + async hover(targetOrX: string | ActionTarget | number, y?: number): Promise { + if (typeof targetOrX === "number") { + if (typeof y !== "number") throw new Error("hover(x, y) requires both numeric coordinates"); + await this.page.hover(targetOrX, y); + return; + } + + const target = + typeof targetOrX === "string" ? ({ kind: "selector", value: targetOrX } as const) : targetOrX; + switch (target.kind) { + case "selector": + await this.page.locator(target.value).hover(); + return; + case "coords": + await this.page.hover(target.x, target.y); + return; + default: + throw new Error(`stagehand_code does not support hover target kind "${target.kind}" yet`); + } + } + + async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { + await this.page.scroll(x, y, deltaX, deltaY); + } + + async type( + targetOrText: string | ActionTarget | { kind: "focused" }, + text?: string, + ): Promise { + if (typeof targetOrText === "string" && text === undefined) { + await this.page.type(targetOrText); + return; + } + if (typeof text !== "string") throw new Error("type(target, text) requires text"); + + const target = + typeof targetOrText === "string" + ? ({ kind: "selector", value: targetOrText } as const) + : targetOrText; + switch (target.kind) { + case "focused": + await this.page.type(text); + return; + case "selector": + await this.page.locator(target.value).type(text); + return; + case "coords": + await this.page.click(target.x, target.y); + await this.page.type(text); + return; + default: + throw new Error(`stagehand_code does not support type target kind "${target.kind}" yet`); + } + } + + async press( + targetOrKey: string | ActionTarget | { kind: "focused" }, + key?: string, + ): Promise { + if (typeof targetOrKey === "string" && key === undefined) { + await this.page.keyPress(targetOrKey); + return; + } + if (typeof key !== "string") throw new Error("press(target, key) requires key"); + + const target = + typeof targetOrKey === "string" + ? ({ kind: "selector", value: targetOrKey } as const) + : targetOrKey; + switch (target.kind) { + case "focused": + await this.page.keyPress(key); + return; + case "selector": + await this.page.locator(target.value).click(); + await this.page.keyPress(key); + return; + case "coords": + await this.page.click(target.x, target.y); + await this.page.keyPress(key); + return; + default: + throw new Error(`stagehand_code does not support press target kind "${target.kind}" yet`); + } + } + + async represent(opts?: { includeIframes?: boolean }): Promise { + const snapshot = await this.page.snapshot({ includeIframes: opts?.includeIframes }); + const content = snapshot.formattedTree; + return { + kind: "snapshot_refs", + content, + metadata: { + bytes: Buffer.byteLength(content, "utf8"), + tokenEstimate: Math.ceil(content.length / 4), + refCount: Object.keys(snapshot.xpathMap).length, + }, + raw: snapshot, + }; + } +} + +class StagehandCodeSession implements CoreSession { + private readonly handles = new Map(); + private closed = false; + + constructor(private readonly sdk: StagehandInitResult) {} + + private wrap(page: Page): StagehandPageHandle { + const existing = this.handles.get(page.pageId); + if (existing) return existing; + const handle = new StagehandPageHandle(page); + this.handles.set(page.pageId, handle); + return handle; + } + + async listPages(): Promise { + return (await this.sdk.stagehand.browser.context.pages()).map((page) => this.wrap(page)); + } + + async activePage(): Promise { + const page = await this.sdk.stagehand.browser.context.activePage(); + if (page) return this.wrap(page); + const pages = await this.sdk.stagehand.browser.context.pages(); + if (pages.length === 0) throw new Error("No active page available"); + return this.wrap(pages[0]); + } + + async newPage(url?: string): Promise { + return this.wrap(await this.sdk.stagehand.browser.context.newPage(url)); + } + + async selectPage(pageId: string): Promise { + const page = (await this.sdk.stagehand.browser.context.pages()).find( + (candidate) => candidate.pageId === pageId, + ); + if (!page) throw new Error(`Unknown page id "${pageId}"`); + await this.sdk.stagehand.browser.context.setActivePage(page); + } + + async closePage(pageId: string): Promise { + const page = (await this.sdk.stagehand.browser.context.pages()).find( + (candidate) => candidate.pageId === pageId, + ); + if (!page) throw new Error(`Unknown page id "${pageId}"`); + await page.close(); + this.handles.delete(pageId); + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + try { + await this.sdk.stagehand.close(); + } finally { + await this.sdk.stagehand.browser.close(); + } + } + + async getArtifacts(): Promise { + return []; + } + + async getRawMetrics(): Promise> { + return { + ...(await this.sdk.stagehand.metrics()), + browserProvider: this.sdk.stagehand.browser.provider, + browserOrigin: this.sdk.stagehand.browser.origin, + }; + } +} + +async function captureStagehandEvidence(stagehand: Stagehand): Promise { + const page = await stagehand.browser.context.activePage().catch((): undefined => undefined); + if (!page) return {}; + + const evidence: ProbeEvidence = {}; + try { + evidence.screenshot = await page.screenshot(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.url = await page.url(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.ariaTree = (await page.snapshot({ includeIframes: true })).formattedTree; + } catch { + // Best effort: preserve other evidence modalities. + } + return evidence; +} + +function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMode { + return startupProfile === "tool_create_browserbase" ? "browserbase_native" : "launch"; +} + +export class StagehandCodeTool implements CoreTool { + readonly id = "stagehand_code"; + readonly surface = "code"; + readonly family = "stagehand"; + readonly supportedStartupProfiles: StartupProfile[] = [ + "tool_launch_local", + "tool_create_browserbase", + ]; + readonly supportedCapabilities: CoreCapability[] = [...SUPPORTED_CAPABILITIES]; + readonly supportedTargetKinds: TargetKind[] = ["selector", "coords", "focused"]; + + async start(input: ToolStartInput): Promise { + if (!this.supportedStartupProfiles.includes(input.startupProfile)) { + throw new Error( + `stagehand_code does not support startup profile "${input.startupProfile}" yet`, + ); + } + + const sdk = await initStagehand({ + logger: input.logger, + modelName: SURFACE_MODEL, + environment: input.startupProfile === "tool_create_browserbase" ? "BROWSERBASE" : "LOCAL", + }); + const session = new StagehandCodeSession(sdk); + + input.logger.log({ + category: "stagehand_code", + message: "Initialized stagehand_code Stagehand SDK runtime.", + level: 1, + auxiliary: { + startupProfile: { value: input.startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + return { + session, + agentMount: { + via: "handles", + handles: { stagehand: sdk.stagehand, page: sdk.page, z }, + promptInstructions: buildStagehandCodePromptInstructions(), + runTool: { + description: [ + "Execute JavaScript against the initialized Stagehand SDK.", + "The snippet runs inside an async function with stagehand, page, startUrl, task, z (zod), and console in scope.", + "Use await directly. Return a JSON-serializable value when useful.", + ].join(" "), + codeParamDescription: + "JavaScript function body to execute. stagehand/page/startUrl/task/z are already in scope.", + denyMessage: `Use Bash for inspection and ${AGENT_RUN_TOOL_NAME} for browser automation.`, + }, + }, + captureEvidence: () => captureStagehandEvidence(sdk.stagehand), + cleanup: () => session.close(), + metadata: { + environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", + browserOwnership: "tool", + connectionMode: connectionModeFromProfile(input.startupProfile), + startupProfile: input.startupProfile, + }, + }; + } +} + +export function buildStagehandCodePromptInstructions(): string { + return [ + "Browser tool surface: stagehand_code (Stagehand SDK).", + `Use the ${AGENT_RUN_TOOL_NAME} tool for browser automation. It exposes an initialized Stagehand client (stagehand), its initial page, startUrl, and task object.`, + "AI methods live on the client: await stagehand.act('instruction'), await stagehand.observe('instruction'), await stagehand.extract('instruction', zodSchema) — a zod `z` is in scope for extract schemas (use single-word keys).", + "Deterministic methods live on the page: await page.goto(url), await page.locator(selector).click(), await page.locator(selector).fill(value), await page.locator(selector).type(text), await page.url(), await page.title(), await page.screenshot().", + "Page accessors are async RPCs — always await them.", + "The first browser action should usually be: await page.goto(startUrl, { waitUntil: 'domcontentloaded' }).", + "Use Bash for inspection and lightweight scripting. Do not create a separate browser process.", + "Do not edit repository files.", + "Return useful JSON-serializable values from run snippets so you can inspect progress.", + ].join("\n"); +} diff --git a/packages/evals/tests/core/tool-registry.test.ts b/packages/evals/tests/core/tool-registry.test.ts index 0d9c7dbf89..6e9be39171 100644 --- a/packages/evals/tests/core/tool-registry.test.ts +++ b/packages/evals/tests/core/tool-registry.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { getCoreTool, listCoreTools } from "../../core/tools/registry.js"; +import { buildStagehandCodePromptInstructions } from "../../core/tools/stagehand_code.js"; describe("core tool registry", () => { it("lists extended tool surfaces", () => { @@ -12,5 +13,14 @@ describe("core tool registry", () => { expect(getCoreTool("playwright_mcp").id).toBe("playwright_mcp"); expect(getCoreTool("chrome_devtools_mcp").id).toBe("chrome_devtools_mcp"); expect(getCoreTool("browse_cli").id).toBe("browse_cli"); + expect(getCoreTool("stagehand_code").id).toBe("stagehand_code"); + }); + + it("shows awaited Stagehand locator actions to coding agents", () => { + const prompt = buildStagehandCodePromptInstructions(); + + expect(prompt).toContain("await page.locator(selector).click()"); + expect(prompt).toContain("await page.locator(selector).fill(value)"); + expect(prompt).toContain("await page.locator(selector).type(text)"); }); });