diff --git a/Makefile b/Makefile index 6bb42c3c..247aea8e 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,8 @@ sync: _check-ansible --rsync-path="sudo -u anton rsync" \ -e "ssh $$SSH_OPTS" \ "$(REPO_ROOT)/" "$$USER@$$IP:$(REMOTE_REPO)/" 2>&1; \ + echo " ○ Installing remote dependencies..."; \ + ssh $$SSH_OPTS "$$USER@$$IP" "cd $(REMOTE_REPO) && sudo -u anton bash -c 'pnpm --config.confirmModulesPurge=false --filter=\"./packages/*\" --filter=\"!@anton/desktop\" --filter=\"!@anton/mobile\" install --frozen-lockfile'" 2>&1 || exit 1; \ echo " ○ Building on remote..."; \ ssh $$SSH_OPTS "$$USER@$$IP" "cd $(REMOTE_REPO) && sudo -u anton bash -c 'pnpm -r --filter=\"./packages/*\" --filter=\"!@anton/desktop\" --filter=\"!@anton/mobile\" build'" 2>&1 | tail -10; \ echo " ○ Rebuilding native modules on remote..."; \ diff --git a/package.json b/package.json index 75b30d8f..e4f1d44d 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ ], "pnpm": { "onlyBuiltDependencies": [ + "agent-browser", "better-sqlite3" ] }, diff --git a/packages/agent-config/prompts/system.md b/packages/agent-config/prompts/system.md index 4e336e9c..4cded7cb 100644 --- a/packages/agent-config/prompts/system.md +++ b/packages/agent-config/prompts/system.md @@ -12,8 +12,8 @@ You are a doer, and a describer. When the user asks you to do something, use you - **glob**: Find files by pattern (e.g. "*.ts", "**/*.tsx"). **Always use instead of shell find/ls.** - **list**: List directory contents or show directory tree structure. - **browser**: Browse and interact with web pages. Two modes: - - **fetch/extract**: Fast, lightweight content retrieval (no JS). Use for reading articles, docs, APIs behind the scenes. - - **open** (+ snapshot/click/fill/scroll/screenshot/get/wait/close): Full browser automation with live screenshots shown in the user's sidebar. **Use `open` when the user asks to visit, browse, scrape, or interact with a website** — this shows the browser UI live. Chromium auto-installs on first use. + - **fetch/extract**: Fast Lightpanda engine for reading and extracting pages behind the scenes. + - **open** (+ snapshot/click/fill/scroll/screenshot/get/wait/close): Visible Chromium browser with persistent Anton cookies/profile and live stream in the user's Browser pane. **Use `open` when the user asks to visit, browse, preview localhost, scrape an app, or interact with a website** — this shows the browser UI live. - **web_search**: Fast single-pass web search (Exa). Use for **single-fact lookups**, **finding a specific URL**, **quick time-sensitive checks** (price, score, "is X live"), or when you already know what you're searching for. **Do NOT loop this tool** to answer research questions — switch to `web_research`. - **web_research**: Deep multi-hop research (Parallel). Runs several queries in parallel, fetches pages, synthesises excerpts, returns research-grade results with citations. **PREFER THIS** whenever the user asks for: "give me a brief on X", "overview / background / writeup of X", "research / investigate / look into X", "due diligence on X", "find me reliable sources on X", "what's known about X", "what's the latest on X", "compare X and Y", "X vs Y", or any question you'd otherwise answer with 3+ back-to-back `web_search` calls. One `web_research` call replaces an entire research loop. If not configured, guide the user to enable the Deep Research connector in Settings → Connectors — do NOT silently fall back to looping `web_search`. diff --git a/packages/agent-core/package.json b/packages/agent-core/package.json index 0d7db94b..b790e770 100644 --- a/packages/agent-core/package.json +++ b/packages/agent-core/package.json @@ -33,18 +33,14 @@ "@anton/protocol": "workspace:*", "@mariozechner/pi-agent-core": "^0.60.0", "@mariozechner/pi-ai": "^0.60.0", - "@mozilla/readability": "^0.6.0", "@sinclair/typebox": "^0.34.0", + "agent-browser": "0.26.0", "autoevals": "^0.0.80", "braintrust": "^0.0.182", - "linkedom": "^0.18.12", - "marked": "^18.0.0", - "playwright": "^1.52.0", - "turndown": "^7.2.2" + "marked": "^18.0.0" }, "devDependencies": { "@types/node": "^22.0.0", - "@types/turndown": "^5.0.6", "tsx": "^4.0.0", "typescript": "^5.6.0" } diff --git a/packages/agent-core/src/agent.ts b/packages/agent-core/src/agent.ts index d17443e1..e5e63b88 100644 --- a/packages/agent-core/src/agent.ts +++ b/packages/agent-core/src/agent.ts @@ -219,6 +219,8 @@ export interface ToolCallbacks { screenshot?: string lastAction: import('@anton/protocol').BrowserAction elementCount?: number + stream?: import('@anton/protocol').BrowserStreamState + engine?: import('@anton/protocol').BrowserEngine }) => void /** Callback when the browser is closed. */ onBrowserClose?: () => void @@ -468,7 +470,7 @@ export function buildTools( defineTool({ name: BROWSER_TOOL_NAME, label: 'Browser', - description: `Web browsing and browser automation. Two modes:\n• **fetch/extract** — Fast, lightweight. Use for reading articles, docs, APIs behind the scenes. No JS execution.\n• **open/snapshot/click/fill/scroll/screenshot/get/wait/close** — Full browser with live screenshots shown in the user sidebar. Use \`open\` when the user asks to visit, browse, scrape, or interact with a website. Chromium auto-installs on first use.\nFor local files, use the ${READ_TOOL_NAME} tool.`, + description: `Web browsing and browser automation. Two modes:\n• **fetch/extract** — Fast Lightpanda engine for reading and extracting pages behind the scenes.\n• **open/snapshot/click/fill/scroll/screenshot/get/wait/close** — Visible Chromium browser with persistent Anton cookies/profile and live stream in the desktop Browser pane. Use \`open\` when the user asks to visit, browse, preview localhost, scrape an app, or interact with a website.\nFor local files, use the ${READ_TOOL_NAME} tool.`, parameters: Type.Object({ operation: Type.Union( [ diff --git a/packages/agent-core/src/harness/codex-harness-session.ts b/packages/agent-core/src/harness/codex-harness-session.ts index 6eb9b5da..8a2f6134 100644 --- a/packages/agent-core/src/harness/codex-harness-session.ts +++ b/packages/agent-core/src/harness/codex-harness-session.ts @@ -879,6 +879,8 @@ export class CodexHarnessSession { screenshot?: string lastAction: import('@anton/protocol').BrowserAction elementCount?: number + stream?: import('@anton/protocol').BrowserStreamState + engine?: import('@anton/protocol').BrowserEngine }) { this.emit({ type: 'browser_state', ...state }) } diff --git a/packages/agent-core/src/harness/harness-session.ts b/packages/agent-core/src/harness/harness-session.ts index 0b17f8b8..50ab0868 100644 --- a/packages/agent-core/src/harness/harness-session.ts +++ b/packages/agent-core/src/harness/harness-session.ts @@ -453,6 +453,8 @@ export class HarnessSession { screenshot?: string lastAction: import('@anton/protocol').BrowserAction elementCount?: number + stream?: import('@anton/protocol').BrowserStreamState + engine?: import('@anton/protocol').BrowserEngine }) { this.pushEvent?.({ type: 'browser_state', ...state }) } diff --git a/packages/agent-core/src/index.ts b/packages/agent-core/src/index.ts index 2e3c69a0..11b93b1a 100644 --- a/packages/agent-core/src/index.ts +++ b/packages/agent-core/src/index.ts @@ -71,7 +71,15 @@ export { type ResolvedProviderToken, } from './tools/factories.js' export { initTracing, flushTraces, hashPromptVersion, logSpanFeedback } from './tracing.js' -export { closeBrowserSession } from './tools/browser.js' +export { + closeBrowserSession, + executeBrowser, + getBrowserRuntimeStatus, + installBrowserRuntime, + refreshVisibleBrowserState, + setVisibleBrowserViewport, + type BrowserCallbacks, +} from './tools/browser.js' export { type HarnessAdapter, ClaudeAdapter, diff --git a/packages/agent-core/src/session.ts b/packages/agent-core/src/session.ts index 91347f6a..94de0e04 100644 --- a/packages/agent-core/src/session.ts +++ b/packages/agent-core/src/session.ts @@ -458,6 +458,8 @@ export type SessionEvent = screenshot?: string lastAction: import('@anton/protocol').BrowserAction elementCount?: number + stream?: import('@anton/protocol').BrowserStreamState + engine?: import('@anton/protocol').BrowserEngine } | { type: 'browser_close' } @@ -981,6 +983,8 @@ export class Session { screenshot?: string lastAction: import('@anton/protocol').BrowserAction elementCount?: number + stream?: import('@anton/protocol').BrowserStreamState + engine?: import('@anton/protocol').BrowserEngine }) { this.pushEvent?.({ type: 'browser_state', ...state }) } diff --git a/packages/agent-core/src/tools/browser-factory.ts b/packages/agent-core/src/tools/browser-factory.ts index f271d9c2..8abc4a7a 100644 --- a/packages/agent-core/src/tools/browser-factory.ts +++ b/packages/agent-core/src/tools/browser-factory.ts @@ -1,18 +1,18 @@ /** - * `browser` — fetch / extract / Playwright automation. Lifted out of + * `browser` — Lightpanda-backed fetch / extract plus visible Chrome automation. + * Lifted out of * agent.ts so the harness MCP shim can hand it to Codex / Claude Code. * - * The lightweight `fetch` / `extract` operations don't need any - * callbacks. The full-browser operations (`open` / `snapshot` / + * The lightweight `fetch` / `extract` operations use agent-browser's + * Lightpanda engine and don't need callbacks. The full-browser operations (`open` / `snapshot` / * `click` / `fill` / `scroll` / `screenshot` / `get` / `wait` / - * `close`) drive `onBrowserState` to push live screenshots into the - * desktop sidebar — same callback shape Pi SDK uses. + * `close`) use agent-browser's Chrome engine with an Anton-owned persistent + * profile and drive `onBrowserState` to push live state into the desktop + * browser pane — same callback shape Pi SDK uses. * - * Note on per-session scoping: the underlying Playwright instance in - * `tools/browser.ts` is process-scoped today, just like in Pi SDK. If - * we ever run multiple harness sessions concurrently driving a real - * browser, we'll need to scope it per-session. For now the constraint - * matches Pi SDK's, so behavior is identical. + * Note on per-session scoping: agent-browser sessions are named. The default + * visible browser uses `anton-visible`; the background Lightpanda browser uses + * `anton-lightpanda`. */ import type { AgentTool } from '@mariozechner/pi-agent-core' @@ -26,8 +26,8 @@ export function buildBrowserTool(callbacks?: BrowserCallbacks): AgentTool { label: 'Browser', description: 'Web browsing and browser automation. Two modes:\n' + - '• fetch/extract — Fast, lightweight. Use for reading articles, docs, APIs behind the scenes. No JS execution.\n' + - '• open/snapshot/click/fill/scroll/screenshot/get/wait/close — Full browser with live screenshots shown in the user sidebar. Use `open` when the user asks to visit, browse, scrape, or interact with a website. Chromium auto-installs on first use.\n' + + '• fetch/extract — Fast Lightpanda engine for reading and extracting pages behind the scenes.\n' + + '• open/snapshot/click/fill/scroll/screenshot/get/wait/close — Visible Chromium browser with persistent Anton cookies/profile and live stream in the desktop Browser pane. Use `open` when the user asks to visit, browse, preview localhost, scrape an app, or interact with a website.\n' + 'For local files, use the read tool instead.', parameters: Type.Object({ operation: Type.Union( diff --git a/packages/agent-core/src/tools/browser.ts b/packages/agent-core/src/tools/browser.ts index 67fdb90d..2241fdee 100644 --- a/packages/agent-core/src/tools/browser.ts +++ b/packages/agent-core/src/tools/browser.ts @@ -1,13 +1,56 @@ -import { execFile, execSync } from 'node:child_process' +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + constants, + accessSync, + chmodSync, + existsSync, + mkdirSync, + renameSync, + rmSync, +} from 'node:fs' +import { writeFile } from 'node:fs/promises' +import { createRequire } from 'node:module' +import { arch, platform } from 'node:os' +import { join } from 'node:path' import { promisify } from 'node:util' +import { getAntonDir } from '@anton/agent-config' import { createLogger } from '@anton/logger' +import type { + BrowserAction, + BrowserEngine, + BrowserRuntimeComponent, + BrowserRuntimeInstallTarget, + BrowserRuntimeStatus, + BrowserStreamState, +} from '@anton/protocol' const execFileAsync = promisify(execFile) +const require = createRequire(import.meta.url) const log = createLogger('browser') -import type { BrowserAction } from '@anton/protocol' -import { Readability } from '@mozilla/readability' -import { parseHTML } from 'linkedom' -import TurndownService from 'turndown' + +const VISIBLE_SESSION = 'anton-visible' +const BACKGROUND_SESSION = 'anton-lightpanda' +const VISIBLE_PROFILE = join(getAntonDir(), 'browser', 'profiles', 'default') +const DEFAULT_VISIBLE_URL = 'https://antoncomputer.in' +const DEFAULT_VISIBLE_VIEWPORT_WIDTH = 1440 +const DEFAULT_VISIBLE_VIEWPORT_HEIGHT = 1100 +const VISIBLE_VIEWPORT_SCALE = 1.5 +const MANAGED_LIGHTPANDA_DIR = join(getAntonDir(), 'browser', 'lightpanda') +const MANAGED_LIGHTPANDA_EXECUTABLE_PATH = join(MANAGED_LIGHTPANDA_DIR, 'lightpanda') +// Keep Lightpanda upgrades explicit and reviewable instead of tracking nightly. +const LIGHTPANDA_RELEASE_TAG = '0.2.9' +const LIGHTPANDA_RELEASE_API_URL = `https://api.github.com/repos/lightpanda-io/browser/releases/tags/${LIGHTPANDA_RELEASE_TAG}` +const LIGHTPANDA_ASSET_SUFFIXES = { + darwin: { + arm64: 'aarch64-macos', + x64: 'x86_64-macos', + }, + linux: { + arm64: 'aarch64-linux', + x64: 'x86_64-linux', + }, +} as const export interface BrowserToolInput { operation: @@ -22,6 +65,9 @@ export interface BrowserToolInput { | 'get' | 'wait' | 'close' + | 'back' + | 'forward' + | 'reload' url?: string ref?: string text?: string @@ -38,294 +84,622 @@ export interface BrowserCallbacks { screenshot?: string lastAction: BrowserAction elementCount?: number + stream?: BrowserStreamState + engine?: BrowserEngine }) => void onBrowserClose?: () => void } -const turndown = new TurndownService({ - headingStyle: 'atx', - codeBlockStyle: 'fenced', - bulletListMarker: '-', -}) +interface AgentBrowserRunOpts { + engine: BrowserEngine + session: string + profile?: string + json?: boolean + timeoutMs?: number +} -// Remove script/style/nav/footer tags -turndown.remove(['script', 'style', 'nav', 'footer', 'header', 'noscript', 'iframe']) +function makeAction(action: string, target?: string, value?: string): BrowserAction { + return { action, target, value, timestamp: Date.now() } +} -// ── Lightweight fetch helpers (no browser needed) ──────────────────── +function normalizeUrl(url: string): string { + const trimmed = url.trim() + if (!trimmed) return DEFAULT_VISIBLE_URL + if (/^(https?:|file:|about:)/i.test(trimmed)) return trimmed + if (/^(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?(\/|$)/i.test(trimmed)) { + return `http://${trimmed}` + } + return trimmed +} -function fetchHtml(url: string, maxBytes = 500_000): string { - return execSync(`curl -sL --max-time 15 --max-filesize 5000000 "${url}" | head -c ${maxBytes}`, { - encoding: 'utf-8', - timeout: 20_000, - }) +function formatAgentBrowserError(err: unknown, engine: BrowserEngine): string { + const message = (err as Error).message || String(err) + if (engine === 'lightpanda') { + return [ + 'agent-browser Lightpanda failed.', + 'Install or repair the Browser runtime from Customize → Connectors → Browser.', + '', + message, + ].join('\n') + } + return [ + 'agent-browser Chrome failed.', + 'Install or repair the Browser runtime from Customize → Connectors → Browser.', + '', + message, + ].join('\n') } -function htmlToMarkdown(html: string, _url: string): string { - const { document } = parseHTML(html) - const reader = new Readability(document, { charThreshold: 100 }) - const article = reader.parse() +function shouldCloseAndRetryChrome(message: string): boolean { + return ( + message.includes('No usable sandbox') || + message.includes('DevToolsActivePort') || + message.includes('Chrome exited early') || + message.includes('error while loading shared libraries') + ) +} - if (article?.content) { - const { document: cleanDoc } = parseHTML(article.content) - let md = turndown.turndown(cleanDoc.toString()) - if (article.title) { - md = `# ${article.title}\n\n${md}` +function shouldInstallChromeDeps(message: string): boolean { + return message.includes('error while loading shared libraries') +} + +function browserChildEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { ...process.env } + // Anton owns browser runtime configuration through code defaults and CLI args. + // Inherit normal service env only, so systemd does not need browser-specific vars. + for (const key of Object.keys(env)) { + if (key.startsWith('AGENT_BROWSER_') || key === 'LIGHTPANDA_EXECUTABLE_PATH') { + delete env[key] } - return md.slice(0, 80_000) } + return env +} - const body = document.querySelector('body') - if (body) { - return turndown.turndown(body.innerHTML || body.toString()).slice(0, 80_000) +function resolveAgentBrowserBin(): string | null { + try { + return require.resolve('agent-browser/bin/agent-browser.js') + } catch { + return null } - return html.slice(0, 50_000) } -// ── Playwright browser session ─────────────────────────────────────── +function resolveAgentBrowserPackageJson(): string | null { + try { + return require.resolve('agent-browser/package.json') + } catch { + return null + } +} -import type { Browser, BrowserContext, CDPSession, Page } from 'playwright' +function getAgentBrowserBin(): string { + const binPath = resolveAgentBrowserBin() + if (binPath) return binPath + throw new Error( + 'agent-browser is not installed in this Anton deployment. Run pnpm install with the current lockfile and redeploy.', + ) +} -interface BrowserSession { - browser: Browser - context: BrowserContext - page: Page - cdp: CDPSession - /** Cached element refs from last snapshot: @e1 → Locator selector */ - refs: Map +function lightpandaExecutableReady(): boolean { + if (!existsSync(MANAGED_LIGHTPANDA_EXECUTABLE_PATH)) return false + try { + accessSync(MANAGED_LIGHTPANDA_EXECUTABLE_PATH, constants.X_OK) + return true + } catch { + return false + } } -/** Single shared browser session (one at a time per agent-core process). */ -let session: BrowserSession | null = null -/** Set to true during first launch if chromium needs installing — lets tool result inform the user. */ -let chromiumJustInstalled = false +function getLightpandaAssetName(): string { + const currentPlatform = platform() + const currentArch = arch() + const suffix = + currentPlatform === 'darwin' || currentPlatform === 'linux' + ? LIGHTPANDA_ASSET_SUFFIXES[currentPlatform][ + currentArch as keyof (typeof LIGHTPANDA_ASSET_SUFFIXES)[typeof currentPlatform] + ] + : undefined + + if (!suffix) { + throw new Error(`Lightpanda is not available for ${currentPlatform}/${currentArch}`) + } + return `lightpanda-${suffix}` +} -/** Idle timeout — auto-close browser after 5 minutes of no activity. */ -const BROWSER_IDLE_TIMEOUT_MS = 5 * 60 * 1000 -let idleTimer: ReturnType | null = null +interface LightpandaReleaseAsset { + name?: string + browser_download_url?: string + digest?: string +} -function resetIdleTimer(): void { - if (idleTimer) clearTimeout(idleTimer) - idleTimer = setTimeout(() => { - if (session) { - log.info('auto-closing browser after 5 minutes idle') - closeBrowser().catch(() => {}) - } - }, BROWSER_IDLE_TIMEOUT_MS) +async function fetchLightpandaReleaseAsset(): Promise<{ + assetName: string + downloadUrl: string + expectedDigest: string +}> { + const assetName = getLightpandaAssetName() + const response = await fetch(LIGHTPANDA_RELEASE_API_URL, { + headers: { + Accept: 'application/vnd.github+json', + 'User-Agent': 'Anton-Browser-Runtime', + }, + }) + if (!response.ok) { + throw new Error(`Failed to read Lightpanda release metadata: HTTP ${response.status}`) + } + + const release = (await response.json()) as { assets?: LightpandaReleaseAsset[] } + const asset = release.assets?.find((candidate) => candidate.name === assetName) + if (!asset?.browser_download_url) { + throw new Error(`Lightpanda release ${LIGHTPANDA_RELEASE_TAG} is missing ${assetName}`) + } + if (!asset.digest?.startsWith('sha256:')) { + throw new Error(`Lightpanda release ${assetName} is missing a SHA-256 digest`) + } + return { + assetName, + downloadUrl: asset.browser_download_url, + expectedDigest: asset.digest, + } } -async function ensureBrowser(): Promise { - if (session) { - resetIdleTimer() - return session +async function downloadLightpandaAsset(downloadUrl: string, expectedDigest: string): Promise { + const response = await fetch(downloadUrl, { + headers: { + 'User-Agent': 'Anton-Browser-Runtime', + }, + }) + if (!response.ok) { + throw new Error(`Failed to download Lightpanda: HTTP ${response.status}`) } - // Dynamic import — playwright is heavy, only load when needed - const pw = await import('playwright') + const bytes = Buffer.from(await response.arrayBuffer()) + const actualDigest = `sha256:${createHash('sha256').update(bytes).digest('hex')}` + if (actualDigest !== expectedDigest) { + throw new Error(`Lightpanda checksum mismatch: expected ${expectedDigest}, got ${actualDigest}`) + } - // Auto-install chromium if not found - let browser: Browser + mkdirSync(MANAGED_LIGHTPANDA_DIR, { recursive: true }) + const tempPath = join(MANAGED_LIGHTPANDA_DIR, `lightpanda.${process.pid}.${Date.now()}.tmp`) try { - browser = await pw.chromium.launch({ - headless: true, - args: [ - '--disable-blink-features=AutomationControlled', - '--no-first-run', - '--no-default-browser-check', - '--disable-dev-shm-usage', - ], - }) - } catch (launchErr: unknown) { - const msg = (launchErr as Error).message || '' - if (msg.includes("Executable doesn't exist") || msg.includes('browserType.launch')) { - // Chromium not installed — install it async using playwright's own CLI - log.info('Chromium not found, installing') - chromiumJustInstalled = true - // Use playwright's CLI from the installed package (not npx) - const playwrightCli = require.resolve('playwright/cli') - await execFileAsync(process.execPath, [playwrightCli, 'install', 'chromium'], { - timeout: 120_000, - }) - log.info('Chromium installed successfully') - // Retry launch after install - browser = await pw.chromium.launch({ - headless: true, - args: [ - '--disable-blink-features=AutomationControlled', - '--no-first-run', - '--no-default-browser-check', - '--disable-dev-shm-usage', - ], - }) - } else { - throw launchErr - } + await writeFile(tempPath, bytes, { mode: 0o700 }) + chmodSync(tempPath, 0o700) + renameSync(tempPath, MANAGED_LIGHTPANDA_EXECUTABLE_PATH) + } catch (err) { + rmSync(tempPath, { force: true }) + throw err } +} - const context = await browser.newContext({ - viewport: { width: 1280, height: 800 }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', - }) - const page = await context.newPage() - const cdp = await page.context().newCDPSession(page) - await cdp.send('Accessibility.enable') - session = { browser, context, page, cdp, refs: new Map() } - resetIdleTimer() - return session -} - -async function closeBrowser(): Promise { - if (idleTimer) { - clearTimeout(idleTimer) - idleTimer = null - } - if (!session) return +async function runAgentBrowserCommand(args: string[], opts?: { timeoutMs?: number }) { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [getAgentBrowserBin(), ...args], + { + encoding: 'utf8', + timeout: opts?.timeoutMs ?? 30_000, + maxBuffer: 8 * 1024 * 1024, + env: browserChildEnv(), + }, + ) + return (stdout || stderr).trim() +} + +async function missingLinuxSharedLibraries(executablePath?: string): Promise { + if (!executablePath || platform() !== 'linux') return [] try { - await session.browser.close() + const { stdout, stderr } = await execFileAsync('ldd', [executablePath], { + encoding: 'utf8', + timeout: 10_000, + maxBuffer: 1024 * 1024, + }) + const output = `${stdout}\n${stderr}` + return Array.from( + new Set( + [...output.matchAll(/^\s*(\S+)\s+=>\s+not found\s*$/gm)] + .map((match) => match[1]) + .filter(Boolean), + ), + ) } catch { - // Best-effort + return [] } - session = null } -/** - * Close the browser if open. Called during server/session shutdown. - * Safe to call multiple times or when no browser is open. - */ -export async function closeBrowserSession(): Promise { - await closeBrowser() -} - -// ── Accessibility tree → refs ──────────────────────────────────────── - -/** CDP Accessibility.AXNode shape (subset of fields we use). */ -interface CDPAXNode { - nodeId: string - role: { value: string } - name?: { value: string } - value?: { value: string } - description?: { value: string } - properties?: Array<{ name: string; value: { type: string; value?: unknown } }> - childIds?: string[] - backendDOMNodeId?: number -} - -const INTERACTIVE_ROLES = new Set([ - 'link', - 'button', - 'textbox', - 'searchbox', - 'combobox', - 'checkbox', - 'radio', - 'switch', - 'slider', - 'spinbutton', - 'tab', - 'menuitem', - 'menuitemcheckbox', - 'menuitemradio', - 'option', - 'treeitem', -]) - -/** - * Get accessibility tree via CDP and extract interactive elements with refs. - * Returns lines like: `@e1 button "Submit"` - * Also populates the session ref map for later click/fill. - */ -async function buildRefSnapshot(s: BrowserSession): Promise<{ text: string; count: number }> { - s.refs.clear() - - // Use CDP to get the full accessibility tree - const { nodes } = (await s.cdp.send('Accessibility.getFullAXTree')) as { - nodes: CDPAXNode[] +async function runAgentBrowser(args: string[], opts: AgentBrowserRunOpts): Promise { + const cliArgs: string[] = ['--engine', opts.engine, '--session', opts.session] + if (opts.engine === 'lightpanda' && lightpandaExecutableReady()) { + cliArgs.push('--executable-path', MANAGED_LIGHTPANDA_EXECUTABLE_PATH) } + if (opts.engine === 'chrome' && platform() === 'linux') { + cliArgs.push('--args', '--no-sandbox,--disable-dev-shm-usage') + } + cliArgs.push('--screenshot-format', 'jpeg', '--screenshot-quality', '88') + if (opts.profile) cliArgs.push('--profile', opts.profile) + cliArgs.push(...args) + if (opts.json) cliArgs.push('--json') - const lines: string[] = [] - let counter = 1 - - for (const node of nodes) { - const role = node.role?.value - const name = node.name?.value - if (!role || !name || !INTERACTIVE_ROLES.has(role)) continue - - const refId = `@e${counter++}` - // Build a Playwright locator using getByRole - s.refs.set(refId, `role=${role}[name="${name.replace(/"/g, '\\"')}"]`) - - let line = `${refId} ${role} "${name}"` - if (node.value?.value) line += ` value="${node.value.value}"` - - // Check properties for checked/disabled/expanded - if (node.properties) { - for (const prop of node.properties) { - if (prop.name === 'checked' && prop.value.value !== undefined) { - line += ` checked=${prop.value.value}` - } else if (prop.name === 'disabled' && prop.value.value) { - line += ' disabled' - } else if (prop.name === 'expanded' && prop.value.value !== undefined) { - line += ` expanded=${prop.value.value}` - } - } - } - - lines.push(line) + try { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [getAgentBrowserBin(), ...cliArgs], + { + encoding: 'utf8', + timeout: opts.timeoutMs ?? 30_000, + maxBuffer: 8 * 1024 * 1024, + env: browserChildEnv(), + }, + ) + return (stdout || stderr).trim() + } catch (err: unknown) { + throw new Error(formatAgentBrowserError(err, opts.engine)) } +} - const text = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)' - return { text, count: s.refs.size } +async function runVisible(args: string[], opts?: { json?: boolean; timeoutMs?: number }) { + mkdirSync(VISIBLE_PROFILE, { recursive: true }) + return runAgentBrowser(args, { + engine: 'chrome', + session: VISIBLE_SESSION, + profile: VISIBLE_PROFILE, + json: opts?.json, + timeoutMs: opts?.timeoutMs, + }) } -// ── State emission ─────────────────────────────────────────────────── +async function runBackground(args: string[], opts?: { json?: boolean; timeoutMs?: number }) { + return runAgentBrowser(args, { + engine: 'lightpanda', + session: BACKGROUND_SESSION, + json: opts?.json, + timeoutMs: opts?.timeoutMs, + }) +} -function makeAction(action: string, target?: string, value?: string): BrowserAction { - return { action, target, value, timestamp: Date.now() } +async function getVisibleProperty(property: 'url' | 'title'): Promise { + return runVisible(['get', property], { timeoutMs: 10_000 }).catch(() => '') +} + +async function getVisibleStream(): Promise { + const stream: BrowserStreamState = { session: VISIBLE_SESSION, engine: 'chrome' } + const parseStatus = (raw: string) => { + const parsed = JSON.parse(raw) as { + success?: boolean + data?: { + enabled?: boolean + port?: number + connected?: boolean + screencasting?: boolean + } + enabled?: boolean + port?: number + connected?: boolean + screencasting?: boolean + } + return parsed.data ?? parsed + } + try { + let raw = await runVisible(['stream', 'status'], { json: true, timeoutMs: 10_000 }) + let parsed = parseStatus(raw) + if (!parsed.enabled || !parsed.port) { + await runVisible(['stream', 'enable'], { timeoutMs: 10_000 }).catch(() => '') + raw = await runVisible(['stream', 'status'], { json: true, timeoutMs: 10_000 }) + parsed = parseStatus(raw) + } + return { ...stream, ...parsed } + } catch { + return stream + } } -async function emitState( +async function emitVisibleState( action: BrowserAction, callbacks?: BrowserCallbacks, elementCount?: number, ) { - if (!callbacks?.onBrowserState || !session) return + if (!callbacks?.onBrowserState) return + const [url, title, stream] = await Promise.all([ + getVisibleProperty('url'), + getVisibleProperty('title'), + getVisibleStream(), + ]) + callbacks.onBrowserState({ + url, + title, + lastAction: action, + elementCount, + stream, + engine: 'chrome', + }) +} + +function clampViewportSize(width: number, height: number): { width: number; height: number } { + const safeWidth = Number.isFinite(width) ? Math.round(width) : DEFAULT_VISIBLE_VIEWPORT_WIDTH + const safeHeight = Number.isFinite(height) ? Math.round(height) : DEFAULT_VISIBLE_VIEWPORT_HEIGHT + return { + width: Math.min(1920, Math.max(800, safeWidth)), + height: Math.min(1400, Math.max(600, safeHeight)), + } +} + +async function setVisibleViewport(width: number, height: number): Promise { + const viewport = clampViewportSize(width, height) + await runVisible( + [ + 'set', + 'viewport', + String(viewport.width), + String(viewport.height), + String(VISIBLE_VIEWPORT_SCALE), + ], + { + timeoutMs: 10_000, + }, + ) +} + +async function openVisible(url: string, callbacks?: BrowserCallbacks): Promise { + const target = normalizeUrl(url) + let output: string try { - const url = session.page.url() - const title = await session.page.title() - // Capture JPEG screenshot, base64 encoded, max 800px wide for efficiency - const screenshotBuf = await session.page.screenshot({ - type: 'jpeg', - quality: 60, - scale: 'css', + await setVisibleViewport(DEFAULT_VISIBLE_VIEWPORT_WIDTH, DEFAULT_VISIBLE_VIEWPORT_HEIGHT).catch( + () => '', + ) + output = await runVisible(['open', target], { timeoutMs: 45_000 }) + } catch (err) { + const message = (err as Error).message || String(err) + if (!shouldCloseAndRetryChrome(message)) throw err + + await runAgentBrowserCommand(['close', '--all'], { timeoutMs: 30_000 }).catch(() => '') + if (shouldInstallChromeDeps(message)) { + await installChromeRuntime() + } + await setVisibleViewport(DEFAULT_VISIBLE_VIEWPORT_WIDTH, DEFAULT_VISIBLE_VIEWPORT_HEIGHT).catch( + () => '', + ) + output = await runVisible(['open', target], { timeoutMs: 45_000 }) + } + await emitVisibleState(makeAction('open', target), callbacks) + return output || `Opened ${target}` +} + +export async function refreshVisibleBrowserState( + callbacks?: BrowserCallbacks, + action: BrowserAction = makeAction('refresh'), +): Promise { + await emitVisibleState(action, callbacks) +} + +export async function setVisibleBrowserViewport( + width: number, + height: number, + callbacks?: BrowserCallbacks, +): Promise { + const viewport = clampViewportSize(width, height) + await setVisibleViewport(viewport.width, viewport.height) + await emitVisibleState( + makeAction('viewport', `${viewport.width}x${viewport.height}`), + callbacks, + ) +} + +async function ensureLightpandaRuntime(): Promise { + if (lightpandaExecutableReady()) return + await installLightpandaRuntime() +} + +async function getLightpandaText(url: string, selector?: string): Promise { + await ensureLightpandaRuntime() + const target = normalizeUrl(url) + await runBackground(['open', target], { timeoutMs: 30_000 }) + const args = ['get', 'text', selector ?? 'body'] + return runBackground(args, { timeoutMs: 30_000 }) +} + +async function getLightpandaHtml(url: string, selector?: string): Promise { + await ensureLightpandaRuntime() + const target = normalizeUrl(url) + await runBackground(['open', target], { timeoutMs: 30_000 }) + const args = ['get', 'html', selector ?? 'html'] + return runBackground(args, { timeoutMs: 30_000 }) +} + +function browserRuntimeOverall( + components: BrowserRuntimeComponent[], +): BrowserRuntimeStatus['overall'] { + if (components.some((c) => c.status === 'installing')) return 'installing' + const required = components.filter((c) => c.required) + if (required.every((c) => c.status === 'ready')) return 'ready' + if (required.some((c) => c.status === 'error')) return 'error' + if (required.some((c) => c.status === 'ready')) return 'partial' + return 'missing' +} + +function agentBrowserComponent(): BrowserRuntimeComponent { + const binPath = resolveAgentBrowserBin() + const pkgPath = resolveAgentBrowserPackageJson() + if (!binPath || !pkgPath) { + return { + id: 'agent-browser', + label: 'agent-browser', + status: 'missing', + required: true, + installable: false, + detail: + 'agent-browser is missing from this deployment. Run pnpm install with the current lockfile and redeploy.', + } + } + + try { + const pkg = require(pkgPath) as { version?: string } + return { + id: 'agent-browser', + label: 'agent-browser', + status: 'ready', + required: true, + installable: false, + path: binPath, + version: pkg.version, + detail: 'Pinned with Anton', + } + } catch (err) { + return { + id: 'agent-browser', + label: 'agent-browser', + status: 'error', + required: true, + installable: false, + detail: (err as Error).message, + } + } +} + +async function chromeComponent(): Promise { + try { + const raw = await runAgentBrowserCommand(['doctor', '--offline', '--quick', '--json'], { + timeoutMs: 20_000, }) - const screenshot = screenshotBuf.toString('base64') - callbacks.onBrowserState({ url, title, screenshot, lastAction: action, elementCount }) - } catch { - // Best-effort — don't fail the tool call + const parsed = JSON.parse(raw) as { + checks?: Array<{ id?: string; status?: string; message?: string }> + } + const check = parsed.checks?.find((c) => c.id === 'chrome.installed') + if (check?.status === 'pass') { + const detail = check.message ?? 'Chrome is available' + const chromePath = detail.match(/Chrome at ([^\s]+)(?:\s+\(|$)/)?.[1] + const antonManaged = + detail.includes('/.agent-browser/browsers/') || + detail.includes('.agent-browser/browsers/') || + detail.includes('Chrome for Testing') + if (!antonManaged) { + return { + id: 'chrome', + label: 'Chrome for Anton', + status: 'missing', + required: true, + installable: true, + path: chromePath, + detail: 'Install Anton-managed Chrome to keep browser sessions isolated.', + } + } + const missingLibraries = await missingLinuxSharedLibraries(chromePath) + if (missingLibraries.length > 0) { + return { + id: 'chrome', + label: 'Chrome for Anton', + status: 'error', + required: true, + installable: true, + path: chromePath, + detail: `Chrome is installed, but Linux system libraries are missing: ${missingLibraries.join(', ')}. Run Repair to install Chrome dependencies.`, + } + } + return { + id: 'chrome', + label: 'Chrome for Anton', + status: 'ready', + required: true, + installable: true, + path: chromePath, + detail: 'Managed Chrome is installed', + } + } + return { + id: 'chrome', + label: 'Chrome for Anton', + status: check?.status === 'fail' ? 'missing' : 'unknown', + required: true, + installable: true, + detail: check?.message ?? 'Chrome status is unknown', + } + } catch (err) { + return { + id: 'chrome', + label: 'Chrome for Anton', + status: 'error', + required: true, + installable: true, + detail: (err as Error).message, + } + } +} + +function lightpandaComponent(): BrowserRuntimeComponent { + const executablePath = MANAGED_LIGHTPANDA_EXECUTABLE_PATH + const exists = existsSync(executablePath) + const installed = lightpandaExecutableReady() + return { + id: 'lightpanda', + label: 'Lightpanda', + status: installed ? 'ready' : exists ? 'error' : 'missing', + required: true, + installable: true, + path: installed ? executablePath : undefined, + detail: installed + ? 'Installed in Anton-managed runtime directory' + : exists + ? `Lightpanda exists but is not executable at ${executablePath}. Run Repair.` + : 'Install Lightpanda for fast background browsing', } } -// ── Ref resolution ─────────────────────────────────────────────────── +export async function getBrowserRuntimeStatus(): Promise { + const components = [agentBrowserComponent(), await chromeComponent(), lightpandaComponent()] + return { + overall: browserRuntimeOverall(components), + profileDir: VISIBLE_PROFILE, + components, + checkedAt: Date.now(), + } +} -function resolveRef(ref: string): string { - if (!session) throw new Error('Browser not open. Use operation: "open" first.') - const selector = session.refs.get(ref) - if (!selector) { - throw new Error( - `Unknown ref "${ref}". Run operation: "snapshot" first to see available elements.`, - ) +async function installChromeRuntime(): Promise { + const args = ['install'] + if (platform() === 'linux') args.push('--with-deps') + await runAgentBrowserCommand(args, { timeoutMs: 300_000 }) +} + +async function installLightpandaRuntime(): Promise { + const asset = await fetchLightpandaReleaseAsset() + log.info( + { assetName: asset.assetName, path: MANAGED_LIGHTPANDA_EXECUTABLE_PATH }, + 'installing Anton-managed Lightpanda runtime', + ) + await downloadLightpandaAsset(asset.downloadUrl, asset.expectedDigest) +} + +export async function installBrowserRuntime( + target: BrowserRuntimeInstallTarget, + onProgress?: (stage: 'checking' | 'installing' | 'verifying' | 'done', message: string) => void, +): Promise { + onProgress?.('checking', 'Checking browser runtime') + + if (target === 'chrome' || target === 'all' || target === 'repair') { + onProgress?.('installing', 'Installing Chrome for Anton') + if (target === 'repair') { + await runAgentBrowserCommand(['close', '--all'], { timeoutMs: 30_000 }).catch(() => '') + await installChromeRuntime() + await runAgentBrowserCommand(['doctor', '--fix'], { timeoutMs: 180_000 }) + } else { + await installChromeRuntime() + } + } + + if (target === 'lightpanda' || target === 'all' || target === 'repair') { + onProgress?.('installing', 'Installing Lightpanda for Anton') + await installLightpandaRuntime() } - return selector + + onProgress?.('verifying', 'Verifying browser runtime') + const status = await getBrowserRuntimeStatus() + return status } -// ── Main tool executor ─────────────────────────────────────────────── +export async function closeBrowserSession(): Promise { + await Promise.allSettled([ + runVisible(['close'], { timeoutMs: 10_000 }), + runBackground(['close'], { timeoutMs: 10_000 }), + ]) +} -/** - * Browser tool: fetch web pages (lightweight) or automate real browser (Playwright). - * - * fetch/extract: Fast, no JS, uses curl + Readability. - * open/snapshot/click/fill/screenshot/scroll/get/wait/close: Full browser via Playwright. - */ export async function executeBrowser( input: BrowserToolInput, callbacks?: BrowserCallbacks, @@ -334,142 +708,86 @@ export async function executeBrowser( try { switch (operation) { - // ── Lightweight (no real browser) ────────────────────────────── - case 'fetch': { if (!url) return 'Error: url is required for fetch' - const html = fetchHtml(url) - if (!html) return '(empty response)' - return htmlToMarkdown(html, url) + return await getLightpandaText(url) } case 'extract': { if (!url) return 'Error: url is required for extract' - const html = fetchHtml(url, 200_000) - - if (selector) { - const { document } = parseHTML(html) - const elements = document.querySelectorAll(selector) - if (elements.length === 0) { - return `No elements found matching selector: ${selector}` - } - - const extracted = Array.from(elements) - .map((el: Element) => turndown.turndown(el.innerHTML || el.textContent || '')) - .join('\n\n---\n\n') - - return `Extracted ${elements.length} element(s) from ${url} (selector: ${selector}):\n\n${extracted.slice(0, 50_000)}` - } - - return htmlToMarkdown(html, url) + return property === 'html' + ? await getLightpandaHtml(url, selector) + : await getLightpandaText(url, selector) } - // ── Full browser automation (Playwright) ────────────────────── - case 'open': { - if (!url) return 'Error: url is required for open' - const s = await ensureBrowser() - const wasInstalled = chromiumJustInstalled - chromiumJustInstalled = false - await s.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }) - // Wait a bit for JS to settle - await s.page.waitForLoadState('networkidle').catch(() => {}) - const action = makeAction('open', url) - await emitState(action, callbacks) - const prefix = wasInstalled ? '(Chromium was auto-installed on first use.) ' : '' - return `${prefix}Opened ${url} — title: "${await s.page.title()}"` + return openVisible(url || DEFAULT_VISIBLE_URL, callbacks) } case 'snapshot': { - if (!session) return 'Error: Browser not open. Use operation: "open" first.' - const { text: snapText, count: snapCount } = await buildRefSnapshot(session) - await emitState(makeAction('snapshot'), callbacks, snapCount) - return `${snapCount} interactive elements:\n\n${snapText}` + const output = await runVisible(['snapshot', '-i'], { timeoutMs: 30_000 }) + const match = output.match(/\[ref=/g) + await emitVisibleState(makeAction('snapshot'), callbacks, match?.length) + return output } case 'click': { if (!ref) return 'Error: ref is required for click (e.g. @e1)' - const sel = resolveRef(ref) - await session!.page.locator(sel).first().click({ timeout: 10_000 }) - // Wait for navigation or network activity to settle - await session!.page.waitForLoadState('networkidle').catch(() => {}) - await emitState(makeAction('click', ref), callbacks) - return `Clicked ${ref}` + const output = await runVisible(['click', ref], { timeoutMs: 30_000 }) + await emitVisibleState(makeAction('click', ref), callbacks) + return output || `Clicked ${ref}` } case 'fill': { if (!ref) return 'Error: ref is required for fill' if (text === undefined) return 'Error: text is required for fill' - const sel = resolveRef(ref) - await session!.page.locator(sel).first().fill(text, { timeout: 10_000 }) - await emitState(makeAction('fill', ref, text), callbacks) - return `Filled ${ref} with "${text}"` + const output = await runVisible(['fill', ref, text], { timeoutMs: 30_000 }) + await emitVisibleState(makeAction('fill', ref, text), callbacks) + return output || `Filled ${ref}` } case 'screenshot': { - if (!session) return 'Error: Browser not open. Use operation: "open" first.' - const buf = await session.page.screenshot({ type: 'jpeg', quality: 70 }) - const b64 = buf.toString('base64') - await emitState(makeAction('screenshot'), callbacks) - return `Screenshot captured (${Math.round(b64.length / 1024)}KB base64)` + const output = await runVisible(['screenshot'], { timeoutMs: 30_000 }) + await emitVisibleState(makeAction('screenshot'), callbacks) + return output || 'Screenshot captured' } case 'scroll': { - if (!session) return 'Error: Browser not open. Use operation: "open" first.' const dir = direction || 'down' - const px = amount || 500 - const delta = dir === 'up' ? -px : px - await session.page.mouse.wheel(0, delta) - // Small delay for content to render - await session.page.waitForTimeout(300) - await emitState(makeAction('scroll', dir, String(px)), callbacks) - return `Scrolled ${dir} ${px}px` + const px = String(amount || 500) + const output = await runVisible(['scroll', dir, px], { timeoutMs: 30_000 }) + await emitVisibleState(makeAction('scroll', dir, px), callbacks) + return output || `Scrolled ${dir} ${px}px` } case 'get': { - if (!session) return 'Error: Browser not open. Use operation: "open" first.' const prop = property || 'text' - switch (prop) { - case 'url': - return session.page.url() - case 'title': - return await session.page.title() - case 'html': { - if (ref) { - const sel = resolveRef(ref) - return await session.page.locator(sel).first().innerHTML({ timeout: 5_000 }) - } - const html = await session.page.content() - return html.slice(0, 50_000) - } - default: { - if (ref) { - const sel = resolveRef(ref) - return await session.page.locator(sel).first().innerText({ timeout: 5_000 }) - } - // Full page text - const bodyText = await session.page - .locator('body') - .innerText({ timeout: 5_000 }) - .catch(() => '(could not read page text)') - return bodyText.slice(0, 50_000) - } - } + const defaultSelector = prop === 'html' ? 'html' : prop === 'text' ? 'body' : undefined + const args = ref + ? ['get', prop, ref] + : defaultSelector + ? ['get', prop, defaultSelector] + : ['get', prop] + return await runVisible(args, { timeoutMs: 30_000 }) } case 'wait': { - if (!session) return 'Error: Browser not open. Use operation: "open" first.' - if (ref) { - const sel = resolveRef(ref) - await session.page.locator(sel).first().waitFor({ state: 'visible', timeout: 30_000 }) - return `Element ${ref} is visible` - } - await session.page.waitForLoadState('networkidle', { timeout: 30_000 }).catch(() => {}) - return 'Page loaded (network idle)' + const target = ref || String(amount || 1000) + const output = await runVisible(['wait', target], { timeoutMs: 35_000 }) + await emitVisibleState(makeAction('wait', target), callbacks) + return output || `Waited for ${target}` + } + + case 'back': + case 'forward': + case 'reload': { + const output = await runVisible([operation], { timeoutMs: 30_000 }) + await emitVisibleState(makeAction(operation), callbacks) + return output || `Browser ${operation}` } case 'close': { - await closeBrowser() + await closeBrowserSession() callbacks?.onBrowserClose?.() return 'Browser closed' } @@ -478,11 +796,7 @@ export async function executeBrowser( return `Unknown operation: ${operation}` } } catch (err: unknown) { - const msg = (err as Error).message || String(err) - // If browser crashed, clean up - if (msg.includes('Target closed') || msg.includes('Browser has been closed')) { - session = null - } - return `Error: ${msg}` + log.warn({ err, operation }, 'browser operation failed') + return `Error: ${(err as Error).message || String(err)}` } } diff --git a/packages/agent-core/src/tools/factories.ts b/packages/agent-core/src/tools/factories.ts index b06ed226..da14debb 100644 --- a/packages/agent-core/src/tools/factories.ts +++ b/packages/agent-core/src/tools/factories.ts @@ -175,9 +175,10 @@ export function buildAntonCoreTools(ctx: AntonCoreToolContext = {}): AgentTool[] // Read / write the system clipboard. Trivial but lets the model // act on "paste what I just copied". buildClipboardTool(), - // Web browsing + Playwright automation. Same callbacks Pi SDK - // uses to drive the desktop browser sidebar — when undefined, the - // tool still supports the lightweight fetch/extract operations. + // Web browsing through agent-browser: Lightpanda for background + // fetch/extract, Chrome for visible automation. Same callbacks Pi + // SDK uses to drive the desktop browser pane — when undefined, the + // tool still supports the Lightpanda fetch/extract operations. buildBrowserTool(ctx.browserCallbacks), // Loads full SKILL.md instructions on demand. The prompt layer only // carries a compact metadata listing, so this tool is the bridge from diff --git a/packages/agent-server/src/index.ts b/packages/agent-server/src/index.ts index cae4c53e..5b6756df 100644 --- a/packages/agent-server/src/index.ts +++ b/packages/agent-server/src/index.ts @@ -87,7 +87,7 @@ async function main() { scheduler.stop() agentManager.shutdown() await server.shutdown() // Stop MCP servers, kill PTYs, release resources - await closeBrowserSession() // Close Playwright browser if open + await closeBrowserSession() // Close agent-browser sessions if open await flushTraces() process.exit(0) } diff --git a/packages/agent-server/src/server.ts b/packages/agent-server/src/server.ts index 0eb9795f..0c0fe5b2 100644 --- a/packages/agent-server/src/server.ts +++ b/packages/agent-server/src/server.ts @@ -112,18 +112,23 @@ import { createMcpIpcServer, createSession, ensureHarnessSessionInit, + executeBrowser, executePublish, extractHarnessMemoriesFromMirror, + getBrowserRuntimeStatus, getModelContextSize, hashPromptVersion, + installBrowserRuntime, isHarnessSession, matchesSurface, probeMcpShim, + refreshVisibleBrowserState, readHarnessArtifacts, readHarnessHistory, readLastUserFromHarness, resolveModel, resumeSession, + setVisibleBrowserViewport, synthesizeHarnessTurn, writeHarnessSessionTitle, } from '@anton/agent-core' @@ -138,6 +143,11 @@ import { } from '@anton/protocol' import type { AiMessage, + BrowserAction, + BrowserEngine, + BrowserInputEvent, + BrowserRuntimeInstallTarget, + BrowserStreamState, ChannelId, ContextBreakdown, ControlMessage, @@ -173,6 +183,15 @@ const log = createLogger('server') const DEFAULT_SESSION_ID = 'default' +interface VisibleBrowserSessionState { + url: string + title: string + lastAction: BrowserAction + stream?: BrowserStreamState + engine?: BrowserEngine + updatedAt: number +} + const IMAGE_MIME_BY_EXT: Record = { jpg: 'image/jpeg', jpeg: 'image/jpeg', @@ -356,6 +375,15 @@ export class AgentServer { */ private initialMcpProbe: Promise | null = null private activeClient: WebSocket | null = null + private browserStreamWs: WebSocket | null = null + private browserStreamPort: number | null = null + private browserStreamSessionId: string | null = null + private browserRuntimeInstallInFlight: Promise | null = null + private browserRuntimeInstallInFlightTarget: BrowserRuntimeInstallTarget | null = null + private visibleBrowserStates = new Map() + private browserStateRefreshTimers = new Map() + private browserViewportTimers = new Map() + private browserViewportBySession = new Map() private pendingSessionCreates = new Map>() // Track pending interactive prompts so they can be re-sent on client reconnect private pendingPrompts: Map }> = @@ -502,6 +530,7 @@ export class AgentServer { /** Graceful shutdown: stop MCP servers, close connections, release resources. */ async shutdown(): Promise { + this.closeBrowserStreamProxy() try { await this.mcpManager.stopAll() log.info('MCP servers stopped') @@ -2159,6 +2188,30 @@ export class AgentServer { this.handleConnectorSetToolPermission(msg) break + // ── Browser pane ── + case 'browser_open': + await this.handleBrowserOpen(msg.sessionId || DEFAULT_SESSION_ID, msg.url) + break + case 'browser_navigate': + await this.handleBrowserOpen(msg.sessionId || DEFAULT_SESSION_ID, msg.url) + break + case 'browser_command': + await this.handleBrowserCommand(msg.sessionId || DEFAULT_SESSION_ID, msg.command) + break + case 'browser_input': + this.forwardBrowserInput(msg.event) + this.refreshVisibleBrowserStateSoon(msg.sessionId || DEFAULT_SESSION_ID, msg.event) + break + case 'browser_viewport': + this.handleBrowserViewport(msg.sessionId || DEFAULT_SESSION_ID, msg.width, msg.height) + break + case 'browser_runtime_status': + await this.handleBrowserRuntimeStatus() + break + case 'browser_runtime_install': + this.handleBrowserRuntimeInstall(msg.target) + break + // ── Publish artifacts ── case 'publish_artifact': this.handlePublishArtifact(msg) @@ -2685,14 +2738,16 @@ export class AgentServer { skills: this.getActiveSkillsForPrompt(), userMessage, }) + const browserContext = this.visibleBrowserContextForTurn(id) + const baseWithBrowserContext = browserContext ? `${base}\n\n${browserContext}` : base // Inject the replay seed ONCE, on the first turn only. From turn // 1 onward the CLI's own --resume tape carries the history. if (turnIndex === 0 && replaySeedForFirstTurn && !replaySeedConsumed) { replaySeedConsumed = true - return `${base}${replaySeedForFirstTurn}` + return `${baseWithBrowserContext}${replaySeedForFirstTurn}` } - return base + return baseWithBrowserContext } // Ensure meta.json + empty messages.jsonl exist @@ -5537,6 +5592,21 @@ export class AgentServer { 'do not loop quick searches. Cite sources.]' chatContent = `${RESEARCH_TURN_HINT}\n\n${msg.content}` } + const browserRefreshSessionId = + this.visibleBrowserStates.has(sessionId) + ? sessionId + : this.latestVisibleBrowserStateSessionId() + if (browserRefreshSessionId) { + await this.refreshVisibleBrowserStateForSession(browserRefreshSessionId, { + action: 'context', + timestamp: Date.now(), + }).catch((err) => { + log.debug( + { err: (err as Error).message, sessionId: browserRefreshSessionId }, + 'browser context refresh failed', + ) + }) + } // Auto-create default session if it doesn't exist let session = this.sessions.get(sessionId) @@ -5673,6 +5743,13 @@ export class AgentServer { return 0 } + if (!isHarnessSession(session)) { + const browserContext = this.visibleBrowserContextForTurn(sessionId) + if (browserContext) { + chatContent = `${chatContent}\n\n${browserContext}` + } + } + this.activeTurns.add(sessionId) let eventCount = 0 @@ -5892,6 +5969,20 @@ export class AgentServer { sessionId, }) } + } else if (event.type === 'browser_state') { + const browserEvent = event as { + url: string + title: string + lastAction: BrowserAction + stream?: BrowserStreamState + engine?: BrowserEngine + } + this.rememberVisibleBrowserState(sessionId, browserEvent) + if (browserEvent.stream?.port) { + this.ensureBrowserStreamProxy(browserEvent.stream.port, sessionId) + } + } else if (event.type === 'browser_close') { + this.closeBrowserStreamProxy() } // Stamp `done` with the per-turn assistantMessageId so the client @@ -6250,6 +6341,410 @@ export class AgentServer { session.setAskUserHandler(this.buildAskUserHandlerForSession(session.id)) } + // ── Browser stream proxy ───────────────────────────────────────── + + private rememberVisibleBrowserState( + sessionId: string, + state: { + url: string + title: string + lastAction: BrowserAction + stream?: BrowserStreamState + engine?: BrowserEngine + }, + ): void { + if (!state.url) return + this.visibleBrowserStates.set(sessionId, { + url: state.url, + title: state.title, + lastAction: state.lastAction, + stream: state.stream, + engine: state.engine, + updatedAt: Date.now(), + }) + } + + private visibleBrowserContextForTurn(sessionId: string): string | null { + const latestSessionId = this.latestVisibleBrowserStateSessionId() + const state = + this.visibleBrowserStates.get(sessionId) ?? + (latestSessionId ? this.visibleBrowserStates.get(latestSessionId) : undefined) + if (!state?.url) return null + const title = state.title?.trim() + const ageSeconds = Math.max(0, Math.round((Date.now() - state.updatedAt) / 1000)) + return [ + `[Visible browser: Anton's browser pane is open at ${state.url}`, + title ? ` with title "${title}"` : '', + `. Last refreshed ${ageSeconds}s ago. If the user refers to this page, this site, or the browser, inspect the current browser session with the browser tool before answering; do not infer page contents from this hint alone.]`, + ].join('') + } + + private latestVisibleBrowserStateSessionId(): string | null { + let latest: { sessionId: string; updatedAt: number } | null = null + for (const [stateSessionId, state] of this.visibleBrowserStates.entries()) { + if (!latest || state.updatedAt > latest.updatedAt) { + latest = { sessionId: stateSessionId, updatedAt: state.updatedAt } + } + } + return latest?.sessionId ?? null + } + + private browserCallbacksForSession( + sessionId: string, + ): import('@anton/agent-core').BrowserCallbacks { + return { + onBrowserState: (state) => this.emitBrowserStateToClient(sessionId, state), + onBrowserClose: () => { + this.closeBrowserStreamProxy() + this.visibleBrowserStates.delete(sessionId) + this.sendToClient(Channel.AI, { type: 'browser_close', sessionId }) + }, + } + } + + private async refreshVisibleBrowserStateForSession( + sessionId: string, + action: BrowserAction, + ): Promise { + await refreshVisibleBrowserState(this.browserCallbacksForSession(sessionId), action) + } + + private refreshVisibleBrowserStateSoon(sessionId: string, event: BrowserInputEvent): void { + const existing = this.browserStateRefreshTimers.get(sessionId) + if (existing) clearTimeout(existing) + + this.browserStateRefreshTimers.set( + sessionId, + setTimeout(() => { + this.browserStateRefreshTimers.delete(sessionId) + const action = event.type === 'input_keyboard' ? 'keyboard' : 'mouse' + const refresh = (phase: string) => + this.refreshVisibleBrowserStateForSession(sessionId, { + action: `${action}_${phase}`, + timestamp: Date.now(), + }).catch((err) => { + log.debug({ err: (err as Error).message, sessionId }, 'browser state refresh failed') + }) + refresh('input') + setTimeout(() => refresh('settled'), 1200) + }, 350), + ) + } + + private handleBrowserViewport(sessionId: string, width: number, height: number): void { + if (!Number.isFinite(width) || !Number.isFinite(height)) return + const rounded = { + width: Math.round(width), + height: Math.round(height), + } + if (rounded.width < 320 || rounded.height < 240) return + + const previous = this.browserViewportBySession.get(sessionId) + if ( + previous && + Math.abs(previous.width - rounded.width) < 24 && + Math.abs(previous.height - rounded.height) < 24 + ) { + return + } + this.browserViewportBySession.set(sessionId, rounded) + + const existing = this.browserViewportTimers.get(sessionId) + if (existing) clearTimeout(existing) + this.browserViewportTimers.set( + sessionId, + setTimeout(() => { + this.browserViewportTimers.delete(sessionId) + setVisibleBrowserViewport( + rounded.width, + rounded.height, + this.browserCallbacksForSession(sessionId), + ).catch((err) => { + log.debug({ err: (err as Error).message, sessionId }, 'browser viewport resize failed') + }) + }, 200), + ) + } + + private emitBrowserStateToClient( + sessionId: string, + state: { + url: string + title: string + screenshot?: string + lastAction: BrowserAction + elementCount?: number + stream?: BrowserStreamState + engine?: BrowserEngine + }, + ): void { + this.rememberVisibleBrowserState(sessionId, state) + if (state.stream?.port) { + this.ensureBrowserStreamProxy(state.stream.port, sessionId) + } + this.sendToClient(Channel.AI, { type: 'browser_state', ...state, sessionId }) + } + + private ensureBrowserStreamProxy(port: number, sessionId: string): void { + if ( + this.browserStreamWs && + this.browserStreamPort === port && + this.browserStreamWs.readyState === WebSocket.OPEN + ) { + this.browserStreamSessionId = sessionId + return + } + + this.closeBrowserStreamProxy() + this.browserStreamPort = port + this.browserStreamSessionId = sessionId + + const ws = new WebSocket(`ws://127.0.0.1:${port}`) + this.browserStreamWs = ws + const streamSessionId = sessionId + const targetSessionId = () => + this.browserStreamWs === ws + ? (this.browserStreamSessionId ?? streamSessionId) + : streamSessionId + + ws.on('open', () => { + this.sendToClient(Channel.AI, { + type: 'browser_stream_status', + sessionId: targetSessionId(), + connected: true, + }) + }) + + ws.on('message', (data) => { + let parsed: unknown + try { + parsed = JSON.parse(data.toString()) + } catch { + return + } + + if (!parsed || typeof parsed !== 'object') return + const message = parsed as Record + if (message.type === 'frame' && typeof message.data === 'string') { + this.sendToClient(Channel.AI, { + type: 'browser_frame', + sessionId: targetSessionId(), + data: message.data, + metadata: + message.metadata && typeof message.metadata === 'object' ? message.metadata : undefined, + }) + } else if (message.type === 'status') { + this.sendToClient(Channel.AI, { + type: 'browser_stream_status', + sessionId: targetSessionId(), + connected: Boolean(message.connected), + screencasting: + typeof message.screencasting === 'boolean' ? message.screencasting : undefined, + viewportWidth: + typeof message.viewportWidth === 'number' ? message.viewportWidth : undefined, + viewportHeight: + typeof message.viewportHeight === 'number' ? message.viewportHeight : undefined, + }) + } + }) + + ws.on('close', () => { + const wasCurrent = this.browserStreamWs === ws + if (!wasCurrent) return + const closedSessionId = targetSessionId() + this.clearBrowserStreamProxy() + this.sendToClient(Channel.AI, { + type: 'browser_stream_status', + sessionId: closedSessionId, + connected: false, + }) + }) + + ws.on('error', (err) => { + log.warn({ err, port }, 'browser stream proxy failed') + }) + } + + private closeBrowserStreamProxy(): void { + if (this.browserStreamWs) { + try { + this.browserStreamWs.close() + } catch { + // Best effort. + } + } + this.clearBrowserStreamProxy() + } + + private clearBrowserStreamProxy(): void { + this.browserStreamWs = null + this.browserStreamPort = null + this.browserStreamSessionId = null + } + + private forwardBrowserInput(event: BrowserInputEvent): void { + if (!this.browserStreamWs || this.browserStreamWs.readyState !== WebSocket.OPEN) return + this.browserStreamWs.send(JSON.stringify(event)) + } + + private async handleBrowserRuntimeStatus(): Promise { + try { + const status = await getBrowserRuntimeStatus() + this.sendToClient(Channel.AI, { type: 'browser_runtime_status_response', status }) + } catch (err) { + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target: 'repair', + stage: 'error', + message: `Failed to check browser runtime: ${(err as Error).message}`, + }) + } + } + + private handleBrowserRuntimeInstall(target: BrowserRuntimeInstallTarget): void { + void this.runBrowserRuntimeInstall(target) + } + + private browserRuntimeInstallCovers( + active: BrowserRuntimeInstallTarget, + requested: BrowserRuntimeInstallTarget, + ): boolean { + if (active === requested) return true + if (active === 'all' || active === 'repair') return true + return false + } + + private runBrowserRuntimeInstall(target: BrowserRuntimeInstallTarget): Promise { + if (this.browserRuntimeInstallInFlight) { + const activeTarget = this.browserRuntimeInstallInFlightTarget + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target, + stage: 'installing', + message: activeTarget + ? `Waiting for ${activeTarget} browser runtime install` + : 'Browser runtime install is already running', + }) + if (activeTarget && this.browserRuntimeInstallCovers(activeTarget, target)) { + return this.browserRuntimeInstallInFlight + } + return this.browserRuntimeInstallInFlight.then(() => this.runBrowserRuntimeInstall(target)) + } + + this.browserRuntimeInstallInFlightTarget = target + this.browserRuntimeInstallInFlight = installBrowserRuntime(target, (stage, message) => { + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target, + stage, + message, + }) + }) + .then((status) => { + this.sendToClient(Channel.AI, { + type: 'browser_runtime_status_response', + status, + }) + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target, + stage: 'done', + message: 'Browser runtime ready', + status, + }) + }) + .catch((err) => { + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target, + stage: 'error', + message: `Browser runtime install failed: ${(err as Error).message}`, + }) + }) + .finally(() => { + this.browserRuntimeInstallInFlight = null + this.browserRuntimeInstallInFlightTarget = null + }) + + return this.browserRuntimeInstallInFlight + } + + private async ensureVisibleBrowserRuntime(): Promise { + const status = await getBrowserRuntimeStatus() + const chrome = status.components.find((component) => component.id === 'chrome') + const agentBrowser = status.components.find((component) => component.id === 'agent-browser') + if (chrome?.status === 'ready' && agentBrowser?.status === 'ready') return true + + this.sendToClient(Channel.AI, { type: 'browser_runtime_status_response', status }) + await this.runBrowserRuntimeInstall('chrome') + + const nextStatus = await getBrowserRuntimeStatus() + this.sendToClient(Channel.AI, { type: 'browser_runtime_status_response', status: nextStatus }) + const nextChrome = nextStatus.components.find((component) => component.id === 'chrome') + const nextAgentBrowser = nextStatus.components.find( + (component) => component.id === 'agent-browser', + ) + return nextChrome?.status === 'ready' && nextAgentBrowser?.status === 'ready' + } + + private browserOpenFailureMessage(output: string): string { + if (output.includes('error while loading shared libraries') || output.includes('not found')) { + return 'Chrome is missing Linux system dependencies. Run Browser repair to install them.' + } + if (output.includes('DevToolsActivePort') || output.includes('Chrome exited early')) { + return 'Chrome could not start on this server. Run Browser repair.' + } + return output.replace(/^Error:\s*/, '').split('\n')[0] || 'Browser failed to open.' + } + + private async handleBrowserOpen(sessionId: string, url?: string): Promise { + const runtimeReady = await this.ensureVisibleBrowserRuntime() + if (!runtimeReady) { + return + } + + const output = await executeBrowser( + { operation: 'open', ...(url ? { url } : {}) }, + this.browserCallbacksForSession(sessionId), + ) + if (output.startsWith('Error:')) { + this.sendToClient(Channel.AI, { + type: 'browser_runtime_install_progress', + target: 'repair', + stage: 'error', + message: this.browserOpenFailureMessage(output), + }) + await this.handleBrowserRuntimeStatus() + } + } + + private async handleBrowserCommand( + sessionId: string, + command: 'back' | 'forward' | 'reload', + ): Promise { + const runtimeReady = await this.ensureVisibleBrowserRuntime() + if (!runtimeReady) { + return + } + + const output = await executeBrowser( + { operation: command }, + this.browserCallbacksForSession(sessionId), + ) + if (output.startsWith('Error:')) { + log.debug({ output, command, sessionId }, 'browser command failed') + await this.refreshVisibleBrowserStateForSession(sessionId, { + action: `${command}_failed`, + timestamp: Date.now(), + }).catch((err) => { + log.debug( + { err: (err as Error).message, command, sessionId }, + 'browser state refresh failed', + ) + }) + } + } + /** * Harness-side counterpart to `wireAskUserHandler`. The codex / claude * harness can't call `setAskUserHandler` (no Pi SDK Session), so the @@ -6276,10 +6771,16 @@ export class AgentServer { ): import('@anton/agent-core').HarnessSessionContext['browserCallbacks'] { return { onBrowserState: (state) => { + this.rememberVisibleBrowserState(sessionId, state) + if (state.stream?.port) { + this.ensureBrowserStreamProxy(state.stream.port, sessionId) + } const session = this.sessions.get(sessionId) if (session && isHarnessSession(session)) session.emitBrowserState(state) }, onBrowserClose: () => { + this.closeBrowserStreamProxy() + this.visibleBrowserStates.delete(sessionId) const session = this.sessions.get(sessionId) if (session && isHarnessSession(session)) session.emitBrowserClose() }, diff --git a/packages/desktop/src/App.tsx b/packages/desktop/src/App.tsx index 40f25d9f..7db6457f 100644 --- a/packages/desktop/src/App.tsx +++ b/packages/desktop/src/App.tsx @@ -1,5 +1,5 @@ import { AnimatePresence } from 'framer-motion' -import { Code, Ticket } from 'lucide-react' +import { AppWindow, Code, Ticket } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { ActivityDock } from './components/ActivityDock.js' import { CommandPalette } from './components/CommandPalette.js' @@ -61,7 +61,7 @@ export function App() { const hasMessages = (activeConv?.messages?.length || 0) > 0 const artifactPanelOpen = artifactStore((s) => s.artifactPanelOpen) uiStore((s) => s.sidebarCollapsed) - uiStore((s) => s.toggleSidebar) + const setSidebarCollapsed = uiStore((s) => s.setSidebarCollapsed) const updateStage = updateStore((s) => s.updateStage) const sidePanelOpen = artifactPanelOpen const projects = projectStore((s) => s.projects) @@ -72,6 +72,7 @@ export function App() { const tourCompleted = uiStore((s) => s.tourCompleted) const setArtifactPanelOpen = artifactStore((s) => s.setArtifactPanelOpen) const setSidePanelView = uiStore((s) => s.setSidePanelView) + const sidePanelView = uiStore((s) => s.sidePanelView) const tasksHidden = uiStore((s) => s.tasksHidden) const toggleTasksHidden = uiStore((s) => s.toggleTasksHidden) const currentTasks = useActiveSessionState((s) => s.tasks) @@ -246,6 +247,12 @@ export function App() { setConnected(false) } + const openBrowserPanel = useCallback(() => { + setSidePanelView('browser') + setArtifactPanelOpen(true) + setSidebarCollapsed(true) + }, [setArtifactPanelOpen, setSidePanelView, setSidebarCollapsed]) + if (!connected) { return setConnected(true)} /> } @@ -381,6 +388,17 @@ export function App() { )} {activeView === 'chat' && hasMessages && ( <> + {!(sidePanelOpen && sidePanelView === 'browser') && ( + + )} {sessionUsage && ( - - )} - {/* Single-view close button for devmode */} {!showTabs && activeView === 'devmode' && (
@@ -196,7 +248,7 @@ export function SidePanel() { {/* Panel content */}
- {activeView === 'browser' && } + {activeView === 'browser' && } {activeView === 'artifacts' && } {activeView === 'plan' && } {activeView === 'context' && } diff --git a/packages/desktop/src/components/browser/BrowserViewerContent.tsx b/packages/desktop/src/components/browser/BrowserViewerContent.tsx index 28526ce0..7276dccc 100644 --- a/packages/desktop/src/components/browser/BrowserViewerContent.tsx +++ b/packages/desktop/src/components/browser/BrowserViewerContent.tsx @@ -1,109 +1,468 @@ -import { Globe, MousePointer, Navigation, PenLine, ScrollText } from 'lucide-react' -import { useEffect, useRef } from 'react' +import type { BrowserRuntimeInstallTarget, BrowserRuntimeStatus } from '@anton/protocol' +import { + AlertTriangle, + AppWindow, + ChevronLeft, + ChevronRight, + Loader2, + Maximize2, + Minimize2, + RefreshCw, + X, +} from 'lucide-react' +import type React from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' +import { connection } from '../../lib/connection.js' +import { useStore } from '../../lib/store.js' import { artifactStore } from '../../lib/store/artifactStore.js' -function formatAction(action: { action: string; target?: string; value?: string }): string { - switch (action.action) { - case 'open': - return `Navigated to ${action.target || 'page'}` - case 'click': - return `Clicked ${action.target || 'element'}` - case 'fill': - return `Typed "${action.value || ''}" into ${action.target || 'input'}` - case 'scroll': - return `Scrolled ${action.target || 'down'} ${action.value || ''}px` - case 'snapshot': - return 'Read page elements' - case 'screenshot': - return 'Captured screenshot' - case 'wait': - return `Waited for ${action.target || 'page load'}` - case 'get': - return `Read ${action.target || 'page info'}` - default: - return action.action - } +function keyboardModifiers(e: React.KeyboardEvent): number { + return (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0) } -function actionIcon(action: string) { - switch (action) { - case 'open': - return - case 'click': - return - case 'fill': - return - case 'snapshot': - return - default: - return - } +function visibleRuntimeReady(status: BrowserRuntimeStatus | null): boolean { + if (!status) return false + const chrome = status.components.find((component) => component.id === 'chrome') + const agentBrowser = status.components.find((component) => component.id === 'agent-browser') + return chrome?.status === 'ready' && agentBrowser?.status === 'ready' +} + +function visibleRuntimeInstallTarget( + status: BrowserRuntimeStatus | null, +): BrowserRuntimeInstallTarget | null { + if (!status || visibleRuntimeReady(status)) return null + const agentBrowser = status.components.find((component) => component.id === 'agent-browser') + const chrome = status.components.find((component) => component.id === 'chrome') + if (agentBrowser?.status !== 'ready') return null + return chrome?.status === 'error' ? 'repair' : 'chrome' +} + +function runtimeIssueText(status: BrowserRuntimeStatus | null): string { + if (!status) return 'Checking browser runtime' + const agentBrowser = status.components.find((component) => component.id === 'agent-browser') + const chrome = status.components.find((component) => component.id === 'chrome') + if (agentBrowser?.status !== 'ready') return 'Browser package is missing from this deployment.' + if (chrome?.status === 'error') return 'Chrome for Anton needs repair before it can open.' + return 'Chrome for Anton needs setup before it can open.' } -export function BrowserViewerContent() { +export function BrowserViewerContent({ showTopBar = true }: { showTopBar?: boolean }) { const browserState = artifactStore((s) => s.browserState) - const actionCount = browserState?.actions.length ?? 0 - const logEndRef = useRef(null) + const browserRuntimeStatus = artifactStore((s) => s.browserRuntimeStatus) + const browserRuntimeProgress = artifactStore((s) => s.browserRuntimeProgress) + const browserRuntimeInstalling = artifactStore((s) => s.browserRuntimeInstalling) + const setArtifactPanelOpen = artifactStore((s) => s.setArtifactPanelOpen) + const setBrowserRuntimeProgress = artifactStore((s) => s.setBrowserRuntimeProgress) + const setBrowserRuntimeInstalling = artifactStore((s) => s.setBrowserRuntimeInstalling) + const activeConv = useStore((s) => s.getActiveConversation()) + const sessionId = activeConv?.sessionId + const viewportRef = useRef(null) + const imageRef = useRef(null) + const pointerDownRef = useRef(false) + const autoOpenRef = useRef(false) + const [urlValue, setUrlValue] = useState('') + const [fullscreen, setFullscreen] = useState(false) - // Auto-scroll action log when new actions arrive - // biome-ignore lint/correctness/useExhaustiveDependencies: actionCount triggers scroll on new actions useEffect(() => { - logEndRef.current?.scrollIntoView({ behavior: 'smooth' }) - }, [actionCount]) - - if (!browserState) { - return ( -
- -

Browser not active

-
- ) + connection.sendBrowserRuntimeStatus() + }, []) + + useEffect(() => { + setUrlValue(browserState?.url || '') + }, [browserState?.url]) + + const startBrowser = useCallback(() => { + setBrowserRuntimeInstalling('chrome') + setBrowserRuntimeProgress({ + type: 'browser_runtime_install_progress', + target: 'chrome', + stage: 'checking', + message: 'Opening browser', + }) + connection.sendBrowserOpen(sessionId) + }, [sessionId, setBrowserRuntimeInstalling, setBrowserRuntimeProgress]) + + const startRuntimeInstall = useCallback( + (target: BrowserRuntimeInstallTarget) => { + setBrowserRuntimeInstalling(target) + setBrowserRuntimeProgress({ + type: 'browser_runtime_install_progress', + target, + stage: 'checking', + message: 'Checking browser runtime', + }) + connection.sendBrowserRuntimeInstall(target) + }, + [setBrowserRuntimeInstalling, setBrowserRuntimeProgress], + ) + + const navigate = useCallback( + (url: string) => { + const target = url.trim() + if (!target) return + connection.sendBrowserNavigate(target, sessionId) + }, + [sessionId], + ) + + const sendCommand = useCallback( + (command: 'back' | 'forward' | 'reload') => { + if (!browserState) return + connection.sendBrowserCommand(command, sessionId) + }, + [browserState, sessionId], + ) + + useEffect(() => { + if (browserState || autoOpenRef.current) return + autoOpenRef.current = true + startBrowser() + }, [browserState, startBrowser]) + + useEffect(() => { + if (!browserState) return + const viewport = viewportRef.current + if (!viewport) return + + let frameId: number | null = null + const sendViewport = () => { + if (frameId !== null) cancelAnimationFrame(frameId) + frameId = requestAnimationFrame(() => { + frameId = null + const rect = viewport.getBoundingClientRect() + if (rect.width < 320 || rect.height < 240) return + connection.sendBrowserViewport(Math.round(rect.width), Math.round(rect.height), sessionId) + }) + } + + const observer = new ResizeObserver(sendViewport) + observer.observe(viewport) + sendViewport() + return () => { + observer.disconnect() + if (frameId !== null) cancelAnimationFrame(frameId) + } + }, [browserState, sessionId]) + + useEffect(() => { + if (!fullscreen) return + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') setFullscreen(false) + } + document.addEventListener('keydown', onKeyDown) + return () => document.removeEventListener('keydown', onKeyDown) + }, [fullscreen]) + + const pointForEvent = useCallback( + (e: React.PointerEvent | React.WheelEvent) => { + const rect = imageRef.current?.getBoundingClientRect() + if (!rect || !browserState) return null + if ( + e.clientX < rect.left || + e.clientX > rect.right || + e.clientY < rect.top || + e.clientY > rect.bottom + ) { + return null + } + const width = browserState.frameMetadata?.deviceWidth || browserState.viewportWidth || 1280 + const height = browserState.frameMetadata?.deviceHeight || browserState.viewportHeight || 720 + return { + x: Math.max(0, Math.min(width, ((e.clientX - rect.left) / rect.width) * width)), + y: Math.max(0, Math.min(height, ((e.clientY - rect.top) / rect.height) * height)), + } + }, + [browserState], + ) + + const sendMouse = useCallback( + ( + e: React.PointerEvent | React.WheelEvent, + eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel', + extra: Record = {}, + ) => { + const point = pointForEvent(e) + if (!point) return + connection.sendBrowserInput( + { + type: 'input_mouse', + eventType, + x: Math.round(point.x), + y: Math.round(point.y), + ...extra, + }, + sessionId, + ) + }, + [pointForEvent, sessionId], + ) + + const image = browserState?.frame || browserState?.screenshot + const installActive = + browserRuntimeInstalling !== null && + browserRuntimeProgress?.stage !== 'done' && + browserRuntimeProgress?.stage !== 'error' + const runtimeProgressError = !browserState && browserRuntimeProgress?.stage === 'error' + const runtimeKnown = browserRuntimeStatus !== null + const runtimeIssue = + !browserState && + !installActive && + (runtimeProgressError || (runtimeKnown && !visibleRuntimeReady(browserRuntimeStatus))) + const installTarget = runtimeProgressError + ? 'repair' + : visibleRuntimeInstallTarget(browserRuntimeStatus) + const addressText = browserState + ? urlValue + : installActive + ? (browserRuntimeProgress?.message ?? 'Preparing browser') + : runtimeIssue + ? 'Setup required' + : 'No preview available' + const emptyMessage = installActive + ? (browserRuntimeProgress?.message ?? 'Preparing browser') + : runtimeProgressError + ? (browserRuntimeProgress?.message ?? 'Browser failed to open.') + : runtimeIssue + ? runtimeIssueText(browserRuntimeStatus) + : 'All running apps and browser use activity will appear here.' + const primaryLabel = installActive + ? 'Preparing browser' + : runtimeIssue + ? installTarget === 'repair' + ? 'Repair browser' + : installTarget + ? 'Set up browser' + : 'Check runtime' + : 'Open browser' + + const handlePrimaryAction = () => { + if (installActive) return + if (runtimeIssue) { + if (installTarget) { + startRuntimeInstall(installTarget) + } else { + connection.sendBrowserRuntimeStatus() + } + return + } + startBrowser() } return ( -
- {/* URL Bar */} -
- - - {browserState.url} - -
+
+ {showTopBar && ( +
+
+ +
+ +
+ )} - {/* Screenshot */} -
- {browserState.screenshot ? ( - {browserState.title - ) : ( -
- - Waiting for screenshot... +
+
+
+ + +
- )} -
- {/* Action Log */} -
-
Activity
-
- {browserState.actions.map((action, i) => ( -
- {actionIcon(action.action)} - {formatAction(action)} - - {new Date(action.timestamp).toLocaleTimeString([], { - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - })} - + {browserState ? ( +
{ + e.preventDefault() + navigate(urlValue) + }} + > + + setUrlValue(e.target.value)} + spellCheck={false} + aria-label="Browser URL" + /> + + ) : ( +
+ {installActive ? ( + + ) : runtimeIssue ? ( + + ) : ( + + )} + {addressText}
- ))} -
+ )} + +
+ + +
+ + {browserState ? ( + + ) : ( +
+
+ {emptyMessage} + +
+
+ )}
) diff --git a/packages/desktop/src/components/connectors/ConnectorsView.tsx b/packages/desktop/src/components/connectors/ConnectorsView.tsx index 5d564b08..be970ebc 100644 --- a/packages/desktop/src/components/connectors/ConnectorsView.tsx +++ b/packages/desktop/src/components/connectors/ConnectorsView.tsx @@ -12,16 +12,39 @@ * - 'never' → tool is filtered out of getAllTools(), agent never sees it */ -import { Check, ExternalLink, Loader2, Plug, Plus, Search, Trash2 } from 'lucide-react' +import type { + BrowserRuntimeInstallProgressMessage, + BrowserRuntimeInstallTarget, + BrowserRuntimeStatus, +} from '@anton/protocol' +import { + AlertTriangle, + AppWindow, + Check, + CheckCircle2, + CircleDashed, + Download, + ExternalLink, + Loader2, + Plug, + Plus, + RefreshCw, + Search, + Trash2, + Wrench, + XCircle, +} from 'lucide-react' import { useEffect, useMemo, useRef, useState } from 'react' import { connection } from '../../lib/connection.js' import { useStore } from '../../lib/store.js' +import { artifactStore } from '../../lib/store/artifactStore.js' import { connectorStore } from '../../lib/store/connectorStore.js' import type { ConnectorRegistryInfo, ConnectorStatusInfo } from '../../lib/store/types.js' import { ConnectorIcon } from './ConnectorIcons.js' import { AppSetup } from './ConnectorsPage.js' type ToolPermission = 'auto' | 'ask' | 'never' +const BROWSER_RUNTIME_ID = 'browser-runtime' // Heuristic classification of tool names. The protocol doesn't tell us whether // a tool is read-only or write/delete, so we sniff common verbs from the name. @@ -53,6 +76,10 @@ function getPermission( return perms?.[toolName] ?? 'auto' } +function isBrowserRuntimeReady(status: BrowserRuntimeStatus | null): boolean { + return status?.overall === 'ready' +} + // ── Sidebar item type ───────────────────────────────────────────────── // For multi-account connectors we group all instances under the registry id. @@ -74,6 +101,11 @@ export function ConnectorsView() { const [selectedId, setSelectedId] = useState(null) const [search, setSearch] = useState('') const [connectPopupId, setConnectPopupId] = useState(null) + const browserRuntimeStatus = artifactStore((s) => s.browserRuntimeStatus) + const browserRuntimeProgress = artifactStore((s) => s.browserRuntimeProgress) + const browserRuntimeInstalling = artifactStore((s) => s.browserRuntimeInstalling) + const setBrowserRuntimeProgress = artifactStore((s) => s.setBrowserRuntimeProgress) + const setBrowserRuntimeInstalling = artifactStore((s) => s.setBrowserRuntimeInstalling) // Refresh data when connection comes up const connectionStatus = useStore((s) => s.connectionStatus) @@ -81,6 +113,7 @@ export function ConnectorsView() { if (connectionStatus === 'connected') { connectorStore.getState().listConnectors() connectorStore.getState().listConnectorRegistry() + connection.sendBrowserRuntimeStatus() } }, [connectionStatus]) @@ -172,12 +205,25 @@ export function ConnectorsView() { const q = search.trim().toLowerCase() const filterFn = (name: string) => !q || name.toLowerCase().includes(q) + const browserItem: SidebarItem = { + id: BROWSER_RUNTIME_ID, + name: 'Browser', + instances: [], + connected: isBrowserRuntimeReady(browserRuntimeStatus), + } + const showBrowser = filterFn(browserItem.name) return { - connectedItems: connected.filter((i) => filterFn(i.name)), - notConnectedItems: available.filter((i) => filterFn(i.name)), + connectedItems: + showBrowser && browserItem.connected + ? [...connected.filter((i) => filterFn(i.name)), browserItem] + : connected.filter((i) => filterFn(i.name)), + notConnectedItems: + showBrowser && !browserItem.connected + ? [browserItem, ...available.filter((i) => filterFn(i.name))] + : available.filter((i) => filterFn(i.name)), } - }, [connectors, registry, search]) + }, [connectors, registry, search, browserRuntimeStatus]) // Auto-select the first connected connector if nothing is selected useEffect(() => { @@ -254,7 +300,24 @@ export function ConnectorsView() { {/* ── Right detail pane ────────────────────────────── */}
- {selected ? ( + {selected?.id === BROWSER_RUNTIME_ID ? ( + connection.sendBrowserRuntimeStatus()} + onInstall={(target) => { + setBrowserRuntimeInstalling(target) + setBrowserRuntimeProgress({ + type: 'browser_runtime_install_progress', + target, + stage: 'checking', + message: 'Checking browser runtime', + }) + connection.sendBrowserRuntimeInstall(target) + }} + /> + ) : selected ? ( - + {id === BROWSER_RUNTIME_ID ? ( + + ) : ( + + )} {name} {accountCount > 1 && ( @@ -653,6 +720,261 @@ function ConnectorDetail({ ) } +// ── Browser runtime ─────────────────────────────────────────────────── + +type RuntimeComponent = BrowserRuntimeStatus['components'][number] +type RuntimeOverall = BrowserRuntimeStatus['overall'] + +function runtimeHeadline(status: BrowserRuntimeStatus | null): string { + switch (status?.overall) { + case 'ready': + return 'Ready on this server' + case 'partial': + return 'Setup incomplete' + case 'missing': + return 'Setup required' + case 'installing': + return 'Setting up browser' + case 'error': + return 'Repair required' + default: + return 'Checking browser runtime' + } +} + +function runtimeSubtext(status: BrowserRuntimeStatus | null): string { + switch (status?.overall) { + case 'ready': + return 'Visible Chrome and background browsing are available.' + case 'partial': + return 'One browser engine still needs setup.' + case 'missing': + return 'Install the server-side browser runtime.' + case 'installing': + return 'Installing browser components on the Anton server.' + case 'error': + return 'One component failed verification.' + default: + return 'Anton is checking the server runtime.' + } +} + +function componentStatusLabel(component: RuntimeComponent): string { + switch (component.status) { + case 'ready': + return 'Ready' + case 'missing': + return 'Missing' + case 'installing': + return 'Installing' + case 'error': + return 'Needs repair' + default: + return 'Checking' + } +} + +function componentDetail(component: RuntimeComponent): string { + if (component.status === 'ready') { + switch (component.id) { + case 'agent-browser': + return 'Automation runtime is available.' + case 'chrome': + return 'Visible browser engine is available.' + case 'lightpanda': + return 'Background browser engine is available.' + } + } + if (component.detail?.startsWith('Command failed:')) { + return 'Verification failed. Run Repair to reinstall this component.' + } + return component.detail ?? 'Waiting for status.' +} + +function RuntimeStatusIcon({ + status, + size = 16, +}: { + status?: RuntimeOverall | RuntimeComponent['status'] + size?: number +}) { + if (status === 'ready') return + if (status === 'error') return + if (status === 'installing') { + return + } + if (status === 'missing' || status === 'partial') { + return + } + return +} + +function BrowserRuntimeDetail({ + status, + progress, + installing, + onRefresh, + onInstall, +}: { + status: BrowserRuntimeStatus | null + progress: BrowserRuntimeInstallProgressMessage | null + installing: BrowserRuntimeInstallTarget | null + onRefresh: () => void + onInstall: (target: BrowserRuntimeInstallTarget) => void +}) { + const components = + status?.components ?? + ([ + { + id: 'agent-browser', + label: 'agent-browser', + status: 'unknown', + required: true, + installable: false, + detail: 'Checking bundled CLI', + }, + { + id: 'chrome', + label: 'Chrome for Anton', + status: 'unknown', + required: true, + installable: true, + detail: 'Checking visible browser runtime', + }, + { + id: 'lightpanda', + label: 'Lightpanda', + status: 'unknown', + required: true, + installable: true, + detail: 'Checking background browser runtime', + }, + ] satisfies BrowserRuntimeStatus['components']) + + const ready = status?.overall === 'ready' + const isBusy = installing !== null + const visibleProgress = progress && progress.stage !== 'done' ? progress : null + const primaryLabel = isBusy ? 'Setting up' : ready ? 'Repair' : 'Finish setup' + + return ( +
+
+
+
+ +
+
+
Connector
+
Browser
+
+
+
+ + +
+
+ +
+
+
+ +
+
+
{runtimeHeadline(status)}
+
{runtimeSubtext(status)}
+
+
+ {status?.profileDir && ( +
+ Profile {status.profileDir} +
+ )} +
+ + {visibleProgress && ( +
+ {visibleProgress.stage !== 'error' && } + {visibleProgress.message} +
+ )} + +
+ {components.map((component) => { + const installTarget = + component.id === 'chrome' || component.id === 'lightpanda' ? component.id : null + return ( +
+
+ + + +
+
+ {component.label} + {component.version && ( + {component.version} + )} +
+
+ {componentDetail(component)} +
+
+
+ + {componentStatusLabel(component)} + + {installTarget && component.status !== 'ready' && ( + + )} +
+ ) + })} +
+
+ ) +} + // ── Tool group ───────────────────────────────────────────────────────── function ToolGroup({ diff --git a/packages/desktop/src/components/home/StreamHome.tsx b/packages/desktop/src/components/home/StreamHome.tsx index d7ed81d5..fc6282b3 100644 --- a/packages/desktop/src/components/home/StreamHome.tsx +++ b/packages/desktop/src/components/home/StreamHome.tsx @@ -1,4 +1,4 @@ -import { BookOpen, Code2, Mail, Pencil, Sparkles } from 'lucide-react' +import { AppWindow, BookOpen, Code2, Mail, Pencil, Sparkles } from 'lucide-react' import { useMemo, useRef, useState } from 'react' import { sanitizeTitle } from '../../lib/conversations.js' import { ensureSessionReadyForSend } from '../../lib/sessionReadiness.js' @@ -13,6 +13,7 @@ import { ChatInput } from '../chat/ChatInput.js' interface Props { onSkillSelect: (skill: Skill) => void + onOpenBrowser: () => void } const categories = [ @@ -68,7 +69,7 @@ function pickTail(): string { return TAIL_PHRASES[Math.floor(Math.random() * TAIL_PHRASES.length)] ?? '' } -export function StreamHome({ onSkillSelect }: Props) { +export function StreamHome({ onSkillSelect, onOpenBrowser }: Props) { const [draft, setDraft] = useState('') const newConversation = useStore((s) => s.newConversation) const activeConversationId = useStore((s) => s.activeConversationId) @@ -170,6 +171,17 @@ export function StreamHome({ onSkillSelect }: Props) { return (
+
+ +
diff --git a/packages/desktop/src/index.css b/packages/desktop/src/index.css index 2789fa44..81715915 100644 --- a/packages/desktop/src/index.css +++ b/packages/desktop/src/index.css @@ -1224,8 +1224,10 @@ button { .workspace-body { min-height: 0; + min-width: 0; flex: 1; display: flex; + overflow: hidden; } .customize-placeholder { @@ -2937,7 +2939,41 @@ button { flex: 1; overflow-y: auto; display: flex; + position: relative; +} + +.home-actions { + position: absolute; + top: 18px; + right: 22px; + z-index: 2; + display: flex; + align-items: center; +} + +.home-browser-btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 5px; + height: 30px; + padding: 0 8px; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--text-3); + font-family: inherit; + font-size: 12px; + font-weight: 500; + cursor: pointer; + transition: background 0.12s, color 0.12s; +} + +.home-browser-btn:hover { + background: var(--bg-elev-2); + color: var(--text); } + .home { max-width: 720px; margin: 0 auto; @@ -10975,7 +11011,7 @@ button { .side-panel { min-width: 320px; - max-width: 900px; + max-width: 2400px; flex-shrink: 0; display: flex; flex-direction: column; @@ -10986,6 +11022,15 @@ button { position: relative; } +.side-panel[data-view="browser"] { + max-width: 2400px; + background: var(--bg); +} + +.side-panel[data-view="browser"] .side-panel__body { + background: var(--bg); +} + .side-panel__resize-handle { position: absolute; left: -3px; @@ -11077,6 +11122,7 @@ button { .side-panel__body { flex: 1; min-height: 0; + min-width: 0; display: flex; flex-direction: column; overflow: hidden; @@ -11088,113 +11134,267 @@ button { display: flex; flex-direction: column; height: 100%; + min-height: 0; + min-width: 0; + width: 100%; overflow: hidden; - gap: 0; + padding: 0; + background: var(--bg); +} + +.browser-viewer--fullscreen { + position: fixed; + inset: 0; + z-index: 1000; + background: var(--bg); +} + +.browser-viewer__topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + min-height: 38px; + margin-bottom: 12px; + flex-shrink: 0; +} + +.browser-viewer__tabs { + display: inline-flex; + align-items: center; + padding: 3px; + border-radius: 999px; + background: rgba(var(--overlay), 0.04); + border: 1px solid rgba(var(--overlay), 0.04); +} + +.browser-viewer__tab { + display: inline-flex; + align-items: center; + gap: 7px; + height: 30px; + padding: 0 14px; + border: 0; + border-radius: 999px; + background: transparent; + color: var(--text-muted); + font-size: 13px; + font-weight: 500; +} + +.browser-viewer__tab--active { + color: var(--text); + background: var(--bg-elevated); + box-shadow: 0 1px 5px rgba(0, 0, 0, 0.14); } -.browser-viewer--empty { +.browser-viewer__topbar-btn { + display: inline-flex; align-items: center; justify-content: center; - gap: 8px; + width: 30px; + height: 30px; + border: 0; + border-radius: 8px; + background: transparent; + color: var(--text-muted); + cursor: pointer; +} + +.browser-viewer__topbar-btn:hover { + background: rgba(var(--overlay), 0.06); + color: var(--text); } -.browser-viewer__url-bar { +.browser-viewer__frame { + flex: 1; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; + overflow: hidden; + border: 0; + border-radius: 0; + background: var(--bg); + box-shadow: none; +} + +.browser-viewer__chrome { display: flex; align-items: center; - gap: 8px; - padding: 8px 12px; + gap: 9px; + min-height: 54px; + min-width: 0; + padding: 10px 14px; + flex-shrink: 0; + background: var(--bg-elevated); border-bottom: 1px solid var(--border); - background: rgba(var(--overlay), 0.02); - min-height: 36px; } -.browser-viewer__url { - font-size: 12px; +.browser-viewer__nav { + display: flex; + align-items: center; + gap: 3px; + flex-shrink: 0; +} + +.browser-viewer__chrome-actions { + display: flex; + align-items: center; + gap: 3px; + flex-shrink: 0; +} + +.browser-viewer__nav-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 26px; + height: 26px; + border-radius: 7px; + border: 0; + background: transparent; + color: var(--text-muted); + cursor: pointer; +} + +.browser-viewer__nav-btn:disabled { + opacity: 0.35; + cursor: default; +} + +.browser-viewer__nav-btn:not(:disabled):hover { + background: rgba(var(--overlay), 0.07); + color: var(--text); +} + +.browser-viewer__address { + min-width: 180px; + flex: 1; + display: flex; + align-items: center; + justify-content: center; + gap: 9px; + height: 32px; + padding: 0 14px; + border: 0; + border-radius: 999px; + background: rgba(var(--overlay), 0.045); color: var(--text-secondary); +} + +.browser-viewer__address--empty { + color: var(--text-muted); + font-size: 13px; + font-weight: 500; +} + +.browser-viewer__address--empty span { + min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} + +.browser-viewer__url-input { + min-width: 0; flex: 1; - font-family: var(--font-mono, monospace); + border: 0; + outline: none; + background: transparent; + color: var(--text-secondary); + font-size: 13px; + text-align: center; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-family: var(--font-mono, ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace); } -.browser-viewer__screenshot { +.browser-viewer__viewport, +.browser-viewer__empty-surface { flex: 1; min-height: 0; + min-width: 0; overflow: hidden; + border: 0; + border-top: 1px solid var(--border); + background: var(--bg); +} + +.browser-viewer__viewport { display: flex; align-items: flex-start; justify-content: center; - background: rgba(0, 0, 0, 0.15); - border-bottom: 1px solid var(--border); + padding: 0; + outline: none; + cursor: default; + user-select: none; + touch-action: none; +} + +.browser-viewer__viewport:focus { + box-shadow: inset 0 0 0 1px color-mix(in oklch, var(--accent) 42%, transparent); } -.browser-viewer__screenshot img { +.browser-viewer__viewport img { width: 100%; height: auto; + max-width: 100%; + flex-shrink: 0; object-fit: contain; display: block; + pointer-events: none; } -.browser-viewer__screenshot-placeholder { +.browser-viewer__loading, +.browser-viewer__empty-surface { display: flex; - flex-direction: column; align-items: center; justify-content: center; - gap: 8px; - height: 100%; - min-height: 120px; -} - -.browser-viewer__actions { - flex: 0 0 auto; - max-height: 200px; - display: flex; - flex-direction: column; - overflow: hidden; } -.browser-viewer__actions-header { - padding: 8px 12px 4px; - font-size: 11px; - font-weight: 500; - color: var(--text-secondary); - text-transform: uppercase; - letter-spacing: 0.05em; -} - -.browser-viewer__actions-list { - flex: 1; - overflow-y: auto; - padding: 0 12px 8px; +.browser-viewer__empty-surface { + position: relative; } -.browser-viewer__action-item { +.browser-viewer__empty-copy { display: flex; + flex-direction: column; align-items: center; - gap: 6px; - padding: 3px 0; - font-size: 12px; - color: var(--text-secondary); + gap: 14px; + width: min(480px, calc(100% - 48px)); + color: var(--text-muted); + font-size: 14px; + line-height: 1.45; + text-align: center; + transform: translateY(18%); } -.browser-viewer__action-item svg { - opacity: 0.5; - flex-shrink: 0; +.browser-viewer__empty-action { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + height: 36px; + padding: 0 15px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--bg-elevated); + color: var(--text); + font-size: 13px; + font-weight: 500; + cursor: pointer; } -.browser-viewer__action-item span:first-of-type { - flex: 1; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; +.browser-viewer__empty-action:not(:disabled):hover { + border-color: var(--border-strong); + background: rgba(var(--overlay), 0.06); } -.browser-viewer__action-time { - font-size: 10px; - opacity: 0.4; - flex-shrink: 0; - font-family: var(--font-mono, monospace); +.browser-viewer__empty-action:disabled { + cursor: default; + opacity: 0.7; } /* ── Plan panel ────────────────────────────────────────────────────── */ @@ -14185,6 +14385,374 @@ button { margin: 0; } +/* ── Browser runtime connector ── */ + +.browser-runtime { + max-width: 880px; +} + +.browser-runtime__topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + margin-bottom: 18px; +} + +.browser-runtime__heading { + display: flex; + align-items: center; + gap: 12px; + min-width: 0; +} + +.browser-runtime__icon { + display: flex; + align-items: center; + justify-content: center; + width: 38px; + height: 38px; + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text); + background: rgba(var(--overlay), 0.04); +} + +.browser-runtime__title-stack { + min-width: 0; +} + +.browser-runtime__eyebrow { + margin-bottom: 2px; + color: var(--text-muted); + font-size: 11px; + font-weight: 500; + text-transform: uppercase; +} + +.browser-runtime__title { + color: var(--text); + font-size: 22px; + font-weight: 650; +} + +.browser-runtime__header-actions { + display: flex; + align-items: center; + gap: 8px; + flex-shrink: 0; +} + +.browser-runtime__action, +.browser-runtime__component-action { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; + height: 34px; + padding: 0 13px; + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text); + background: transparent; + font-size: 12px; + font-weight: 500; + cursor: pointer; + transition: background 0.15s ease, border-color 0.15s ease, opacity 0.15s ease; +} + +.browser-runtime__action--secondary:hover, +.browser-runtime__component-action:hover { + border-color: var(--border-strong); + background: rgba(var(--overlay), 0.06); +} + +.browser-runtime__action--primary { + border-color: var(--text); + background: var(--text); + color: var(--bg); +} + +.browser-runtime__action--primary:hover { + opacity: 0.88; +} + +.browser-runtime__action:disabled, +.browser-runtime__component-action:disabled { + cursor: default; + opacity: 0.55; +} + +.browser-runtime__overview { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + min-height: 78px; + padding: 16px; + border: 1px solid var(--border); + border-radius: 8px; + background: rgba(var(--overlay), 0.025); + margin-bottom: 14px; +} + +.browser-runtime__overview--ready { + border-color: rgba(52, 211, 153, 0.28); + background: rgba(52, 211, 153, 0.055); +} + +.browser-runtime__overview--partial, +.browser-runtime__overview--missing { + border-color: rgba(251, 191, 36, 0.28); + background: rgba(251, 191, 36, 0.055); +} + +.browser-runtime__overview--error { + border-color: rgba(248, 113, 113, 0.28); + background: rgba(248, 113, 113, 0.055); +} + +.browser-runtime__overview-main { + display: flex; + align-items: center; + gap: 12px; + min-width: 0; +} + +.browser-runtime__overview-icon { + display: flex; + align-items: center; + justify-content: center; + width: 30px; + height: 30px; + border-radius: 8px; + color: var(--text); + background: rgba(var(--overlay), 0.06); + flex-shrink: 0; +} + +.browser-runtime__overview--ready .browser-runtime__overview-icon { + color: #34d399; + background: rgba(52, 211, 153, 0.1); +} + +.browser-runtime__overview--partial .browser-runtime__overview-icon, +.browser-runtime__overview--missing .browser-runtime__overview-icon { + color: #fbbf24; + background: rgba(251, 191, 36, 0.1); +} + +.browser-runtime__overview--error .browser-runtime__overview-icon { + color: #f87171; + background: rgba(248, 113, 113, 0.1); +} + +.browser-runtime__overview-copy { + min-width: 0; +} + +.browser-runtime__overview-title { + color: var(--text); + font-size: 14px; + font-weight: 600; +} + +.browser-runtime__overview-subtitle { + margin-top: 3px; + color: var(--text-muted); + font-size: 12px; + line-height: 1.4; +} + +.browser-runtime__profile { + display: flex; + align-items: center; + gap: 8px; + min-width: 220px; + max-width: 360px; + color: var(--text-muted); + font-size: 11px; + flex-shrink: 1; +} + +.browser-runtime__profile span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-family: var(--font-mono, ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace); +} + +.browser-runtime__progress { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 14px; + padding: 10px 12px; + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text-muted); + font-size: 12px; + background: rgba(var(--overlay), 0.03); +} + +.browser-runtime__progress--error { + color: #f87171; + border-color: rgba(248, 113, 113, 0.3); + background: rgba(248, 113, 113, 0.06); +} + +.browser-runtime__components { + display: flex; + flex-direction: column; + gap: 2px; +} + +.browser-runtime__component { + display: grid; + grid-template-columns: minmax(0, 1fr) auto auto; + align-items: center; + gap: 12px; + min-height: 70px; + padding: 12px 0; + border-bottom: 1px solid var(--border); +} + +.browser-runtime__component:last-child { + border-bottom: 0; +} + +.browser-runtime__component-main { + min-width: 0; + display: flex; + align-items: center; + gap: 11px; +} + +.browser-runtime__component-icon { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + border-radius: 8px; + color: var(--text-muted); + background: rgba(var(--overlay), 0.04); + flex-shrink: 0; +} + +.browser-runtime__component-icon--ready { + color: #34d399; + background: rgba(52, 211, 153, 0.08); +} + +.browser-runtime__component-icon--missing, +.browser-runtime__component-icon--unknown { + color: #fbbf24; + background: rgba(251, 191, 36, 0.08); +} + +.browser-runtime__component-icon--error { + color: #f87171; + background: rgba(248, 113, 113, 0.08); +} + +.browser-runtime__component-icon--installing { + color: #60a5fa; + background: rgba(96, 165, 250, 0.08); +} + +.browser-runtime__component-copy { + min-width: 0; +} + +.browser-runtime__component-title { + display: flex; + align-items: baseline; + gap: 8px; + color: var(--text); + font-size: 13px; + font-weight: 600; +} + +.browser-runtime__version { + color: var(--text-muted); + font-size: 11px; + font-weight: 400; +} + +.browser-runtime__component-detail { + margin-top: 3px; + color: var(--text-muted); + font-size: 11px; + line-height: 1.4; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.browser-runtime__component-status { + justify-self: end; + min-width: 74px; + color: var(--text-muted); + font-size: 11px; + text-align: right; +} + +.browser-runtime__component-status--ready { + color: #34d399; +} + +.browser-runtime__component-status--missing, +.browser-runtime__component-status--unknown { + color: #fbbf24; +} + +.browser-runtime__component-status--error { + color: #f87171; +} + +.browser-runtime__component-status--installing { + color: #60a5fa; +} + +.browser-runtime__component-action { + height: 32px; + padding: 0 11px; + font-size: 11px; +} + +@media (max-width: 760px) { + .browser-runtime__topbar, + .browser-runtime__overview { + align-items: stretch; + flex-direction: column; + } + + .browser-runtime__header-actions { + width: 100%; + } + + .browser-runtime__action { + flex: 1; + } + + .browser-runtime__profile { + max-width: 100%; + min-width: 0; + } + + .browser-runtime__component { + grid-template-columns: minmax(0, 1fr); + align-items: stretch; + } + + .browser-runtime__component-status, + .browser-runtime__component-action { + justify-self: start; + } +} + /* ── Permissions section ── */ .connectors-view__perms { diff --git a/packages/desktop/src/lib/connection.ts b/packages/desktop/src/lib/connection.ts index 3177d7f0..31709cea 100644 --- a/packages/desktop/src/lib/connection.ts +++ b/packages/desktop/src/lib/connection.ts @@ -10,6 +10,8 @@ import { type AiMessage, + type BrowserInputEvent, + type BrowserRuntimeInstallTarget, Channel, type ChatImageAttachmentInput, type ControlMessage, @@ -273,6 +275,43 @@ export class Connection { this.send(Channel.AI, { type: 'cancel_turn', sessionId }) } + sendBrowserOpen(sessionId?: string, url?: string) { + this.send(Channel.AI, { + type: 'browser_open', + ...(sessionId && { sessionId }), + ...(url && { url }), + }) + } + + sendBrowserNavigate(url: string, sessionId?: string) { + this.send(Channel.AI, { type: 'browser_navigate', url, ...(sessionId && { sessionId }) }) + } + + sendBrowserCommand(command: 'back' | 'forward' | 'reload', sessionId?: string) { + this.send(Channel.AI, { type: 'browser_command', command, ...(sessionId && { sessionId }) }) + } + + sendBrowserInput(event: BrowserInputEvent, sessionId?: string) { + this.send(Channel.AI, { type: 'browser_input', event, ...(sessionId && { sessionId }) }) + } + + sendBrowserViewport(width: number, height: number, sessionId?: string) { + this.send(Channel.AI, { + type: 'browser_viewport', + width, + height, + ...(sessionId && { sessionId }), + }) + } + + sendBrowserRuntimeStatus() { + this.send(Channel.AI, { type: 'browser_runtime_status' }) + } + + sendBrowserRuntimeInstall(target: BrowserRuntimeInstallTarget) { + this.send(Channel.AI, { type: 'browser_runtime_install', target }) + } + // ── Provider management ───────────────────────────────────────── sendProvidersList() { diff --git a/packages/desktop/src/lib/store/artifactStore.ts b/packages/desktop/src/lib/store/artifactStore.ts index 428c4aef..59e4f919 100644 --- a/packages/desktop/src/lib/store/artifactStore.ts +++ b/packages/desktop/src/lib/store/artifactStore.ts @@ -2,7 +2,14 @@ * Artifact domain store — code/file artifacts + browser viewer state. */ -import { Channel } from '@anton/protocol' +import { + type BrowserFrameMetadata, + type BrowserRuntimeInstallProgressMessage, + type BrowserRuntimeInstallTarget, + type BrowserRuntimeStatus, + type BrowserStreamState, + Channel, +} from '@anton/protocol' import { create } from 'zustand' import type { Artifact, ArtifactRenderType } from '../artifacts.js' import { connection } from '../connection.js' @@ -11,8 +18,14 @@ interface BrowserState { url: string title: string screenshot: string | null + frame: string | null + frameMetadata?: BrowserFrameMetadata actions: Array<{ action: string; target?: string; value?: string; timestamp: number }> active: boolean + stream?: BrowserStreamState + streamConnected?: boolean + viewportWidth?: number + viewportHeight?: number } interface ArtifactState { @@ -28,6 +41,9 @@ interface ArtifactState { // Browser viewer browserState: BrowserState | null + browserRuntimeStatus: BrowserRuntimeStatus | null + browserRuntimeProgress: BrowserRuntimeInstallProgressMessage | null + browserRuntimeInstalling: BrowserRuntimeInstallTarget | null // Artifact actions addArtifact: (artifact: Artifact) => void @@ -47,7 +63,19 @@ interface ArtifactState { screenshot?: string lastAction: { action: string; target?: string; value?: string; timestamp: number } elementCount?: number + stream?: BrowserStreamState + engine?: string }) => void + setBrowserFrame: (frame: string, metadata?: BrowserFrameMetadata) => void + setBrowserStreamStatus: (status: { + connected: boolean + screencasting?: boolean + viewportWidth?: number + viewportHeight?: number + }) => void + setBrowserRuntimeStatus: (status: BrowserRuntimeStatus) => void + setBrowserRuntimeProgress: (progress: BrowserRuntimeInstallProgressMessage) => void + setBrowserRuntimeInstalling: (target: BrowserRuntimeInstallTarget | null) => void clearBrowserState: () => void // Publish modal @@ -81,6 +109,9 @@ export const artifactStore = create((set, get) => ({ artifactViewMode: 'list', artifactTabs: [], browserState: null, + browserRuntimeStatus: null, + browserRuntimeProgress: null, + browserRuntimeInstalling: null, publishModalOpen: false, publishModalArtifactId: null, publishError: null, @@ -151,12 +182,76 @@ export const artifactStore = create((set, get) => ({ url: state.url, title: state.title, screenshot: state.screenshot ?? current?.screenshot ?? null, + frame: current?.frame ?? null, + frameMetadata: current?.frameMetadata, actions: newActions, active: true, + stream: state.stream ?? current?.stream, + streamConnected: state.stream?.connected ?? current?.streamConnected, + viewportWidth: current?.viewportWidth, + viewportHeight: current?.viewportHeight, }, + browserRuntimeProgress: null, + browserRuntimeInstalling: null, }) }, + setBrowserFrame: (frame, metadata) => + set((state) => { + if (!state.browserState) return {} + return { + browserState: { + ...state.browserState, + frame, + frameMetadata: metadata, + streamConnected: true, + viewportWidth: metadata?.deviceWidth ?? state.browserState.viewportWidth, + viewportHeight: metadata?.deviceHeight ?? state.browserState.viewportHeight, + }, + } + }), + + setBrowserStreamStatus: (status) => + set((state) => { + if (!state.browserState) return {} + return { + browserState: { + ...state.browserState, + streamConnected: status.connected, + viewportWidth: status.viewportWidth ?? state.browserState.viewportWidth, + viewportHeight: status.viewportHeight ?? state.browserState.viewportHeight, + }, + } + }), + + setBrowserRuntimeStatus: (status) => + set((state) => { + const keepProgress = + state.browserRuntimeProgress?.stage === 'error' || + state.browserRuntimeProgress?.message === 'Opening browser' + return { + browserRuntimeStatus: status, + browserRuntimeInstalling: + status.overall === 'ready' && !keepProgress ? null : state.browserRuntimeInstalling, + browserRuntimeProgress: + !keepProgress && (status.overall === 'ready' || !state.browserRuntimeInstalling) + ? null + : state.browserRuntimeProgress, + } + }), + + setBrowserRuntimeProgress: (progress) => + set((state) => ({ + browserRuntimeProgress: progress.stage === 'done' ? null : progress, + browserRuntimeInstalling: + progress.stage === 'done' || progress.stage === 'error' + ? null + : (state.browserRuntimeInstalling ?? progress.target), + browserRuntimeStatus: progress.status ?? state.browserRuntimeStatus, + })), + + setBrowserRuntimeInstalling: (target) => set({ browserRuntimeInstalling: target }), + clearBrowserState: () => set({ browserState: null }), openPublishModal: (artifactId) => @@ -187,6 +282,9 @@ export const artifactStore = create((set, get) => ({ artifactViewMode: 'list', artifactTabs: [], browserState: null, + browserRuntimeStatus: null, + browserRuntimeProgress: null, + browserRuntimeInstalling: null, publishModalOpen: false, publishModalArtifactId: null, publishError: null, diff --git a/packages/desktop/src/lib/store/handlers/interactionHandler.ts b/packages/desktop/src/lib/store/handlers/interactionHandler.ts index 7f5ec67a..dc91aec3 100644 --- a/packages/desktop/src/lib/store/handlers/interactionHandler.ts +++ b/packages/desktop/src/lib/store/handlers/interactionHandler.ts @@ -173,6 +173,8 @@ export function handleInteractionMessage(msg: AiMessage, ctx: MessageContext): b screenshot: msg.screenshot, lastAction: msg.lastAction, elementCount: msg.elementCount, + stream: msg.stream, + engine: msg.engine, }) if (!wasActive) { uiStore.setState({ sidePanelView: 'browser' }) @@ -182,6 +184,35 @@ export function handleInteractionMessage(msg: AiMessage, ctx: MessageContext): b return true } + case 'browser_frame': { + if (ctx.isForActiveSession) { + artifactStore.getState().setBrowserFrame(msg.data, msg.metadata) + } + return true + } + + case 'browser_stream_status': { + if (ctx.isForActiveSession) { + artifactStore.getState().setBrowserStreamStatus({ + connected: msg.connected, + screencasting: msg.screencasting, + viewportWidth: msg.viewportWidth, + viewportHeight: msg.viewportHeight, + }) + } + return true + } + + case 'browser_runtime_status_response': { + artifactStore.getState().setBrowserRuntimeStatus(msg.status) + return true + } + + case 'browser_runtime_install_progress': { + artifactStore.getState().setBrowserRuntimeProgress(msg) + return true + } + case 'browser_close': { if (ctx.isForActiveSession) { artifactStore.getState().clearBrowserState() diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 12bfc209..9a239ba2 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -690,6 +690,8 @@ export interface AiBrowserStateMessage { screenshot?: string // base64 JPEG lastAction: BrowserAction elementCount?: number + stream?: BrowserStreamState + engine?: BrowserEngine } export interface AiBrowserCloseMessage { @@ -697,6 +699,138 @@ export interface AiBrowserCloseMessage { sessionId?: string } +export type BrowserEngine = 'chrome' | 'lightpanda' + +export interface BrowserStreamState { + session: string + engine: BrowserEngine + port?: number + connected?: boolean + screencasting?: boolean +} + +export interface BrowserFrameMetadata { + deviceWidth?: number + deviceHeight?: number + pageScaleFactor?: number + offsetTop?: number + scrollOffsetX?: number + scrollOffsetY?: number +} + +export interface AiBrowserFrameMessage { + type: 'browser_frame' + sessionId?: string + data: string + metadata?: BrowserFrameMetadata +} + +export interface AiBrowserStreamStatusMessage { + type: 'browser_stream_status' + sessionId?: string + connected: boolean + screencasting?: boolean + viewportWidth?: number + viewportHeight?: number +} + +export interface BrowserOpenMessage { + type: 'browser_open' + sessionId?: string + url?: string + profile?: string +} + +export interface BrowserNavigateMessage { + type: 'browser_navigate' + sessionId?: string + url: string +} + +export interface BrowserCommandMessage { + type: 'browser_command' + sessionId?: string + command: 'back' | 'forward' | 'reload' +} + +export type BrowserInputEvent = + | { + type: 'input_mouse' + eventType: 'mousePressed' | 'mouseReleased' | 'mouseMoved' | 'mouseWheel' + x: number + y: number + button?: 'left' | 'right' | 'middle' + clickCount?: number + deltaX?: number + deltaY?: number + } + | { + type: 'input_keyboard' + eventType: 'keyDown' | 'keyUp' | 'char' + key?: string + code?: string + text?: string + modifiers?: number + } + +export interface BrowserInputMessage { + type: 'browser_input' + sessionId?: string + event: BrowserInputEvent +} + +export interface BrowserViewportMessage { + type: 'browser_viewport' + sessionId?: string + width: number + height: number +} + +export type BrowserRuntimeComponentId = 'agent-browser' | 'chrome' | 'lightpanda' +export type BrowserRuntimeComponentStatus = 'ready' | 'missing' | 'installing' | 'error' | 'unknown' +export type BrowserRuntimeOverallStatus = 'ready' | 'partial' | 'missing' | 'installing' | 'error' +export type BrowserRuntimeInstallTarget = 'chrome' | 'lightpanda' | 'all' | 'repair' + +export interface BrowserRuntimeComponent { + id: BrowserRuntimeComponentId + label: string + status: BrowserRuntimeComponentStatus + required: boolean + installable: boolean + detail?: string + path?: string + version?: string +} + +export interface BrowserRuntimeStatus { + overall: BrowserRuntimeOverallStatus + profileDir: string + components: BrowserRuntimeComponent[] + checkedAt: number +} + +export interface BrowserRuntimeStatusMessage { + type: 'browser_runtime_status' +} + +export interface BrowserRuntimeInstallMessage { + type: 'browser_runtime_install' + target: BrowserRuntimeInstallTarget +} + +export interface BrowserRuntimeStatusResponse { + type: 'browser_runtime_status_response' + status: BrowserRuntimeStatus +} + +export interface BrowserRuntimeInstallProgressMessage { + type: 'browser_runtime_install_progress' + target: BrowserRuntimeInstallTarget + stage: 'checking' | 'installing' | 'verifying' | 'done' | 'error' + message: string + status?: BrowserRuntimeStatus +} + // ── Task tracker (Claude Code–style todo list) ────────────────────── export type TaskStatus = 'pending' | 'in_progress' | 'completed' @@ -1667,8 +1801,19 @@ export type AiMessage = | ConnectorOAuthCompleteMessage | ConnectorOAuthDisconnectMessage // Browser automation + | BrowserOpenMessage + | BrowserNavigateMessage + | BrowserCommandMessage + | BrowserInputMessage + | BrowserViewportMessage | AiBrowserStateMessage + | AiBrowserFrameMessage + | AiBrowserStreamStatusMessage | AiBrowserCloseMessage + | BrowserRuntimeStatusMessage + | BrowserRuntimeInstallMessage + | BrowserRuntimeStatusResponse + | BrowserRuntimeInstallProgressMessage // Publish | PublishArtifactMessage | PublishArtifactResponse diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8eb933ee..90ab9642 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -67,37 +67,25 @@ importers: '@mariozechner/pi-ai': specifier: ^0.60.0 version: 0.60.0(ws@8.19.0)(zod@3.25.76) - '@mozilla/readability': - specifier: ^0.6.0 - version: 0.6.0 '@sinclair/typebox': specifier: ^0.34.0 version: 0.34.48 + agent-browser: + specifier: 0.26.0 + version: 0.26.0 autoevals: specifier: ^0.0.80 version: 0.0.80 braintrust: specifier: ^0.0.182 version: 0.0.182(@aws-sdk/credential-provider-web-identity@3.972.21)(openai@4.47.1)(react@19.2.4)(sswr@2.2.0(svelte@5.55.0))(svelte@5.55.0)(vue@3.5.31(typescript@5.9.3))(zod@3.25.76) - linkedom: - specifier: ^0.18.12 - version: 0.18.12 marked: specifier: ^18.0.0 version: 18.0.0 - playwright: - specifier: ^1.52.0 - version: 1.58.2 - turndown: - specifier: ^7.2.2 - version: 7.2.2 devDependencies: '@types/node': specifier: ^22.0.0 version: 22.19.15 - '@types/turndown': - specifier: ^5.0.6 - version: 5.0.6 tsx: specifier: ^4.0.0 version: 4.21.0 @@ -1874,13 +1862,6 @@ packages: '@mistralai/mistralai@1.14.1': resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==} - '@mixmark-io/domino@2.2.0': - resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} - - '@mozilla/readability@0.6.0': - resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} - engines: {node: '>=14.0.0'} - '@next/env@14.2.35': resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==} @@ -2928,9 +2909,6 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} - '@types/turndown@5.0.6': - resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} - '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -3069,6 +3047,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agent-browser@0.26.0: + resolution: {integrity: sha512-pdqSfjwbFSp+qnwlb2g23e9wXveIOfMi19xpPA9xZUbzEAUp6W4YBZj6Ybj8z4M7WkcbGDDYc+oDIHDt9R3EDQ==} + hasBin: true + agentkeepalive@4.6.0: resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} engines: {node: '>= 8.0.0'} @@ -3615,9 +3597,6 @@ packages: resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==} engines: {node: '>= 6'} - cssom@0.5.0: - resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==} - csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} @@ -4195,11 +4174,6 @@ packages: fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} - fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -4341,18 +4315,12 @@ packages: resolution: {integrity: sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==} engines: {node: ^16.14.0 || >=18.0.0} - html-escaper@3.0.3: - resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} - html-url-attributes@3.0.1: resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==} html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} - htmlparser2@10.1.0: - resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==} - http-errors@2.0.1: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} @@ -4706,15 +4674,6 @@ packages: linear-sum-assignment@1.0.9: resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==} - linkedom@0.18.12: - resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==} - engines: {node: '>=16'} - peerDependencies: - canvas: '>= 2' - peerDependenciesMeta: - canvas: - optional: true - linkify-it@2.2.0: resolution: {integrity: sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==} @@ -5418,16 +5377,6 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} - playwright-core@1.58.2: - resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} - engines: {node: '>=18'} - hasBin: true - - playwright@1.58.2: - resolution: {integrity: sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==} - engines: {node: '>=18'} - hasBin: true - plist@3.1.0: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} @@ -6202,9 +6151,6 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turndown@7.2.2: - resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} - type-detect@4.0.8: resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} engines: {node: '>=4'} @@ -6233,9 +6179,6 @@ packages: uc.micro@1.0.6: resolution: {integrity: sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==} - uhyphen@0.2.0: - resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==} - undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} @@ -8444,10 +8387,6 @@ snapshots: - bufferutil - utf-8-validate - '@mixmark-io/domino@2.2.0': {} - - '@mozilla/readability@0.6.0': {} - '@next/env@14.2.35': {} '@opentelemetry/api@1.9.0': {} @@ -9626,8 +9565,6 @@ snapshots: '@types/trusted-types@2.0.7': {} - '@types/turndown@5.0.6': {} - '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -9790,6 +9727,8 @@ snapshots: agent-base@7.1.4: {} + agent-browser@0.26.0: {} + agentkeepalive@4.6.0: dependencies: humanize-ms: 1.2.1 @@ -10425,8 +10364,6 @@ snapshots: css-what@6.2.2: {} - cssom@0.5.0: {} - csstype@3.2.3: {} data-uri-to-buffer@4.0.1: {} @@ -11044,9 +10981,6 @@ snapshots: fs.realpath@1.0.0: {} - fsevents@2.3.2: - optional: true - fsevents@2.3.3: optional: true @@ -11226,19 +11160,10 @@ snapshots: dependencies: lru-cache: 10.4.3 - html-escaper@3.0.3: {} - html-url-attributes@3.0.1: {} html-void-elements@3.0.0: {} - htmlparser2@10.1.0: - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.2.2 - entities: 7.0.1 - http-errors@2.0.1: dependencies: depd: 2.0.0 @@ -11609,14 +11534,6 @@ snapshots: ml-matrix: 6.12.1 ml-spectra-processing: 14.22.0 - linkedom@0.18.12: - dependencies: - css-select: 5.2.2 - cssom: 0.5.0 - html-escaper: 3.0.3 - htmlparser2: 10.1.0 - uhyphen: 0.2.0 - linkify-it@2.2.0: dependencies: uc.micro: 1.0.6 @@ -12743,14 +12660,6 @@ snapshots: pirates@4.0.7: {} - playwright-core@1.58.2: {} - - playwright@1.58.2: - dependencies: - playwright-core: 1.58.2 - optionalDependencies: - fsevents: 2.3.2 - plist@3.1.0: dependencies: '@xmldom/xmldom': 0.8.12 @@ -13668,10 +13577,6 @@ snapshots: dependencies: safe-buffer: 5.2.1 - turndown@7.2.2: - dependencies: - '@mixmark-io/domino': 2.2.0 - type-detect@4.0.8: {} type-fest@0.21.3: {} @@ -13686,8 +13591,6 @@ snapshots: uc.micro@1.0.6: {} - uhyphen@0.2.0: {} - undici-types@5.26.5: {} undici-types@6.21.0: {} diff --git a/scripts/esbuild.config.js b/scripts/esbuild.config.js index 0ddd5649..85bb755d 100644 --- a/scripts/esbuild.config.js +++ b/scripts/esbuild.config.js @@ -13,7 +13,7 @@ const config = { agent: { - externals: ['node-pty', 'chokidar', 'playwright-core', 'playwright', 'chromium-bidi'], + externals: ['node-pty', 'chokidar'], }, cli: { externals: ['node-pty'],