diff --git a/docs/plans/computer-use-refactor-plan.md b/docs/plans/computer-use-refactor-plan.md index b342634ac2..694514d804 100644 --- a/docs/plans/computer-use-refactor-plan.md +++ b/docs/plans/computer-use-refactor-plan.md @@ -100,7 +100,8 @@ ``` ┌────────────────────────────────────────────────────────────────┐ │ L3 模式与配置面 │ -│ · 单一开关面: ai.computer_use_enabled ⊇ browser_control │ +│ · 双独立开关: ai.computer_use_enabled (桌面) 与 │ +│ ai.browser_control_enabled (浏览器, 默认开) 互不牵连 │ │ · permission intents: computer_use + browser_control │ │ · 每回合工具组装 (仿 codex spec_plan): 按模型能力/平台/远程裁剪 │ │ · deny 表单一真源 (Rust 导出 + contract test 三端对齐) │ @@ -148,7 +149,7 @@ **决策 7:动作即观察。** 执行器在每个 mutating 动作后:settle 延迟(桌面固定/浏览器等 network idle)→ 自动截图或快照 diff → 打包进同一 result(仿 playwright-mcp Response 聚合器 + cua post-action screenshot)。配套截图 retention(只留最近 N 张,按块修剪保护 prompt cache)。 -**决策 8:能力开关收敛为两级真实门控。** 删除装饰性 cargo feature 层;ControlHub 实现真实 `is_enabled()`(服从 `ai.computer_use_enabled` 或新增 `ai.browser_control_enabled`,按 DeliveryProfile/远程会话裁剪);新增 `browser_control` permission intent 进后端枚举与 `GlobalPermissionRulesDialog.tsx`(对应 C5)。 +**决策 8:能力开关收敛为两个独立的真实门控。** 删除装饰性 cargo feature 层;桌面控制与浏览器控制是两个独立能力:`ai.computer_use_enabled` 只门控 ComputerUse 桌面工具(现状已如此);新增独立的 `ai.browser_control_enabled`(默认开)门控 ControlHub browser 域,关闭 computer use 不影响浏览器控制(产品决策确认,2026-07-26)。ControlHub 实现真实 `is_enabled()` 服从后者,并按 DeliveryProfile/远程会话裁剪;新增 `browser_control` permission intent 进后端枚举与 `GlobalPermissionRulesDialog.tsx`(对应 C5 的可管辖性诉求)。 --- @@ -201,7 +202,7 @@ ### 阶段 6:配置/权限/Peer 面(~2 周) - **文件**:`control_hub_tool.rs`(真实 `is_enabled`)、`GlobalPermissionRulesDialog.tsx` + 后端 intent 枚举(新增 `browser_control`)、`session-config.json` 文案修正、`SessionConfig.tsx`(拆 personalization/permissions 两组件、状态命令走传输适配层或标注本机/远端)、`peer-device-adapter.ts`/`peer_host_invoke.rs`/`cli/peer_host/deny.rs`(Rust 单一真源导出 + contract test,`browser_control_*` 补 deny)、`ChatInput.tsx`/`AgentsScene.tsx`(抽 `useComputerUseEnabled()` hook,slash 路径补门禁,门禁移后端 `get_available_modes`)、`agents.rs`/`agentVisibility.ts`(统一 ComputerUse 身份与命名)。 -- 风险:低中。验证:deny 表 contract test 三端对齐;Peer 场景手测开关/权限弹窗归属;关闭 computer use 后确认 ControlHub browser 域同步禁用。 +- 风险:低中。验证:deny 表 contract test 三端对齐;Peer 场景手测开关/权限弹窗归属;关闭 computer use 后确认 ControlHub browser 域**不受影响**(两开关独立),关闭 browser_control 后确认 ControlHub browser 域禁用且 ComputerUse 不受影响。 --- diff --git a/src/apps/desktop/src/computer_use/desktop_host/mod.rs b/src/apps/desktop/src/computer_use/desktop_host/mod.rs index 24aa8325a8..929303c55b 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/mod.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/mod.rs @@ -109,6 +109,45 @@ mod visual_grid_tests { } } +#[cfg(all(test, target_os = "windows"))] +mod windows_foreground_tests { + use super::*; + + #[test] + fn foreground_app_reports_executable_separately_from_window_title() { + let app = DesktopComputerUseHost::windows_foreground_application( + "Search".to_string(), + 4242, + Some("explorer.exe".to_string()), + ); + + assert_eq!(app.name.as_deref(), Some("Search")); + assert_eq!(app.process_name.as_deref(), Some("explorer.exe")); + assert_eq!(app.process_id, Some(4242)); + } + + #[test] + fn foreground_app_falls_back_to_title_only_when_process_lookup_fails() { + let app = + DesktopComputerUseHost::windows_foreground_application("Search".to_string(), 4242, None); + + assert_eq!(app.name.as_deref(), Some("Search")); + assert_eq!(app.process_name, None); + } + + #[test] + fn foreground_app_drops_empty_title_and_empty_executable() { + let app = DesktopComputerUseHost::windows_foreground_application( + String::new(), + 0, + Some(String::new()), + ); + + assert_eq!(app.name, None); + assert_eq!(app.process_name, None); + } +} + /// Unified mutable session state for computer use — one mutex instead of five. /// State transitions are applied centrally after each action (screenshot, pointer move, click, etc.). #[derive(Debug)] @@ -123,7 +162,7 @@ struct ComputerUseSessionMutableState { navigation_focus: Option, /// Cached full-screen screenshot for fast consecutive crops. screenshot_cache: Option, - /// After `screenshot`, block `pointer_move_rel` / `ComputerUseMouseStep` until an absolute move + /// After `screenshot`, block `pointer_move_rel` until an absolute move /// from AX/OCR/globals (`mouse_move`, `move_to_text`, `click_element`) clears this. block_vision_pixel_nudge_after_screenshot: bool, /// After click / key / type / scroll / drag: recommend a **`screenshot`** to confirm UI state (Cowork verify). @@ -394,6 +433,9 @@ end tell"#]) let bundle = parts.get(2).map(|x| x.trim()).filter(|x| !x.is_empty()); Some(ComputerUseForegroundApplication { name: Some(name.to_string()), + // `name of p` from System Events is already the process name, not a + // window title, so it doubles as the process identity here. + process_name: Some(name.to_string()), bundle_id: bundle.map(|b| b.to_string()), process_id: Some(pid), }) @@ -430,11 +472,12 @@ end tell"#]) } else { String::new() }; - Some(ComputerUseForegroundApplication { - name: if title.is_empty() { None } else { Some(title) }, - bundle_id: None, - process_id: Some(pid as i32), - }) + let exe_basename = if pid == 0 { + None + } else { + crate::computer_use::windows_list_apps::exe_basename_for_pid(pid) + }; + Some(Self::windows_foreground_application(title, pid, exe_basename)) }; ComputerUseSessionSnapshot { @@ -444,6 +487,29 @@ end tell"#]) } } + /// Build the Windows foreground-app identity from the window title and the + /// owning process's executable basename. + /// + /// `name` stays the window title; `process_name` carries the *process* + /// identity. Callers that classify the frontmost app (browser detection) + /// must match on `process_name`: window titles collide with app names by + /// substring (a "Search" window contains "arc"). When the process cannot be + /// opened (access denied, exited), `process_name` is `None` and callers see + /// the pre-existing title-only shape. + #[cfg(target_os = "windows")] + fn windows_foreground_application( + title: String, + pid: u32, + exe_basename: Option, + ) -> ComputerUseForegroundApplication { + ComputerUseForegroundApplication { + name: if title.is_empty() { None } else { Some(title) }, + process_name: exe_basename.filter(|s| !s.is_empty()), + bundle_id: None, + process_id: Some(pid as i32), + } + } + #[cfg(target_os = "linux")] fn session_snapshot_linux() -> ComputerUseSessionSnapshot { // Best-effort: no standard API across Wayland/X11 without extra deps. diff --git a/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs b/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs index 49819192b3..cd4e1705eb 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs @@ -23,8 +23,8 @@ use enigo::{Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Se use log::debug; use std::time::Duration; -/// Relative nudges (`pointer_move_rel`, `ComputerUseMouseStep`) right after a model-driven screenshot are almost always wrong when deltas are guessed from the image; block until a trusted absolute move. -const VISION_PIXEL_NUDGE_AFTER_SCREENSHOT_MSG: &str = "Computer use refused: do not use `pointer_move_rel` or `ComputerUseMouseStep` immediately after a `screenshot` — nudging from the JPEG is inaccurate. First reposition with `move_to_text`, `click_element`, `locate` + `mouse_move` (`use_screen_coordinates`: true), or `mouse_move` using globals from tool JSON; then relative nudges are allowed if still needed."; +/// Relative nudges (`pointer_move_rel`) right after a model-driven screenshot are almost always wrong when deltas are guessed from the image; block until a trusted absolute move. +const VISION_PIXEL_NUDGE_AFTER_SCREENSHOT_MSG: &str = "Computer use refused: do not use `pointer_move_rel` immediately after a `screenshot` — nudging from the JPEG is inaccurate. First reposition with `move_to_text`, `click_element`, `locate` + `mouse_move` (`use_screen_coordinates`: true), or `mouse_move` using globals from tool JSON; then relative nudges are allowed if still needed."; impl DesktopComputerUseHost { pub(super) fn ensure_input_automation_allowed() -> BitFunResult<()> { @@ -602,7 +602,7 @@ impl DesktopComputerUseHost { .map_err(|e| BitFunError::tool(format!("lock: {}", e)))?; let Some(map) = s.pointer_map else { return Err(BitFunError::tool( - "Run action screenshot first: on macOS, pointer_move_relative / ComputerUseMouseStep convert pixel deltas using the last capture scale." + "Run action screenshot first: on macOS, `pointer_move_rel` converts pixel deltas using the last capture scale." .to_string(), )); }; diff --git a/src/apps/desktop/src/computer_use/windows_list_apps.rs b/src/apps/desktop/src/computer_use/windows_list_apps.rs index b090de651a..d7cf0cdc41 100644 --- a/src/apps/desktop/src/computer_use/windows_list_apps.rs +++ b/src/apps/desktop/src/computer_use/windows_list_apps.rs @@ -169,7 +169,7 @@ unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { } /// Resolve the full image path of `pid` and return its `.exe` basename. -fn exe_basename_for_pid(pid: u32) -> Option { +pub(super) fn exe_basename_for_pid(pid: u32) -> Option { let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; if handle.is_null() { return None; diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md b/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md index 409b975866..b029cbb399 100644 --- a/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md @@ -9,7 +9,7 @@ hidden: true Fast browser automation CLI for AI agents. Chrome/Chromium via CDP with accessibility-tree snapshots and compact `@eN` element refs. -Prefer BitFun's `ControlHub` browser domain when it is available; use this skill only when `ControlHub` is unavailable. The two stacks use separate browser instances, element refs, and login state, so do not mix them within one task. +`ControlHub`'s browser domain is BitFun's default path for ordinary web pages, so this skill is opt-in and you are reading it because it was invoked explicitly — proceed with agent-browser for this task. It is the right tool for Electron desktop apps, Slack workspace automation, and cloud browsers, which `ControlHub` does not cover. The two stacks use separate browser instances, element refs, and login state, so do not mix them within one task. Install only after user approval: `npm i -g agent-browser@0.32.3 && agent-browser install` diff --git a/src/crates/assembly/core/src/agentic/agents/prompts/agentic_mode.md b/src/crates/assembly/core/src/agentic/agents/prompts/agentic_mode.md index 623b66ceea..8f4d3df7ac 100644 --- a/src/crates/assembly/core/src/agentic/agents/prompts/agentic_mode.md +++ b/src/crates/assembly/core/src/agentic/agents/prompts/agentic_mode.md @@ -89,7 +89,7 @@ The user will primarily request you perform software engineering tasks. This inc - When the user explicitly asks to complete work and review it carefully, finish the implementation first, then dispatch at most one independent read-only `CodeReview` Task. Do not fan out `CodeReview` into architecture, performance, security, product, or other invented dimensions: broader coverage belongs to the unified `/review` path, which selects bounded review lenses and owns cost confirmation. Do not launch review by default for every task. - Treat reviewer output as adversarial evidence. The reviewer never fixes its own findings. Apply accepted fixes in the implementation agent, then request a fresh independent review only when the change or risk warrants it. - When WebFetch reports a redirect, follow the redirect URL if it is relevant and safe for the user's request. -- For browser and web-page work, route in this order: (1) reading page content that does not require the user's login state: use WebFetch; (2) pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs); (3) non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions. Prefer `ControlHub` over browser-automation skills such as `agent-browser`; use those skills only when `ControlHub` is unavailable. +- For browser and web-page work, route in this order: (0) only opening or showing a URL for the user, with no page reading or interaction: use `ControlHub` with `domain: "browser"`, `action: "open_builtin"`; (1) reading page content that does not require the user's login state: use WebFetch; (2) pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs) — `connect` drives BitFun's managed browser profile, which is separate from the user's everyday browser; it persists cookies and logins across runs, so if the page shows a login wall, ask the user to sign in once in that window instead of retrying navigation or entering credentials yourself; (3) non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions only when `ComputerUse` appears in your current tool list; if it does not, tell the user the task needs the Computer Use mode (enabled via the Computer use setting) instead of guessing another path or calling an unavailable tool. `ControlHub` covers ordinary web pages; for scenarios it does not support — Electron desktop apps (VS Code/Slack/Discord/Figma), Slack workspace automation, cloud browsers — load `agent-browser` explicitly via `Skill(skill="agent-browser")`, since it is opt-in and not listed in your skills by default. - When multiple tool calls are independent, run them in parallel. Keep dependent operations sequential, and never use placeholders or guess missing parameters. - Use specialized tools for file reads, edits, searches, and deletions because they preserve workspace context and permissions. Use ExecCommand for commands that genuinely need a shell. Do not use shell commands only to communicate with the user. - For security-sensitive tasks, support defensive analysis and remediation only. Refuse malicious code, exploit workflows, credential harvesting, or instructions that would facilitate abuse. diff --git a/src/crates/assembly/core/src/agentic/agents/prompts/claw_mode.md b/src/crates/assembly/core/src/agentic/agents/prompts/claw_mode.md index 72a803daa6..007fe94651 100644 --- a/src/crates/assembly/core/src/agentic/agents/prompts/claw_mode.md +++ b/src/crates/assembly/core/src/agentic/agents/prompts/claw_mode.md @@ -16,19 +16,20 @@ When a first-class tool exists for an action, use the tool directly instead of a Use `ControlHub` for browser automation, terminal signalling, and routing/capability introspection only when it appears in your current tool list: -- `domain: "browser"` for websites and web apps in the user's real browser through CDP. +- `domain: "browser"` for websites and web apps in BitFun's managed browser profile through CDP. - `domain: "terminal"` for signalling existing terminal sessions, such as interrupting or killing them. - `domain: "meta"` for capability and route checks. For browser and web-page work, route in this order: -1. Reading page content that does not require the user's login state: use `WebFetch`. -2. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs). -3. Non-Chromium browsers (Firefox/Safari) or native desktop apps: delegate to a `ComputerUse` session as described below. +1. Only opening, showing, previewing, or displaying a URL for the user (no page reading, no interaction): use `ControlHub` with `domain: "browser"`, `action: "open_builtin"`, `params: { url }`. The page renders in BitFun's built-in right-side browser panel. Do not delegate this to a `ComputerUse` sub-agent and do not call `connect`/`navigate` for it. +2. Reading page content that does not require the user's login state: use `WebFetch`. +3. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs). `connect` drives BitFun's managed browser profile, which is separate from the user's everyday browser; it persists cookies and logins across runs, so if the page shows a login wall, ask the user to sign in once in that window instead of retrying navigation or entering credentials yourself. +4. Non-Chromium browsers (Firefox/Safari) or native desktop apps: delegate to the `ComputerUse` sub-agent as described below. -Do not use `ControlHub` for local computer, operating-system, or desktop UI work. Desktop and system actions have moved to the dedicated `ComputerUse` tool/agent. This includes screenshots, OCR, mouse, keyboard, app state, app launching, opening files or URLs through the OS, clipboard access, OS facts, and local scripts. +Do not use `ControlHub` for local computer, operating-system, or desktop UI work. Desktop and system actions have moved to the dedicated `ComputerUse` tool/agent. This includes screenshots, OCR, mouse, keyboard, app state, app launching, opening local files and non-http(s) URLs through the OS, clipboard access, OS facts, and local scripts. -If the user asks you to operate or inspect the local computer, delegate the task to a `ComputerUse` session via SessionControl/SessionMessage only when both tools appear in your current tool list. Include the user's goal, target app/window/site, safety constraints, and expected verification in the handoff. If delegation is unavailable, explain that the task needs the Computer Use mode. +If the user asks you to operate or inspect the local computer, delegate the task via `Task` with the `ComputerUse` sub-agent, only when that sub-agent is listed among your available `Task` subagent types. Include the user's goal, target app/window/site, safety constraints, and expected verification in the handoff. If delegation is unavailable, explain that the task needs the Computer Use mode. # Session Coordination @@ -47,7 +48,8 @@ Choose the session type intentionally: - `agentic` for implementation, debugging, and code changes. - `Plan` for requirement clarification, scoping, and planning before coding. - `Cowork` for research, documents, presentations, summaries, and other office-related work. -- `ComputerUse` for local computer/system/desktop operation and perception. + +Local computer/desktop work is not a SessionControl session type; delegate it through `Task` with the `ComputerUse` sub-agent when that subagent type is available. Operational rules: diff --git a/src/crates/assembly/core/src/agentic/agents/prompts/computer_use_mode.md b/src/crates/assembly/core/src/agentic/agents/prompts/computer_use_mode.md index de82fa4362..2c4d1ab20c 100644 --- a/src/crates/assembly/core/src/agentic/agents/prompts/computer_use_mode.md +++ b/src/crates/assembly/core/src/agentic/agents/prompts/computer_use_mode.md @@ -18,8 +18,8 @@ Work in a tight observe -> act -> verify loop. Before acting on a desktop UI, ob Prefer the smallest reliable control surface: -1. When `ControlHub` appears in your current tool list, use it with `domain: "browser"` for websites and web apps in the user's real browser. -2. Use `ComputerUse` for third-party desktop apps, OS dialogs, system-wide keyboard and mouse, accessibility, OCR, screenshots, app state, app/file/url opening, clipboard access, OS facts, and local scripts. +1. When `ControlHub` appears in your current tool list, use it with `domain: "browser"` for websites and web apps in BitFun's managed browser profile. +2. Use `ComputerUse` for third-party desktop apps, OS dialogs, system-wide keyboard and mouse, accessibility, OCR, screenshots, app state, app/file opening, clipboard access, OS facts, and local scripts. Use it for URL opening only when the page must land in the system default browser; for display-only http(s) URLs prefer `ControlHub` `browser.open_builtin`. 3. Use `ExecCommand` for local shell commands when that is the clearest path and does not bypass desktop safety expectations. 4. When available, use `ControlHub` with `domain: "meta"` to inspect non-desktop control capabilities before long or uncertain automation flows. @@ -45,7 +45,7 @@ Use `control`, `alt`, `shift`, and usually `meta`/`super`. Prefer shell tools an Never assume focus, display, or cursor position. For multi-display setups, inspect display state and pin a display before actions that must happen on a specific screen. -Do not click or press Enter blindly. If the UI state is unknown, call `ComputerUse` with an observation action such as `get_app_state`, `build_interactive_view`, `screenshot`, `list_apps`, or `locate`. +Do not click or press Enter blindly. If the UI state is unknown, call `ComputerUse` with an observation action such as `get_app_state`, `describe_screen`, `list_apps`, `locate`, or — only when the primary model supports images — `screenshot` / `build_interactive_view`. Use paste for any multi-line text, CJK/Japanese/Korean/Arabic text, emoji, long text, file paths, messages, or search queries. Use type_text only for short Latin text into a known focused field when paste is unavailable or inappropriate. @@ -57,7 +57,7 @@ If the same GUI tactic fails twice, switch strategy: use keyboard navigation, ap # Text-Only Operation (when the primary model cannot view screenshots) -When Runtime Context indicates the primary model does not support image understanding, the `screenshot` action returns no image (`screenshot_unavailable: true`). Do NOT retry `screenshot` and do NOT call it to verify — it cannot help you see. Instead: +When Runtime Context indicates the primary model does not support image understanding, the vision-only actions — `screenshot`, `build_interactive_view`, `interactive_click`, `build_visual_mark_view`, `visual_click` — are unavailable: they are absent from your tool schema, `screenshot` returns no image (`screenshot_unavailable: true`), and the other four return NOT_AVAILABLE. Do NOT retry them and do NOT call them to verify — they cannot help you see. Instead: - **Observe with `describe_screen`** — it returns a text snapshot (frontmost app, `ax_tree_text` with `node_idx`s, `ui_tree_text`, pointer, displays) with no image. This is your eyes. Call it before acting when state is unknown, and after an action to verify `ax_state_digest` changed. - **Target with AX / OCR, never guessed coordinates** — `click_element`/`app_click` with `node_idx`/`text_contains`/`title_contains`/`role_substring`; `move_to_text`/`click_target` with `target_text` (+ `move_to_text_match_index` when several OCR hits are returned as text candidates). @@ -69,9 +69,10 @@ When Runtime Context indicates the primary model does not support image understa For websites and web apps, route in this order: -1. Reading page content that does not require the user's login state: use `WebFetch` when it is available. -2. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs) so cookies, login state, and extensions are preserved. -3. Non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions. +1. Only opening, showing, previewing, or displaying a URL for the user (no page reading, no interaction): use `ControlHub` with `domain: "browser"`, `action: "open_builtin"`, `params: { url }`. The page renders in BitFun's built-in right-side browser panel. Do not call `connect`/`navigate` for this. +2. Reading page content that does not require the user's login state: use `WebFetch` when it is available. +3. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs). `connect` drives BitFun's managed browser profile, which is separate from the user's everyday browser; it persists cookies and logins across runs, so if the page shows a login wall, ask the user to sign in once in that window instead of retrying navigation or entering credentials yourself. +4. Non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions. If `ControlHub` is unavailable, do not claim browser-domain automation; use `ComputerUse` only for browser chrome or OS-level interaction that it can actually observe and verify. diff --git a/src/crates/assembly/core/src/agentic/agents/prompts/cowork_mode.md b/src/crates/assembly/core/src/agentic/agents/prompts/cowork_mode.md index 6a7cb8f019..4e04cdcbcf 100644 --- a/src/crates/assembly/core/src/agentic/agents/prompts/cowork_mode.md +++ b/src/crates/assembly/core/src/agentic/agents/prompts/cowork_mode.md @@ -69,8 +69,20 @@ Cowork mode includes a Task tool for spawning subagents. Use subagents when dele If an answer relies on linkable MCP content such as Slack, Asana, or Box records, include a concise "Sources:" section using the tool's preferred citation format when available, otherwise [Title](URL). For WebSearch or WebFetch results, cite the sources used when claims depend on retrieved web content. # Computer Use + +Use `ControlHub` with `domain: "browser"` for browser and web-page work, only when it appears in your current tool list. + +For browser and web-page work, route in this order: + +1. Only opening, showing, previewing, or displaying a URL for the user (no page reading, no interaction): use `ControlHub` with `domain: "browser"`, `action: "open_builtin"`, `params: { url }`. +2. Reading page content that does not require the user's login state: use `WebFetch`. +3. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs). `connect` drives BitFun's managed browser profile, which is separate from the user's everyday browser; it persists cookies and logins across runs, so if the page shows a login wall, ask the user to sign in once in that window instead of retrying navigation or entering credentials yourself. +4. Non-Chromium browsers (Firefox/Safari) or native desktop apps: Cowork cannot drive these — explain the limitation and suggest Computer Use mode instead. + +Do not use `ControlHub` for local computer, operating-system, or desktop UI work, and do not substitute a browser-automation skill for it. + # Skills -Use the Skill tool when a relevant domain-specific workflow would improve the result, such as presentations, spreadsheets, documents, PDFs, browser automation, UI/UX work, or other enabled skill areas. Review the loaded skill's requirements before making files or running complex workflows. Multiple skills can be combined when they are genuinely useful. +Use the Skill tool when a relevant domain-specific workflow would improve the result, such as presentations, spreadsheets, documents, PDFs, UI/UX work, or other enabled skill areas. Browser automation is handled by the `ControlHub` browser domain, not by a skill; do not load browser-automation skills such as `agent-browser`. Review the loaded skill's requirements before making files or running complex workflows. Multiple skills can be combined when they are genuinely useful. # File Creation Advice @@ -161,4 +173,4 @@ Example decisions: # Additional Skills Reminder -For computer-use tasks, proactively use relevant skills when a domain-specific workflow is involved and the skill is available. Load skills by name, and combine them only when that adds clear value. +For computer-use tasks, proactively use relevant skills when a domain-specific workflow is involved and the skill is available. Load skills by name, and combine them only when that adds clear value. Browser work is not one of these: route it through `ControlHub` as described above. diff --git a/src/crates/assembly/core/src/agentic/tools/browser_control/actions.rs b/src/crates/assembly/core/src/agentic/tools/browser_control/actions.rs index 05cfca10ea..887d3d2abf 100644 --- a/src/crates/assembly/core/src/agentic/tools/browser_control/actions.rs +++ b/src/crates/assembly/core/src/agentic/tools/browser_control/actions.rs @@ -1,6 +1,7 @@ //! Atomic browser actions implemented via CDP commands. use super::cdp_client::{CdpClient, CdpEvent}; +use crate::agentic::tools::implementations::control_hub::{coded_tool_error, ErrorCode}; use crate::util::errors::{BitFunError, BitFunResult}; use serde_json::{json, Value}; use std::collections::BTreeMap; @@ -67,6 +68,274 @@ async fn wait_for_lifecycle( } } +// ── Structured errors ────────────────────────────────────────────────── +// +// High-frequency failure points build the `[CODE] message\nHints: a | b` +// wire format at the source, so ControlHub's `map_dispatch_error` recovers a +// stable `error.code` plus recovery hints through structured parsing instead +// of the fragile phrase-matching fallback. + +/// Build a structured error in the `[CODE] message\nHints: a | b` shape. +fn structured_error( + code: ErrorCode, + message: impl std::fmt::Display, + hints: &[&str], +) -> BitFunError { + if hints.is_empty() { + coded_tool_error(code, message) + } else { + coded_tool_error(code, format!("{}\nHints: {}", message, hints.join(" | "))) + } +} + +/// Classify a JS exception reported by `Runtime.evaluate` into a structured +/// error. `Element not found` originates from `resolve_element_js` and is by +/// far the most common interaction failure, so it gets a dedicated +/// `NOT_FOUND` code with a snapshot-recovery instruction for the model. +pub(crate) fn classify_evaluate_exception(message: &str) -> BitFunError { + if message.contains("Element not found") { + // `resolve_element_js` appends the cross-origin iframe count to its + // throw message: those frames are invisible to both `snapshot` and + // the resolver, so without saying so the model reads "not found" as + // "not on the page" and retries forever. + let mut hints = vec!["Element not found — take a new snapshot and use a fresh @eN ref"]; + if message.contains("cross-origin iframe") { + hints.push("The page contains cross-origin iframes whose contents cannot be inspected or clicked — an element inside one is unreachable; work with the top-level document instead"); + } + structured_error(ErrorCode::NotFound, format!("JS error: {}", message), &hints) + } else { + structured_error( + ErrorCode::Internal, + format!("JS error: {}", message), + &["JavaScript threw during evaluation — fix the expression, or take a fresh snapshot to re-check page state"], + ) + } +} + +/// Classify a CDP transport failure (send/receive level). A dead WebSocket or +/// closed target means the session is unusable and must be re-attached; a CDP +/// timeout means the page did not answer. Anything else passes through so the +/// heuristic fallback in `map_dispatch_error` still applies. +pub(crate) fn classify_transport_error(err: BitFunError) -> BitFunError { + let raw = err.to_string(); + let message = raw.strip_prefix("Tool error: ").unwrap_or(raw.as_str()); + if message.contains("CDP send failed") + || message.contains("CDP response channel closed") + || message.contains("Target closed") + { + structured_error( + ErrorCode::WrongTab, + message, + &["The browser session is dead (tab closed or browser quit) — call browser.connect or switch_page to attach a live tab, then retry"], + ) + } else if message.contains("CDP timeout") { + structured_error( + ErrorCode::Timeout, + message, + &["The page did not answer in time — take a snapshot to check its state, or reload and retry"], + ) + } else { + err + } +} + +/// Error for a click/hover target whose center point is covered by another +/// element. Dispatching the mouse event anyway would act on the overlay while +/// reporting success for the intended element — the worst possible outcome, +/// so the action is refused instead. +pub(crate) fn occluded_element_error(selector: &str, blocker: &str) -> BitFunError { + structured_error( + ErrorCode::GuardRejected, + format!( + "Element '{}' is not clickable at its center point: it is covered by {}.", + selector, blocker + ), + &["Clear what covers it (close the modal / cookie banner, or press Escape), or scroll it fully into view, then take a fresh snapshot and retry"], + ) +} + +/// Error for an element that resolved inside a cross-origin iframe: its +/// coordinates cannot be translated to the top-level viewport, so +/// coordinate-based actions (click/hover) cannot reach it. +pub(crate) fn cross_origin_frame_error(selector: &str) -> BitFunError { + structured_error( + ErrorCode::NotAvailable, + format!( + "Element '{}' sits inside a cross-origin iframe; its coordinates cannot be mapped to the top-level viewport, so coordinate-based actions (click/hover) cannot reach it.", + selector + ), + &["Take a snapshot and target an element in the top document or a same-origin frame instead"], + ) +} + +/// CDP fields that make `Input.dispatchKeyEvent` behave like a real key press. +/// +/// Chrome only runs a key's **default action** — Enter submitting a form, Tab +/// moving focus, a character being inserted — when the event carries +/// `windowsVirtualKeyCode` and, for text-producing keys, `text`. A bare +/// `{ type, key }` event still reaches JS listeners, which is why the omission +/// looks like it works right up until the page relies on the default action. +/// Mapping follows the US layout table used by Chrome's own automation +/// clients: only Enter and single-character keys carry `text`. +fn key_event_fields(key: &str) -> Value { + let (name, code, virtual_key, text): (&str, &str, i64, Option<&str>) = match key { + "Enter" | "Return" => ("Enter", "Enter", 13, Some("\r")), + "Tab" => ("Tab", "Tab", 9, None), + "Escape" | "Esc" => ("Escape", "Escape", 27, None), + "Backspace" => ("Backspace", "Backspace", 8, None), + "Delete" => ("Delete", "Delete", 46, None), + "ArrowUp" => ("ArrowUp", "ArrowUp", 38, None), + "ArrowDown" => ("ArrowDown", "ArrowDown", 40, None), + "ArrowLeft" => ("ArrowLeft", "ArrowLeft", 37, None), + "ArrowRight" => ("ArrowRight", "ArrowRight", 39, None), + "Home" => ("Home", "Home", 36, None), + "End" => ("End", "End", 35, None), + "PageUp" => ("PageUp", "PageUp", 33, None), + "PageDown" => ("PageDown", "PageDown", 34, None), + "Space" | " " => (" ", "Space", 32, Some(" ")), + other => { + let mut chars = other.chars(); + return match (chars.next(), chars.next()) { + (Some(ch), None) => { + let virtual_key = ch.to_ascii_uppercase() as i64; + json!({ + "key": other, + "text": other, + "windowsVirtualKeyCode": virtual_key, + "nativeVirtualKeyCode": virtual_key, + }) + } + // Unknown named key: pass it through so the page still sees a + // keydown, rather than guessing a wrong virtual key code. + _ => json!({ "key": other }), + }; + } + }; + let mut fields = json!({ + "key": name, + "code": code, + "windowsVirtualKeyCode": virtual_key, + "nativeVirtualKeyCode": virtual_key, + }); + if let Some(text) = text { + fields["text"] = json!(text); + } + fields +} + +/// Snapshot walker. Kept as a module const so the ordering guarantees it +/// encodes (stale refs cleared **before** renumbering) are unit-testable. +const SNAPSHOT_SCRIPT: &str = r#" + (function() { + const SEL = 'a, button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="combobox"], [role="option"], [tabindex="0"], [contenteditable="true"]'; + const items = []; + let idx = 1; + let offscreen = 0; + let crossOriginFrames = 0; + + function visible(el, win) { + const rect = el.getBoundingClientRect(); + if (rect.width < 2 || rect.height < 2) return null; + if (rect.right < 0 || rect.bottom < 0 || rect.left > win.innerWidth || rect.top > win.innerHeight) { + offscreen++; + return null; + } + const style = win.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return null; + return rect; + } + + function record(el, rect, scope, framePath) { + const text = (el.textContent || '').trim().slice(0, 100); + items.push({ + ref: '@e' + idx, + tag: el.tagName.toLowerCase(), + type: el.getAttribute('type') || '', + name: el.getAttribute('name') || '', + text, + ariaLabel: el.getAttribute('aria-label') || '', + placeholder: el.placeholder || '', + role: el.getAttribute('role') || '', + href: el.href || '', + id: el.id || '', + scope, + frame_path: framePath, + rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) } + }); + try { el.setAttribute('data-cdp-ref', '@e' + idx); } catch (_) {} + idx++; + } + + // Every snapshot renumbers refs from @e1, so refs left behind by + // the previous snapshot MUST be dropped first: an element that + // dropped out of this snapshot would otherwise keep an @eN that + // the new numbering hands to a different element, and + // `click @eN` — which resolves by attribute — would silently hit + // the stale one. + function clearRefs(root) { + try { + root.querySelectorAll('[data-cdp-ref]').forEach(el => el.removeAttribute('data-cdp-ref')); + } catch (_) {} + try { + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) clearRefs(host.shadowRoot); + }); + } catch (_) {} + } + + // Recursive walk: collects from `root` (Document or ShadowRoot) + // and recurses into open shadow roots of every descendant. Iframes + // are handled by the caller because we need the iframe's own + // window for visibility checks. + function walk(root, win, scope, framePath) { + const els = root.querySelectorAll(SEL); + els.forEach(el => { + const rect = visible(el, win); + if (rect) record(el, rect, scope, framePath); + }); + // Open shadow roots + const allHosts = root.querySelectorAll('*'); + allHosts.forEach(h => { + if (h.shadowRoot) { + try { walk(h.shadowRoot, win, 'shadow', framePath); } catch (_) {} + } + }); + } + + // Same-origin iframes only; cross-origin ones are counted so the + // report can state what it could not see. + const frames = []; + document.querySelectorAll('iframe, frame').forEach((frame, fi) => { + let doc = null; + try { doc = frame.contentDocument; } catch (_) {} + if (doc) { + frames.push({ frame, doc, fi }); + } else { + crossOriginFrames++; + } + }); + + clearRefs(document); + frames.forEach(f => clearRefs(f.doc)); + + walk(document, window, 'document', ''); + frames.forEach(({ frame, doc, fi }) => { + const subWin = frame.contentWindow; + const path = `iframe[${fi}]${frame.src ? `[src="${frame.src.slice(0, 80)}"]` : ''}`; + try { walk(doc, subWin, 'iframe', path); } catch (_) {} + }); + + return JSON.stringify({ + url: location.href, + title: document.title, + elements: items, + offscreen_count: offscreen, + cross_origin_frames: crossOriginFrames, + features: { shadow_dom_traversed: true, same_origin_iframes_traversed: true, viewport_only: true }, + }); + })() + "#; + /// High-level browser actions backed by CDP method calls. pub struct BrowserActions<'a> { client: &'a CdpClient, @@ -195,6 +464,12 @@ impl<'a> BrowserActions<'a> { /// `"document" | "shadow" | "iframe"`. The synthetic `data-cdp-ref` /// attribute is set in the host scope so subsequent `click` / `fill` /// can locate it via the same recursive walk. + /// + /// The listing covers the **current viewport only**; elements scrolled + /// out of view and cross-origin iframe contents are excluded but + /// reported (`offscreen_count`, `cross_origin_frames`, plus trailing + /// note lines in `snapshot`) so their absence is never read as "the + /// element does not exist". pub async fn snapshot(&self) -> BitFunResult { self.snapshot_with_options(false).await } @@ -211,83 +486,7 @@ impl<'a> BrowserActions<'a> { /// `backend_node_id` field; pages where `DOM.getDocument` errors out /// (very rare — e.g. about:blank) silently fall back to no ids. pub async fn snapshot_with_options(&self, with_backend_node_ids: bool) -> BitFunResult { - let script = r#" - (function() { - const SEL = 'a, button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="combobox"], [role="option"], [tabindex="0"], [contenteditable="true"]'; - const items = []; - let idx = 1; - - function visible(el, win) { - const rect = el.getBoundingClientRect(); - if (rect.width < 2 || rect.height < 2) return null; - if (rect.right < 0 || rect.bottom < 0 || rect.left > win.innerWidth || rect.top > win.innerHeight) return null; - const style = win.getComputedStyle(el); - if (style.display === 'none' || style.visibility === 'hidden') return null; - return rect; - } - - function record(el, rect, scope, framePath) { - const text = (el.textContent || '').trim().slice(0, 100); - items.push({ - ref: '@e' + idx, - tag: el.tagName.toLowerCase(), - type: el.getAttribute('type') || '', - name: el.getAttribute('name') || '', - text, - ariaLabel: el.getAttribute('aria-label') || '', - placeholder: el.placeholder || '', - role: el.getAttribute('role') || '', - href: el.href || '', - id: el.id || '', - scope, - frame_path: framePath, - rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) } - }); - try { el.setAttribute('data-cdp-ref', '@e' + idx); } catch (_) {} - idx++; - } - - // Recursive walk: collects from `root` (Document or ShadowRoot) - // and recurses into open shadow roots of every descendant. Iframes - // are handled by the caller because we need the iframe's own - // window for visibility checks. - function walk(root, win, scope, framePath) { - const els = root.querySelectorAll(SEL); - els.forEach(el => { - const rect = visible(el, win); - if (rect) record(el, rect, scope, framePath); - }); - // Open shadow roots - const allHosts = root.querySelectorAll('*'); - allHosts.forEach(h => { - if (h.shadowRoot) { - try { walk(h.shadowRoot, win, 'shadow', framePath); } catch (_) {} - } - }); - } - - walk(document, window, 'document', ''); - - // Same-origin iframes - const frames = document.querySelectorAll('iframe, frame'); - frames.forEach((frame, fi) => { - let doc = null; - try { doc = frame.contentDocument; } catch (_) {} - if (!doc) return; // cross-origin: skip silently - const subWin = frame.contentWindow; - const path = `iframe[${fi}]${frame.src ? `[src="${frame.src.slice(0, 80)}"]` : ''}`; - try { walk(doc, subWin, 'iframe', path); } catch (_) {} - }); - - return JSON.stringify({ - url: location.href, - title: document.title, - elements: items, - features: { shadow_dom_traversed: true, same_origin_iframes_traversed: true }, - }); - })() - "#; - let result = self.evaluate(script).await?; + let result = self.evaluate(SNAPSHOT_SCRIPT).await?; let text = result .get("result") .and_then(|r| r.get("value")) @@ -382,6 +581,26 @@ impl<'a> BrowserActions<'a> { refs.insert(reference.to_string(), element.clone()); } } + let offscreen = parsed + .get("offscreen_count") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + if offscreen > 0 { + lines.push(format!( + "- note: {} more interactive element(s) exist outside the current viewport and are NOT listed above; scroll toward them and snapshot again to get refs for them", + offscreen + )); + } + let cross_origin_frames = parsed + .get("cross_origin_frames") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + if cross_origin_frames > 0 { + lines.push(format!( + "- note: this page contains {} cross-origin iframe(s) whose contents cannot be inspected; elements inside them are absent here and cannot be targeted by @eN refs", + cross_origin_frames + )); + } if let Some(obj) = parsed.as_object_mut() { obj.insert("snapshot".to_string(), json!(lines.join("\n"))); obj.insert("refs".to_string(), json!(refs)); @@ -549,12 +768,23 @@ impl<'a> BrowserActions<'a> { })) } + /// Resolve the element's center in **top-level viewport** coordinates. + /// + /// `getBoundingClientRect` is relative to the element's own document's + /// viewport. For an element inside a same-origin iframe (which + /// `resolve_element_js` can reach) that is the *iframe's* viewport, while + /// `Input.dispatchMouseEvent` expects top-level viewport coordinates — so + /// walk the `window.frameElement` chain upward and add each frame's own + /// bounding rect (plus its border via `clientLeft`/`clientTop`). A + /// cross-origin ancestor throws on `frameElement` access; that case is + /// surfaced as a structured error instead of clicking at a wrong spot. + /// + /// The point is also hit-tested with `elementFromPoint` in the element's + /// own document: a mouse event dispatched at coordinates covered by an + /// overlay lands on the overlay, and reporting that as a successful click + /// on the intended element is the worst failure mode available. async fn element_center(&self, selector: &str) -> BitFunResult<(f64, f64)> { - let js = Self::resolve_element_js(selector); - let center_js = format!( - r#"(function(){{ {} el.scrollIntoView({{ block: 'center', inline: 'center', behavior: 'instant' }}); const rect = el.getBoundingClientRect(); return JSON.stringify({{ x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }}); }})()"#, - js - ); + let center_js = Self::element_center_js(selector); let result = self.evaluate(¢er_js).await?; let coords_str = result .get("result") @@ -562,9 +792,76 @@ impl<'a> BrowserActions<'a> { .and_then(|v| v.as_str()) .unwrap_or("{}"); let coords: Value = serde_json::from_str(coords_str).unwrap_or(json!({})); - let x = coords.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let y = coords.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); - Ok((x, y)) + if coords.get("error").and_then(|v| v.as_str()) == Some("cross_origin_frame") { + return Err(cross_origin_frame_error(selector)); + } + if let Some(blocker) = coords.get("blocked_by").and_then(|v| v.as_str()) { + return Err(occluded_element_error(selector, blocker)); + } + match ( + coords.get("x").and_then(|v| v.as_f64()), + coords.get("y").and_then(|v| v.as_f64()), + ) { + (Some(x), Some(y)) => Ok((x, y)), + _ => Err(structured_error( + ErrorCode::Internal, + format!("Failed to compute viewport center for '{}'", selector), + &["Take a fresh snapshot and retry with a new @eN ref"], + )), + } + } + + fn element_center_js(selector: &str) -> String { + format!( + r#"(function(){{ + {js} + el.scrollIntoView({{ block: 'center', inline: 'center', behavior: 'instant' }}); + const rect = el.getBoundingClientRect(); + const localX = rect.x + rect.width / 2; + const localY = rect.y + rect.height / 2; + let x = localX; + let y = localY; + try {{ + let win = el.ownerDocument.defaultView; + while (win && win !== win.top) {{ + const fe = win.frameElement; + if (!fe) break; + const fr = fe.getBoundingClientRect(); + x += fr.x + fe.clientLeft; + y += fr.y + fe.clientTop; + win = win.parent; + }} + }} catch (e) {{ + return JSON.stringify({{ error: 'cross_origin_frame' }}); + }} + let blockedBy = null; + try {{ + let hit = el.ownerDocument.elementFromPoint(localX, localY); + // Document-level elementFromPoint stops at a shadow host, + // and host.contains(shadowChild) is false — without + // descending, every element inside an open shadow root + // (which resolve/snapshot deliberately support) would be + // misreported as occluded by its own host. + while (hit && hit.shadowRoot) {{ + const inner = hit.shadowRoot.elementFromPoint(localX, localY); + if (!inner || inner === hit) break; + hit = inner; + }} + if (!hit) {{ + blockedBy = 'nothing (the point is outside the viewport)'; + }} else if (hit !== el && !el.contains(hit) && !hit.contains(el)) {{ + const hid = hit.id ? '#' + hit.id : ''; + const hcls = (typeof hit.className === 'string' && hit.className.trim()) + ? '.' + hit.className.trim().split(/\s+/).slice(0, 2).join('.') + : ''; + const label = (hit.textContent || '').trim().slice(0, 40); + blockedBy = '<' + hit.tagName.toLowerCase() + hid + hcls + '>' + (label ? ' "' + label + '"' : ''); + }} + }} catch (_) {{}} + return JSON.stringify({{ x: x, y: y, blocked_by: blockedBy }}); + }})()"#, + js = Self::resolve_element_js(selector) + ) } pub async fn hover(&self, selector: &str) -> BitFunResult { @@ -645,20 +942,7 @@ impl<'a> BrowserActions<'a> { /// Select a dropdown option by visible text. pub async fn select(&self, selector: &str, option_text: &str) -> BitFunResult { - let js = format!( - r#"(function(){{ - const sel = document.querySelector('{}'); - if (!sel) return JSON.stringify({{ error: 'Select not found' }}); - const opts = Array.from(sel.options); - const opt = opts.find(o => o.text.includes('{}')); - if (!opt) return JSON.stringify({{ error: 'Option not found', available: opts.map(o => o.text) }}); - sel.value = opt.value; - sel.dispatchEvent(new Event('change', {{ bubbles: true }})); - return JSON.stringify({{ success: true, value: opt.value, text: opt.text }}); - }})()"#, - selector.replace('\'', "\\'"), - option_text.replace('\'', "\\'") - ); + let js = Self::select_option_js(selector, option_text); let result = self.evaluate(&js).await?; let text = result .get("result") @@ -671,36 +955,29 @@ impl<'a> BrowserActions<'a> { /// Press a key (Enter, Escape, Tab, etc.). pub async fn press_key(&self, key: &str) -> BitFunResult { - self.client - .send( - "Input.dispatchKeyEvent", - Some(json!({ - "type": "keyDown", - "key": key, - })), - ) - .await?; - if key.chars().count() == 1 { - self.client - .send( - "Input.dispatchKeyEvent", - Some(json!({ - "type": "char", - "key": key, - "text": key, - })), - ) - .await?; + let fields = key_event_fields(key); + // `keyDown` with `text` is what makes Chrome perform the key's default + // action; keys that produce no text must go out as `rawKeyDown` or the + // renderer drops them. + let event_type = if fields.get("text").is_some() { + "keyDown" + } else { + "rawKeyDown" + }; + let mut down = fields.clone(); + if let Some(obj) = down.as_object_mut() { + obj.insert("type".to_string(), json!(event_type)); } self.client - .send( - "Input.dispatchKeyEvent", - Some(json!({ - "type": "keyUp", - "key": key, - })), - ) + .send("Input.dispatchKeyEvent", Some(down)) .await?; + + let mut up = fields; + if let Some(obj) = up.as_object_mut() { + obj.remove("text"); + obj.insert("type".to_string(), json!("keyUp")); + } + self.client.send("Input.dispatchKeyEvent", Some(up)).await?; Ok(json!({ "success": true, "action": "press_key", "key": key })) } @@ -823,11 +1100,8 @@ impl<'a> BrowserActions<'a> { })); } selector => { + let js = Self::element_exists_js(selector); for _ in 0..30 { - let js = format!( - "!!document.querySelector('{}')", - selector.replace('\'', "\\'") - ); let result = self.evaluate(&js).await?; let found = result .get("result") @@ -841,10 +1115,11 @@ impl<'a> BrowserActions<'a> { } tokio::time::sleep(std::time::Duration::from_millis(500)).await; } - return Err(BitFunError::tool(format!( - "Timeout waiting for element: {}", - cond - ))); + return Err(structured_error( + ErrorCode::Timeout, + format!("Timeout waiting for element: {}", cond), + &["Wait timed out — take a snapshot to check the current page state, or wait on 'load' / 'networkidle' instead of a selector"], + )); } } } @@ -979,7 +1254,7 @@ impl<'a> BrowserActions<'a> { .and_then(|v| v.as_str()) .or_else(|| details.get("text").and_then(|v| v.as_str())) .unwrap_or("Runtime.evaluate failed"); - return Err(BitFunError::tool(format!("JS error: {}", message))); + return Err(classify_evaluate_exception(message)); } return Ok(value); } @@ -997,7 +1272,9 @@ impl<'a> BrowserActions<'a> { } } } - Err(last_error.unwrap_or_else(|| BitFunError::tool("Runtime.evaluate failed".to_string()))) + Err(classify_transport_error(last_error.unwrap_or_else(|| { + BitFunError::tool("Runtime.evaluate failed".to_string()) + }))) } pub async fn get_cookies(&self, urls: Option>) -> BitFunResult { @@ -1236,10 +1513,375 @@ impl<'a> BrowserActions<'a> { }} return null; }} + function __crossOriginFrames() {{ + let n = 0; + document.querySelectorAll('iframe, frame').forEach(f => {{ + let doc = null; + try {{ doc = f.contentDocument; }} catch (_) {{}} + if (!doc) n++; + }}); + return n; + }} const el = __findAnywhere(); - if (!el) throw new Error('Element not found: ' + __sel + ' — take a fresh snapshot or check shadow/iframe scope'); + if (!el) {{ + const __xo = __crossOriginFrames(); + throw new Error('Element not found: ' + __sel + ' — take a fresh snapshot or check shadow/iframe scope' + + (__xo ? ' (page contains ' + __xo + ' cross-origin iframe(s) whose contents cannot be inspected)' : '')); + }} "#, escaped = escaped ) } + + /// JS that reports whether `selector` (CSS **or** `@eN` ref) currently + /// resolves, without throwing. `wait { condition: }` polls it; + /// the raw `document.querySelector` it replaced threw a `SyntaxError` on + /// every `@eN` ref because `@e3` is not valid CSS. + fn element_exists_js(selector: &str) -> String { + format!( + r#"(function(){{ + try {{ + {resolve} + return !!el; + }} catch (_) {{ + return false; + }} + }})()"#, + resolve = Self::resolve_element_js(selector) + ) + } + + /// JS that picks a `: do NOT drive the native dialog — use browser.set_file_input_files { selector, files: [\"/abs/path\"] }. For JS alert/confirm/prompt use browser.dialog", + "For login/cookies/extensions keep using the CDP browser path; do not ask the user to enable a debug port on their everyday browser profile", "For isolated project Web UI testing, use the headless browser flow instead of desktop automation", ]) } - fn is_probably_browser_app(foreground: &ComputerUseForegroundApplication) -> bool { - let name = foreground - .name - .as_deref() - .unwrap_or("") - .to_ascii_lowercase(); - let bundle = foreground + /// Structured app identity: macOS bundle id, or the executable basename on + /// platforms that report one. Deliberately **not** the display name: on + /// Windows `foreground.name` is the foreground *window title* + /// (`GetWindowTextW`), which is user content, not an app identity. + fn app_identity(foreground: &ComputerUseForegroundApplication) -> Option<&str> { + foreground .bundle_id .as_deref() - .unwrap_or("") - .to_ascii_lowercase(); + .or(foreground.process_name.as_deref()) + .map(str::trim) + .filter(|id| !id.is_empty()) + } + + fn identity_is_chromium(identity: &str) -> bool { + let id = identity.trim().to_ascii_lowercase(); + CHROMIUM_APP_IDENTITIES.iter().any(|known| { + id == *known + // Channel variants: com.google.chrome.canary, com.brave.browser.beta … + || (known.contains('.') + && !known.ends_with(".exe") + && id.starts_with(&format!("{known}."))) + }) + } + /// Whole-token match on a human-readable app name or window title, used only + /// when no structured identity is available. Substring matching is not an + /// option here: "Search" contains "arc", "Knowledge Base" contains "edge". + fn name_suggests_chromium(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + let tokens: Vec<&str> = name + .split(|c: char| !c.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .collect(); + tokens.iter().any(|token| CHROMIUM_NAME_TOKENS.contains(token)) + // "edge" needs the product phrase, "arc" the trailing-app-name shape + // Chromium browsers give their windows ("Page title — Arc"). + || tokens.windows(2).any(|pair| matches!(pair, ["microsoft", "edge"])) + || matches!(tokens.last(), Some(&"arc")) + } + + fn is_probably_browser_app(foreground: &ComputerUseForegroundApplication) -> bool { // Only Chromium-family browsers are guarded: they are the only ones the // ControlHub browser domain can drive over CDP. Firefox/Safari (and other // non-Chromium browsers) have no CDP path, so desktop control must stay // allowed for them — blocking both surfaces would leave no control path. - const NAME_HINTS: &[&str] = &["chrome", "chromium", "edge", "brave", "arc"]; - const BUNDLE_HINTS: &[&str] = &["chrome", "chromium", "edge", "brave", "arc"]; + match Self::app_identity(foreground) { + Some(identity) => Self::identity_is_chromium(identity), + None => Self::name_suggests_chromium(foreground.name.as_deref().unwrap_or("")), + } + } + + /// Identifiers carried by an explicit `app` selector. `{"pid":N}` carries + /// none, so it cannot be classified here. + fn selector_labels(app: &Value) -> Vec<&str> { + match app { + Value::String(name) => vec![name.as_str()], + Value::Object(_) => ["name", "bundle_id"] + .iter() + .filter_map(|key| app.get(*key).and_then(Value::as_str)) + .filter(|label| !label.trim().is_empty()) + .collect(), + _ => Vec::new(), + } + } + + fn selector_is_chromium(app: &Value) -> bool { + Self::selector_labels(app) + .iter() + .any(|label| Self::identity_is_chromium(label) || Self::name_suggests_chromium(label)) + } - NAME_HINTS.iter().any(|hint| name.contains(hint)) - || BUNDLE_HINTS.iter().any(|hint| bundle.contains(hint)) + /// `alt+tab` / `command+tab` and their shift variants: the OS app switcher. + /// It is the only way to move focus off a browser with the keyboard, so + /// guarding it would leave a non-browser task with no way to reach its + /// target app. + fn is_focus_switch_chord(action: &str, params: &Value) -> bool { + if action != "key_chord" { + return false; + } + let Some(keys) = params.get("keys").and_then(Value::as_array) else { + return false; + }; + let keys: Vec = keys + .iter() + .filter_map(Value::as_str) + .map(|key| key.trim().to_ascii_lowercase()) + .collect(); + keys.iter().any(|key| key == "tab") + && keys.iter().all(|key| { + matches!( + key.as_str(), + "tab" + | "alt" + | "option" + | "command" + | "cmd" + | "meta" + | "super" + | "shift" + | "control" + | "ctrl" + ) + }) } - /// Rejects physical input actions while a CDP-drivable browser is frontmost. - /// Read-only observation actions (`screenshot`, `locate`, `describe_screen`, …) - /// stay allowed. Called by `ComputerUseTool::call_impl` before dispatch. + /// Rejects physical input actions that would drive a CDP-drivable browser. + /// Read-only observation actions (`screenshot`, `locate`, `describe_screen`, + /// `get_app_state`, `build_*_view`, …) and scripts stay allowed. Called by + /// `ComputerUseTool::call_impl` before dispatch. pub(crate) async fn desktop_action_targets_browser( &self, action: &str, + params: &Value, context: &ToolUseContext, ) -> Option { - let guarded_actions = [ + // Every action that produces physical input, app-scoped and + // interactive/visual variants included: guarding only the frontmost + // primitives would let the model bypass the boundary by renaming the + // same click (`app_click` with an explicit browser selector). + const GUARDED_ACTIONS: &[&str] = &[ "click", "click_target", "click_element", @@ -162,10 +300,29 @@ impl ComputerUseActions { "type_text", "paste", "move_to_text", + "app_click", + "app_type_text", + "app_scroll", + "app_key_chord", + "interactive_click", + "interactive_type_text", + "interactive_scroll", + "visual_click", ]; - if !guarded_actions.contains(&action) { + if !GUARDED_ACTIONS.contains(&action) || Self::is_focus_switch_chord(action, params) { return None; } + if let Some(app) = params.get("app") { + if Self::selector_is_chromium(app) { + return Some(Self::desktop_browser_guard_error(action, None)); + } + // A selector naming another app drives that app whatever is + // frontmost — and answering from the selector alone also skips the + // host round-trip. A pid-only selector names nothing: fall through. + if !Self::selector_labels(app).is_empty() { + return None; + } + } let host = context.computer_use_host.as_ref()?; let snapshot = host.computer_use_session_snapshot().await; let foreground = snapshot.foreground_application.as_ref()?; @@ -846,6 +1003,21 @@ impl ComputerUseActions { ), )); } + // Same gate, different reason: these two act on the `i` index of an + // interactive view, and building that view is itself vision-only. + if text_only && matches!(action, "interactive_type_text" | "interactive_scroll") { + let replacement = if action == "interactive_scroll" { + "app_scroll" + } else { + "app_type_text" + }; + return Err(coded_tool_error( + ErrorCode::NotAvailable, + format!( + "`{action}` addresses elements by the `i` index of an interactive view, which requires a vision-capable primary model. Use `{replacement}` with a `focus` target resolved from `get_app_state` / `describe_screen` instead." + ), + )); + } let bg = host.supports_background_input(); let ax = host.supports_ax_tree(); @@ -1526,8 +1698,16 @@ impl ComputerUseActions { .ok_or_else(|| BitFunError::tool("open_url requires 'url'".to_string()))?; match LocalSystemProvider::new().open_url(url) { Ok(outcome) => Ok(vec![ToolResult::ok( - json!({ "opened": true, "url": url, "method": outcome.method }), - Some(format!("Opened {} in default handler", url)), + json!({ + "opened": true, + "url": url, + "method": outcome.method, + "note": OPEN_URL_ROUTING_NOTE, + }), + Some(format!( + "Opened {} in default handler. {}", + url, OPEN_URL_ROUTING_NOTE + )), )]), Err(e) => Ok(local_system_error_response("system", "open_url", e)), } @@ -1549,11 +1729,16 @@ impl ComputerUseActions { "path": path_str, "with_app": app_name, "method": outcome.method, + "note": OPEN_FILE_ROUTING_NOTE, }), - Some(match app_name { - Some(a) => format!("Opened {} with {}", path_str, a), - None => format!("Opened {} with default handler", path_str), - }), + Some(format!( + "{}. {}", + match app_name { + Some(a) => format!("Opened {} with {}", path_str, a), + None => format!("Opened {} with default handler", path_str), + }, + OPEN_FILE_ROUTING_NOTE + )), )]), Err(e) => Ok(local_system_error_response("system", "open_file", e)), } @@ -1602,7 +1787,9 @@ fn error_code_from_local(code: &str) -> ErrorCode { mod tests { use super::loop_tracker_observe; use super::ComputerUseActions; + use super::{OPEN_FILE_ROUTING_NOTE, OPEN_URL_ROUTING_NOTE}; use crate::agentic::tools::computer_use_host::ComputerUseForegroundApplication; + use serde_json::json; // A unique PID avoids interference with the shared APP_LOOP_TRACKER state // across tests in the same process. @@ -1652,6 +1839,29 @@ mod tests { ComputerUseForegroundApplication { name: Some(name.to_string()), bundle_id: Some(bundle_id.to_string()), + process_name: None, + process_id: Some(1), + } + } + + /// A host that reports no app identity — on Windows `name` is the + /// foreground *window title*, not an application name. + fn titled(window_title: &str) -> ComputerUseForegroundApplication { + ComputerUseForegroundApplication { + name: Some(window_title.to_string()), + bundle_id: None, + process_name: None, + process_id: Some(1), + } + } + + /// Windows shape: window title in `name`, executable basename in + /// `process_name`. + fn windows_app(window_title: &str, exe: &str) -> ComputerUseForegroundApplication { + ComputerUseForegroundApplication { + name: Some(window_title.to_string()), + bundle_id: None, + process_name: Some(exe.to_string()), process_id: Some(1), } } @@ -1669,6 +1879,10 @@ mod tests { "Microsoft Edge", "com.microsoft.edgemac" ))); + assert!(ComputerUseActions::is_probably_browser_app(&foreground( + "Google Chrome Canary", + "com.google.Chrome.canary" + ))); assert!(!ComputerUseActions::is_probably_browser_app(&foreground( "Firefox", "org.mozilla.firefox" @@ -1679,6 +1893,166 @@ mod tests { ))); } + /// The identity wins over the display name: an editor window whose title + /// happens to contain a browser word is still an editor. + #[test] + fn browser_guard_prefers_identity_over_display_name() { + assert!(!ComputerUseActions::is_probably_browser_app(&foreground( + "chrome-devtools.ts — Code", + "com.microsoft.VSCode" + ))); + } + + /// Window titles are user content, not app identities. Substring hints on + /// them locked desktop input out of ordinary Windows apps ("Knowledge Base" + /// contains "edge", "Search Results" contains "arc"). + #[test] + fn browser_guard_ignores_window_titles_that_merely_contain_browser_words() { + for title in [ + "Knowledge Base - Obsidian", + "edge_cases.ts - proj - Visual Studio Code", + "Search Results in Documents", + "Monarch", + "Archive Utility", + "Ledger Live", + ] { + assert!( + !ComputerUseActions::is_probably_browser_app(&titled(title)), + "`{title}` is not a browser" + ); + } + } + + /// Without an identity the window title is the only signal left, so real + /// Chromium windows must still be recognised from it. + #[test] + fn browser_guard_still_matches_chromium_window_titles() { + for title in [ + "Google - Google Chrome", + "Inbox - Microsoft Edge", + "BitFun docs — Arc", + "New Tab - Brave", + ] { + assert!( + ComputerUseActions::is_probably_browser_app(&titled(title)), + "`{title}` is a Chromium browser window" + ); + } + } + + /// Windows/Linux report an executable basename rather than a bundle id. + #[test] + fn browser_guard_matches_executable_basenames() { + assert!(ComputerUseActions::is_probably_browser_app(&foreground( + "Google - Google Chrome", + "chrome.exe" + ))); + assert!(ComputerUseActions::is_probably_browser_app(&foreground( + "Inbox - Microsoft Edge", + "msedge.exe" + ))); + assert!(!ComputerUseActions::is_probably_browser_app(&foreground( + "edge_cases.ts - Visual Studio Code", + "Code.exe" + ))); + assert!(!ComputerUseActions::is_probably_browser_app(&foreground( + "Knowledge Base - Obsidian", + "obsidian.exe" + ))); + } + + /// An explicit `app` selector is classified without asking the host, so + /// `app_click { app: { name: "Google Chrome" } }` cannot be used to reach + /// the browser from the desktop side. + #[test] + fn app_selector_naming_a_chromium_browser_is_recognised() { + assert!(ComputerUseActions::selector_is_chromium( + &json!({ "name": "Google Chrome" }) + )); + assert!(ComputerUseActions::selector_is_chromium( + &json!({ "bundle_id": "com.microsoft.edgemac" }) + )); + assert!(ComputerUseActions::selector_is_chromium(&json!( + "Brave Browser" + ))); + assert!(!ComputerUseActions::selector_is_chromium( + &json!({ "name": "WeChat" }) + )); + // pid-only carries no identity — the frontmost check decides instead. + assert!(!ComputerUseActions::selector_is_chromium( + &json!({ "pid": 123 }) + )); + } + + /// The app switcher must stay callable while a browser is frontmost: it is + /// the escape hatch for tasks whose target is not the browser at all. + #[test] + fn app_switcher_chords_are_never_guarded() { + assert!(ComputerUseActions::is_focus_switch_chord( + "key_chord", + &json!({ "keys": ["alt", "tab"] }) + )); + assert!(ComputerUseActions::is_focus_switch_chord( + "key_chord", + &json!({ "keys": ["command", "shift", "tab"] }) + )); + assert!(!ComputerUseActions::is_focus_switch_chord( + "key_chord", + &json!({ "keys": ["command", "t"] }) + )); + assert!(!ComputerUseActions::is_focus_switch_chord( + "type_text", + &json!({ "keys": ["alt", "tab"] }) + )); + } + + /// The rejection must lead somewhere: a non-browser escape route, the + /// ControlHub actions that own browser chrome / file pickers / dialogs, and + /// no contradiction with `browser.connect`'s "never ask for a debug port". + #[test] + fn browser_guard_hints_offer_an_executable_way_out() { + let error = ComputerUseActions::desktop_browser_guard_error("click", None); + assert!( + error.message.contains("not because your task is browser-related"), + "{}", + error.message + ); + let hints = error.hints.join(" | "); + assert!(hints.contains("browser.connect"), "{hints}"); + assert!(hints.contains("browser.set_file_input_files"), "{hints}"); + assert!(hints.contains("browser.dialog"), "{hints}"); + assert!(hints.contains("browser.navigate"), "{hints}"); + assert!(hints.contains("open_app"), "{hints}"); + assert!(hints.contains("app_click"), "{hints}"); + assert!( + !hints.contains("test port enabled") && !hints.contains("--remote-debugging-port"), + "must not contradict browser.connect's managed-profile rule: {hints}" + ); + } + + /// `open_url` hands the page to the user's default browser, which the + /// agent can neither observe nor control. The success note must say so + /// and route follow-up page work to the ControlHub browser domain — + /// never to desktop clicks (those trip the desktop browser guard). + #[test] + fn open_url_routing_note_points_at_browser_domain() { + assert!(OPEN_URL_ROUTING_NOTE.contains("cannot observe or control")); + assert!(OPEN_URL_ROUTING_NOTE.contains("ControlHub domain=\"browser\"")); + assert!(OPEN_URL_ROUTING_NOTE.contains("browser.connect")); + assert!(OPEN_URL_ROUTING_NOTE.contains("snapshot")); + } + + /// `open_file` opens an external app window; follow-up interaction goes + /// through ComputerUse desktop actions (screenshot first), NOT the + /// browser domain. + #[test] + fn open_file_routing_note_points_at_desktop_actions() { + assert!(OPEN_FILE_ROUTING_NOTE.contains("external application window")); + assert!(OPEN_FILE_ROUTING_NOTE.contains("ComputerUse desktop")); + assert!(OPEN_FILE_ROUTING_NOTE.contains("screenshot")); + assert!(!OPEN_FILE_ROUTING_NOTE.contains("browser")); + } + /// A genuine tree mutation (digest changes) must NOT trigger the warning, /// even on the same target — progress resets the streak. #[test] diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs index 4134fa70ff..cf9ce31d5f 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs @@ -141,7 +141,7 @@ impl ComputerUseTool { The **primary model cannot consume images** in tool results — **do not** use **`screenshot`**.\n\ **OBSERVE & VERIFY (text-only):** Use **`describe_screen`** as your eyes — it returns a text snapshot (frontmost app + AX tree `ax_tree_text` with `node_idx`s + `ui_tree_text` + pointer) with NO image. Call it before acting when UI state is unknown, and after an action to verify the `ax_state_digest` changed. This replaces the `screenshot` observe→act→verify loop for text-only models.\n\ **ACTION PRIORITY (CRITICAL):** Always think in this order:\n\ -1. **Terminal/CLI/System commands first** — Use Bash tool for terminal commands, system scripts (e.g., macOS `osascript`), shell automation. Most efficient.\n\ +1. **Terminal/CLI/System commands first** — Use the **`ExecCommand`** tool for terminal commands, system scripts (e.g., macOS `osascript`), shell automation. Most efficient.\n\ 2. **Keyboard shortcuts second** — Use **`key_chord`** / **`type_text`** for system/app shortcuts, navigation keys. Unsure what shortcut a target app registers for a function (e.g. \"Save\")? Call **`get_app_shortcuts`** first instead of guessing or clicking through menus.\n\ 3. **Precise UI control last** — Only when above fail: **`click_target`** / **`move_to_target`** (AX → OCR → screen coords in one call) → lower-level **`click_element`** / **`move_to_text`** → **`mouse_move`** + **`click`**.\n\ **Rhythm:** one action at a time; use **`wait`** when UI animates. Observe **`interaction_state`** and **`computer_use_context`** in tool JSON.\n\ @@ -219,7 +219,6 @@ The **primary model cannot consume images** in tool results — **do not** use * "target": { "type": "object", "description": "For `app_click`: click target such as `{ \"node_idx\": 3 }`, image/screen coordinates, or OCR text." }, "focus": { "type": ["object", "null"], "description": "For app-scoped text/scroll actions: optional focus target." }, "predicate": { "type": "object", "description": "For `app_wait_for`: wait predicate." }, - "i": { "type": ["integer", "null"], "description": "For interactive/visual actions: element or mark index from the latest view." }, "dx": { "type": "integer", "description": "For app/interactive scroll actions: horizontal delta." }, "dy": { "type": "integer", "description": "For app/interactive scroll actions: vertical delta." }, "mouse_button": { "type": "string", "enum": ["left", "right", "middle"], "description": "For app/interactive/visual click actions." }, @@ -251,8 +250,8 @@ The **primary model cannot consume images** in tool results — **do not** use * let properties = Self::merge_with_shared_properties(json!({ "action": { "type": "string", - "enum": ["click_target", "move_to_target", "click_element", "move_to_text", "click", "mouse_move", "scroll", "drag", "locate", "key_chord", "type_text", "pointer_move_rel", "wait", "list_displays", "focus_display", "paste", "list_apps", "get_app_state", "get_app_shortcuts", "describe_screen", "app_click", "app_type_text", "app_scroll", "app_key_chord", "app_wait_for", "interactive_type_text", "interactive_scroll", "open_app", "open_url", "open_file", "clipboard_get", "clipboard_set", "run_script", "run_apple_script", "get_os_info"], - "description": "The action to perform. **Primary model is text-only — no `screenshot`.** **ACTION PRIORITY:** 1) Use Bash tool for CLI/terminal/system commands first. 2) **`open_app`** to launch apps. **`run_apple_script`** for AppleScript (macOS). 3) Prefer `key_chord` for shortcuts/navigation. Before guessing a shortcut, call **`get_app_shortcuts`** to look up what a target app actually has registered (e.g. \"what triggers Save in this app?\"), then fire it with `key_chord` / `app_key_chord` — avoids trial-and-error mouse clicks. 4) Only when above fail: `click_target` / `move_to_target` (AX → OCR → screen coords in one call), then lower-level `click_element`, `move_to_text`, or `mouse_move` + `click`. Never guess coordinates. **`describe_screen`** is the text-only equivalent of `screenshot`: it returns a structured text snapshot (frontmost app + AX tree + UI tree text + pointer + window geometry) with NO image — use it to observe and verify state when the primary model cannot view screenshots." + "enum": ["click_target", "move_to_target", "click_element", "move_to_text", "click", "mouse_move", "scroll", "drag", "locate", "key_chord", "type_text", "pointer_move_rel", "wait", "list_displays", "focus_display", "paste", "list_apps", "get_app_state", "get_app_shortcuts", "describe_screen", "app_click", "app_type_text", "app_scroll", "app_key_chord", "app_wait_for", "open_app", "open_url", "open_file", "clipboard_get", "clipboard_set", "run_script", "run_apple_script", "get_os_info"], + "description": "The action to perform. **Primary model is text-only — no `screenshot`.** **Browser boundary:** no input action here may drive a Chromium-family browser (Chrome/Edge/Brave/Arc) — use ControlHub domain=\"browser\" for those; switching focus away with `key_chord` [\"alt\",\"tab\"] / [\"command\",\"tab\"] or `open_app` is always allowed. **ACTION PRIORITY:** 1) Use the `ExecCommand` tool for CLI/terminal/system commands first. 2) **`open_app`** to launch apps. **`run_apple_script`** for AppleScript (macOS). 3) Prefer `key_chord` for shortcuts/navigation. Before guessing a shortcut, call **`get_app_shortcuts`** to look up what a target app actually has registered (e.g. \"what triggers Save in this app?\"), then fire it with `key_chord` / `app_key_chord` — avoids trial-and-error mouse clicks. 4) Only when above fail: `click_target` / `move_to_target` (AX → OCR → screen coords in one call), then lower-level `click_element`, `move_to_text`, or `mouse_move` + `click`. Never guess coordinates. **`describe_screen`** is the text-only equivalent of `screenshot`: it returns a structured text snapshot (frontmost app + AX tree + UI tree text + pointer + window geometry) with NO image — use it to observe and verify state when the primary model cannot view screenshots." }, "use_screen_coordinates": { "type": "boolean", "description": "For `mouse_move`, `drag`: **must be true** — global display coordinates from `move_to_text`, `locate`, AX, or `pointer_global`. **Not** for `click`." }, "delta_x": { "type": "integer", "description": "For `pointer_move_rel`: horizontal delta (negative=left); also accepted as `dx`. For `scroll`: horizontal wheel delta." }, @@ -1020,13 +1019,13 @@ impl Tool for ComputerUseTool { Ok(format!( "Desktop automation (host OS: {}). {} All actions in one tool. Send only parameters that apply to the chosen `action`. \ **ACTION PRIORITY (CRITICAL):** Always think in this order before choosing an action:\n\ -1. **Terminal/CLI/System commands first** — Use Bash tool for terminal commands, system scripts (e.g., macOS `osascript`, AppleScript), shell automation. This is the MOST EFFICIENT approach.\n\ +1. **Terminal/CLI/System commands first** — Use the **`ExecCommand`** tool for terminal commands, system scripts (e.g., macOS `osascript`, AppleScript), shell automation. This is the MOST EFFICIENT approach.\n\ 2. **Keyboard shortcuts second** — Use **`key_chord`** for system shortcuts, app shortcuts, navigation keys (Enter, Escape, Tab, Space, Arrow keys). Prefer over mouse when equivalent. Don't know the shortcut for a target app's function? Call **`get_app_shortcuts`** to read its registered menu shortcuts (macOS `AXMenuBar`, Windows UIA menu tree), then fire it with `key_chord` / `app_key_chord` instead of clicking through menus.\n\ 3. **Precise UI control last** — Only when above methods fail: prefer **`click_target`** / **`move_to_target`** (AX → OCR → screen coords in one call). Use lower-level **`click_element`**, **`move_to_text`**, or **`mouse_move`** + **`click`** only when you need manual disambiguation.\n\ **Screenshot usage:** **`screenshot`** is ONLY for observing/confirming UI state and extracting text/information — NEVER use screenshot coordinates to control mouse movement. Always use precise methods (AX, OCR, system coordinates) for targeting.\n\ **Cowork-style loop:** **`screenshot`** (observe) → **one** action → **`screenshot`** (verify). Use **`wait`** if UI animates. When **`interaction_state.recommend_screenshot_to_verify_last_action`** is true, call **`screenshot`** next. \ **`click_target` / `move_to_target`:** Unified target resolver. In one call it tries AX (`node_idx`, `text_contains`, `title_contains`, `role_substring`, `identifier_contains`, or `target_text`) first, then OCR (`target_text` / `text_query`), then explicit global `x`/`y` with `use_screen_coordinates: true`. `click_target` moves and clicks authoritatively, avoiding the multi-step locate → move → screenshot → click loop for common targets. \ -**`click_element`:** Lower-level Accessibility tree (AX/UIA/AT-SPI) locate + click. Provide `title_contains` / `role_substring` / `identifier_contains`. On macOS, **`TextArea`** and **`TextField`** match both `AXTextArea` and `AXTextField` (many chat apps use TextField for compose). If several text fields match, the host deprioritizes known **search** controls (e.g. WeChat `_SC_SEARCH_FIELD`) and prefers **lower** on-screen fields (composer). Bypasses coordinate screenshot guard. \ +**`click_element`:** Lower-level Accessibility tree (AX/UIA/AT-SPI) locate + click. Provide `title_contains` / `role_substring` / `identifier_contains`. On macOS, **`TextArea`** and **`TextField`** match both `AXTextArea` and `AXTextField` (many chat apps use TextField for compose). If several text fields match, the host deprioritizes known **search** controls (e.g. WeChat `_SC_SEARCH_FIELD`) and prefers **lower** on-screen fields (composer). Bypasses coordinate screenshot guard — but **not** the browser boundary: no ComputerUse input action (including `app_click` / `interactive_click` / `visual_click`) may drive a Chromium-family browser; use ControlHub domain=\"browser\" instead. \ **`move_to_text`:** OCR-match visible text (`text_query`) and **move the pointer** to it (no click, no keys); **no prior `screenshot` required for targeting** (host captures **raw** pixels for Vision — no agent screenshot overlays; on macOS defaults to the **frontmost window** unless **`ocr_region_native`** overrides). Matching **strips whitespace** between CJK glyphs and allows **small edit distance** when Vision mis-reads one character. The host **trusts** the resulting globals — **next `click`** does **not** require an extra `screenshot` (same as AX). If **several** hits match, the host returns **preview JPEGs + accessibility** per candidate — pick **`move_to_text_match_index`** (1-based) and call **`move_to_text` again** with the same query/region, or narrow with **`ocr_region_native`**. Use **`click`** afterward if you need a mouse press. Prefer after `click_element` misses when text is visible. \ **`click`:** Press at **current pointer only** — **never** pass `x`, `y`, `coordinate_mode`, or `use_screen_coordinates`. Position first with **`move_to_text`**, **`mouse_move`** (**globals only**), or **`click_element`**. After pointer moves, **`screenshot`** again before the next guarded **`click`** when the host requires it. \ **`mouse_move` / `drag`:** **`use_screen_coordinates`: true** required — global coordinates from **`move_to_text`**, **`locate`**, AX, or **`pointer_global`**; never JPEG pixel guesses. \ @@ -1065,7 +1064,7 @@ impl Tool for ComputerUseTool { "action": { "type": "string", "enum": ["screenshot", "describe_screen", "click_target", "move_to_target", "click_element", "move_to_text", "click", "mouse_move", "scroll", "drag", "locate", "key_chord", "type_text", "pointer_move_rel", "wait", "list_displays", "focus_display", "paste", "list_apps", "get_app_state", "get_app_shortcuts", "app_click", "app_type_text", "app_scroll", "app_key_chord", "app_wait_for", "build_interactive_view", "interactive_click", "interactive_type_text", "interactive_scroll", "build_visual_mark_view", "visual_click", "open_app", "open_url", "open_file", "clipboard_get", "clipboard_set", "run_script", "run_apple_script", "get_os_info"], - "description": "The action to perform. **ACTION PRIORITY:** 1) Use Bash tool for CLI/terminal/system commands (most efficient). 2) **`open_app`** to launch apps by name. **`run_apple_script`** to run AppleScript (macOS). 3) Prefer **`key_chord`** for shortcuts/navigation keys over mouse. Not sure what shortcut a target app uses? Call **`get_app_shortcuts`** first to read its registered menu shortcuts, then fire the winner with `key_chord` / `app_key_chord` instead of clicking through menus. 4) Only when above fail: `click_target` / `move_to_target` (AX → OCR → screen coords in one call) before lower-level `click_element`, `move_to_text`, or `mouse_move` + `click`. **`screenshot`** is for observation/confirmation ONLY — never derive mouse coordinates from screenshots. `click` = press at **current pointer only** (no x/y params). `scroll` supports optional position (`scroll_x`/`scroll_y`). `type_text`, `drag`, `pointer_move_rel`, `wait`, `locate` = standard actions." + "description": "The action to perform. **Browser boundary:** no input action here may drive a Chromium-family browser (Chrome/Edge/Brave/Arc) — use ControlHub domain=\"browser\" for those; switching focus away with `key_chord` [\"alt\",\"tab\"] / [\"command\",\"tab\"] or `open_app` is always allowed. **ACTION PRIORITY:** 1) Use the `ExecCommand` tool for CLI/terminal/system commands (most efficient). 2) **`open_app`** to launch apps by name. **`run_apple_script`** to run AppleScript (macOS). 3) Prefer **`key_chord`** for shortcuts/navigation keys over mouse. Not sure what shortcut a target app uses? Call **`get_app_shortcuts`** first to read its registered menu shortcuts, then fire the winner with `key_chord` / `app_key_chord` instead of clicking through menus. 4) Only when above fail: `click_target` / `move_to_target` (AX → OCR → screen coords in one call) before lower-level `click_element`, `move_to_text`, or `mouse_move` + `click`. **`screenshot`** is for observation/confirmation ONLY — never derive mouse coordinates from screenshots. `click` = press at **current pointer only** (no x/y params). `scroll` supports optional position (`scroll_x`/`scroll_y`). `type_text`, `drag`, `pointer_move_rel`, `wait`, `locate` = standard actions." }, "use_screen_coordinates": { "type": "boolean", "description": "For `mouse_move`, `drag`: **must be true** — global display coordinates (e.g. macOS points) from `move_to_text`, `locate`, AX, or `pointer_global`. **Not** for `click`." }, "delta_x": { "type": "integer", "description": "For `pointer_move_rel`: horizontal delta (negative=left); also accepted as `dx`. **Not** allowed as the first move after `screenshot` (host). For `scroll`: horizontal wheel delta." }, @@ -1096,6 +1095,7 @@ impl Tool for ComputerUseTool { "app_name": { "type": "string", "description": "For `open_app`: the application name to launch (e.g. \"Safari\", \"WeChat\", \"Visual Studio Code\")." }, "script": { "type": "string", "description": "For `run_apple_script`: the AppleScript code to execute via `osascript`. macOS only." }, "opts": { "type": "object", "description": "For `build_interactive_view` / `build_visual_mark_view`: optional view options." }, + "i": { "type": ["integer", "null"], "description": "For interactive/visual actions: element or mark index from the latest view." }, "scroll_x": { "type": "integer", "description": "For `scroll`: optional global X coordinate to move pointer before scrolling. Use with `scroll_y`. Requires `use_screen_coordinates`: true." }, "scroll_y": { "type": "integer", "description": "For `scroll`: optional global Y coordinate to move pointer before scrolling. Use with `scroll_x`. Requires `use_screen_coordinates`: true." } })); @@ -1177,7 +1177,7 @@ impl Tool for ComputerUseTool { // desktop side — the ControlHub browser domain owns that surface. // Read-only observation actions pass through. if let Some(err) = super::computer_use_actions::ComputerUseActions::new() - .desktop_action_targets_browser(action, context) + .desktop_action_targets_browser(action, input, context) .await { return Ok(err_response("computer_use", action, err)); @@ -2226,9 +2226,10 @@ mod tests { } } - /// Visual-only actions (their results are marked-up screenshots) must not - /// be advertised to text-only models; their view-options parameter goes - /// with them. + /// Visual-only actions must not be advertised to text-only models: their + /// results are marked-up screenshots, and `interactive_type_text` / + /// `interactive_scroll` address elements by the `i` index of a view only a + /// vision-capable model can build. Their parameters go with them. #[test] fn visual_only_actions_are_absent_from_text_only_schema() { let full_actions = action_enum(&ComputerUseTool::new().input_schema()); @@ -2238,6 +2239,8 @@ mod tests { "interactive_click", "build_visual_mark_view", "visual_click", + "interactive_type_text", + "interactive_scroll", ] { assert!( full_actions.iter().any(|a| a == action), @@ -2253,7 +2256,39 @@ mod tests { !text_only_keys.contains("opts"), "`opts` only configures the removed view-building actions" ); - assert!(property_keys(&ComputerUseTool::new().input_schema()).contains("opts")); + assert!( + !text_only_keys.contains("i"), + "`i` indexes a view no text-only action can build" + ); + let full_keys = property_keys(&ComputerUseTool::new().input_schema()); + assert!(full_keys.contains("opts")); + assert!(full_keys.contains("i")); + } + + /// The `Bash` tool is not registered in the product tool registry + /// (`ExecCommand` is), so naming it as the top-priority action sends the + /// model at a tool that does not exist. + #[tokio::test] + async fn descriptions_and_schemas_never_reference_a_nonexistent_bash_tool() { + let full_description = ComputerUseTool::new() + .description() + .await + .expect("description"); + let text_only_description = ComputerUseTool::description_text_only(); + let full_schema = ComputerUseTool::new().input_schema().to_string(); + let text_only_schema = ComputerUseTool::input_schema_text_only().to_string(); + for blob in [ + full_description.as_str(), + text_only_description.as_str(), + full_schema.as_str(), + text_only_schema.as_str(), + ] { + assert!( + !blob.contains("Bash"), + "ComputerUse text must not name the unregistered Bash tool" + ); + assert!(blob.contains("ExecCommand")); + } } /// Minimal host whose only signal is a Chromium-family frontmost app; @@ -2317,6 +2352,7 @@ mod tests { foreground_application: Some(ComputerUseForegroundApplication { name: Some("Google Chrome".to_string()), bundle_id: Some("com.google.Chrome".to_string()), + process_name: Some("Google Chrome".to_string()), process_id: Some(4242), }), pointer_global: None, @@ -2349,6 +2385,91 @@ mod tests { ); } + /// Renaming the same physical input must not get through the boundary: the + /// app-scoped and interactive/visual variants are guarded too. + #[tokio::test] + async fn app_scoped_input_is_rejected_while_chromium_browser_is_frontmost() { + let mut context = ToolUseContext::for_tool_listing(None, None); + context.computer_use_host = Some(std::sync::Arc::new(ChromeForegroundHost)); + for action in [ + "app_click", + "app_type_text", + "app_scroll", + "app_key_chord", + "interactive_click", + "interactive_type_text", + "interactive_scroll", + "visual_click", + ] { + let results = ComputerUseTool::new() + .call_impl(&json!({ "action": action }), &context) + .await + .expect("guard rejection is a structured envelope"); + assert_eq!( + results[0].content().get("ok").and_then(Value::as_bool), + Some(false), + "`{action}` must be guarded" + ); + } + } + + /// An explicit browser selector is rejected on its own evidence, without + /// asking the host what is frontmost. + #[tokio::test] + async fn app_selector_naming_chromium_is_rejected_without_a_foreground_signal() { + let context = ToolUseContext::for_tool_listing(None, None); + let results = ComputerUseTool::new() + .call_impl( + &json!({ + "action": "app_click", + "app": { "name": "Google Chrome" }, + "target": { "node_idx": 12 } + }), + &context, + ) + .await + .expect("guard rejection is a structured envelope"); + assert_eq!( + results[0].content().get("ok").and_then(Value::as_bool), + Some(false) + ); + } + + /// The guard is positional, not task-related: a task whose target is not + /// the browser must keep a way to reach it while the browser is frontmost. + /// Both escape routes must therefore pass the guard untouched. + #[tokio::test] + async fn guard_leaves_an_escape_route_for_non_browser_targets() { + let mut context = ToolUseContext::for_tool_listing(None, None); + context.computer_use_host = Some(std::sync::Arc::new(ChromeForegroundHost)); + let actions = super::super::computer_use_actions::ComputerUseActions::new(); + for input in [ + // App switcher: the only keyboard way off a browser window. + json!({ "action": "key_chord", "keys": ["command", "tab"] }), + json!({ "action": "key_chord", "keys": ["alt", "tab"] }), + // App-scoped input aimed at a different app. + json!({ "action": "app_type_text", "app": { "name": "WeChat" }, "text": "hi" }), + ] { + let action = input.get("action").and_then(Value::as_str).expect("action"); + assert!( + actions + .desktop_action_targets_browser(action, &input, &context) + .await + .is_none(), + "{input} must not be guarded" + ); + } + // A normal chord in the browser is still rejected. + assert!(actions + .desktop_action_targets_browser( + "key_chord", + &json!({ "action": "key_chord", "keys": ["command", "t"] }), + &context + ) + .await + .is_some()); + } + /// The `action` enum, description, and a handful of other fields are /// deliberately different (richer guidance) between the two schemas. This /// test documents that the shared/override split does not silently diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs index e5b26f531b..87735daa69 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs @@ -22,6 +22,7 @@ use crate::agentic::tools::framework::{ use crate::infrastructure::events::{get_global_event_system, BackendEvent}; use crate::service::config::{get_global_config_service, GlobalConfig}; use crate::util::errors::{BitFunError, BitFunResult}; +use crate::util::types::ToolImageAttachment; use async_trait::async_trait; use bitfun_services_core::system::{truncate_with_marker, LocalSystemProvider}; use serde_json::{json, Value}; @@ -39,6 +40,18 @@ static BROWSER_SESSIONS: std::sync::OnceLock> = const OPEN_BUILT_IN_BROWSER_EVENT: &str = "agentic://open-built-in-browser"; +/// `connect { mode: "headless" }` only attaches, it never launches. It must +/// therefore not default to the port the `default` mode's managed browser +/// occupies: otherwise a session that already ran `connect { mode: "default" }` +/// can never reach a headless browser, because `verify_headless_cdp_browser` +/// hard-rejects the headed browser sitting on that port. +const DEFAULT_HEADLESS_CDP_PORT: u16 = DEFAULT_CDP_PORT + 1; + +/// Computer Use is an independent switch from browser control (`ai.computer_use_enabled` +/// defaults to off, and there is no desktop host outside the desktop app), so ControlHub +/// must never route recovery through a tool that is absent from the agent's tool list. +const COMPUTER_USE_UNAVAILABLE_HINT: &str = "Desktop automation is unavailable in this session (Computer Use is off in Settings, or there is no desktop host). Do not call ComputerUse — it is not in your tool list. Use ControlHub browser.open_builtin for http(s) URLs and ExecCommand for local files and scripts; ask the user to enable Computer Use in Settings if desktop control is genuinely required."; + fn browser_sessions() -> Arc { BROWSER_SESSIONS .get_or_init(|| Arc::new(BrowserSessionRegistry::new())) @@ -69,7 +82,7 @@ impl ControlHubTool { fn default_browser_connect_hints(kind: &BrowserKind, port: u16) -> Vec { let exe = BrowserLauncher::browser_executable(kind); vec![ - "For login/cookies/extensions, use the user's default browser via CDP — never fall back to desktop mouse/keyboard automation.".to_string(), + "Drive pages over CDP rather than desktop mouse/keyboard automation. Note this is BitFun's managed browser profile, not the user's everyday profile: it keeps its own cookies and logins across runs, so on a login wall ask the user to sign in once in that window instead of retrying or typing credentials.".to_string(), format!( "If CDP is not ready on test port {}, retry browser.connect — it starts \"{}\" against BitFun's managed profile with CDP enabled. Do not ask the user to enable a debug port on their everyday browser profile.", port, exe @@ -82,13 +95,30 @@ impl ControlHubTool { vec![ "For project Web UI testing that does not depend on user login state, use the dedicated headless browser flow instead of the user's browser.".to_string(), format!( - "Start or attach a headless test browser on the test port {} and then drive it through browser DOM actions only.", + "browser.connect {{ mode: \"headless\" }} only attaches — it never starts a browser. Start one first, e.g. \"{}\" --headless=new --remote-debugging-port={} --user-data-dir=, then retry.", + BrowserLauncher::browser_executable(&BrowserKind::Chrome), + port + ), + format!( + "If your headless browser listens elsewhere, pass the port explicitly: browser.connect {{ mode: \"headless\", port: }} (default headless test port is {}).", port ), "Do not switch to desktop mouse/keyboard browser control in headless mode.".to_string(), ] } + /// Whether the `ComputerUse` tool is actually present in this session's + /// tool list. Computer Use and browser control are independent switches, + /// so ControlHub stays fully usable while Computer Use is off — but any + /// hint that tells the agent to "use ComputerUse" must be gated on this, + /// otherwise the agent burns turns calling a tool it does not have. + async fn computer_use_available(context: &ToolUseContext) -> bool { + use super::computer_use_tool::ComputerUseTool; + ComputerUseTool::new() + .is_available_in_context(Some(context)) + .await + } + /// `connect { mode: "headless" }` cannot launch a headless browser yet — /// it only attaches to whatever is already listening on the CDP port, /// which may be the user's real logged-in browser. Verify from the @@ -118,7 +148,10 @@ impl ControlHubTool { )) } - fn normalize_builtin_browser_url(raw_url: &str) -> Result { + fn normalize_builtin_browser_url( + raw_url: &str, + computer_use_available: bool, + ) -> Result { let trimmed = raw_url.trim(); if trimmed.is_empty() { return Err(ControlHubError::new( @@ -142,7 +175,11 @@ impl ControlHubTool { ErrorCode::InvalidParams, "Only http and https URLs can be opened in the built-in browser.", ) - .with_hint("Use WebFetch/WebSearch for reading content, or ComputerUse for local files and OS-level URL opening.")); + .with_hint(if computer_use_available { + "Use WebFetch/WebSearch for reading content, or ComputerUse for local files and OS-level URL opening." + } else { + "Use WebFetch/WebSearch for reading content; local files must be handled with ExecCommand or opened by the user (ComputerUse is not available in this session)." + })); } Ok(normalized) @@ -160,17 +197,18 @@ Use this tool via `{ domain, action, params }` for browser automation, terminal * For requests that only open, show, preview, or view a URL, use `open_builtin`. This is the default browser-opening action and keeps the page inside BitFun. * Do not call `connect`, `tab_new`, or `navigate` merely to display a URL. Use the CDP workflow only when the agent must read page content or interact with the DOM. - UI action: - * `open_builtin { url, title?, replace_existing? }` — open an http(s) URL in BitFun's built-in right-side browser panel. This changes the BitFun UI only; it does not fetch page text for reasoning. + * `open_builtin { url, title?, replace_existing? }` — open an http(s) URL in BitFun's built-in right-side browser panel. This changes the BitFun UI only; it does not fetch page text for reasoning. The panel is display-only for the user — the agent cannot snapshot, read, or interact with it; use `connect` + `snapshot` when page content is needed. - Automation modes (external managed browser): - * `connect { mode: "default" }` (default) — start or attach the stable managed browser profile with CDP enabled. - * `connect { mode: "headless" }` — start or attach the stable managed headless browser profile for project Web UI testing that does not depend on user login state. -- Actions: open_builtin, connect, tab_new, navigate, back, forward, reload, snapshot, click, hover, fill, type, check, uncheck, select, press_key, scroll, auto_scroll, wait, get, get_text, get_url, get_title, get_html, screenshot, evaluate, fetch, cookies, set_cookies, set_file_input_files, cdp, network, console, errors, trace, dialog, frame, frame_main, read_article, close, list_pages, tab_query, switch_page, list_sessions. -- Automation workflow: connect -> navigate -> snapshot (returns @e1, @e2 ... refs) -> click/fill using refs. -- Take a fresh snapshot after any DOM mutation; stale refs return `error.code = STALE_REF`. + * `connect { mode: "default" }` (default) — start or attach BitFun's managed browser profile with CDP enabled on port 9222. + * `connect { mode: "headless" }` — attach to an already-running headless browser on the headless test port 9223. This mode never starts a browser; when nothing is listening it returns `NOT_AVAILABLE` together with the exact launch command. + * `params.port` overrides the CDP port for `connect` and for every other CDP action; after `connect`, actions reuse the connected session's port automatically. +- Actions: open_builtin, connect, tab_new, navigate, back, forward, reload, snapshot, click, hover, fill, type, check, uncheck, select, press_key, scroll, auto_scroll, wait, get, get_text, get_url, get_title, get_html, screenshot, evaluate, fetch, cookies, set_cookies, set_file_input_files, cdp, network, console, errors, trace, dialog, read_article, close, list_pages, tab_query, switch_page, list_sessions. +- Automation workflow: connect -> navigate -> snapshot (returns @e1, @e2 ... refs) -> click/fill with `{ "selector": "@e1" }` (the key `ref` is accepted too). +- Take a fresh snapshot after any DOM mutation; a stale `@eN` ref returns `error.code = STALE_REF`, while a selector that matches nothing returns `NOT_FOUND`. ### domain: "terminal" - list_sessions, kill (`terminal_session_id`), interrupt (`terminal_session_id`). -- Use the `Bash` tool to run new commands; this domain only signals existing terminal sessions. +- Use the `ExecCommand` tool to run new commands (stop those with `ExecControl`); this domain only signals pre-existing UI terminal sessions, not `ExecCommand` sessions. ### domain: "meta" - `capabilities` — returns `{ domains: { browser, terminal, meta }, local_client: { os, arch }, workspace_execution: { is_remote }, schema_version }`. @@ -201,6 +239,8 @@ Branch on `ok` and `error.code`, not on English messages. "desktop" => { let hint = if context.is_remote() { "Desktop automation (screenshots, OCR, mouse, keyboard) is not available in remote workspace sessions. Use ExecCommand for shell-based alternatives on the remote SSH host." + } else if !Self::computer_use_available(context).await { + COMPUTER_USE_UNAVAILABLE_HINT } else { "Use the dedicated ComputerUse tool/agent for screenshots, OCR, mouse, keyboard, and desktop app control." }; @@ -214,11 +254,13 @@ Branch on `ok` and `error.code`, not on English messages. .with_hint(hint), )) } - "browser" => self.handle_browser(action, params).await, + "browser" => self.handle_browser(action, params, context).await, "terminal" => self.handle_terminal(action, params, context).await, "system" => { let hint = if context.is_remote() { "System actions (open_app, open_url, clipboard, OS info, local scripts) are not available in remote workspace sessions. Use ExecCommand for shell-based alternatives on the remote SSH host." + } else if !Self::computer_use_available(context).await { + COMPUTER_USE_UNAVAILABLE_HINT } else { "Use the dedicated ComputerUse tool/agent for open_app, open_url, open_file, clipboard, OS info, and local scripts." }; @@ -233,10 +275,28 @@ Branch on `ok` and `error.code`, not on English messages. )) } "meta" => self.handle_meta(action, params, context).await, - other => Err(BitFunError::tool(format!( - "Unknown domain: '{}'. Valid ControlHub domains: browser, terminal, meta. Use ComputerUse for desktop/system actions.", - other - ))), + // Structured rather than `Err`: `map_dispatch_error`'s keyword + // classifier matches nothing in this message and would report a + // routing mistake as INTERNAL, which the model cannot recover from. + other => { + let hint = if !Self::computer_use_available(context).await { + COMPUTER_USE_UNAVAILABLE_HINT + } else { + "Use the dedicated ComputerUse tool/agent for desktop and OS-level actions." + }; + Ok(err_response( + other, + action, + ControlHubError::new( + ErrorCode::UnknownDomain, + format!( + "Unknown domain: '{}'. Valid ControlHub domains: browser, terminal, meta.", + other + ), + ) + .with_hint(hint), + )) + } } } @@ -302,6 +362,7 @@ Branch on `ok` and `error.code`, not on English messages. }) }; + let computer_use = Self::computer_use_available(context).await; let body = json!({ "domains": { "browser": { @@ -326,12 +387,24 @@ Branch on `ok` and `error.code`, not on English messages. "desktop_environment": desktop_env, }, "workspace_execution": workspace_execution, - "schema_version": "1.3", + "computer_use": { + "available": computer_use, + "reason": if computer_use { + Value::Null + } else if is_remote { + json!("Not available in remote workspace sessions") + } else { + json!("Computer Use is disabled (ai.computer_use_enabled = false) or no desktop host is present") + }, + }, + "schema_version": "1.4", }); - Ok(vec![ToolResult::ok( - body, - Some("ControlHub capabilities snapshot".to_string()), - )]) + // The value of a capability probe is entirely in the field + // values, so the assistant-visible text must be the payload + // itself — a one-line summary tells the model nothing. + let assistant = serde_json::to_string_pretty(&body) + .unwrap_or_else(|_| body.to_string()); + Ok(vec![ToolResult::ok(body, Some(assistant))]) } "route_hint" => { // Best-effort heuristic mapping a free-form intent to one @@ -345,12 +418,19 @@ Branch on `ok` and `error.code`, not on English messages. })?; let lower = intent.to_lowercase(); - let mut suggestions: Vec<(&'static str, u32, &'static str)> = vec![]; - let push = |s: &mut Vec<(&'static str, u32, &'static str)>, + // `domain` here is always a real ControlHub domain (or the + // sentinel "unavailable"). A tool name such as ComputerUse is + // reported separately as `tool`, because a model that copies + // `suggested_domain` back into `{ domain: ... }` would + // otherwise send an unroutable request. + let mut suggestions: Vec<(&'static str, Option<&'static str>, u32, &'static str)> = + vec![]; + let push = |s: &mut Vec<(&'static str, Option<&'static str>, u32, &'static str)>, domain: &'static str, + tool: Option<&'static str>, score: u32, why: &'static str| { - s.push((domain, score, why)); + s.push((domain, tool, score, why)); }; let browser_kw = [ @@ -397,6 +477,7 @@ Branch on `ok` and `error.code`, not on English messages. push( &mut suggestions, "browser", + None, 85, "Matches browser/URL keywords; default to browser.open_builtin for opening or showing URLs, and use browser.connect only when DOM reading or interaction is required", ); @@ -404,21 +485,32 @@ Branch on `ok` and `error.code`, not on English messages. } } let is_remote = context.is_remote(); + let computer_use = Self::computer_use_available(context).await; for kw in desktop_kw { if lower.contains(kw) { if is_remote { push( &mut suggestions, "unavailable", + None, 75, "Desktop automation is not available in remote workspace sessions. Use ExecCommand for shell-based alternatives on the remote SSH host.", ); + } else if !computer_use { + push( + &mut suggestions, + "unavailable", + None, + 75, + "Matches local desktop keywords, but Computer Use is off in this session and ComputerUse is not in your tool list. Use ExecCommand for local work, or ask the user to enable Computer Use in Settings.", + ); } else { push( &mut suggestions, - "ComputerUse", + "unavailable", + Some("ComputerUse"), 75, - "Matches local desktop/system keywords; use the ComputerUse tool/agent", + "Matches local desktop/system keywords; this is not a ControlHub domain — call the ComputerUse tool instead", ); } break; @@ -429,6 +521,7 @@ Branch on `ok` and `error.code`, not on English messages. push( &mut suggestions, "terminal", + None, 80, "Matches terminal-signal keywords", ); @@ -441,37 +534,53 @@ Branch on `ok` and `error.code`, not on English messages. push( &mut suggestions, "unavailable", + None, 70, "System actions (open_app, clipboard, OS info, local scripts) are not available in remote workspace sessions. Use ExecCommand for shell-based alternatives on the remote SSH host.", ); + } else if !computer_use { + push( + &mut suggestions, + "unavailable", + None, + 70, + "Matches OS/launch keywords, but Computer Use is off in this session and ComputerUse is not in your tool list. Use ExecCommand for local scripts and file handling, or ask the user to enable Computer Use in Settings.", + ); } else { push( &mut suggestions, - "ComputerUse", + "unavailable", + Some("ComputerUse"), 70, - "Matches OS/launch keywords; use the ComputerUse tool/agent", + "Matches OS/launch keywords; this is not a ControlHub domain — call the ComputerUse tool instead", ); } break; } } - suggestions.sort_by_key(|suggestion| std::cmp::Reverse(suggestion.1)); + suggestions.sort_by_key(|suggestion| std::cmp::Reverse(suggestion.2)); let ranked: Vec = suggestions .iter() - .map(|(d, score, why)| json!({ "domain": d, "score": score, "why": why })) + .map(|(d, tool, score, why)| { + json!({ "domain": d, "tool": tool, "score": score, "why": why }) + }) .collect(); - let suggested = suggestions.first().map(|(d, _, _)| (*d).to_string()); + let top = suggestions.first().copied(); + let suggested = top.map(|(d, _, _, _)| d.to_string()); + let suggested_tool = top.and_then(|(_, tool, _, _)| tool); Ok(vec![ToolResult::ok( json!({ "intent": intent, "suggested_domain": suggested, + "suggested_tool": suggested_tool, "ranked": ranked, - "note": "Heuristic only — confirm by reading meta.capabilities and the domain-specific docs.", + "note": "Heuristic only — confirm by reading meta.capabilities and the domain-specific docs. `suggested_domain` is always a ControlHub domain (or \"unavailable\"); a separate tool to call, if any, is in `suggested_tool`.", }), - Some(match &suggested { - Some(d) => format!("Best guess: domain={}", d), - None => "No confident routing match".to_string(), + Some(match (&suggested, suggested_tool) { + (Some(d), Some(t)) => format!("Best guess: domain={} tool={}", d, t), + (Some(d), None) => format!("Best guess: domain={}", d), + (None, _) => "No confident routing match".to_string(), }), )]) } @@ -508,22 +617,42 @@ Branch on `ok` and `error.code`, not on English messages. ) } - async fn handle_browser(&self, action: &str, params: &Value) -> BitFunResult> { - let port = params - .get("port") - .and_then(|v| v.as_u64()) - .map(|p| p as u16) - .unwrap_or(DEFAULT_CDP_PORT); - + async fn handle_browser( + &self, + action: &str, + params: &Value, + context: &ToolUseContext, + ) -> BitFunResult> { let session_id_param = params .get("session_id") .and_then(|v| v.as_str()) .map(str::to_string); + let port = match params.get("port").and_then(|v| v.as_u64()) { + Some(p) => p as u16, + None if action == "connect" => { + if Self::browser_connect_mode_from_params(params) == "headless" { + DEFAULT_HEADLESS_CDP_PORT + } else { + DEFAULT_CDP_PORT + } + } + // Port-addressed actions after `connect` (list_pages / tab_query / + // tab_new / switch_page) must reuse the connected session's port, + // otherwise a headless session's follow-up calls fall back to the + // headed browser's port. + None => browser_sessions() + .get(session_id_param.as_deref()) + .await + .map(|s| s.port) + .unwrap_or(DEFAULT_CDP_PORT), + }; + match action { "open_builtin" => { let raw_url = params.get("url").and_then(Value::as_str).unwrap_or(""); - let url = match Self::normalize_builtin_browser_url(raw_url) { + let computer_use = Self::computer_use_available(context).await; + let url = match Self::normalize_builtin_browser_url(raw_url, computer_use) { Ok(url) => url, Err(error) => return Ok(err_response("browser", "open_builtin", error)), }; @@ -560,8 +689,16 @@ Branch on `ok` and `error.code`, not on English messages. "url": url, "title": title, "replace_existing": replace_existing, + "observable_by_agent": false, + "note": "The built-in browser panel is display-only for the user; the agent cannot observe or interact with its content.", + "hints": [ + "Do not call snapshot/get_text/click against this panel — it is not a CDP session.", + "To read page content or interact with the DOM, use browser.connect followed by snapshot, then click/fill via @eN refs.", + ], }), - Some(format!("Opened {url} in the built-in browser side panel.")), + Some(format!( + "Opened {url} in the built-in browser side panel (display-only for the user; not observable by the agent — use browser.connect + snapshot to read or interact with a page)." + )), )]) } @@ -807,7 +944,11 @@ Branch on `ok` and `error.code`, not on English messages. "pages": summary, "default_session_id": default_id, }), - Some(format!("{} page(s) found", pages.len())), + Some(format!( + "{} page(s) found (id | title | url):\n{}", + summary.len(), + page_table(&summary) + )), )]) } @@ -875,7 +1016,12 @@ Branch on `ok` and `error.code`, not on English messages. "total": total, "default_session_id": default_id, }), - Some(format!("{} of {} page(s) matched", matched, total)), + Some(format!( + "{} of {} page(s) matched (id | title | url):\n{}", + matched, + total, + page_table(&filtered) + )), )]) } @@ -1016,7 +1162,12 @@ Branch on `ok` and `error.code`, not on English messages. "sessions": ids, "default_session_id": default, }), - Some(format!("{} session(s) tracked", ids.len())), + Some(format!( + "{} session(s) tracked (default={}):\n{}", + ids.len(), + default.as_deref().unwrap_or("-"), + ids.join("\n") + )), )]) } "network" | "network_requests" => { @@ -1044,7 +1195,11 @@ Branch on `ok` and `error.code`, not on English messages. "total_events": total, "requests": requests, }), - Some(format!("Network summary: {} total events", total)), + Some(format!( + "Network summary: {} total events\n{}", + total, + event_list(&requests) + )), )]) } _ => { @@ -1068,7 +1223,11 @@ Branch on `ok` and `error.code`, not on English messages. }; Ok(vec![ToolResult::ok( json!({ "events": events, "count": events.len() }), - Some(format!("{} network event(s)", events.len())), + Some(format!( + "{} network event(s):\n{}", + events.len(), + event_list(&events) + )), )]) } } @@ -1091,7 +1250,11 @@ Branch on `ok` and `error.code`, not on English messages. let events = state.query_console(filter, since, limit).await; Ok(vec![ToolResult::ok( json!({ "events": events, "count": events.len() }), - Some(format!("{} console event(s)", events.len())), + Some(format!( + "{} console event(s):\n{}", + events.len(), + event_list(&events) + )), )]) } "errors" => { @@ -1112,7 +1275,11 @@ Branch on `ok` and `error.code`, not on English messages. let events = state.query_errors(filter, since, limit).await; Ok(vec![ToolResult::ok( json!({ "events": events, "count": events.len() }), - Some(format!("{} JS error event(s)", events.len())), + Some(format!( + "{} JS error event(s):\n{}", + events.len(), + event_list(&events) + )), )]) } "trace" => { @@ -1196,18 +1363,26 @@ Branch on `ok` and `error.code`, not on English messages. .and_then(|v| v.as_array()) .map(|a| a.len()) .unwrap_or(0); - Ok(vec![ToolResult::ok( - result, - Some(format!("Snapshot: {} interactive elements", el_count)), - )]) + // The whole `@eN` workflow lives in the rendered + // snapshot text; a bare element count leaves the model + // with no ref to click or fill. + let snapshot_text = + result.get("snapshot").and_then(|v| v.as_str()).unwrap_or(""); + let url = result.get("url").and_then(|v| v.as_str()).unwrap_or(""); + let title = result.get("title").and_then(|v| v.as_str()).unwrap_or(""); + let assistant = format!( + "Snapshot ({} interactive elements)\nurl: {}\ntitle: {}\n{}", + el_count, url, title, snapshot_text + ); + Ok(vec![ToolResult::ok(result, Some(assistant))]) } "click" => { - let selector = params - .get("selector") - .and_then(|v| v.as_str()) - .ok_or_else(|| { - BitFunError::tool("click requires 'selector'".to_string()) - })?; + let selector = match selector_param(params) { + Some(s) => s, + None => { + return Ok(missing_selector_response("click")); + } + }; let result = actions.click(selector).await?; Ok(vec![ToolResult::ok( result, @@ -1215,13 +1390,12 @@ Branch on `ok` and `error.code`, not on English messages. )]) } "fill" => { - let selector = params - .get("selector") - .or_else(|| params.get("ref")) - .and_then(|v| v.as_str()) - .ok_or_else(|| { - BitFunError::tool("fill requires 'selector'".to_string()) - })?; + let selector = match selector_param(params) { + Some(s) => s, + None => { + return Ok(missing_selector_response("fill")); + } + }; let value = params .get("value") .and_then(|v| v.as_str()) @@ -1245,12 +1419,12 @@ Branch on `ok` and `error.code`, not on English messages. Ok(vec![ToolResult::ok(result, Some("Typed text".to_string()))]) } "select" => { - let selector = params - .get("selector") - .and_then(|v| v.as_str()) - .ok_or_else(|| { - BitFunError::tool("select requires 'selector'".to_string()) - })?; + let selector = match selector_param(params) { + Some(s) => s, + None => { + return Ok(missing_selector_response("select")); + } + }; let option_text = params .get("option_text") .and_then(|v| v.as_str()) @@ -1267,7 +1441,7 @@ Branch on `ok` and `error.code`, not on English messages. let lowered = err_msg.to_lowercase(); let (code, hint) = if lowered.contains("select not found") { ( - ErrorCode::NotFound, + unresolved_selector_code(selector), format!( "No