diff --git a/apps/extension/src/content/HelpRequestOverlay.tsx b/apps/extension/src/content/HelpRequestOverlay.tsx index 7da83fb..e3f867d 100644 --- a/apps/extension/src/content/HelpRequestOverlay.tsx +++ b/apps/extension/src/content/HelpRequestOverlay.tsx @@ -15,6 +15,8 @@ export interface HelpRequestData { prompt: string; /** Custom overlay title; falls back to i18n when omitted. */ title?: string; + /** Full task UI on the subject tab; compact status UI on related tabs. */ + displayMode?: "full" | "compact"; /** CSS selectors to scroll to + flash-highlight. */ selectors: string[]; onContinue: (note: string) => void; @@ -174,6 +176,8 @@ function measure(selectors: string[]): Box[] { export function HelpRequestOverlay({ request }: Props) { const { t } = useTranslation("extension"); + const isCompact = request?.displayMode === "compact"; + const effectiveSelectors = isCompact ? [] : (request?.selectors ?? []); const [note, setNote] = useState(""); const [collapsed, setCollapsed] = useState(false); const [boxes, setBoxes] = useState([]); @@ -207,8 +211,9 @@ export function HelpRequestOverlay({ request }: Props) { // Scroll the first matched target into view once per distinct request id. useLayoutEffect(() => { if (!request) return; + if (isCompact) return; if (lastScrolledIdRef.current === request.id) return; - for (const sel of request.selectors) { + for (const sel of effectiveSelectors) { const el = querySelectorSafe(sel); if (el) { lastScrolledIdRef.current = request.id; @@ -216,7 +221,7 @@ export function HelpRequestOverlay({ request }: Props) { break; } } - }, [request]); + }, [request, isCompact, effectiveSelectors]); // Keep highlight boxes aligned with the page as it scrolls / resizes. useEffect(() => { @@ -225,7 +230,7 @@ export function HelpRequestOverlay({ request }: Props) { return; } const update = () => { - const nextBoxes = measure(request.selectors); + const nextBoxes = measure(effectiveSelectors); setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes)); }; update(); @@ -237,7 +242,7 @@ export function HelpRequestOverlay({ request }: Props) { window.removeEventListener("resize", update); window.clearInterval(interval); }; - }, [request]); + }, [request, effectiveSelectors]); const onHeaderPointerDown = useCallback( (e: ReactPointerEvent) => { @@ -294,8 +299,12 @@ export function HelpRequestOverlay({ request }: Props) { setPanelPos(null); return; } + if (isCompact) { + setPanelPos(null); + return; + } if (userMovedRef.current) return; - const measured = boxes.length > 0 ? boxes : measure(request.selectors); + const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors); const anchor = unionBox(measured); const banner = bannerRef.current; if (!anchor || !banner) { @@ -319,7 +328,7 @@ export function HelpRequestOverlay({ request }: Props) { } const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement); setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos)); - }, [request, boxes, collapsed]); + }, [request, boxes, collapsed, isCompact, effectiveSelectors]); if (!request) return null; @@ -342,6 +351,8 @@ export function HelpRequestOverlay({ request }: Props) { z-index: 2147483647; display: flex; flex-direction: column; + justify-content: flex-start; + min-height: 0; gap: 10px; width: var(--bsk-help-banner-width); max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px); @@ -359,16 +370,25 @@ export function HelpRequestOverlay({ request }: Props) { .bsk-help-banner[data-collapsed="true"] { gap: 0; width: var(--bsk-help-banner-width); - padding: 10px 12px; + padding: 12px; + } + + .bsk-help-banner[data-display-mode="compact"] { + width: min(360px, calc(100vw - ${VIEWPORT_MARGIN * 2}px)); + padding: 12px 14px; + gap: 0; + border-radius: 12px; } .bsk-help-drag-strip { - flex-shrink: 0; + position: absolute; + top: 6px; + left: 0; + right: 0; display: flex; align-items: center; justify-content: center; height: 12px; - margin: -8px 0 -10px; cursor: grab; user-select: none; touch-action: none; @@ -395,15 +415,20 @@ export function HelpRequestOverlay({ request }: Props) { } .bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip { - margin: -6px 0 -8px; + top: 4px; } .bsk-help-header { + flex-shrink: 0; display: flex; align-items: center; gap: 10px; } + .bsk-help-banner[data-display-mode="compact"] .bsk-help-header { + gap: 8px; + } + .bsk-help-title { flex: 1 1 auto; min-width: 0; @@ -414,31 +439,34 @@ export function HelpRequestOverlay({ request }: Props) { color: #111; } + .bsk-help-compact-title { + flex: 1 1 auto; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 13px; + font-weight: 500; + color: #374151; + } + .bsk-help-banner[data-collapsed="true"] .bsk-help-title { white-space: nowrap; } .bsk-help-header-actions { flex-shrink: 0; - display: flex; + display: none; align-items: center; gap: 10px; margin-left: auto; overflow: hidden; - max-width: 0; - opacity: 0; - transform: translateX(8px); - pointer-events: none; - transition: - max-width var(--bsk-help-duration) var(--bsk-help-ease), - opacity 220ms var(--bsk-help-ease), - transform var(--bsk-help-duration) var(--bsk-help-ease); } .bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions { + display: flex; max-width: 280px; opacity: 1; - transform: translateX(0); pointer-events: auto; } @@ -471,15 +499,13 @@ export function HelpRequestOverlay({ request }: Props) { } .bsk-help-body { - display: grid; - grid-template-rows: 1fr; + display: block; + flex-shrink: 0; min-width: 0; - transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease); } .bsk-help-banner[data-collapsed="true"] .bsk-help-body { position: absolute; - grid-template-rows: 0fr; width: 0; height: 0; overflow: hidden; @@ -528,6 +554,7 @@ export function HelpRequestOverlay({ request }: Props) { .bsk-help-footer-actions { display: flex; justify-content: flex-end; + flex-wrap: wrap; gap: 10px; } @@ -539,7 +566,8 @@ export function HelpRequestOverlay({ request }: Props) { font-size: 13px; font-weight: 500; color: #4b5563; - flex-shrink: 0; + min-width: 0; + white-space: normal; } .bsk-help-btn-continue { @@ -550,16 +578,30 @@ export function HelpRequestOverlay({ request }: Props) { font-size: 13px; font-weight: 600; color: #fff; - flex-shrink: 0; + min-width: 0; + white-space: normal; } .bsk-help-header-actions .bsk-help-btn-cancel, .bsk-help-header-actions .bsk-help-btn-continue { padding: 6px 12px; + white-space: nowrap; } .bsk-help-header-actions .bsk-help-btn-continue { padding: 6px 14px; + max-width: 180px; + } + + .bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions { + display: flex; + max-width: none; + } + + .bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-cancel, + .bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-continue { + padding: 6px 10px; + font-size: 12px; } .bsk-help-footer-actions .bsk-help-btn-cancel { @@ -584,29 +626,31 @@ export function HelpRequestOverlay({ request }: Props) { } `} - {boxes.map((b, i) => ( -
- ))} + {!isCompact && + boxes.map((b, i) => ( +
+ ))}
-
- -
+ {!isCompact && ( +
+ +
+ )}
browser-skill - {request.title ?? t("helpRequest.title")} -
- + {isCompact ? ( + {t("helpRequest.compactStatus")} + ) : ( + {request.title ?? t("helpRequest.title")} + )} + {!isCompact && ( +
+ + +
+ )} + {!isCompact && ( -
- + )}
-
-
-
-

{request.prompt}

- setNote(e.target.value)} - /> -
- - + onChange={(e) => setNote(e.target.value)} + /> +
+ + +
-
+ )}
); diff --git a/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx b/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx index 828213e..98eba65 100644 --- a/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx +++ b/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx @@ -38,6 +38,31 @@ describe("HelpRequestOverlay", () => { expect(screen.getByText("Please complete the captcha")).toBeTruthy(); }); + it("renders a compact status without full request controls", () => { + const el = document.createElement("div"); + el.id = "login"; + el.getBoundingClientRect = () => + ({ top: 10, left: 10, width: 100, height: 40, right: 110, bottom: 50 }) as DOMRect; + document.body.append(el); + + const { container } = renderOverlay( + baseRequest({ + displayMode: "compact", + selectors: ["#login"], + }), + ); + + expect(screen.getByText(i18n.t("helpRequest.compactStatus", { ns: "extension" }))).toBeTruthy(); + expect(screen.queryByText("Please complete the captcha")).toBeNull(); + expect(screen.queryByRole("textbox")).toBeNull(); + expect(screen.queryByLabelText(i18n.t("helpRequest.collapse", { ns: "extension" }))).toBeNull(); + expect(container.querySelector("[data-slot='help-continue-button']")).toBeNull(); + expect(container.querySelector("[data-slot='help-cancel-button']")).toBeNull(); + expect(document.querySelectorAll("[data-slot='help-highlight']").length).toBe(0); + + el.remove(); + }); + it("renders a custom title when provided", () => { renderOverlay(baseRequest({ title: "Verify your identity" })); expect(screen.getByText("Verify your identity")).toBeTruthy(); @@ -204,6 +229,15 @@ describe("HelpRequestOverlay", () => { expect(styles).toContain("width: 0"); }); + it("uses natural body height in the expanded layout", () => { + const { container } = renderOverlay(baseRequest()); + const styles = overlayStyles(container); + + expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s); + expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s); + expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s); + }); + it("keeps the same banner width when collapsed", () => { const { container } = renderOverlay(baseRequest()); const styles = overlayStyles(container); diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index d721d70..d92baa8 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -14,10 +14,14 @@ import { type RecordCaptureController, } from "@/content/record-capture"; import { - HELP_RESPONSE, + HELP_ACK, + HELP_FINISH, + HELP_QUERY, + type HelpAckMessage, type HelpCancelMessage, + type HelpFinishMessage, + type HelpQueryResponse, type HelpRequestMessage, - type HelpResponseMessage, isHelpCancelMessage, isHelpRequestMessage, } from "@/lib/help-bridge"; @@ -55,8 +59,6 @@ export default defineContentScript({ if (window.top !== window) return; const overlays = new OverlayController(); - let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null = - null; let recordCapture: RecordCaptureController | null = null; let activeRecordRequestId: string | null = null; let reactRoot: ReactDOM.Root | null = null; @@ -116,8 +118,7 @@ export default defineContentScript({ function resetAgentOverlayState(sessionId: string) { const previousHelp = overlays.resetAgentOverlays(sessionId); if (previousHelp) { - activeHelpRespond?.("cancelled"); - activeHelpRespond = null; + void sendHelpFinish(previousHelp.id, "cancelled"); } recordCapture?.dispose(); recordCapture = null; @@ -156,7 +157,7 @@ export default defineContentScript({ | OverlayAgentOverlayResetMessage | OverlayAutomationBypassMessage, _sender: chrome.runtime.MessageSender, - sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void, + sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void, ) => { if (isRecordContentMessage(message)) { const needsAsync = handleRecordContentMessage( @@ -220,41 +221,30 @@ export default defineContentScript({ if (isHelpCancelMessage(message)) { const state = overlays.snapshot(); if (state.activeHelp && state.activeHelp.id === message.requestId) { - activeHelpRespond?.("cancelled"); + overlays.clearAgentHelpRequest(message.requestId); + renderOverlay(); } return false; } if (isHelpRequestMessage(message)) { const helpMsg = message as HelpRequestMessage; - let responded = false; - const respond = (outcome: "continued" | "cancelled", note?: string) => { - if (responded) return; - responded = true; - const reply: HelpResponseMessage = { - type: HELP_RESPONSE, - outcome, - ...(note ? { note } : {}), - }; - sendResponse(reply); - activeHelpRespond = null; - overlays.clearAgentHelpRequest(helpMsg.requestId); - renderOverlay(); - }; const previousHelp = overlays.setAgentHelpRequest({ id: helpMsg.requestId, prompt: helpMsg.prompt, ...(helpMsg.title ? { title: helpMsg.title } : {}), + ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), selectors: helpMsg.selectors, - onContinue: (note: string) => respond("continued", note.trim() ? note : undefined), - onCancel: () => respond("cancelled"), + onContinue: (note: string) => + void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), + onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), }); - if (previousHelp) { - activeHelpRespond?.("cancelled"); + if (previousHelp && previousHelp.id !== helpMsg.requestId) { + void sendHelpFinish(previousHelp.id, "cancelled"); } - activeHelpRespond = respond; renderOverlay(); - return true; // async sendResponse + sendResponse({ type: HELP_ACK, ok: true }); + return false; } if (message.type === "borrow-request") { @@ -282,9 +272,60 @@ export default defineContentScript({ return false; }; + async function sendHelpFinish( + requestId: string, + outcome: "continued" | "cancelled", + note?: string, + ): Promise { + const msg: HelpFinishMessage = { + type: HELP_FINISH, + requestId, + outcome, + ...(note ? { note } : {}), + }; + overlays.clearAgentHelpRequest(requestId); + renderOverlay(); + await chrome.runtime.sendMessage(msg).catch((err) => { + console.debug("[bsk overlay] help finish failed", err); + }); + } + + function mountHelpRequest(helpMsg: Omit): void { + overlays.setAgentHelpRequest({ + id: helpMsg.requestId, + prompt: helpMsg.prompt, + ...(helpMsg.title ? { title: helpMsg.title } : {}), + ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), + selectors: helpMsg.selectors, + onContinue: (note: string) => + void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), + onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), + }); + } + + async function queryActiveHelpWithRetry(): Promise { + for (let attempt = 0; attempt < 6; attempt += 1) { + try { + const helpQuery = (await chrome.runtime.sendMessage({ + type: HELP_QUERY, + })) as HelpQueryResponse | undefined; + if (helpQuery?.active && helpQuery.request) { + mountHelpRequest(helpQuery.request); + renderOverlay(); + return true; + } + } catch (err) { + console.debug("[bsk overlay] help query failed", err); + } + await new Promise((resolve) => window.setTimeout(resolve, 150)); + } + return false; + } + async function syncAgentOverlay(): Promise { if (!(await anySessionLive())) return; try { + const helpActive = await queryActiveHelpWithRetry(); const reply = (await chrome.runtime.sendMessage({ kind: OVERLAY_MSG_WHO_AM_I, })) as OverlayWhoAmIResponse | undefined; @@ -319,6 +360,10 @@ export default defineContentScript({ }); } + if (!helpActive && overlays.snapshot().activeHelp === null) { + void queryActiveHelpWithRetry(); + } + overlays.activateAgentSession(reply.sessionId); renderOverlay(); } catch (err) { diff --git a/apps/extension/src/lib/__tests__/help-bridge.test.ts b/apps/extension/src/lib/__tests__/help-bridge.test.ts index 1a14283..50b4b81 100644 --- a/apps/extension/src/lib/__tests__/help-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/help-bridge.test.ts @@ -1,10 +1,17 @@ import { describe, expect, it } from "vitest"; import { + HELP_ACK, HELP_CANCEL, + HELP_FINISH, + HELP_QUERY, HELP_REQUEST, HELP_RESPONSE, + isHelpAckMessage, isHelpCancelMessage, + isHelpFinishMessage, + isHelpQueryMessage, isHelpRequestMessage, + isHelpResponseMessage, } from "../help-bridge"; describe("help-bridge", () => { @@ -12,6 +19,9 @@ describe("help-bridge", () => { expect(HELP_REQUEST).toBe("bsk-help-request"); expect(HELP_RESPONSE).toBe("bsk-help-response"); expect(HELP_CANCEL).toBe("bsk-help-cancel"); + expect(HELP_ACK).toBe("bsk-help-ack"); + expect(HELP_QUERY).toBe("bsk-help-query"); + expect(HELP_FINISH).toBe("bsk-help-finish"); }); it("type-guards a help-request message", () => { @@ -20,6 +30,7 @@ describe("help-bridge", () => { type: HELP_REQUEST, requestId: "r1", prompt: "log in", + displayMode: "compact", selectors: ["#login"], timeoutMs: 1000, }), @@ -55,4 +66,18 @@ describe("help-bridge", () => { expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false); expect(isHelpCancelMessage(null)).toBe(false); }); + + it("type-guards lifecycle messages", () => { + expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true); + expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true); + expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe( + true, + ); + expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe( + true, + ); + expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe( + false, + ); + }); }); diff --git a/apps/extension/src/lib/help-bridge.ts b/apps/extension/src/lib/help-bridge.ts index 6f82b10..b73b45c 100644 --- a/apps/extension/src/lib/help-bridge.ts +++ b/apps/extension/src/lib/help-bridge.ts @@ -6,12 +6,15 @@ * Mirrors the borrow-confirmation message style (see * `@/tools/borrow-confirmation`). The background asks a specific tab to * enter "help mode" (render HelpRequestOverlay + hide ControlOverlay); - * the content script replies once the user clicks Continue / Cancel. + * the content script acks display, then later reports Done / Cancel. */ export const HELP_REQUEST = "bsk-help-request"; export const HELP_RESPONSE = "bsk-help-response"; export const HELP_CANCEL = "bsk-help-cancel"; +export const HELP_ACK = "bsk-help-ack"; +export const HELP_QUERY = "bsk-help-query"; +export const HELP_FINISH = "bsk-help-finish"; export interface HelpRequestMessage { type: typeof HELP_REQUEST; @@ -19,11 +22,18 @@ export interface HelpRequestMessage { prompt: string; /** Custom overlay title; omitted when the extension should use its default. */ title?: string; + /** Full task UI on the subject tab; compact status UI on related tabs. */ + displayMode?: "full" | "compact"; /** CSS selectors to scroll to + flash-highlight (may be empty). */ selectors: string[]; timeoutMs: number; } +export interface HelpAckMessage { + type: typeof HELP_ACK; + ok: true; +} + export interface HelpResponseMessage { type: typeof HELP_RESPONSE; outcome: "continued" | "cancelled"; @@ -35,6 +45,22 @@ export interface HelpCancelMessage { requestId: string; } +export interface HelpQueryMessage { + type: typeof HELP_QUERY; +} + +export interface HelpQueryResponse { + active: boolean; + request?: Omit; +} + +export interface HelpFinishMessage { + type: typeof HELP_FINISH; + requestId: string; + outcome: "continued" | "cancelled"; + note?: string; +} + export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage { if (typeof msg !== "object" || msg === null) { return false; @@ -45,6 +71,7 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage { typeof m.requestId === "string" && typeof m.prompt === "string" && (m.title === undefined || typeof m.title === "string") && + (m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") && Array.isArray(m.selectors) && m.selectors.every((selector) => typeof selector === "string") && typeof m.timeoutMs === "number" @@ -58,3 +85,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage { const m = msg as Record; return m.type === HELP_CANCEL && typeof m.requestId === "string"; } + +export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return ( + m.type === HELP_RESPONSE && + (m.outcome === "continued" || m.outcome === "cancelled") && + (m.note === undefined || typeof m.note === "string") + ); +} + +export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === HELP_ACK && m.ok === true; +} + +export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === HELP_QUERY; +} + +export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return ( + m.type === HELP_FINISH && + typeof m.requestId === "string" && + (m.outcome === "continued" || m.outcome === "cancelled") && + (m.note === undefined || typeof m.note === "string") + ); +} diff --git a/apps/extension/src/tools/__tests__/human-loop.test.ts b/apps/extension/src/tools/__tests__/human-loop.test.ts index 9375f54..9f6e9c7 100644 --- a/apps/extension/src/tools/__tests__/human-loop.test.ts +++ b/apps/extension/src/tools/__tests__/human-loop.test.ts @@ -1,7 +1,53 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { RequestHelpParams } from "@/transport/types"; -import { handleRequestHelp, type RequestHelpDeps } from "../human-loop"; +import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop"; + +function chromeEvent unknown>() { + const listeners = new Set(); + return { + addListener: vi.fn((listener: T) => { + listeners.add(listener); + }), + removeListener: vi.fn((listener: T) => { + listeners.delete(listener); + }), + emit: (...args: Parameters) => { + for (const listener of [...listeners]) listener(...args); + }, + }; +} + +function installHelpLifecycleChrome() { + const runtimeOnMessage = + chromeEvent< + ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: unknown) => void, + ) => unknown + >(); + const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>(); + const tabsOnCreated = chromeEvent<(tab: chrome.tabs.Tab) => unknown>(); + const tabsOnUpdated = + chromeEvent<(tabId: number, changeInfo: chrome.tabs.TabChangeInfo) => unknown>(); + const webNavigationOnCompleted = + chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>(); + + vi.stubGlobal("chrome", { + runtime: { onMessage: runtimeOnMessage }, + tabs: { onActivated: tabsOnActivated, onCreated: tabsOnCreated, onUpdated: tabsOnUpdated }, + webNavigation: { onCompleted: webNavigationOnCompleted }, + }); + + return { + runtimeOnMessage, + tabsOnActivated, + tabsOnCreated, + tabsOnUpdated, + webNavigationOnCompleted, + }; +} function fakeManager(sessionId: string, agentWindowId: number, tabId: number) { const mgr = { @@ -27,14 +73,19 @@ function baseDeps(over: Partial = {}): RequestHelpDeps { windows: { update: vi.fn(async () => ({}) as never) }, activateTab: vi.fn(async () => {}), sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })), - watchTabNavigation: () => () => {}, cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"], notifications: null, + autoAttachLifecycle: false, ...over, }; } describe("handleRequestHelp", () => { + afterEach(() => { + resetHelpLifecycleForTests(); + vi.unstubAllGlobals(); + }); + it("rejects unknown session", async () => { const res = await handleRequestHelp( fakeManager("abcd", 99, 5), @@ -78,22 +129,29 @@ describe("handleRequestHelp", () => { expect(sentMsg.title).toBeUndefined(); }); - it("returns navigated when the tab navigates during the wait", async () => { - const unwatch = vi.fn(); + it("does not complete merely because the tab navigates during the wait", async () => { + vi.useFakeTimers(); const deps = baseDeps({ sendToTab: vi.fn(() => new Promise(() => {})), - watchTabNavigation: vi.fn((_tabId, cb) => { - queueMicrotask(() => cb()); - return unwatch; - }), }); - const res = await handleRequestHelp( - fakeManager("abcd", 99, 5), - baseParams({ tab_id: 5 }), - deps, - ); - expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 }); - expect(unwatch).toHaveBeenCalled(); + try { + const pending = handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ tab_id: 5, timeout_ms: 10 }), + deps, + ); + await vi.advanceTimersByTimeAsync(5); + let settled = false; + void pending.then(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(10); + await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 }); + } finally { + vi.useRealTimers(); + } }); it("returns timed_out when the wait expires", async () => { @@ -115,6 +173,160 @@ describe("handleRequestHelp", () => { } }); + it("returns completed when explicit completion criteria match", async () => { + const deps = baseDeps({ + sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })), + cdp: { + send: vi.fn(async (_tabId: number, method: string) => { + if (method === "Runtime.evaluate") return { result: { value: true } }; + return {}; + }), + } as unknown as RequestHelpDeps["cdp"], + }); + const res = await handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ + tab_id: 5, + completion_criteria: { + any: [{ selector_exists: "#account-menu" }], + stable_for_ms: 0, + }, + }), + deps, + ); + expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 }); + }); + + it("keeps user control active on new tabs without moving completion off the primary tab", async () => { + vi.useFakeTimers(); + const chromeEvents = installHelpLifecycleChrome(); + const sendToTab = vi.fn(async () => ({ type: "bsk-help-ack", ok: true })); + const deps = baseDeps({ + autoAttachLifecycle: undefined, + sendToTab, + tabsApi: { + get: vi.fn(async (tabId: number) => ({ + id: tabId, + windowId: 99, + active: tabId === 6, + url: tabId === 6 ? "https://app.example/reset/success" : "https://app.example/login", + })) as never, + query: vi.fn(async () => [{ id: 5, windowId: 99, active: true }] as never), + }, + }); + + try { + const pending = handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ + tab_id: 5, + timeout_ms: 1_000, + completion_criteria: { + any: [{ url_contains: "/reset/success" }], + stable_for_ms: 0, + }, + }), + deps, + ); + await vi.waitFor(() => expect(sendToTab).toHaveBeenCalledWith(5, expect.anything())); + + chromeEvents.tabsOnActivated.emit({ tabId: 6, windowId: 99 }); + await vi.advanceTimersByTimeAsync(200); + + await vi.waitFor(() => + expect(sendToTab).toHaveBeenCalledWith( + 6, + expect.objectContaining({ + type: "bsk-help-request", + displayMode: "compact", + selectors: [], + }), + ), + ); + + await vi.advanceTimersByTimeAsync(1_000); + await expect(pending).resolves.toMatchObject({ + outcome: "timed_out", + tab_id: 5, + }); + expect(sendToTab).toHaveBeenCalledWith( + 6, + expect.objectContaining({ type: "bsk-help-cancel" }), + ); + } finally { + vi.useRealTimers(); + } + }); + + it("lets same-window tabs query active help after content-script load", async () => { + const chromeEvents = installHelpLifecycleChrome(); + const ac = new AbortController(); + const cdpSend = vi.fn(async () => ({ root: { nodeId: 1 }, nodeIds: [] })); + const deps = baseDeps({ + autoAttachLifecycle: undefined, + signal: ac.signal, + sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })), + cdp: { + send: cdpSend, + } as unknown as RequestHelpDeps["cdp"], + }); + + const pending = handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ tab_id: 5, timeout_ms: 60_000 }), + deps, + ); + await vi.waitFor(() => expect(deps.sendToTab).toHaveBeenCalledWith(5, expect.anything())); + + const sameWindowResponse = vi.fn(); + chromeEvents.runtimeOnMessage.emit( + { type: "bsk-help-query" }, + { tab: { id: 6 } as chrome.tabs.Tab }, + sameWindowResponse, + ); + await vi.waitFor(() => + expect(sameWindowResponse).toHaveBeenCalledWith({ + active: true, + request: expect.objectContaining({ + requestId: expect.any(String), + prompt: "log in", + displayMode: "compact", + selectors: [], + }), + }), + ); + + const subjectResponse = vi.fn(); + chromeEvents.runtimeOnMessage.emit( + { type: "bsk-help-query" }, + { tab: { id: 5 } as chrome.tabs.Tab }, + subjectResponse, + ); + await vi.waitFor(() => + expect(subjectResponse).toHaveBeenCalledWith({ + active: true, + request: expect.objectContaining({ + requestId: expect.any(String), + prompt: "log in", + displayMode: "full", + }), + }), + ); + + ac.abort(); + await expect(pending).resolves.toMatchObject({ code: "cancelled" }); + expect(cdpSend).toHaveBeenCalledWith( + 5, + "DOM.querySelectorAll", + expect.objectContaining({ selector: "[data-bsk-help]" }), + ); + expect(cdpSend).toHaveBeenCalledWith( + 6, + "DOM.querySelectorAll", + expect.objectContaining({ selector: "[data-bsk-help]" }), + ); + }); + it("tags ref targets via CDP and reports them matched", async () => { const mgr = { get: (id: string) => @@ -190,38 +402,40 @@ describe("handleRequestHelp", () => { expect("code" in res).toBe(false); }); - it("returns protocol_error when sendToTab rejects", async () => { + it("keeps waiting when the initial overlay delivery rejects", async () => { + vi.useFakeTimers(); const deps = baseDeps({ sendToTab: vi.fn(async () => { throw new Error("no receiver"); }), }); - const res = await handleRequestHelp( - fakeManager("abcd", 99, 5), - baseParams({ tab_id: 5 }), - deps, - ); - expect("code" in res && res.code).toBe("protocol_error"); + try { + const res = handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ tab_id: 5, timeout_ms: 10 }), + deps, + ); + await vi.advanceTimersByTimeAsync(20); + await expect(res).resolves.toMatchObject({ outcome: "timed_out" }); + } finally { + vi.useRealTimers(); + } }); - it("returns protocol_error for a malformed help response", async () => { - const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) }); - const res1 = await handleRequestHelp( - fakeManager("abcd", 99, 5), - baseParams({ tab_id: 5 }), - undefinedReply, - ); - expect("code" in res1 && res1.code).toBe("protocol_error"); - - const badOutcome = baseDeps({ - sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })), - }); - const res2 = await handleRequestHelp( - fakeManager("abcd", 99, 5), - baseParams({ tab_id: 5 }), - badOutcome, - ); - expect("code" in res2 && res2.code).toBe("protocol_error"); + it("keeps waiting for explicit completion after a malformed help response", async () => { + vi.useFakeTimers(); + const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) }); + try { + const res = handleRequestHelp( + fakeManager("abcd", 99, 5), + baseParams({ tab_id: 5, timeout_ms: 10 }), + deps, + ); + await vi.advanceTimersByTimeAsync(20); + await expect(res).resolves.toMatchObject({ outcome: "timed_out" }); + } finally { + vi.useRealTimers(); + } }); it("reports selector match status from CDP", async () => { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 5091141..17f5456 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -29,7 +29,7 @@ import type { import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; import { handleEvaluate } from "./evaluate"; -import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop"; +import { handleRequestHelp } from "./human-loop"; import { handleClick, handleFill, handlePress, handleSelect } from "./interaction"; import { handleNavigate, @@ -394,7 +394,6 @@ export class ToolDispatcher { await chrome.tabs.update(tabId, { active: true }); }, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), - watchTabNavigation: defaultWatchTabNavigation, ...(this.cdp ? { cdp: this.cdp } : {}), notifications: makeHelpNotifications(), notificationCopy: this.helpNotificationCopy?.(), diff --git a/apps/extension/src/tools/human-loop.ts b/apps/extension/src/tools/human-loop.ts index aeb6c9d..9b65325 100644 --- a/apps/extension/src/tools/human-loop.ts +++ b/apps/extension/src/tools/human-loop.ts @@ -1,21 +1,24 @@ -// `tool.request_help` — pause and ask the human to act on a tab. -// -// Brings the target tab to the foreground, resolves `ref` targets to a -// temporary attribute selector via CDP, asks the content script to enter -// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks -// until the user clicks Continue / Cancel, the wait times out, or the -// daemon cancels the RPC. +// `tool.request_help` — pause automation and let the human control a tab. import { + HELP_ACK, HELP_CANCEL, + HELP_FINISH, + HELP_QUERY, HELP_REQUEST, - HELP_RESPONSE, type HelpCancelMessage, + type HelpFinishMessage, + type HelpQueryResponse, type HelpRequestMessage, - type HelpResponseMessage, + isHelpAckMessage, + isHelpFinishMessage, + isHelpQueryMessage, + isHelpResponseMessage, } from "@/lib/help-bridge"; -import type { SessionManager } from "@/session-manager/manager"; +import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { + HelpCompletionCondition, + HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult, @@ -34,53 +37,59 @@ import { import { lookupSnapshotRef } from "./snapshot-ref"; const DEFAULT_HELP_TIMEOUT_MS = 300_000; -/** Attribute the overlay highlights for resolved `ref` targets. */ const HELP_ATTR = "data-bsk-help"; +const HELP_SEND_RETRIES = 3; +const HELP_SEND_RETRY_DELAY_MS = 350; +const HELP_REARM_DEBOUNCE_MS = 150; +const HELP_REARM_MAX_ATTEMPTS = 12; +const HELP_REARM_RETRY_DELAY_MS = 400; +const DEFAULT_COMPLETION_STABLE_MS = 1_000; +const COMPLETION_POLL_MS = 500; export interface RequestHelpNotifications { create(id: string, options: chrome.notifications.NotificationOptions): Promise; clear(id: string): Promise; } -export type TabNavigationUnsubscribe = () => void; - export interface RequestHelpDeps { tabsApi: ChromeTabsApi; windows: { update(windowId: number, info: { focused?: boolean }): Promise; }; - /** Activate a tab inside its window. */ activateTab(tabId: number): Promise; - /** Send the help-request to the tab's content script and await reply. */ sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise; - /** - * Watch for navigation on `tabId` (full page load or SPA URL change). - * Invokes `onNavigated` once; returns unsubscribe. - */ - watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe; cdp?: CdpRunner; - /** Pass `null` to skip OS notifications (tests). */ notifications: RequestHelpNotifications | null; notificationCopy?: { title: string; body: string }; signal?: AbortSignal; + autoAttachLifecycle?: boolean; } -/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */ -export function defaultWatchTabNavigation( - tabId: number, - onNavigated: () => void, -): TabNavigationUnsubscribe { - const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => { - if (updatedTabId !== tabId) return; - if (changeInfo.url !== undefined || changeInfo.status === "loading") { - chrome.tabs.onUpdated.removeListener(listener); - onNavigated(); - } - }; - chrome.tabs.onUpdated.addListener(listener); - return () => chrome.tabs.onUpdated.removeListener(listener); +interface ActiveHelpRequest { + ctx: SessionContext; + requestId: string; + primaryTabId: number; + overlayTabIds: Set; + prompt: string; + title?: string; + targets: HelpTarget[]; + selectors: string[]; + timeoutMs: number; + notificationId: string; + resolvedTargets?: ResolvedTarget[]; + completionCriteria?: HelpCompletionCriteria; + deps: RequestHelpDeps; + settled: boolean; + resolve: (value: RequestHelpResult | RpcError) => void; + timeoutTimer: ReturnType; + completionTimer: ReturnType | null; + completionMatchedSince: number | null; + abortHandler: (() => void) | null; } +const activeHelpRequests = new Map(); +const helpRearmTimers = new Map>(); + let defaultDeps: RequestHelpDeps | null = null; function getDefaultDeps(): RequestHelpDeps { if (!defaultDeps) { @@ -91,7 +100,6 @@ function getDefaultDeps(): RequestHelpDeps { await chrome.tabs.update(tabId, { active: true }); }, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), - watchTabNavigation: defaultWatchTabNavigation, notifications: { create: (id, opts) => new Promise((resolve, reject) => @@ -113,11 +121,50 @@ function makeRequestId(tabId: number): string { return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; } -/** - * Tag a `ref` target's DOM node with `data-bsk-help=""` via CDP so the - * content overlay can highlight it with a plain attribute selector. - * Returns the selector on success, or null when the ref can't resolve. - */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequestMessage { + const isPrimaryTab = tabId === help.primaryTabId; + return { + type: HELP_REQUEST, + requestId: help.requestId, + prompt: help.prompt, + ...(help.title ? { title: help.title } : {}), + displayMode: isPrimaryTab ? "full" : "compact", + selectors: isPrimaryTab ? help.selectors : [], + timeoutMs: help.timeoutMs, + }; +} + +async function sendHelpRequestWithAck( + tabId: number, + msg: HelpRequestMessage, + deps: RequestHelpDeps, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) { + try { + const response = await deps.sendToTab(tabId, msg); + if (isHelpAckMessage(response)) return null; + if (isHelpResponseMessage(response)) { + return { + type: HELP_FINISH, + requestId: msg.requestId, + outcome: response.outcome, + ...(response.note ? { note: response.note } : {}), + }; + } + lastError = new Error("content script did not ack HELP_REQUEST"); + } catch (err) { + lastError = err; + } + if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS); + } + throw lastError ?? new Error("failed to show help overlay"); +} + async function tagRefTarget( cdp: CdpRunner, tabId: number, @@ -145,7 +192,6 @@ async function tagRefTarget( } } -/** Returns true when `selector` matches a live element in the page. */ async function selectorExists( cdp: CdpRunner, tabId: number, @@ -163,7 +209,6 @@ async function selectorExists( } } -/** Remove every help attribute we added (best-effort cleanup). */ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise { if (!cdp) return; try { @@ -180,34 +225,16 @@ async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise< await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {}); } } catch { - // best-effort + // Best-effort cleanup. } } -export async function handleRequestHelp( - manager: SessionManager, - params: RequestHelpParams, - deps: RequestHelpDeps = getDefaultDeps(), -): Promise { - const ctxOrErr = lookupSession(manager, params, "request_help"); - if (isRpcError(ctxOrErr)) return ctxOrErr; - const ctx = ctxOrErr; - if (!params.prompt || typeof params.prompt !== "string") { - return { code: "invalid_params", message: "request_help requires a prompt" }; - } - if (deps.signal?.aborted) { - return { code: "cancelled", message: "request_help aborted" }; - } - - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); - if (isRpcError(target)) return target; - const denied = enforceAgentWindow(ctx, target, "request_help"); - if (denied) return denied; - const tabId = target.tabId; - - // Resolve targets → selectors. `selector` passes through; `ref` is - // tagged with an attribute via CDP and reported as its attr selector. - const targets = params.targets ?? []; +async function resolveHelpTargets( + ctx: ReturnType, + tabId: number, + targets: HelpTarget[], + deps: RequestHelpDeps, +): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> { const selectors: string[] = []; const resolved: ResolvedTarget[] = []; let rootNodeId: number | undefined; @@ -221,22 +248,20 @@ export async function handleRequestHelp( rootNodeId = undefined; } } - for (let i = 0; i < targets.length; i++) { - const tgt = targets[i] as HelpTarget; + + for (let i = 0; i < targets.length; i += 1) { + const tgt = targets[i]; if (tgt.selector) { selectors.push(tgt.selector); - let matched: boolean; - if (deps.cdp) { - matched = - rootNodeId !== undefined - ? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector) - : false; - } else { - matched = true; - } + const matched = + deps.cdp && rootNodeId !== undefined + ? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector) + : deps.cdp + ? false + : true; resolved.push({ matched, selector: tgt.selector }); } else if (tgt.ref) { - const looked = lookupSnapshotRef(ctx, tgt.ref, tabId); + const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null; const backendNodeId = looked?.backendNodeId ?? null; let sel: string | null = null; if (backendNodeId !== null && deps.cdp) { @@ -247,7 +272,407 @@ export async function handleRequestHelp( } } - // Bring the tab to the foreground. + return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined }; +} + +async function refreshHelpTargets(help: ActiveHelpRequest): Promise { + await clearRefTags(help.deps.cdp, help.primaryTabId); + const { selectors, resolvedTargets } = await resolveHelpTargets( + help.ctx, + help.primaryTabId, + help.targets, + help.deps, + ); + help.selectors = selectors; + help.resolvedTargets = resolvedTargets; +} + +async function cleanupHelp(help: ActiveHelpRequest): Promise { + if (help.deps.notifications) { + await help.deps.notifications.clear(help.notificationId).catch(() => {}); + } + const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]); + await Promise.all( + [...tabsToCancel].map((tabId) => + Promise.all([ + clearRefTags(help.deps.cdp, tabId), + help.deps + .sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }) + .catch(() => {}), + ]), + ), + ); +} + +function finishHelp( + help: ActiveHelpRequest, + value: RequestHelpResult | RpcError, + notifyContent = true, +): void { + if (help.settled) return; + help.settled = true; + activeHelpRequests.delete(help.requestId); + clearTimeout(help.timeoutTimer); + if (help.completionTimer) clearInterval(help.completionTimer); + if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler); + clearRearmTimer(help.primaryTabId); + for (const tabId of help.overlayTabIds) clearRearmTimer(tabId); + if (notifyContent) void cleanupHelp(help); + help.resolve(value); +} + +function findHelpByKnownTabId(tabId: number): ActiveHelpRequest | null { + for (const help of activeHelpRequests.values()) { + if (!help.settled && (help.primaryTabId === tabId || help.overlayTabIds.has(tabId))) { + return help; + } + } + return null; +} + +async function findHelpForTab( + tabId: number, + deps: RequestHelpDeps, +): Promise { + const direct = findHelpByKnownTabId(tabId); + if (direct) return direct; + + try { + const tab = await deps.tabsApi.get(tabId); + const windowId = tab.windowId; + if (typeof windowId !== "number") return null; + for (const help of activeHelpRequests.values()) { + if (!help.settled && help.ctx.agentWindowId === windowId) return help; + } + } catch { + return null; + } + return null; +} + +function clearRearmTimer(tabId: number): void { + const timer = helpRearmTimers.get(tabId); + if (!timer) return; + clearTimeout(timer); + helpRearmTimers.delete(tabId); +} + +async function sendCurrentHelpOverlay(help: ActiveHelpRequest): Promise { + const legacyFinish = await sendHelpRequestWithAck( + help.primaryTabId, + helpRequestMessage(help, help.primaryTabId), + help.deps, + ); + help.overlayTabIds.add(help.primaryTabId); + return legacyFinish; +} + +async function refreshAndSendHelpOverlay( + help: ActiveHelpRequest, + tabId: number, +): Promise { + if (tabId === help.primaryTabId) await refreshHelpTargets(help); + const legacyFinish = await sendHelpRequestWithAck( + tabId, + helpRequestMessage(help, tabId), + help.deps, + ); + help.overlayTabIds.add(tabId); + return legacyFinish; +} + +async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise { + for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) { + try { + const legacyFinish = await refreshAndSendHelpOverlay(help, tabId); + if (legacyFinish && !help.settled) { + finishHelp(help, { + outcome: legacyFinish.outcome, + ...(legacyFinish.note ? { note: legacyFinish.note } : {}), + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, + }); + } + return true; + } catch { + if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS); + } + } + return false; +} + +function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void { + const existing = helpRearmTimers.get(tabId); + if (existing) clearTimeout(existing); + helpRearmTimers.set( + tabId, + setTimeout(() => { + helpRearmTimers.delete(tabId); + void (async () => { + const help = await findHelpForTab(tabId, deps); + if (!help) return; + await rearmHelp(help, tabId); + })(); + }, HELP_REARM_DEBOUNCE_MS), + ); +} + +let detachHelpObservation: (() => void) | null = null; + +export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void { + if (detachHelpObservation) return; + const detachRuntime = attachHelpRuntimeListener(deps); + const detachTabs = attachHelpTabListener(deps); + const detachNav = attachHelpNavigationListener(deps); + detachHelpObservation = () => { + detachRuntime(); + detachTabs(); + detachNav(); + }; +} + +export function resetHelpLifecycleForTests(): void { + detachHelpObservation?.(); + detachHelpObservation = null; + for (const timer of helpRearmTimers.values()) clearTimeout(timer); + helpRearmTimers.clear(); + for (const help of activeHelpRequests.values()) { + clearTimeout(help.timeoutTimer); + if (help.completionTimer) clearInterval(help.completionTimer); + } + activeHelpRequests.clear(); +} + +function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void { + const listener = ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: HelpQueryResponse) => void, + ) => { + if (isHelpFinishMessage(message)) { + void finishHelpFromContent(message, sender, deps); + return false; + } + + if (isHelpQueryMessage(message)) { + const tabId = sender.tab?.id; + if (tabId === undefined) { + sendResponse({ active: false }); + return false; + } + void (async () => { + const help = await findHelpForTab(tabId, deps); + if (!help) { + sendResponse({ active: false }); + return; + } + if (tabId === help.primaryTabId) await refreshHelpTargets(help); + help.overlayTabIds.add(tabId); + sendResponse({ + active: true, + request: { + requestId: help.requestId, + prompt: help.prompt, + ...(help.title ? { title: help.title } : {}), + displayMode: tabId === help.primaryTabId ? "full" : "compact", + selectors: tabId === help.primaryTabId ? help.selectors : [], + timeoutMs: help.timeoutMs, + }, + }); + })(); + return true; + } + + return false; + }; + chrome.runtime.onMessage.addListener(listener); + return () => chrome.runtime.onMessage.removeListener(listener); +} + +async function finishHelpFromContent( + message: HelpFinishMessage, + sender: chrome.runtime.MessageSender, + deps: RequestHelpDeps, +): Promise { + const help = activeHelpRequests.get(message.requestId); + if (!help || help.settled) return; + const tabId = sender.tab?.id; + if (tabId !== undefined) { + const match = await findHelpForTab(tabId, deps); + if (match !== help) return; + } + finishHelp(help, { + outcome: message.outcome, + ...(message.note ? { note: message.note } : {}), + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, + }); +} + +function attachHelpTabListener(deps: RequestHelpDeps): () => void { + const onCreated = (tab: chrome.tabs.Tab) => { + if (typeof tab.id !== "number") return; + scheduleRearmForTab(tab.id, deps); + }; + const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { + scheduleRearmForTab(activeInfo.tabId, deps); + }; + chrome.tabs.onCreated?.addListener(onCreated); + chrome.tabs.onActivated?.addListener(onActivated); + return () => { + chrome.tabs.onCreated?.removeListener(onCreated); + chrome.tabs.onActivated?.removeListener(onActivated); + }; +} + +function attachHelpNavigationListener(deps: RequestHelpDeps): () => void { + const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps); + + if (chrome.webNavigation?.onCompleted) { + const completedListener = ( + details: chrome.webNavigation.WebNavigationFramedCallbackDetails, + ) => { + if (details.frameId !== 0) return; + onMainFrameComplete(details.tabId); + }; + chrome.webNavigation.onCompleted.addListener(completedListener); + return () => chrome.webNavigation.onCompleted.removeListener(completedListener); + } + + const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => { + if (info.status !== "complete") return; + onMainFrameComplete(tabId); + }; + chrome.tabs.onUpdated?.addListener(listener); + return () => chrome.tabs.onUpdated?.removeListener(listener); +} + +async function evaluateCompletionCondition( + condition: HelpCompletionCondition, + help: ActiveHelpRequest, +): Promise { + if (condition.url_contains || condition.url_matches) { + let url = ""; + try { + const tab = await help.deps.tabsApi.get(help.primaryTabId); + url = tab.url ?? ""; + } catch { + return false; + } + if (condition.url_contains && !url.includes(condition.url_contains)) return false; + if (condition.url_matches) { + try { + if (!new RegExp(condition.url_matches).test(url)) return false; + } catch { + return false; + } + } + } + + if ( + condition.selector_exists || + condition.selector_missing || + condition.text_exists || + condition.text_missing + ) { + if (!help.deps.cdp) return false; + const expression = `(() => { + const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)}; + const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)}; + const textExists = ${JSON.stringify(condition.text_exists ?? null)}; + const textMissing = ${JSON.stringify(condition.text_missing ?? null)}; + if (selectorExists && !document.querySelector(selectorExists)) return false; + if (selectorMissing && document.querySelector(selectorMissing)) return false; + const text = document.body ? document.body.innerText || "" : ""; + if (textExists && !text.includes(textExists)) return false; + if (textMissing && text.includes(textMissing)) return false; + return true; + })()`; + try { + const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>( + help.primaryTabId, + "Runtime.evaluate", + { expression, returnByValue: true }, + ); + if (result.result?.value !== true) return false; + } catch { + return false; + } + } + + return true; +} + +async function completionCriteriaMatches(help: ActiveHelpRequest): Promise { + const criteria = help.completionCriteria; + if (!criteria) return false; + const any = criteria.any ?? []; + const all = criteria.all ?? []; + const hasAny = any.length > 0; + const hasAll = all.length > 0; + if (!hasAny && !hasAll) return false; + + if (hasAll) { + for (const condition of all) { + if (!(await evaluateCompletionCondition(condition, help))) return false; + } + } + if (hasAny) { + for (const condition of any) { + if (await evaluateCompletionCondition(condition, help)) return true; + } + return false; + } + return true; +} + +function startCompletionPolling(help: ActiveHelpRequest): void { + if (!help.completionCriteria) return; + const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS; + const check = async () => { + if (help.settled) return; + const matched = await completionCriteriaMatches(help); + const now = Date.now(); + if (!matched) { + help.completionMatchedSince = null; + return; + } + help.completionMatchedSince ??= now; + if (now - help.completionMatchedSince < stableMs) return; + finishHelp(help, { + outcome: "completed", + completed_by: "system", + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, + }); + }; + help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS); + void check(); +} + +export async function handleRequestHelp( + manager: SessionManager, + params: RequestHelpParams, + deps: RequestHelpDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "request_help"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (!params.prompt || typeof params.prompt !== "string") { + return { code: "invalid_params", message: "request_help requires a prompt" }; + } + if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" }; + + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "request_help"); + if (denied) return denied; + + if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps); + const tabId = target.tabId; + const initialTargets = params.targets ?? []; + const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps); + await deps.windows.update(target.windowId, { focused: true }).catch(() => {}); await deps.activateTab(tabId).catch(() => {}); @@ -255,7 +680,6 @@ export async function handleRequestHelp( const notificationId = `bsk-help:${requestId}`; const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS; - // OS notification (best-effort). if (deps.notifications) { const copy = deps.notificationCopy ?? { title: "BrowserSkill: Agent needs your help", @@ -272,85 +696,59 @@ export async function handleRequestHelp( .catch(() => {}); } - const cleanup = async () => { - await clearRefTags(deps.cdp, tabId); - if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {}); - // Retract the overlay in case we resolved via timeout / abort. - await deps - .sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage) - .catch(() => {}); - }; - - const request: HelpRequestMessage = { - type: HELP_REQUEST, - requestId, - prompt: params.prompt, - ...(params.title ? { title: params.title } : {}), - selectors, - timeoutMs, - }; - - const resolvedTargets = resolved.length > 0 ? resolved : undefined; - - return new Promise((resolveOuter) => { - let settled = false; - const unwatchNav = deps.watchTabNavigation(tabId, () => { - finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets }); - }); + return new Promise((resolve) => { + const timeoutTimer = setTimeout(() => { + finishHelp(help, { + outcome: "timed_out", + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, + }); + }, timeoutMs); - const finish = (value: RequestHelpResult | RpcError) => { - if (settled) return; - settled = true; - if (timer !== null) clearTimeout(timer); - unwatchNav(); - deps.signal?.removeEventListener("abort", onAbort); - void cleanup(); - resolveOuter(value); + const help: ActiveHelpRequest = { + ctx, + requestId, + primaryTabId: tabId, + overlayTabIds: new Set(), + prompt: params.prompt, + ...(params.title ? { title: params.title } : {}), + targets: initialTargets, + selectors, + timeoutMs, + notificationId, + resolvedTargets, + completionCriteria: params.completion_criteria, + deps, + settled: false, + resolve, + timeoutTimer, + completionTimer: null, + completionMatchedSince: null, + abortHandler: null, }; - const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" }); - - const timer = setTimeout(() => { - finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets }); - }, timeoutMs) as unknown as number; - - if (deps.signal) { - if (deps.signal.aborted) { - onAbort(); - return; - } - deps.signal.addEventListener("abort", onAbort, { once: true }); + const onAbort = () => finishHelp(help, { code: "cancelled", message: "request_help aborted" }); + help.abortHandler = onAbort; + activeHelpRequests.set(requestId, help); + if (deps.signal?.aborted) { + onAbort(); + return; } + deps.signal?.addEventListener("abort", onAbort, { once: true }); + startCompletionPolling(help); - deps - .sendToTab(tabId, request) - .then((reply) => { - const res = reply as HelpResponseMessage | undefined; - if ( - !res || - res.type !== HELP_RESPONSE || - (res.outcome !== "continued" && res.outcome !== "cancelled") - ) { - finish({ - code: "protocol_error", - message: `invalid help response from tab ${tabId}`, - }); - return; - } - finish({ - outcome: res.outcome, - ...(res.note ? { note: res.note } : {}), - tab_id: tabId, - resolved_targets: resolvedTargets, + void sendCurrentHelpOverlay(help) + .then((legacyFinish) => { + if (!legacyFinish || help.settled) return; + finishHelp(help, { + outcome: legacyFinish.outcome, + ...(legacyFinish.note ? { note: legacyFinish.note } : {}), + tab_id: help.primaryTabId, + resolved_targets: help.resolvedTargets, }); }) - .catch((err) => { - finish({ - code: "protocol_error", - message: `failed to show help overlay on tab ${tabId}: ${ - err instanceof Error ? err.message : String(err) - }`, - }); + .catch(() => { + scheduleRearmForTab(tabId, deps); }); }); } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index bcff0e3..6ed8106 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -514,7 +514,22 @@ export interface HelpTarget { selector?: string; } -export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "navigated"; +export type HelpOutcome = "continued" | "cancelled" | "timed_out" | "completed" | "navigated"; + +export interface HelpCompletionCondition { + url_contains?: string; + url_matches?: string; + selector_exists?: string; + selector_missing?: string; + text_exists?: string; + text_missing?: string; +} + +export interface HelpCompletionCriteria { + any?: HelpCompletionCondition[]; + all?: HelpCompletionCondition[]; + stable_for_ms?: number; +} export interface ResolvedTarget { matched: boolean; @@ -528,11 +543,13 @@ export interface RequestHelpParams { prompt: string; title?: string; targets?: HelpTarget[]; + completion_criteria?: HelpCompletionCriteria; timeout_ms?: number; } export interface RequestHelpResult { outcome: HelpOutcome; + completed_by?: "system"; note?: string; tab_id: number; resolved_targets?: ResolvedTarget[]; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index ab32ff7..3e5599a 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -178,7 +178,7 @@ Details and flags: **`bsk --help`** When a step needs a human (captcha, login, OTP) or you want the user to confirm an important action, pause and ask: - bsk request-help --session --prompt "Solve the captcha, then click Continue" \ + bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - `--prompt` (required): what the user should do. @@ -193,15 +193,21 @@ confirm an important action, pause and ask: prompt with no `--target` for cases where there is genuinely no specific element to point at (e.g. "wait for the page to finish loading"). - `--timeout` (default `5m`): how long to wait. +- `--completion-criteria` (optional): JSON success detector. Use it only + when there is a concrete post-help success signal, e.g. + `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. The target tab is brought to the foreground; the page stays interactive while the agent control mask is hidden. The call blocks until the user -acts. The result `outcome` is one of: +explicitly acts, the timeout expires, cancellation arrives, or explicit +completion criteria match. Page reloads, SPA route changes, and captcha +refreshes do not return control by themselves. The result `outcome` is one of: -- `continued` — the user finished and clicked Continue (treat as confirm). +- `continued` — the user finished and clicked Done / return control (treat as confirm). - `cancelled` — the user clicked Cancel (treat as reject/abort). - `timed_out` — nobody acted within the timeout. -- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again. +- `completed` — the explicit `--completion-criteria` matched while the user had control. +- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element. diff --git a/crates/bsk-cli/src/cli/human_loop.rs b/crates/bsk-cli/src/cli/human_loop.rs index 1c6bb8e..deda217 100644 --- a/crates/bsk-cli/src/cli/human_loop.rs +++ b/crates/bsk-cli/src/cli/human_loop.rs @@ -7,7 +7,9 @@ use std::time::Duration; use anyhow::Context; use bsk_protocol::Method; -use bsk_protocol::tools::{HelpTarget, RequestHelpParams, RequestHelpResult}; +use bsk_protocol::tools::{ + HelpCompletionCriteria, HelpTarget, RequestHelpParams, RequestHelpResult, +}; use clap::Args; use crate::cli::ensure_daemon::ensure_daemon; @@ -41,6 +43,10 @@ pub struct RequestHelpArgs { /// Hard timeout (default 5m). Accepts `5m`, `300s`, `300000ms`. #[arg(long, default_value = "5m", value_parser = parse_timeout_ms)] pub timeout: u32, + + /// JSON success detector, e.g. '{"any":[{"url_contains":"/dashboard"}],"stable_for_ms":1000}'. + #[arg(long = "completion-criteria")] + pub completion_criteria: Option, } /// Classify a `--target` value as a ref or a selector. Refs are `@eN` @@ -68,6 +74,12 @@ pub fn parse_target(raw: &str) -> HelpTarget { pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> { let info = ensure_daemon().context("ensure daemon is running")?; let targets: Vec = args.target.iter().map(|t| parse_target(t)).collect(); + let completion_criteria = args + .completion_criteria + .as_deref() + .map(serde_json::from_str::) + .transpose() + .context("parse --completion-criteria JSON")?; let params = RequestHelpParams { session_id: args.session, tab_id: args.tab_id, @@ -78,6 +90,7 @@ pub fn dispatch(args: RequestHelpArgs, format: Format) -> Result<(), CliError> { } else { Some(targets) }, + completion_criteria, timeout_ms: Some(args.timeout), }; let reply = call(info.sock_path, params, args.timeout)?; diff --git a/crates/bsk-protocol/schema/tool_request_help_params.json b/crates/bsk-protocol/schema/tool_request_help_params.json index cf50ad5..9faad8a 100644 --- a/crates/bsk-protocol/schema/tool_request_help_params.json +++ b/crates/bsk-protocol/schema/tool_request_help_params.json @@ -7,6 +7,17 @@ "session_id" ], "properties": { + "completion_criteria": { + "description": "Optional explicit success detector. Navigation by itself is not a completion signal; these criteria must match before the tool can auto-complete without the user clicking Done.", + "anyOf": [ + { + "$ref": "#/definitions/HelpCompletionCriteria" + }, + { + "type": "null" + } + ] + }, "prompt": { "description": "Message shown to the user explaining what to do.", "type": "string" @@ -50,6 +61,78 @@ } }, "definitions": { + "HelpCompletionCondition": { + "type": "object", + "properties": { + "selector_exists": { + "type": [ + "string", + "null" + ] + }, + "selector_missing": { + "type": [ + "string", + "null" + ] + }, + "text_exists": { + "type": [ + "string", + "null" + ] + }, + "text_missing": { + "type": [ + "string", + "null" + ] + }, + "url_contains": { + "type": [ + "string", + "null" + ] + }, + "url_matches": { + "type": [ + "string", + "null" + ] + } + } + }, + "HelpCompletionCriteria": { + "type": "object", + "properties": { + "all": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/HelpCompletionCondition" + } + }, + "any": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/HelpCompletionCondition" + } + }, + "stable_for_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + } + }, "HelpTarget": { "description": "One component the user is asked to look at / interact with. Exactly one of `ref` / `selector` should be supplied (mirrors the click/fill convention). When neither matches a live element the overlay still shows the prompt and reports the miss via [`ResolvedTarget`].", "type": "object", diff --git a/crates/bsk-protocol/schema/tool_request_help_result.json b/crates/bsk-protocol/schema/tool_request_help_result.json index cea8d70..1b6ad4b 100644 --- a/crates/bsk-protocol/schema/tool_request_help_result.json +++ b/crates/bsk-protocol/schema/tool_request_help_result.json @@ -7,6 +7,16 @@ "tab_id" ], "properties": { + "completed_by": { + "anyOf": [ + { + "$ref": "#/definitions/HelpCompletedBy" + }, + { + "type": "null" + } + ] + }, "note": { "type": [ "string", @@ -31,10 +41,16 @@ } }, "definitions": { + "HelpCompletedBy": { + "type": "string", + "enum": [ + "system" + ] + }, "HelpOutcome": { "oneOf": [ { - "description": "User completed the step and clicked Continue.", + "description": "User completed the step and clicked Done / return control.", "type": "string", "enum": [ "continued" @@ -55,7 +71,14 @@ ] }, { - "description": "The page navigated while waiting (full reload or SPA URL change).", + "description": "Explicit success criteria matched while the user had control.", + "type": "string", + "enum": [ + "completed" + ] + }, + { + "description": "The page navigated while waiting (full reload or SPA URL change). Deprecated: navigation alone should not end a help request.", "type": "string", "enum": [ "navigated" diff --git a/crates/bsk-protocol/src/tools/human_loop.rs b/crates/bsk-protocol/src/tools/human_loop.rs index 534cb18..7afbfc1 100644 --- a/crates/bsk-protocol/src/tools/human_loop.rs +++ b/crates/bsk-protocol/src/tools/human_loop.rs @@ -3,7 +3,8 @@ //! Lets an agent pause and ask the human to complete an in-page step //! (captcha, login, confirmation). The extension brings the target tab //! to the foreground, highlights the requested components, and blocks -//! until the user clicks Continue / Cancel (or the call times out). +//! until the user clicks Done / Cancel, explicit completion criteria match, +//! or the call times out. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -41,6 +42,11 @@ pub struct RequestHelpParams { /// Optional components to scroll to + flash-highlight. #[serde(default, skip_serializing_if = "Option::is_none")] pub targets: Option>, + /// Optional explicit success detector. Navigation by itself is not a + /// completion signal; these criteria must match before the tool can + /// auto-complete without the user clicking Done. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_criteria: Option, /// Hard upper bound on the wait. Defaults to 300000 (5 min), /// enforced daemon-side via the generic tool timeout. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -48,16 +54,46 @@ pub struct RequestHelpParams { pub timeout_ms: Option, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct HelpCompletionCriteria { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub all: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0))] + pub stable_for_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct HelpCompletionCondition { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url_contains: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url_matches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector_exists: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector_missing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_exists: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_missing: Option, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum HelpOutcome { - /// User completed the step and clicked Continue. + /// User completed the step and clicked Done / return control. Continued, /// User declined / aborted via the overlay's Cancel button. Cancelled, /// The wait expired before the user acted. TimedOut, + /// Explicit success criteria matched while the user had control. + Completed, /// The page navigated while waiting (full reload or SPA URL change). + /// Deprecated: navigation alone should not end a help request. Navigated, } @@ -67,11 +103,18 @@ impl HelpOutcome { Self::Continued => "continued", Self::Cancelled => "cancelled", Self::TimedOut => "timed_out", + Self::Completed => "completed", Self::Navigated => "navigated", } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum HelpCompletedBy { + System, +} + /// Per-target resolution status echoed back so the agent can tell which /// `ref` / `selector` actually matched a live element. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -87,6 +130,8 @@ pub struct ResolvedTarget { pub struct RequestHelpResult { pub outcome: HelpOutcome, #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_by: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub note: Option, pub tab_id: i64, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -123,6 +168,7 @@ mod tests { prompt: "log in".into(), title: None, targets: None, + completion_criteria: None, timeout_ms: None, }; let v = serde_json::to_value(&p).unwrap(); @@ -140,6 +186,10 @@ mod tests { serde_json::to_value(HelpOutcome::TimedOut).unwrap(), json!("timed_out") ); + assert_eq!( + serde_json::to_value(HelpOutcome::Completed).unwrap(), + json!("completed") + ); assert_eq!( serde_json::to_value(HelpOutcome::Continued).unwrap(), json!("continued") @@ -158,6 +208,7 @@ mod tests { fn result_round_trips() { let r = RequestHelpResult { outcome: HelpOutcome::Continued, + completed_by: None, note: Some("done".into()), tab_id: 7, resolved_targets: Some(vec![ResolvedTarget { diff --git a/packages/i18n/src/locales/en-US/extension.json b/packages/i18n/src/locales/en-US/extension.json index c597f9a..b3c5663 100644 --- a/packages/i18n/src/locales/en-US/extension.json +++ b/packages/i18n/src/locales/en-US/extension.json @@ -74,10 +74,11 @@ }, "helpRequest": { "title": "Agent needs your help", + "compactStatus": "Agent is still waiting for you to finish this step", "noteLabel": "Optional note back to the agent", "notePlaceholder": "Add a note for the agent (optional)", - "continue": "Continue", - "cancel": "Cancel", + "continue": "Done, return control", + "cancel": "Cancel request", "collapse": "Collapse", "expand": "Expand", "dragHandle": "Drag to move", diff --git a/packages/i18n/src/locales/zh-CN/extension.json b/packages/i18n/src/locales/zh-CN/extension.json index 18e85c6..7f6f509 100644 --- a/packages/i18n/src/locales/zh-CN/extension.json +++ b/packages/i18n/src/locales/zh-CN/extension.json @@ -74,10 +74,11 @@ }, "helpRequest": { "title": "Agent 需要你的帮助", + "compactStatus": "当前 Agent 仍在等待你完成此步骤", "noteLabel": "给 Agent 的备注(可选)", "notePlaceholder": "给 Agent 留言(可选)", - "continue": "继续", - "cancel": "取消", + "continue": "完成并交还控制权", + "cancel": "取消请求", "collapse": "折叠", "expand": "展开", "dragHandle": "拖动以移动", diff --git a/skill/SKILL.md b/skill/SKILL.md index ab32ff7..3e5599a 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -178,7 +178,7 @@ Details and flags: **`bsk --help`** When a step needs a human (captcha, login, OTP) or you want the user to confirm an important action, pause and ask: - bsk request-help --session --prompt "Solve the captcha, then click Continue" \ + bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - `--prompt` (required): what the user should do. @@ -193,15 +193,21 @@ confirm an important action, pause and ask: prompt with no `--target` for cases where there is genuinely no specific element to point at (e.g. "wait for the page to finish loading"). - `--timeout` (default `5m`): how long to wait. +- `--completion-criteria` (optional): JSON success detector. Use it only + when there is a concrete post-help success signal, e.g. + `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. The target tab is brought to the foreground; the page stays interactive while the agent control mask is hidden. The call blocks until the user -acts. The result `outcome` is one of: +explicitly acts, the timeout expires, cancellation arrives, or explicit +completion criteria match. Page reloads, SPA route changes, and captcha +refreshes do not return control by themselves. The result `outcome` is one of: -- `continued` — the user finished and clicked Continue (treat as confirm). +- `continued` — the user finished and clicked Done / return control (treat as confirm). - `cancelled` — the user clicked Cancel (treat as reject/abort). - `timed_out` — nobody acted within the timeout. -- `navigated` — the page navigated while waiting (full reload or SPA URL change). Snapshot refs are stale; run `bsk snapshot` on the new page, then decide whether to call `bsk request-help` again. +- `completed` — the explicit `--completion-criteria` matched while the user had control. +- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element.