From ac19c5f92f0daded1d9abc2b0a12b89110b1e0c1 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Thu, 23 Jul 2026 17:07:18 +0800 Subject: [PATCH 1/9] feat(browser): add native PiP overlay --- .../nativekit-agent-browser-pip/plan.md | 294 +++++++++ .../nativekit-agent-browser-pip/spec.md | 578 ++++++++++++++++++ .../nativekit-agent-browser-pip/tasks.md | 103 ++++ docs/features/agent-browser-pip/spec.md | 17 +- electron-builder.yml | 1 + package.json | 1 + pnpm-workspace.yaml | 1 + scripts/afterPack.js | 43 ++ .../browser/AgentBrowserNativeOverlay.ts | 504 +++++++++++++++ .../desktop/browser/YoBrowserPresenter.ts | 177 ++++-- src/main/desktop/routes.ts | 6 +- src/renderer/api/BrowserClient.ts | 13 +- .../components/browser/AgentBrowserPiP.vue | 56 +- src/shared/contracts/events.ts | 2 + src/shared/contracts/events/browser.events.ts | 10 + src/shared/contracts/routes/browser.routes.ts | 3 +- src/shared/types/browser.ts | 9 + src/shared/types/desktop.ts | 6 +- test/main/build/electronBuilderConfig.test.ts | 7 + .../browser/AgentBrowserNativeOverlay.test.ts | 232 +++++++ .../browser/YoBrowserPresenter.test.ts | 115 +++- test/main/routes/contracts.test.ts | 19 + test/main/scripts/afterPack.test.ts | 97 +++ test/renderer/api/clients.test.ts | 17 +- .../components/AgentBrowserPiP.test.ts | 88 ++- test/setup.ts | 3 +- 26 files changed, 2337 insertions(+), 65 deletions(-) create mode 100644 docs/architecture/nativekit-agent-browser-pip/plan.md create mode 100644 docs/architecture/nativekit-agent-browser-pip/spec.md create mode 100644 docs/architecture/nativekit-agent-browser-pip/tasks.md create mode 100644 src/main/desktop/browser/AgentBrowserNativeOverlay.ts create mode 100644 test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts diff --git a/docs/architecture/nativekit-agent-browser-pip/plan.md b/docs/architecture/nativekit-agent-browser-pip/plan.md new file mode 100644 index 0000000000..684656b7fe --- /dev/null +++ b/docs/architecture/nativekit-agent-browser-pip/plan.md @@ -0,0 +1,294 @@ +# NativeKit Agent Browser PiP Migration Plan + +## Status + +Implemented on 2026-07-23 through Phase 4. The macOS arm64 packaging and native-addon smoke portion +of Phase 5 is complete; physical cross-platform interaction and performance measurements remain +open release QA. + +This plan implements the contract in `spec.md` with the smallest stable change: one main-process +NativeKit adapter, one surface-selection branch in `YoBrowserPresenter`, two typed contract +extensions, and the existing Canvas retained intact as fallback. + +## Selected Approach + +Adopt NativeKit only at the floating-window seam. + +```text +existing page/session/capture owner + | + v + YoBrowserPresenter + | + completed JPEG frame + | + +------+------+ + | | + v v +NativeKit adapter existing renderer event +supported runtime compatibility fallback +``` + +DeepChat does not need a second browser page, a second Electron window, a new preload capability, +or a generic native-services layer. NativeKit receives the same bounded JPEG that currently feeds +the renderer Canvas and owns only the native panel. + +## Change Boundaries + +### Dependency and packaging + +- Pin `"@zerob13/nativekit": "0.5.2"` in `package.json` and refresh `pnpm-lock.yaml`. +- Add `@zerob13/nativekit: false` to `pnpm-workspace.yaml` `allowBuilds`; the published prebuild + should load directly, and an unsupported target should not silently become a local source build. +- Keep the package external to the Electron main bundle. +- Add `node_modules/@zerob13/nativekit/prebuilds/**/*` to `asarUnpack`. +- Extend the packaging validation to require the matching prebuild only for supported target + tuples. +- Treat Windows arm64 as intentionally unsupported by NativeKit 0.5.2 rather than a packaging + error. + +### Main-process adapter + +Add `src/main/desktop/browser/AgentBrowserNativeOverlay.ts`. The adapter owns: + +- lazy dynamic import, process-stable capability state, and one-time warning; +- `start`, first-host validation, host refresh, visibility, image replacement, image removal, and + shutdown; +- the single active logical target; +- NativeKit `activate` and `visibilityRequest` listener registration and removal; +- JPEG-to-data-URL conversion at the final native boundary; +- active-session selection before showing the current presentation; +- one-time active-session/show selection so steady-state image refresh only replaces the same + presentation; +- timing around synchronous `pushImage()` calls. + +The adapter exposes a narrow DeepChat-facing API: + +```ts +type NativeOverlayTarget = { + windowId: number + sessionId: string + runId: string + captureEpoch: number +} + +type NativeOverlayCapability = 'available' | 'unavailable' + +interface AgentBrowserNativeOverlay { + initialize(): Promise + attachHost(window: BrowserWindow): boolean + updateTarget(target: NativeOverlayTarget): void + present(frame: Buffer): boolean + setVisible(visible: boolean): boolean + removeTarget(): void + detachHost(windowId: number): void + shutdown(): void +} +``` + +Exact internal method names may follow repository conventions, but the ownership boundary must not +expand. + +### Presenter integration + +Extend `YoBrowserPresenter` preview state with: + +- selected `BrowserPreviewSurface`; +- owning `windowId`; +- native logical target and visibility; +- existing capture epoch as the stale-frame guard. + +The presenter performs these operations in order: + +1. Resolve and validate the owning `BrowserWindow`. +2. Select the process-stable surface. +3. Keep the existing page in the 1280 x 800 render host. +4. Capture, resize, and encode with one frame in flight. +5. Revalidate session, run, window, mode, surface, and epoch. +6. Present to exactly one destination. +7. Schedule the next capture only after presentation returns. + +For the first native frame, `present()` runs while hidden and `setVisible(true)` follows only after +success. Temporary ineligibility hides the panel without removing the presentation. Terminal +lifecycle edges remove it. + +### Shared contracts and renderer + +Change `browser.setPreviewMode` from a boolean-only result to: + +```ts +type BrowserPreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' + +type BrowserPreviewModeResult = { + updated: boolean + surface: BrowserPreviewSurface +} +``` + +Add a typed main-to-renderer event: + +```ts +type BrowserPreviewActionPayload = { + action: 'activate' | 'dismiss' + windowId: number + sessionId: string + runId: string +} +``` + +`AgentBrowserPiP.vue` keeps its current eligibility calculation and request coalescing. It records +the selected surface for the latest request: + +- `native-overlay`: render no PiP DOM and ignore preview-frame events; +- `renderer-canvas`: retain current frame decode, Canvas, toolbar, drag, and dismissal behavior; +- `none`: render nothing. + +The renderer accepts a native action only when `windowId`, `sessionId`, and `runId` still match its +current state. Activation opens the existing Browser panel. Dismissal updates the existing +run-scoped dismissal state. + +### Host and lifecycle synchronization + +- Attach with `BrowserWindow.getContentBounds()` and `getNativeWindowHandle()`. +- Refresh host bounds after move, resize, and display changes through one debounced callback. +- Hide synchronously on blur, hide, minimize, panel open, or preview stop. +- Preserve the allocated presentation across temporary hide so manual placement survives. +- Remove the presentation on run terminal, target replacement, page/session destruction, or host + close. +- Detach closed hosts and call `overlay.stop()` exactly once during presenter shutdown. + +## Implementation Phases + +### Phase 1: Capability and packaging + +1. Add the exact dependency and lockfile entry. +2. Disable install-time source build for NativeKit. +3. Add ASAR unpack configuration. +4. Extend configuration and `afterPack` tests for the published prebuild matrix. +5. Verify dynamic import in an unpackaged development build. + +Exit condition: unsupported runtimes select fallback without startup failure, while supported +packaged targets fail validation if their prebuild is absent. + +### Phase 2: Native adapter + +1. Implement capability detection and lifecycle. +2. Implement one-host, one-presentation state. +3. Implement push-before-show and idempotent cleanup. +4. Keep steady-state refresh to one same-identity `pushImage()` call so manual placement remains + platform-owned. +5. Map unscoped NativeKit events through the current logical target. +6. Add adapter unit tests with a mocked NativeKit module and fake `BrowserWindow`. + +Exit condition: the adapter never exposes NativeKit outside main and every failure is contained. + +### Phase 3: Presenter routing + +1. Initialize the adapter with `YoBrowserPresenter`. +2. return the selected surface from `setPreviewMode`. +3. Route completed frames directly to native or renderer, never both. +4. Add host visibility and bounds synchronization. +5. Preserve capture epoch and one-in-flight scheduling across surface changes. +6. Start with the existing 4 FPS active interval; raise to at most 8 FPS only after profiling. + +Exit condition: supported runtime traffic contains no renderer frame payload on the native path. + +### Phase 4: Renderer coordination + +1. Extend route, client, and event contracts. +2. Make `AgentBrowserPiP.vue` conditional on the acknowledged surface. +3. Handle activate and dismiss actions with exact target validation. +4. Keep the Canvas path behavior and tests intact. + +Exit condition: native activation opens the same Browser panel and native hide dismisses only the +current run. + +### Phase 5: Performance and platform validation + +1. Measure native drag on 60 Hz and 120 Hz macOS reference displays. +2. Verify full out-of-window movement and work-area clamping. +3. Measure first-frame latency, frame age, `pushImage()` duration, and main-thread long tasks. +4. Repeat interaction smoke tests on Windows x64 and Linux X11/XWayland. +5. Verify Canvas fallback on Windows arm64 and native Wayland. +6. Build packaged artifacts and inspect the unpacked prebuild. + +Exit condition: every acceptance criterion in `spec.md` has recorded evidence. + +## Expected File Map + +Likely touched files: + +- `package.json` +- `pnpm-lock.yaml` +- `pnpm-workspace.yaml` +- `electron-builder.yml` +- `scripts/afterPack.js` +- `src/main/desktop/browser/AgentBrowserNativeOverlay.ts` +- `src/main/desktop/browser/YoBrowserPresenter.ts` +- `src/main/desktop/routes.ts` +- `src/shared/contracts/routes/browser.routes.ts` +- `src/shared/contracts/events/browser.events.ts` +- `src/shared/types/desktop.ts` +- `src/renderer/api/BrowserClient.ts` +- `src/renderer/src/components/browser/AgentBrowserPiP.vue` +- focused main, renderer, contract, and packaging tests under `test/` + +No preload file, persisted-settings schema, database migration, or new renderer window is expected. + +## Validation Plan + +### Automated + +- adapter capability, lifecycle, stale-target, failure, and callback tests; +- presenter native/fallback selection and exclusive frame-routing tests; +- route and event schema tests; +- renderer native/no-DOM, Canvas fallback, activate, and run-scoped dismiss tests; +- ASAR configuration and platform-aware `afterPack` tests; +- existing `YoBrowserPresenter` and `AgentBrowserPiP` regression suites. + +Required implementation checks: + +```text +pnpm run format +pnpm run i18n +pnpm run lint +pnpm run typecheck +pnpm run test:main +pnpm run test:renderer +pnpm run build +``` + +### Manual + +For each supported native platform: + +1. Start an Agent-owned YoBrowser run while the Browser panel is closed. +2. Confirm one native PiP appears only after a current frame exists. +3. Drag it fully beyond every DeepChat window edge. +4. Continue dragging while the page snapshot updates. +5. Double-click and verify the same page opens in the Browser panel without reload. +6. Close the panel and verify native preview resumes without a stale flash. +7. Use hide and verify only the current run stays dismissed. +8. Start a later run and verify PiP can appear again. +9. Exercise blur, minimize, display change, session switch, page close, and app quit. + +For each fallback platform, repeat the product flow and verify the current in-chat Canvas remains. + +## Rollout and Rollback + +Land the adapter and Canvas fallback together. Do not remove the renderer frame path in this +migration. + +If performance or native stability misses the gate, keep surface selection on +`renderer-canvas`. Full rollback removes the adapter, dependency, and packaging rule without +touching page ownership or stored data. + +## Complexity Limits + +- Do not introduce a native-provider registry. +- Do not expose NativeKit through preload. +- Do not call NativeKit window/app discovery APIs. +- Do not add an Electron toolbar companion window. +- Do not add a frame queue, worker pipeline, shared texture experiment, or new video dependency. +- Do not force X11 on native Wayland sessions. +- Do not absorb the deferred browser multi-tab or Fit-desktop work into this migration. diff --git a/docs/architecture/nativekit-agent-browser-pip/spec.md b/docs/architecture/nativekit-agent-browser-pip/spec.md new file mode 100644 index 0000000000..8feb6d283a --- /dev/null +++ b/docs/architecture/nativekit-agent-browser-pip/spec.md @@ -0,0 +1,578 @@ +# NativeKit Agent Browser PiP Migration Spec + +## Status + +Implemented on 2026-07-23. DeepChat now prefers NativeKit for the Agent browser PiP on supported +runtimes and retains the renderer Canvas as the compatibility fallback. + +The dependency was upgraded from 0.5.1 to 0.5.2 on 2026-07-23. Version 0.5.1's published macOS +arm64 and x64 binaries declared macOS 15.0 as their deployment target. Version 0.5.2 corrects both +prebuilds to macOS 12.0, matching DeepChat and Electron 40's Monterey floor without changing the +NativeKit API or published architecture matrix. + +Automated integration, a real NativeKit addon smoke test, and a packaged macOS arm64 build are +complete. Cross-platform interaction runs and the 60/120 Hz latency gates remain release QA work; +they are tracked explicitly in `tasks.md` and are not implied by implementation completion. + +The migration pins `@zerob13/nativekit` to exactly `0.5.2`. The reviewed source is tag +[`v0.5.2`](https://github.com/zerob13/nativekit/tree/v0.5.2), commit `7e46dfd`. The npm registry +exposes `0.5.2` with Electron `>=28.0.0` and Node `>=18` compatibility. + +This architecture supersedes only the user-facing PiP surface and frame-delivery path in +`docs/features/agent-browser-pip/spec.md`. The existing Agent-page ownership, fixed background +viewport, read-only preview, panel handoff, run-scoped dismissal, and single-page identity +contracts remain in force. The current renderer Canvas remains the compatibility fallback. + +GitHub issue sync was not requested and was not performed. + +## Counterpoint + +The NativeKit path will make panel dragging fluid and allow the panel to leave the DeepChat window. +The reviewed implementations move the native window directly inside AppKit, Win32, or XCB and do +not route drag samples through renderer state or IPC. + +That confirmed window-motion result is separate from the remote page's content refresh rate. +NativeKit 0.5.2 still accepts complete PNG/JPEG data URLs through a synchronous +`overlay.pushImage()` call, decodes each image natively, and exposes no shared texture, raw-buffer +stream, partial update, or animation API. + +The migration therefore guarantees native, out-of-window movement while describing page freshness +honestly as a bounded snapshot stream rather than 60 FPS browser video. + +## Current Engineering Brief + +### Target + +- User-visible behavior: show a read-only Agent browser preview without forcing the Browser panel + open, and let the user move or dismiss it without disrupting the Agent. +- Current renderer owner: `AgentBrowserPiP.vue`, mounted by `ChatTabView.vue`. +- Current main-process owner: `YoBrowserPresenter`. +- Trigger path: renderer session, side-panel, and window state derive + `capturing | rendering | stopped`, then call `browser.setPreviewMode`. +- Existing native page: one `WebContentsView` reparented into a transparent off-screen + `BaseWindow` at 1280 x 800 while the panel is closed. + +### Current Hot Path + +```text +Agent WebContentsView at 1280 x 800 + -> webContents.capturePage() + -> NativeImage.resize(480 x 300) + -> JPEG quality 72 Buffer + -> typed main-to-renderer event + -> structured clone / Uint8Array + -> Blob + -> createImageBitmap() + -> Canvas resize + drawImage() + -> Vue DOM card, toolbar, halo, and pointer drag +``` + +Active capture is currently capped at 4 FPS and idle capture at 1 FPS. There is one capture in +flight, so frames do not queue. + +### Diagnosis + +The expensive and fragile part is not the background page or `capturePage()`. It is the second +half of the path: + +- every frame crosses into the chat renderer; +- the renderer allocates and decodes another image object; +- Canvas replacement participates in the chat renderer's paint/compositing work; +- pointer dragging updates Vue state and layout inside the conversation; +- the PiP disappears with the app content region and cannot use native work-area clamping. + +The correct ownership layer for the floating surface is the Electron main process plus NativeKit. +The renderer should retain only eligibility coordination and the compatibility Canvas. + +## User Need + +When an Agent operates YoBrowser in the background, the user needs a preview that: + +- moves with direct native input instead of renderer pointer events; +- does not compete with chat rendering and scrolling; +- keeps the same live Agent page and CDP target; +- remains read-only and cannot forward input to the remote page; +- opens the existing Browser panel on deliberate activation; +- dismisses only the current run; +- fails safely on platforms where NativeKit 0.5.2 cannot provide an overlay. + +## Goals + +- Use exact dependency version `@zerob13/nativekit@0.5.2`. +- Make NativeKit the preferred PiP surface on its supported runtime matrix. +- Allow the PiP to be dragged anywhere inside the current display work area, including completely + outside the DeepChat window. +- Keep the existing 1280 x 800 focusless render host and the same page `WebContentsView`. +- Send completed JPEG frames directly from `YoBrowserPresenter` to NativeKit without renderer frame + delivery on the native path. +- Let AppKit, Win32, or XCB own drag, work-area clamping, z-order, and native panel controls. +- Preserve one in-flight capture, epoch/run validation, last-valid-frame retention, and bounded + image size. +- Preserve the current Canvas PiP as a graceful fallback instead of forcing Linux onto X11 or + breaking Windows arm64. +- Hide the native panel synchronously on stale session, panel visibility, host blur/hide/minimize, + run terminal state, page destruction, or app shutdown. +- Add packaging validation so a supported packaged build cannot silently omit its `.node` prebuild. +- Measure drag behavior, first-frame latency, frame freshness, and synchronous native decode cost + before calling the migration complete. + +## Non-Goals + +- Full-frame-rate video, shared textures, WebRTC, or GPU surface sharing. +- Making the remote page interactive inside PiP. +- Placing the live `WebContentsView` inside the native overlay. +- Modifying, forking, or patching NativeKit 0.5.2. +- Recreating the current Vue toolbar as a second transparent window above the native panel. +- Forcing all Linux users to launch Electron with `--ozone-platform=x11`. +- Adding multiple simultaneous PiP panels. +- Persisting native panel position across application restarts. +- Completing the deferred multi-tab or Fit-desktop work from the original feature SDD. +- Adding a generic native-capability framework around one NativeKit consumer. + +## NativeKit 0.5.2 Contract + +### Useful API + +| API | DeepChat use | +| --- | --- | +| `overlay.start()` | Prewarm the native platform renderer during `YoBrowserPresenter.initialize()` | +| `overlay.attachHost()` | Bind one chat `BrowserWindow` by content bounds and native handle | +| `overlay.setMaxSize(400)` | Render the 480 x 300 source at a sharper 400 x 250 DIP panel | +| `overlay.pushImage()` | Create or replace the active Agent browser JPEG | +| `overlay.setActiveSession()` | Keep the current logical Agent session first before showing | +| `overlay.setVisible()` | Hide without deleting the current presentation during temporary ineligibility | +| `overlay.removeImage()` | Clear a terminal, destroyed, or replaced presentation | +| `overlay.detachHost()` | Release a closed chat window | +| `overlay.stop()` | Release all native resources during presenter shutdown | +| `activate` | Double-click intent to focus DeepChat and open the existing Browser panel | +| `visibilityRequest` | Native hide-control intent, mapped to current-run PiP dismissal | + +`suppressSessions`, `completeSession`, app-icon lookup, and system-window query are not required for +this migration. One visible PiP and one current target make them unnecessary. + +### Runtime Characteristics + +- Main-process only; the package must never be imported by renderer or preload code. +- Node-API v8 prebuilds; Electron 40.10.5 is within the declared peer range. +- `pushImage()` accepts a PNG/JPEG base64 data URL, not a `Buffer`. +- Calls are synchronous at the JavaScript boundary. +- macOS decodes and updates an `NSPanel` on the main thread. +- Windows synchronously marshals the update to its STA overlay thread, decodes through WIC, and + presents with `UpdateLayeredWindow`. +- Linux synchronously updates its dedicated XCB overlay thread and decodes through GdkPixbuf. +- Dragging stays inside the platform implementation and emits no renderer `mousemove` IPC. +- Movement and transitions are immediate; 0.5.2 has no animation API. +- The native hide event is global and carries no presentation ID. This is safe only while DeepChat + enforces the one-visible-PiP invariant. + +### Published Capability Matrix + +| Runtime | Preferred surface | Reason | +| --- | --- | --- | +| macOS arm64/x64 | Native overlay | Published prebuild; non-activating `NSPanel` | +| Windows x64 | Native overlay | Published prebuild; owned layered topmost `HWND` | +| Windows arm64 | Canvas fallback | NativeKit 0.5.2 publishes no win32-arm64 prebuild | +| Linux x64/arm64 under X11/XWayland | Native overlay | Published prebuild and XCB window model | +| Linux native Wayland | Canvas fallback | No global positioning or compatible X11 window handle | +| Missing/corrupt addon or native startup failure | Canvas fallback | App startup and Agent tools must remain usable | + +DeepChat will not change the user's Linux display backend merely to enable PiP. + +## Target Architecture + +```text + one live page / one CDP target + | + v +focusless render-host BaseWindow -> Agent WebContentsView at 1280 x 800 + | + v + capturePage (one in flight) + | + v + resize 480 x 300 / JPEG 72 + | + +-------------------+-------------------+ + | | + native capability fallback capability + | | + v v + JPEG Buffer -> base64 data URL typed preview-frame event + | | + v v + @zerob13/nativekit overlay existing Vue Canvas PiP + | + AppKit / Win32 / XCB panel +``` + +### Ownership + +#### `YoBrowserPresenter` + +- remains authoritative for page, run, render-host, capture epoch, and preview mode; +- validates the sender's `BrowserWindow` and current Agent run; +- selects `native-overlay`, `renderer-canvas`, or `none`; +- branches completed frames to exactly one surface; +- stops capture and clears the surface on every existing lifecycle edge. + +#### `AgentBrowserNativeOverlay` + +A focused main-process adapter owns only: + +- dynamic package loading and one-time capability detection; +- `overlay.start()` / `stop()` lifecycle; +- one active host and one active presentation; +- host move/resize/close synchronization; +- JPEG `Buffer` to data-URL conversion; +- prepaint-before-show ordering; +- mapping NativeKit `activate` and `visibilityRequest` to the current logical + `{ sessionId, runId, windowId }`; +- timing counters around synchronous native calls. + +It is not a general Presenter, registry, plugin system, or cross-native abstraction. + +#### Renderer + +`AgentBrowserPiP.vue` keeps: + +- current-session, panel, run, and window eligibility derivation; +- the existing `setPreviewMode` request coalescing; +- current Canvas rendering and drag only when main selects `renderer-canvas`; +- handling typed native actions for open-panel and run dismissal. + +On `native-overlay`, it renders no PiP DOM and receives no frame bytes. + +## Surface Selection Contract + +`browser.setPreviewMode` returns: + +```ts +type BrowserPreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' + +type BrowserPreviewModeResult = { + updated: boolean + surface: BrowserPreviewSurface +} +``` + +Selection is process-stable after NativeKit initialization: + +1. Dynamically import `@zerob13/nativekit`. +2. Start the overlay and keep it globally hidden. +3. On the first eligible host, validate `attachHost()` with the real + `BrowserWindow.getNativeWindowHandle()`. +4. Use `native-overlay` after both steps succeed. +5. Otherwise record one redacted capability warning and use `renderer-canvas` for the process. + +A transient bad frame does not switch surfaces. NativeKit keeps the previous valid presentation, +and the next bounded capture retries. + +## Native Presentation Identity + +Only one presentation may be visible: + +```text +hostId = chat-window: +presentationId = agent-browser:: +native session = agent-browser: +logical target = { windowId, sessionId, runId, captureEpoch } +``` + +The logical target, not NativeKit's unscoped callback, identifies activate and dismiss actions. +Changing window or session removes the previous presentation before attaching the next host. + +The presentation remains allocated but hidden while the Browser panel is temporarily visible or +the host briefly loses eligibility, preserving NativeKit's manual drag position. Terminal run, +session/page destruction, host close, and shutdown remove it. + +## Frame Delivery + +The existing capture safety rules remain: + +- 1280 x 800 source viewport; +- 480 x 300 output; +- 400 x 250 maximum native panel size, preserving the Canvas display size while downsampling the + 480 x 300 frame; +- JPEG quality 72; +- 512 KiB DeepChat frame ceiling, well below NativeKit's 32 MiB input ceiling; +- one capture in flight; +- schedule the next capture only after capture, resize, encode, and presentation finish; +- validate mode, run ID, target window, and epoch immediately before presentation; +- drop stale results without showing them; +- retain the last valid native image when capture or decode fails. + +For the native branch: + +```text +JPEG Buffer + -> data:image/jpeg;base64, + -> overlay.pushImage() +``` + +No frame is sent to the renderer on this branch. + +The overlay starts hidden. On first show or resume, DeepChat pushes a current frame while hidden and +calls `setVisible(true)` only after `pushImage()` succeeds. This avoids flashing a stale or empty +panel. + +The first successful frame also selects the active NativeKit session once. Later image refreshes +call only `pushImage()` with the same host, presentation, and session IDs. They do not repeat +`attachHost()`, `setActiveSession()`, `setVisible(true)`, or any removal operation. NativeKit owns +the manual frame by stable `presentationId`, so an image replacement changes pixels and size only; +it cannot reset a user-dragged origin. Host move/resize synchronization remains a separate, +debounced path. + +## Interaction Contract + +NativeKit 0.5.2 defines the native interaction surface: + +- drag any image area outside the controls; +- double-click the image to activate; +- click the hide control to dismiss the preview for the current Agent run; +- click relocate to clear manual placement and cycle the host anchor; +- the panel never becomes the key/main window and never forwards input to the remote page. + +DeepChat maps activation to: + +1. show and focus the owning chat window; +2. publish a typed `browser.preview.action` event for the exact window/session/run; +3. let the current renderer open the existing Browser panel; +4. stop native capture and reparent the same page View into stable panel bounds. + +DeepChat maps hide to: + +1. mark the logical run dismissed; +2. stop capture and keep the page rendering in the hidden render host; +3. publish the same typed action event so renderer eligibility agrees; +4. allow a later run to show PiP again. + +The native path intentionally uses NativeKit's own hide/relocate controls and double-click +activation. It does not preserve the Canvas-only title toolbar, centered drag affordance, or +activity halo. Recreating those as another overlay would reintroduce the focus, z-order, and +cross-window coordination this migration removes. + +NativeKit 0.5.2 exposes no generic single-click or pointer event. The Canvas-only single click +therefore cannot toggle a native toolbar. The equivalent product actions remain controllable: +native body drag moves the PiP, native double-click performs **Open in panel**, and the native hide +control performs **Close** for the current run. Both surfaces remain read-only and never forward +clicks into the remote page. + +## UI Layout + +### Before: renderer-owned card inside the conversation + +```text ++----------------------------------------------------------------+ +| DeepChat | +| | +| Conversation +----------------------+ | +| | Vue toolbar | | +| | Canvas page mirror | | +| | drag / open / close | | +| +----------------------+ | ++----------------------------------------------------------------+ +``` + +### After: native panel in the desktop work area + +```text ++------------------------------------------+ +----------------------+ +| DeepChat | | Native PiP | +| | | | +| Conversation | | Agent page snapshot | +| Browser panel remains closed | | [↻][◉]| +| | | drag; double-click | ++------------------------------------------+ +----------------------+ + AppKit/Win32/XCB +``` + +The actual NativeKit symbols are platform-native. The diagram shows structure, not exact icons. + +### Compatibility fallback + +```text +Windows arm64 / native Wayland / unavailable addon + -> keep the existing in-conversation Canvas card and controls +``` + +## Lifecycle Matrix + +| Event | Native action | Page action | +| --- | --- | --- | +| Eligible capture starts | Attach/update host; push current frame; show | Keep in 1280 x 800 render host | +| New frame | Replace same presentation only; preserve manual origin | No reparent | +| Browser panel opens | Hide presentation; stop capture | Reparent same View into panel | +| Browser panel closes during eligible run | Push current frame before show | Reparent into render host | +| Native hide | Hide, remember run dismissal | Keep render host; stop capture | +| Native double-click | Focus host; request Browser panel | Reparent after stable bounds | +| Host moves/resizes/display changes | Debounced `attachHost()` refresh | No page change | +| Host blur/hide/minimize | Hide synchronously | Keep rendering only if Agent still needs it | +| Host refocus | Re-evaluate; push-before-show | No reload | +| Run terminal | Remove presentation | Stop capture; release render host | +| Session/page destroyed | Remove presentation | Destroy existing page resources | +| Host closed | Remove presentation; detach host | Clear target | +| App shutdown | `overlay.stop()` once | Existing presenter shutdown | + +Although macOS NativeKit panels can join all Spaces, DeepChat preserves the current product policy: +the Agent page preview is hidden when its owning chat window is not foreground. Making it persist +over unrelated applications is a separate product decision. + +## Performance Contract + +Native feel is split into two measurable promises. + +### Native movement + +- no renderer pointer-move handler or drag-position IPC on the native path; +- drag, clamping, and z-order are handled inside NativeKit; +- the panel can cross every DeepChat window edge and remains constrained only by the display work + area; +- capture work must not produce visible drag hitching on 60 Hz or 120 Hz reference displays. + +### Page freshness + +- active Agent activity: target at most 8 FPS, scheduled 125 ms after the previous full cycle; +- idle page: 1 FPS; +- never queue or overlap captures; +- warm eligible-to-first-visible-frame p95: at most 300 ms; +- active end-to-end frame age p95: at most 250 ms; +- `overlay.pushImage()` p95: at most 8 ms and p99: at most 25 ms on the supported reference + platform; +- no main-process task longer than 50 ms attributable to PiP frame presentation. + +The initial implementation may fall back to the current 4 FPS active cap if the synchronous +NativeKit decode/present gate fails. It must not raise the cap above 8 FPS with NativeKit 0.5.2. + +These are release gates, not runtime promises for arbitrary hardware. The visible claim is +"native movement with a fresh read-only preview," not "native-rate browser video." + +## Security and Privacy + +- NativeKit is imported only in the Electron main process. +- No NativeKit object, native handle, raw IPC channel, or generic capability is exposed through + preload. +- Remote page execution remains sandboxed in its existing YoBrowser session. +- The native panel receives only a downscaled JPEG data URL. +- Frames remain in memory and are never logged, cached, persisted, or written to disk. +- Logs contain platform, surface, duration, dimensions, and redacted lifecycle reason only; they do + not contain image bytes, URL, title, DOM, or session content. +- Main validates the target `BrowserWindow`, session, run ID, mode, and epoch before every show. +- A stale capture cannot replace the current session's panel. +- Host blur/hide/minimize and session deactivation hide the native panel in main, without waiting + for renderer cleanup. + +## Packaging and Compatibility + +- Add exact dependency `"@zerob13/nativekit": "0.5.2"`. +- Mark `@zerob13/nativekit` as a disallowed install-time build in `pnpm-workspace.yaml`; published + prebuilds are resolved at runtime, and unsupported targets must fall back instead of compiling a + local addon. +- Keep it external to the Electron main bundle so `node-gyp-build` resolves the packaged native + addon. +- Unpack `node_modules/@zerob13/nativekit/prebuilds/**/*` from ASAR. +- Verify the expected `nativekit.napi.node` after packaging for darwin-arm64, darwin-x64, + win32-x64, linux-x64, and linux-arm64. +- Do not make win32-arm64 packaging fail; that target intentionally uses Canvas fallback with + version 0.5.2. +- Do not source-build the addon as part of a normal DeepChat release. +- A missing supported prebuild is a packaging failure. An unsupported runtime is a capability + fallback. + +No persisted data or settings schema changes are required. + +## Rollback + +Rollback is code-local: + +1. select `renderer-canvas` unconditionally; +2. stop loading NativeKit; +3. remove the exact dependency and ASAR rule after confirming no other consumer exists. + +The browser page, routes, render host, capture format, and Canvas implementation remain compatible, +so rollback does not navigate pages, migrate user data, or change stored settings. + +## Failure Semantics + +- Dynamic import or `overlay.start()` failure: log once and use Canvas for the process. +- First real-host `attachHost()` failure: mark the native capability unavailable and use Canvas. +- Frame capture/resize/encode failure: retain the previous frame and retry on the next bounded tick. +- `pushImage()` failure: NativeKit transactionally retains its previous presentation; log a + rate-limited warning and retry. +- Stale frame after mode/run/window change: drop before `pushImage()`. +- Native action with no current logical target: ignore. +- Panel activation timeout: keep or restore the native PiP; never lose the live page. +- Host close or addon shutdown: cleanup is idempotent. +- Fallback Canvas failure: preserve the existing last-frame and placeholder behavior. + +## Acceptance Criteria + +- Supported platforms load exactly NativeKit 0.5.2 and use its native overlay. +- Windows arm64, native Wayland, and unavailable-addon cases keep the existing Canvas PiP. +- The preferred path sends no `browser.preview.frame` payload to the renderer. +- The remote page retains the same `WebContents`, `WebContentsView`, session, URL, DOM state, + scroll state, cookies, and CDP target across native PiP and Browser panel handoff. +- The native panel is read-only, draggable, work-area-clamped, and non-activating. +- The native panel can be dragged completely outside the DeepChat window without snapping back to + conversation bounds. +- Replacing the image under the same presentation does not reattach, reactivate, re-show, remove, + or reset the panel's manual position. +- Double-click opens the existing Browser panel; hide dismisses only the current run; relocate uses + NativeKit's anchor cycle. +- Exactly one native PiP may be visible process-wide. +- First show and resume push a current frame before making the panel visible. +- No stale session/run frame is shown after route, window, or run changes. +- Host blur/hide/minimize, panel open, run terminal, page destruction, and shutdown hide or remove + the panel deterministically. +- Dragging performs no renderer `mousemove` IPC and passes the native movement QA gate. +- Frame delivery passes the stated latency and synchronous-call budgets, or active capture remains + capped at 4 FPS. +- The packaged app contains the correct prebuild on every supported target. +- Renderer/preload security boundaries remain unchanged. +- Existing Canvas PiP tests remain as fallback coverage; native adapter, presenter selection, + action mapping, packaging, and packaged smoke tests are added. +- `pnpm run format`, `pnpm run i18n`, `pnpm run lint`, typecheck, focused tests, build, and packaged + platform checks pass after implementation. + +## Implementation Evidence + +Recorded on 2026-07-23: + +- `@zerob13/nativekit` is pinned to `0.5.2`, remains external to the main bundle, and its prebuilds + are unpacked from ASAR. +- The published macOS arm64 and x64 binaries both report `minos 12.0`; the previous 0.5.1 + prebuilds reported `minos 15.0`. +- A real Electron smoke test loaded NativeKit 0.5.2 and completed + `start -> attachHost -> pushImage -> removeImage -> detachHost -> stop`. +- The upstream v0.5.2 Electron demo was reviewed and its smoke run passed. DeepChat follows its + main-process sequence of `start -> attachHost -> pushImage -> setActiveSession -> setVisible`, + refreshes the host after move/resize, handles `activate` and `visibilityRequest`, and stops the + overlay on shutdown. DeepChat runs the selection/show tail only for the first frame; later + refreshes use only the demo's stable-identity `pushImage` update. DeepChat reuses its existing + typed route/event bridge instead of adding the demo's dedicated preload API. +- A macOS arm64 directory package completed, and + `app.asar.unpacked/node_modules/@zerob13/nativekit/prebuilds/darwin-arm64/nativekit.napi.node` + was present, byte-identical to the installed 0.5.2 prebuild, and reported `minos 12.0`. The + packaged DeepChat executable and `LSMinimumSystemVersion` also report macOS 12.0. +- Focused adapter, presenter, contract, client, and renderer tests pass: 69 main tests and 45 + renderer tests, including push-before-show, native-only frame routing, exact action targeting, + fallback selection, and native no-DOM behavior. +- The full main suite passes: 378 files and 4,299 tests, with 19 files and 222 tests skipped. +- `pnpm run format`, `pnpm run i18n`, `pnpm run lint`, `pnpm run typecheck`, and + `pnpm run build` pass. +- The full renderer suite reaches 173 passing files and 1,328 passing tests, but its existing + `App.startup.test.ts` mock returns `undefined` from `initAppStores()` while production chains + `.then()`. That baseline mismatch leaves 15 unrelated startup tests failing and is outside this + migration. +- Windows, Linux, macOS x64, native Wayland fallback, out-of-window visual interaction, and + 60/120 Hz performance measurements remain unverified on physical target environments. + +## Resolved Decisions + +- NativeKit version is exactly 0.5.2. +- Native movement is the primary smoothness win; the page remains a bounded snapshot stream. +- The current Canvas is retained only as a compatibility fallback. +- Linux display backend is not forced. +- NativeKit's own controls are used; no companion Vue/native toolbar window is added. +- One visible PiP makes NativeKit's unscoped action callbacks safe. +- Existing foreground-window visibility policy is preserved. +- No implementation-blocking clarification markers remain. diff --git a/docs/architecture/nativekit-agent-browser-pip/tasks.md b/docs/architecture/nativekit-agent-browser-pip/tasks.md new file mode 100644 index 0000000000..514f91b4f3 --- /dev/null +++ b/docs/architecture/nativekit-agent-browser-pip/tasks.md @@ -0,0 +1,103 @@ +# NativeKit Agent Browser PiP Migration Tasks + +## Status + +Implementation complete on 2026-07-23. Physical cross-platform interaction and performance QA +remain open. + +## Documentation + +- [x] Inspect the current PiP renderer, presenter, route, event, test, and packaging paths. +- [x] Review local NativeKit source at tag `v0.5.2`, commit `7e46dfd`. +- [x] Review and run the upstream v0.5.2 Electron demo control flow. +- [x] Verify both published macOS prebuilds target macOS 12.0. +- [x] Confirm the published prebuild and runtime compatibility matrix. +- [x] Record the native movement, out-of-window drag, snapshot-stream, fallback, and lifecycle + contracts. +- [x] Resolve implementation-blocking architecture decisions. +- [x] Update implementation evidence and status after the migration lands. + +## Dependency and packaging + +- [x] Add exact dependency `@zerob13/nativekit@0.5.2`. +- [x] Refresh `pnpm-lock.yaml`. +- [x] Disable NativeKit install-time source builds in `pnpm-workspace.yaml`. +- [x] Keep NativeKit external to the Electron main bundle. +- [x] Add the NativeKit prebuild glob to `asarUnpack`. +- [x] Extend `afterPack` validation for supported target tuples. +- [x] Keep Windows arm64 packaging valid with Canvas fallback. +- [x] Add or update packaging configuration tests. + +## Native overlay adapter + +- [x] Add focused `AgentBrowserNativeOverlay` main-process adapter. +- [x] Dynamically import and start NativeKit without blocking app startup. +- [x] Detect unsupported runtime, import failure, startup failure, and first-host attach failure. +- [x] Enforce one active host, presentation, and logical target. +- [x] Convert bounded JPEG buffers to data URLs only at the native boundary. +- [x] Select the current logical session after push and before show, matching the upstream demo. +- [x] Select/show only on the first frame and keep later refreshes to same-identity `pushImage()`. +- [x] Render the 480 x 300 source in a 400 x 250 DIP native panel. +- [x] Push a current frame before first show and resume. +- [x] Preserve the previous valid image after capture or push failure. +- [x] Map `activate` and `visibilityRequest` through the exact logical target. +- [x] Make hide, remove, detach, listener cleanup, and shutdown idempotent. +- [x] Add synchronous-call timing and rate-limited redacted logging. +- [x] Add adapter unit tests. +- [x] Cover the no-reattach/no-remove/no-re-show steady-state refresh contract. + +## Presenter integration + +- [x] Initialize and shut down the adapter with `YoBrowserPresenter`. +- [x] Select `native-overlay`, `renderer-canvas`, or `none` process-stably. +- [x] Return the selected surface from `setPreviewMode`. +- [x] Route each completed frame to exactly one surface. +- [x] Revalidate window, session, run, mode, surface, and epoch before presentation. +- [x] Keep one capture in flight and schedule only after presentation finishes. +- [x] Attach the validated owning `BrowserWindow`. +- [x] Debounce host move, resize, and display-bound refresh. +- [x] Hide on blur, hide, minimize, panel open, and preview stop. +- [x] Remove on run terminal, target replacement, page/session destruction, and host close. +- [x] Retain the fixed 1280 x 800 render host and same live page identity. +- [x] Add presenter native/fallback routing and lifecycle tests. + +## Contracts and renderer + +- [x] Add the shared `BrowserPreviewSurface` result contract. +- [x] Add the typed `browser.preview.action` event contract. +- [x] Update desktop routes and `BrowserClient`. +- [x] Track the acknowledged surface in `AgentBrowserPiP.vue`. +- [x] Render no PiP DOM and decode no frame on the native path. +- [x] Keep the current Canvas UI and frame path unchanged as fallback. +- [x] Validate exact window/session/run before native activate or dismiss handling. +- [x] Open the existing Browser panel on activate. +- [x] Reuse current run-scoped dismissal on hide. +- [x] Add contract, client, and renderer tests. + +## Performance and platform QA + +- [x] Verify direct native drag has no renderer pointer-move IPC by ownership and native-path tests. +- [ ] Verify the panel can move completely outside DeepChat and remains work-area-clamped. +- [ ] Verify a 4 FPS image stream does not move a manually dragged panel. +- [ ] Measure drag behavior on 60 Hz and 120 Hz macOS displays. +- [ ] Measure warm first-frame p95 at or below 300 ms. +- [ ] Measure active frame-age p95 at or below 250 ms. +- [ ] Measure `pushImage()` p95 at or below 8 ms and p99 at or below 25 ms. +- [ ] Verify no PiP presentation task longer than 50 ms. +- [x] Keep active capture at 4 FPS unless profiling clears an increase, capped at 8 FPS. +- [x] Smoke-test macOS arm64 through the upstream demo and DeepChat addon sequence. +- [ ] Smoke-test macOS x64, Windows x64, and Linux X11/XWayland. +- [x] Verify Canvas fallback selection for Windows arm64 in automated coverage. +- [ ] Verify Canvas fallback on a physical native Wayland session. +- [x] Inspect the macOS arm64 packaged artifact for the correct unpacked prebuild. + +## Final validation + +- [x] Run `pnpm run format`. +- [x] Run `pnpm run i18n`. +- [x] Run `pnpm run lint`. +- [x] Run `pnpm run typecheck`. +- [x] Run focused main and renderer tests. +- [x] Run `pnpm run build`. +- [x] Record automated and macOS arm64 packaged-platform evidence. +- [ ] Confirm all acceptance criteria in `spec.md`. diff --git a/docs/features/agent-browser-pip/spec.md b/docs/features/agent-browser-pip/spec.md index 601e6494e3..aa8f313631 100644 --- a/docs/features/agent-browser-pip/spec.md +++ b/docs/features/agent-browser-pip/spec.md @@ -8,6 +8,14 @@ focus/blur feedback loop. The revised contract keeps the Agent page at a fixed 1 size and shows a low-frame-rate, read-only Canvas mirror in PiP. A visible multi-tab strip and Fit-desktop emulation remain deferred. Packaged Windows and Linux validation remains open. +On 2026-07-23, the +[NativeKit 0.5.2 surface migration](../../architecture/nativekit-agent-browser-pip/spec.md) was +implemented. That architecture supersedes this document's renderer-owned PiP surface and +frame-delivery path: supported runtimes use a native, out-of-window draggable panel, while the +Canvas described here remains the compatibility fallback. This document's page ownership, fixed +background viewport, read-only preview, panel handoff, and run-scoped dismissal contracts remain +authoritative. + The referenced screenshot was not available in the current task context, so the interaction and layout are specified here while exact visual styling remains provisional. @@ -45,7 +53,11 @@ The requested behavior is: - give desktop-oriented pages a deliberate wide/fit escape hatch instead of leaving them trapped in an unusable narrow viewport. -## Critical Architecture Correction +## V1 Critical Architecture Correction + +This section records the shipped V1 Canvas architecture. The implemented NativeKit migration +supersedes the surface constraint below on supported runtimes; it does not change the one-page, +read-only mirror invariant. The user-facing PiP is not another operating-system window and never contains the remote page's native View. Electron `View` does not expose a reliable view-level ignore-mouse-input contract, and @@ -92,7 +104,8 @@ parent and bounds. It does not navigate, clone cookies, recreate CDP, or copy pa ## Non-Goals - Native video picture-in-picture or `documentPictureInPicture`. -- A free-floating OS window outside the DeepChat window. +- A free-floating OS window outside the DeepChat window in V1; the implemented NativeKit migration + supersedes this surface constraint on supported runtimes. - Multiple simultaneous PiP cards. - Floating pages created and navigated only by the user. - Persisting PiP position or size across application restarts in V1. diff --git a/electron-builder.yml b/electron-builder.yml index 4bb3320be3..28c4b5ae33 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -40,6 +40,7 @@ asarUnpack: - '**/node_modules/@parcel/watcher/**/*' - '**/node_modules/@parcel/watcher-*/**/*' - '**/node_modules/@arcships/light-ocr*/**/*' + - '**/node_modules/@zerob13/nativekit/prebuilds/**/*' extraResources: - from: ./runtime/ to: app.asar.unpacked/runtime diff --git a/package.json b/package.json index fa4b406814..abdb31892c 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "@larksuiteoapi/node-sdk": "^1.64.0", "@modelcontextprotocol/sdk": "^1.29.0", "@parcel/watcher": "^2.5.6", + "@zerob13/nativekit": "0.5.2", "ai": "^7.0.19", "axios": "^1.16.1", "better-sqlite3-multiple-ciphers": "12.9.0", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3b8ca3ef89..13834cf3cb 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,7 @@ publicHoistPattern: - '@img/sharp-*' allowBuilds: + '@zerob13/nativekit': false '@parcel/watcher': false '@tailwindcss/oxide': true better-sqlite3-multiple-ciphers: false diff --git a/scripts/afterPack.js b/scripts/afterPack.js index e71820f857..8022c49514 100644 --- a/scripts/afterPack.js +++ b/scripts/afterPack.js @@ -169,6 +169,48 @@ function getResourcesDir(context) { return path.join(appOutDir, 'resources') } +function getNativeKitPrebuilds(platform, arch) { + const archName = getArchName(arch) + + if (platform === 'darwin' && archName === 'universal') { + return ['darwin-x64', 'darwin-arm64'] + } + + switch (`${platform}:${archName}`) { + case 'darwin:x64': + case 'darwin:arm64': + case 'win32:x64': + case 'linux:x64': + case 'linux:arm64': + return [`${platform}-${archName}`] + default: + return [] + } +} + +async function validateNativeKitPrebuilds(context) { + const prebuilds = getNativeKitPrebuilds(context.electronPlatformName, context.arch) + if (prebuilds.length === 0) { + return + } + + const nativeKitDir = path.join( + getResourcesDir(context), + 'app.asar.unpacked', + 'node_modules', + '@zerob13', + 'nativekit' + ) + for (const prebuild of prebuilds) { + const binaryPath = path.join(nativeKitDir, 'prebuilds', prebuild, 'nativekit.napi.node') + if (!(await pathExists(binaryPath))) { + throw new Error( + `Missing NativeKit prebuild at ${binaryPath}. Check electron-builder asarUnpack configuration.` + ) + } + } +} + async function copyFffNativePackages(context) { const { arch, electronPlatformName, packager } = context const packageNames = getFffBinaryPackages(electronPlatformName, arch) @@ -634,6 +676,7 @@ async function afterPack(context) { await copyParcelWatcherNativePackages(context) await copyOpendalNativePackages(context) await packageLightOcrAssets(context) + await validateNativeKitPrebuilds(context) await encodeMacVssExtension(context) if (isLinux(targets)) { diff --git a/src/main/desktop/browser/AgentBrowserNativeOverlay.ts b/src/main/desktop/browser/AgentBrowserNativeOverlay.ts new file mode 100644 index 0000000000..c2d0414714 --- /dev/null +++ b/src/main/desktop/browser/AgentBrowserNativeOverlay.ts @@ -0,0 +1,504 @@ +import { app, screen, type BrowserWindow, type Rectangle } from 'electron' +import { performance } from 'node:perf_hooks' +import logger from '@shared/logger' + +type NativeKitOverlay = (typeof import('@zerob13/nativekit'))['overlay'] + +export type AgentBrowserNativeOverlayTarget = { + windowId: number + sessionId: string + runId: string + captureEpoch: number +} + +export type AgentBrowserNativeOverlayAction = 'activate' | 'dismiss' + +type NativeOverlayActionHandler = ( + action: AgentBrowserNativeOverlayAction, + target: AgentBrowserNativeOverlayTarget +) => void + +type NativeOverlayTargetRef = { + sessionId: string + runId?: string +} + +type HostListeners = { + focus: () => void + blur: () => void + show: () => void + hide: () => void + minimize: () => void + restore: () => void + move: () => void + resize: () => void + closed: () => void +} + +const PREVIEW_MAX_EDGE = 400 +const HOST_ANCHOR_OFFSET = 16 +const HOST_SYNC_DELAY_MS = 50 +const SLOW_PUSH_WARNING_MS = 25 +const SLOW_PUSH_WARNING_INTERVAL_MS = 60_000 + +export class AgentBrowserNativeOverlay { + private overlay: NativeKitOverlay | null = null + private available = false + private unavailable = false + private initialization: Promise | null = null + private host: BrowserWindow | null = null + private hostListeners: HostListeners | null = null + private hostSyncTimer: ReturnType | null = null + private target: AgentBrowserNativeOverlayTarget | null = null + private desiredVisible = false + private overlayVisible: boolean | null = null + private activeSessionId: string | null = null + private lastSlowPushWarningAt = 0 + private lastPushFailureWarningAt = 0 + private listeningForDisplays = false + + private readonly handleActivate = () => { + this.emitAction('activate') + } + + private readonly handleVisibilityRequest = (visible: boolean) => { + this.overlayVisible = visible + if (!visible) { + this.emitAction('dismiss') + } + } + + private readonly handleDisplayChange = () => { + this.scheduleHostSync() + } + + constructor(private readonly onAction: NativeOverlayActionHandler) {} + + async initialize(): Promise { + if (this.available) { + return true + } + if (this.unavailable) { + return false + } + if (this.initialization) { + return await this.initialization + } + + this.initialization = this.start() + return await this.initialization + } + + isAvailable(): boolean { + return this.available + } + + prepare(target: AgentBrowserNativeOverlayTarget, host: BrowserWindow): boolean { + if (!this.available || !this.overlay || host.isDestroyed()) { + return false + } + + const presentationChanged = + this.target != null && this.presentationId(this.target) !== this.presentationId(target) + if (presentationChanged) { + this.desiredVisible = false + this.setOverlayVisible(false) + this.removeCurrentPresentation() + } + + if (!this.attachHost(host)) { + this.disable('host_attach_failed') + return false + } + + this.target = { ...target } + this.desiredVisible = true + return true + } + + present(target: AgentBrowserNativeOverlayTarget, jpeg: Buffer): boolean { + const overlay = this.overlay + if (!this.available || !overlay || !this.matchesTarget(target) || !this.host) { + return false + } + + const startedAt = performance.now() + try { + const pushed = overlay.pushImage({ + hostId: this.hostId(target.windowId), + presentationId: this.presentationId(target), + sessionId: this.nativeSessionId(target.sessionId), + imageData: `data:image/jpeg;base64,${jpeg.toString('base64')}` + }) + this.recordPushDuration(performance.now() - startedAt) + if (!pushed) { + this.warnFramePush(new Error('pushImage returned false')) + return false + } + + const nativeSessionId = this.nativeSessionId(target.sessionId) + if (this.activeSessionId !== nativeSessionId) { + if (overlay.setActiveSession(nativeSessionId)) { + this.activeSessionId = nativeSessionId + } else { + this.warn('active_session_update_failed', new Error('setActiveSession returned false')) + } + } + + if (this.desiredVisible && this.isHostEligible(this.host)) { + this.setOverlayVisible(true) + } + return true + } catch (error) { + this.recordPushDuration(performance.now() - startedAt) + this.warnFramePush(error) + return false + } + } + + hide(target?: NativeOverlayTargetRef): void { + if (target && !this.matchesRun(target)) { + return + } + this.desiredVisible = false + this.setOverlayVisible(false) + } + + removeTarget(target?: NativeOverlayTargetRef): void { + if (target && !this.matchesRun(target)) { + return + } + this.desiredVisible = false + this.setOverlayVisible(false) + this.removeCurrentPresentation() + this.target = null + } + + shutdown(): void { + this.removeTarget() + this.unbindHost() + this.stopDisplayListeners() + + const overlay = this.overlay + this.overlay = null + this.available = false + this.unavailable = true + this.overlayVisible = null + this.activeSessionId = null + this.initialization = null + if (!overlay) { + return + } + + overlay.removeListener('activate', this.handleActivate) + overlay.removeListener('visibilityRequest', this.handleVisibilityRequest) + try { + overlay.stop() + } catch (error) { + this.warn('shutdown_failed', error) + } + } + + private async start(): Promise { + if (!this.isPublishedTarget()) { + this.unavailable = true + this.logUnavailable('unsupported_platform') + return false + } + + try { + const nativekit = await import('@zerob13/nativekit') + const overlay = nativekit.overlay + if (!overlay.start() || !overlay.setMaxSize(PREVIEW_MAX_EDGE) || !overlay.setVisible(false)) { + try { + overlay.stop() + } catch { + // Startup already failed; there is no useful cleanup error to surface. + } + this.unavailable = true + this.logUnavailable('startup_failed') + return false + } + + overlay.on('activate', this.handleActivate) + overlay.on('visibilityRequest', this.handleVisibilityRequest) + this.overlay = overlay + this.overlayVisible = false + this.available = true + this.startDisplayListeners() + return true + } catch (error) { + this.shutdown() + this.logUnavailable('load_failed', error) + return false + } + } + + private attachHost(host: BrowserWindow): boolean { + const overlay = this.overlay + if (!overlay || host.isDestroyed()) { + return false + } + + if (this.host && this.host.id !== host.id) { + this.unbindHost() + } + + try { + const attached = overlay.attachHost({ + id: this.hostId(host.id), + title: host.getTitle().trim() || app.getName() || 'DeepChat', + bounds: this.normalizeBounds(host.getContentBounds()), + windowHandle: host.getNativeWindowHandle(), + anchor: { + edge: 'trailing', + offset: HOST_ANCHOR_OFFSET + } + }) + if (!attached) { + return false + } + + this.host = host + this.bindHost(host) + return true + } catch (error) { + this.warn('host_attach_failed', error) + return false + } + } + + private bindHost(host: BrowserWindow): void { + if (this.hostListeners) { + return + } + + const hide = () => this.setOverlayVisible(false) + const scheduleSync = () => this.scheduleHostSync() + const closed = () => { + this.removeTarget() + this.unbindHost() + } + const listeners: HostListeners = { + focus: scheduleSync, + blur: hide, + show: scheduleSync, + hide, + minimize: hide, + restore: scheduleSync, + move: scheduleSync, + resize: scheduleSync, + closed + } + + this.hostListeners = listeners + host.on('focus', listeners.focus) + host.on('blur', listeners.blur) + host.on('show', listeners.show) + host.on('hide', listeners.hide) + host.on('minimize', listeners.minimize) + host.on('restore', listeners.restore) + host.on('move', listeners.move) + host.on('resize', listeners.resize) + host.on('closed', listeners.closed) + } + + private unbindHost(): void { + this.clearHostSyncTimer() + const host = this.host + const listeners = this.hostListeners + this.host = null + this.hostListeners = null + + if (host && listeners && !host.isDestroyed()) { + host.removeListener('focus', listeners.focus) + host.removeListener('blur', listeners.blur) + host.removeListener('show', listeners.show) + host.removeListener('hide', listeners.hide) + host.removeListener('minimize', listeners.minimize) + host.removeListener('restore', listeners.restore) + host.removeListener('move', listeners.move) + host.removeListener('resize', listeners.resize) + host.removeListener('closed', listeners.closed) + } + + if (host && this.overlay) { + try { + this.overlay.detachHost(this.hostId(host.id)) + } catch (error) { + this.warn('host_detach_failed', error) + } + } + } + + private scheduleHostSync(): void { + this.clearHostSyncTimer() + this.hostSyncTimer = setTimeout(() => { + this.hostSyncTimer = null + const host = this.host + if (host && !host.isDestroyed()) { + this.attachHost(host) + } + }, HOST_SYNC_DELAY_MS) + this.hostSyncTimer.unref?.() + } + + private clearHostSyncTimer(): void { + if (this.hostSyncTimer) { + clearTimeout(this.hostSyncTimer) + this.hostSyncTimer = null + } + } + + private startDisplayListeners(): void { + if (this.listeningForDisplays) { + return + } + this.listeningForDisplays = true + screen.on('display-added', this.handleDisplayChange) + screen.on('display-removed', this.handleDisplayChange) + screen.on('display-metrics-changed', this.handleDisplayChange) + } + + private stopDisplayListeners(): void { + if (!this.listeningForDisplays) { + return + } + this.listeningForDisplays = false + screen.removeListener('display-added', this.handleDisplayChange) + screen.removeListener('display-removed', this.handleDisplayChange) + screen.removeListener('display-metrics-changed', this.handleDisplayChange) + } + + private emitAction(action: AgentBrowserNativeOverlayAction): void { + const target = this.target + if (!target) { + return + } + this.desiredVisible = false + this.setOverlayVisible(false) + this.onAction(action, { ...target }) + } + + private setOverlayVisible(visible: boolean): void { + if (!this.overlay || this.overlayVisible === visible) { + return + } + try { + if (this.overlay.setVisible(visible)) { + this.overlayVisible = visible + } else { + this.warn('visibility_update_failed', new Error('setVisible returned false')) + } + } catch (error) { + this.warn('visibility_update_failed', error) + } + } + + private removeCurrentPresentation(): void { + if (!this.overlay || !this.target) { + return + } + try { + this.overlay.removeImage(this.presentationId(this.target)) + this.activeSessionId = null + } catch (error) { + this.warn('presentation_remove_failed', error) + } + } + + private matchesTarget(target: AgentBrowserNativeOverlayTarget): boolean { + return ( + this.target?.windowId === target.windowId && + this.target.sessionId === target.sessionId && + this.target.runId === target.runId && + this.target.captureEpoch === target.captureEpoch + ) + } + + private matchesRun(target: NativeOverlayTargetRef): boolean { + return ( + this.target?.sessionId === target.sessionId && + (target.runId == null || this.target.runId === target.runId) + ) + } + + private isHostEligible(host: BrowserWindow): boolean { + return !host.isDestroyed() && host.isVisible() && host.isFocused() && !host.isMinimized() + } + + private isPublishedTarget(): boolean { + const target = `${process.platform}:${process.arch}` + return ( + target === 'darwin:arm64' || + target === 'darwin:x64' || + target === 'win32:x64' || + target === 'linux:arm64' || + target === 'linux:x64' + ) + } + + private hostId(windowId: number): string { + return `chat-window:${windowId}` + } + + private presentationId(target: AgentBrowserNativeOverlayTarget): string { + return `agent-browser:${target.windowId}:${target.sessionId}` + } + + private nativeSessionId(sessionId: string): string { + return `agent-browser:${sessionId}` + } + + private normalizeBounds(bounds: Rectangle): Rectangle { + return { + x: Math.round(bounds.x), + y: Math.round(bounds.y), + width: Math.max(1, Math.round(bounds.width)), + height: Math.max(1, Math.round(bounds.height)) + } + } + + private disable(reason: string): void { + this.unavailable = true + this.logUnavailable(reason) + this.shutdown() + } + + private recordPushDuration(durationMs: number): void { + if ( + durationMs <= SLOW_PUSH_WARNING_MS || + Date.now() - this.lastSlowPushWarningAt < SLOW_PUSH_WARNING_INTERVAL_MS + ) { + return + } + this.lastSlowPushWarningAt = Date.now() + logger.warn('[AgentBrowserNativeOverlay] Slow frame presentation', { + durationMs: Math.round(durationMs) + }) + } + + private warnFramePush(error: unknown): void { + if (Date.now() - this.lastPushFailureWarningAt < SLOW_PUSH_WARNING_INTERVAL_MS) { + return + } + this.lastPushFailureWarningAt = Date.now() + this.warn('frame_push_failed', error) + } + + private logUnavailable(reason: string, error?: unknown): void { + logger.info('[AgentBrowserNativeOverlay] Native overlay unavailable', { + platform: process.platform, + arch: process.arch, + reason, + ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}) + }) + } + + private warn(reason: string, error: unknown): void { + logger.warn('[AgentBrowserNativeOverlay] Native overlay operation failed', { + reason, + error: error instanceof Error ? error.message : String(error) + }) + } +} diff --git a/src/main/desktop/browser/YoBrowserPresenter.ts b/src/main/desktop/browser/YoBrowserPresenter.ts index e11f50beac..e81eb3de22 100644 --- a/src/main/desktop/browser/YoBrowserPresenter.ts +++ b/src/main/desktop/browser/YoBrowserPresenter.ts @@ -8,6 +8,9 @@ import logger from '@shared/logger' import { BrowserPageStatus, type BrowserPageInfo, + type BrowserPreviewMode, + type BrowserPreviewModeResult, + type BrowserPreviewSurface, type ScreenshotOptions, type YoBrowserActivityAction, type YoBrowserActivityDirection, @@ -19,6 +22,11 @@ import { } from '@shared/types/browser' import type { DownloadInfo } from '@shared/types/browser' import type { IWindowPresenter, IYoBrowserPresenter } from '@shared/types/desktop' +import { + AgentBrowserNativeOverlay, + type AgentBrowserNativeOverlayAction, + type AgentBrowserNativeOverlayTarget +} from './AgentBrowserNativeOverlay' import { BrowserTab as BrowserPage } from './BrowserTab' import { BrowserProfileImportService } from './BrowserProfileImportService' import { CDPManager } from './CDPManager' @@ -55,7 +63,8 @@ type SessionBrowserState = { owner: 'agent' | 'user' agentRunId?: string previewHost: BaseWindow | null - previewMode: 'capturing' | 'rendering' | 'stopped' + previewMode: BrowserPreviewMode + previewSurface: BrowserPreviewSurface previewTargetWindowId: number | null previewTimer: ReturnType | null previewCapture: Promise | null @@ -92,6 +101,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { ) private browserDataMutationActive = false private readonly windowPresenter: IWindowPresenter + private readonly nativeOverlay: AgentBrowserNativeOverlay readonly toolHandler: YoBrowserToolHandler constructor( @@ -99,11 +109,14 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { private readonly publishEvent: DeepchatEventPublisher ) { this.windowPresenter = windowPresenter + this.nativeOverlay = new AgentBrowserNativeOverlay((action, target) => { + this.handleNativePreviewAction(action, target) + }) this.toolHandler = new YoBrowserToolHandler(this) } async initialize(): Promise { - // Lazy initialization only. + await this.nativeOverlay.initialize() } async getBrowserStatus(sessionId: string): Promise { @@ -179,6 +192,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return false } + this.nativeOverlay.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) this.detachOtherSessionBrowsers(hostWindowId, sessionId) @@ -257,51 +271,76 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { async setPreviewMode( sessionId: string, - mode: 'capturing' | 'rendering' | 'stopped', + mode: BrowserPreviewMode, hostWindowId?: number, runId?: string - ): Promise { + ): Promise { const state = this.sessionBrowsers.get(sessionId) if (!state) { - return false + return { updated: false, surface: 'none' } } if (mode === 'stopped') { if (runId != null && state.agentRunId != null && runId !== state.agentRunId) { - return false + return { updated: false, surface: 'none' } } + this.nativeOverlay.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) - return true + this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + state.previewSurface = 'none' + return { updated: true, surface: 'none' } } if ( state.owner !== 'agent' || !state.agentRunId || - (runId != null && runId !== state.agentRunId) || - state.visible + (runId != null && runId !== state.agentRunId) ) { - return false + return { updated: false, surface: 'none' } } - if (mode === 'capturing') { - const targetWindow = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) - if (!targetWindow || targetWindow.isDestroyed()) { - return false - } + await this.nativeOverlay.initialize() + + const targetWindow = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) + if (hostWindowId != null && (!targetWindow || targetWindow.isDestroyed())) { + return { updated: false, surface: 'none' } } + if (mode === 'rendering') { + this.nativeOverlay.hide(this.nativeTargetRef(state)) + } await this.stopPreviewCapture(state) - if (!this.ensurePreviewHost(state)) { - return false + if (mode === 'capturing' && (!targetWindow || state.visible)) { + return { updated: false, surface: 'none' } } + if (!state.visible && !this.ensurePreviewHost(state)) { + return { updated: false, surface: 'none' } + } + state.previewMode = mode state.previewTargetWindowId = hostWindowId ?? null state.previewEpoch += 1 - if (mode === 'capturing') { - this.schedulePreviewCapture(state, 0, state.previewEpoch) + if (mode === 'rendering') { + this.nativeOverlay.hide(this.nativeTargetRef(state)) + state.previewSurface = this.nativeOverlay.isAvailable() ? 'native-overlay' : 'renderer-canvas' + return { updated: true, surface: state.previewSurface } + } + + if (!targetWindow) { + return { updated: false, surface: 'none' } + } + const nativeTarget = this.nativeTarget(state) + state.previewSurface = + nativeTarget && this.nativeOverlay.prepare(nativeTarget, targetWindow) + ? 'native-overlay' + : 'renderer-canvas' + + this.schedulePreviewCapture(state, 0, state.previewEpoch) + return { + updated: true, + surface: state.previewSurface } - return true } async destroySessionBrowser(sessionId: string): Promise { @@ -310,6 +349,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } + this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) await this.releasePreviewHost(state) await this.detachSessionBrowser(sessionId) state.page.destroy() @@ -477,6 +517,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { for (const sessionId of Array.from(this.sessionBrowsers.keys())) { await this.destroySessionBrowser(sessionId) } + this.nativeOverlay.shutdown() } private ensureSessionBrowserState(sessionId: string): SessionBrowserState { @@ -511,6 +552,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { owner: 'user', previewHost: null, previewMode: 'stopped', + previewSurface: 'none', previewTargetWindowId: null, previewTimer: null, previewCapture: null, @@ -633,6 +675,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } + this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) this.disposePreviewHost(state) state.page.destroy() state.overlay.destroy() @@ -1277,8 +1320,10 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewHost = host host.once('closed', () => { if (state.previewHost === host) { + this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) state.previewHost = null state.previewMode = 'stopped' + state.previewSurface = 'none' state.previewTargetWindowId = null state.previewEpoch += 1 state.view.setVisible(false) @@ -1334,6 +1379,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewTimer = null } state.previewMode = 'stopped' + state.previewSurface = 'none' state.previewTargetWindowId = null const host = state.previewHost state.previewHost = null @@ -1390,20 +1436,27 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } const data = image.resize(PREVIEW_FRAME).toJPEG(72) if (data.byteLength <= PREVIEW_MAX_BYTES) { - state.previewSequence += 1 - this.windowPresenter.sendToWindow( - state.previewTargetWindowId, - DEEPCHAT_EVENT_CHANNEL, - createDeepchatEventEnvelope('browser.preview.frame', { - sessionId: state.sessionId, - runId: state.agentRunId, - sequence: state.previewSequence, - ...PREVIEW_FRAME, - mimeType: 'image/jpeg', - data, - timestamp: Date.now() - }) - ) + if (state.previewSurface === 'native-overlay') { + const target = this.nativeTarget(state) + if (target && target.captureEpoch === epoch) { + this.nativeOverlay.present(target, data) + } + } else if (state.previewSurface === 'renderer-canvas') { + state.previewSequence += 1 + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.frame', { + sessionId: state.sessionId, + runId: state.agentRunId, + sequence: state.previewSequence, + ...PREVIEW_FRAME, + mimeType: 'image/jpeg', + data, + timestamp: Date.now() + }) + ) + } } } catch (error) { logger.warn('[YoBrowser] Preview capture failed', { @@ -1419,4 +1472,60 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { epoch ) } + + private nativeTarget(state: SessionBrowserState): AgentBrowserNativeOverlayTarget | null { + if (state.previewTargetWindowId == null || !state.agentRunId) { + return null + } + return { + windowId: state.previewTargetWindowId, + sessionId: state.sessionId, + runId: state.agentRunId, + captureEpoch: state.previewEpoch + } + } + + private nativeTargetRef(state: SessionBrowserState): { + sessionId: string + runId?: string + } { + return { + sessionId: state.sessionId, + ...(state.agentRunId ? { runId: state.agentRunId } : {}) + } + } + + private handleNativePreviewAction( + action: AgentBrowserNativeOverlayAction, + target: AgentBrowserNativeOverlayTarget + ): void { + const state = this.sessionBrowsers.get(target.sessionId) + if ( + !state || + state.previewSurface !== 'native-overlay' || + state.previewTargetWindowId !== target.windowId || + state.agentRunId !== target.runId || + state.previewEpoch !== target.captureEpoch + ) { + return + } + + state.previewMode = 'rendering' + void this.stopPreviewCapture(state) + + if (action === 'activate') { + this.windowPresenter.show(target.windowId, true) + } + + this.windowPresenter.sendToWindow( + target.windowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.action', { + action, + windowId: target.windowId, + sessionId: target.sessionId, + runId: target.runId + }) + ) + } } diff --git a/src/main/desktop/routes.ts b/src/main/desktop/routes.ts index c4c42b4aed..e53f3c924e 100644 --- a/src/main/desktop/routes.ts +++ b/src/main/desktop/routes.ts @@ -392,14 +392,14 @@ export function createDesktopRoutes(deps: { browserSetPreviewModeRoute.name, async (rawInput, context) => { const input = browserSetPreviewModeRoute.input.parse(rawInput) - return browserSetPreviewModeRoute.output.parse({ - updated: await browserPresenter.setPreviewMode( + return browserSetPreviewModeRoute.output.parse( + await browserPresenter.setPreviewMode( input.sessionId, input.mode, context.windowId ?? undefined, input.runId ) - }) + ) } ], [ diff --git a/src/renderer/api/BrowserClient.ts b/src/renderer/api/BrowserClient.ts index 38325cd905..5358e8bd55 100644 --- a/src/renderer/api/BrowserClient.ts +++ b/src/renderer/api/BrowserClient.ts @@ -2,6 +2,7 @@ import type { DeepchatBridge } from '@shared/contracts/bridge' import { browserActivityChangedEvent, browserOpenRequestedEvent, + browserPreviewActionEvent, browserPreviewFrameEvent, browserStatusChangedEvent, type DeepchatEventPayload @@ -84,8 +85,7 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() mode: 'capturing' | 'rendering' | 'stopped', runId?: string ) { - const result = await bridge.invoke(browserSetPreviewModeRoute.name, { sessionId, mode, runId }) - return result.updated + return await bridge.invoke(browserSetPreviewModeRoute.name, { sessionId, mode, runId }) } async function destroy(sessionId: string) { @@ -205,6 +205,12 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return bridge.on(browserPreviewFrameEvent.name, listener) } + function onPreviewAction( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(browserPreviewActionEvent.name, listener) + } + return { getStatus, loadUrl, @@ -225,7 +231,8 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() onOpenRequestedForCurrentWindow, onStatusChanged, onActivityChanged, - onPreviewFrame + onPreviewFrame, + onPreviewAction } } diff --git a/src/renderer/src/components/browser/AgentBrowserPiP.vue b/src/renderer/src/components/browser/AgentBrowserPiP.vue index e1d7140248..342a788dd8 100644 --- a/src/renderer/src/components/browser/AgentBrowserPiP.vue +++ b/src/renderer/src/components/browser/AgentBrowserPiP.vue @@ -1,7 +1,7 @@