From 3a61d32eb286be2a23a0b7a5a482cb9b56d31a81 Mon Sep 17 00:00:00 2001 From: Pierre Date: Thu, 16 Jul 2026 06:17:03 +0200 Subject: [PATCH] fix: harden orchestration performance and recovery --- .github/workflows/ci.yml | 7 +- .github/workflows/desktop-release.yml | 6 +- .github/workflows/release-checks.yml | 6 +- .github/workflows/release.yml | 7 +- .../v2/components/search/SearchOverlay.tsx | 4 +- .../v2/components/top-nav/GlobalSearch.tsx | 1 + .../src/v2/components/ui/DropdownMenu.tsx | 89 ++- .../hooks/__tests__/use-focus-trap.test.tsx | 23 +- dashboard/src/v2/hooks/use-focus-trap.ts | 6 +- .../src/v2/lib/sprint-menu-positioning.ts | 14 +- .../src/v2/pages/sprints/SprintsPage.tsx | 128 ++++- .../execution-invocation-tracking.md | 8 +- .../high-concurrency-orchestration.md | 46 +- ...itecture-execution-invocation-tracking.mdx | 8 +- ...tecture-high-concurrency-orchestration.mdx | 46 +- .../docs/developer-building-from-source.mdx | 4 +- .../developer-orchestration-debugging.mdx | 22 +- docs-web/content/docs/developer-testing.mdx | 4 +- .../docs/developer-websocket-realtime.mdx | 13 + .../docs/operations-security-hardening.mdx | 4 +- .../docs/settings-database-settings.mdx | 12 +- .../docs/settings-quality-assurance.mdx | 6 + .../docs/settings-restart-behavior.mdx | 4 +- .../content/docs/user-dashboard-sprints.mdx | 2 + .../docs/user-sprint-orchestration.mdx | 9 + .../content/docs/user-troubleshooting.mdx | 19 +- docs-web/developer/building-from-source.md | 4 +- docs-web/developer/orchestration-debugging.md | 22 +- docs-web/developer/testing.md | 4 +- docs-web/developer/websocket-realtime.md | 13 + docs-web/operations/security-hardening.md | 4 +- docs-web/settings/database-settings.md | 12 +- docs-web/settings/quality-assurance.md | 6 + docs-web/settings/restart-behavior.md | 4 +- docs-web/user/dashboard/sprints.md | 2 + docs-web/user/sprint-orchestration.md | 9 + docs-web/user/troubleshooting.md | 19 +- .../code-quality-performance-contracts.md | 17 +- .../dashboard-realtime-foundation.md | 5 +- .../execution-dashboard-controls.md | 2 +- .../execution-invocation-tracking.md | 4 +- .../high-concurrency-orchestration.md | 97 +++- .../architecture/usage-telemetry-and-stats.md | 6 + docs/dashboard/design-system-sprints.md | 2 +- docs/deployment/electron-desktop.md | 8 +- .../mockup-sprint-pentest-scenarios.md | 14 +- docs/development/mockup-sprint-pentest.md | 14 +- .../rapid-orchestration-debugging.md | 24 +- docs/development/testing-and-quality.md | 2 +- docs/operations/logging-and-correlation.md | 8 +- docs/operations/runbook.md | 16 +- docs/operations/security-hardening.md | 4 +- docs/settings/configuration-and-storage.md | 6 +- docs/settings/quality-assurance.md | 6 + docs/settings/restart-behavior.md | 4 +- docs/sprint-loop/atomic-loop.md | 14 +- electron-builder.config.cjs | 1 + package.json | 5 +- playwright.config.ts | 2 +- pnpm-lock.yaml | 3 + scripts/e2e/mock-provider-cli.mjs | 18 +- .../e2e/mockup-sprint-pentest-scenarios.mjs | 360 +++++++++++- scripts/e2e/run-mockup-sprint-pentest.mjs | 542 +++++++++++++++++- scripts/measure-live-snapshot.ts | 24 +- scripts/prepare-electron-runtime-deps.mjs | 65 ++- scripts/smoke-installed-electron.mjs | 231 ++++++++ src/app/dependency-factory/sprint-factory.ts | 1 + .../dashboard-snapshot-cache-policy.ts | 3 + src/app/lifecycle/dashboard-snapshot-cache.ts | 73 ++- src/domain/sprint/ci/feature-pr-gate.ts | 68 ++- .../sprint/orchestrator/cycle-runner.ts | 176 +++++- .../orchestrator/sprint-action-runner.ts | 6 + .../sprint/orchestrator/watch-loop-runner.ts | 50 +- src/electron/main.ts | 23 + src/electron/startup-smoke.ts | 62 ++ src/infrastructure/git/local-merge.ts | 160 ++---- .../providers/cli/docker-helper-pool.ts | 359 +++++++++--- .../providers/cli/docker-runner.ts | 94 ++- .../providers/cli/provider-execution-loop.ts | 18 +- .../provider-logs/claude-code-log-parser.ts | 292 ++++++---- .../cli/provider-logs/codex-log-parser.ts | 14 +- .../providers/cli/provider-runner.ts | 8 +- .../cli/provider-telemetry-watcher.ts | 46 +- .../providers/cli/provider-usage.ts | 52 +- .../providers/cli/workspace-manager.ts | 138 +++-- .../providers/cli/workspace-volume-helper.ts | 393 +++++++++++-- src/integrations/jules-api-client.ts | 160 +++++- src/repositories/app-db-storage.ts | 1 + src/repositories/db/app-db-migrations.ts | 20 + src/repositories/db/app-db-schema.ts | 1 + src/repositories/execution-repository.ts | 72 ++- src/repositories/guardrail-repository.ts | 53 +- .../runtime-status-projection.ts | 108 +++- src/repositories/qa-review-repository.ts | 81 +++ .../session-tracking-repository.ts | 34 +- src/server/activity-cache-service.ts | 31 +- src/server/code-ux-server.ts | 93 ++- .../dashboard-realtime-websocket-server.ts | 93 ++- src/server/terminal-routes.ts | 19 +- src/services/activity-write-coalescer.ts | 18 +- src/services/cli-process-runner.ts | 2 + src/services/cli-workflow-service.ts | 26 +- .../cli-workflow/pipeline/cleanup-stage.ts | 5 + src/services/custom-dashboard-docker-plan.ts | 3 + .../custom-node-runtime-service.ts | 2 + src/services/dashboard-realtime-service.ts | 18 + src/services/database-maintenance-service.ts | 6 +- src/services/docker-asset-prune-service.ts | 47 +- src/services/docker-orphan-cleanup-utils.ts | 3 +- src/services/git-status-service.ts | 51 +- src/services/guardrail-service.ts | 25 + src/services/planning-agent-service.ts | 176 +++++- src/services/playwright-browser-manager.ts | 3 + src/services/provider-execution-service.ts | 4 + src/services/provider-tool-manager.ts | 3 + src/services/quality-assurance-service.ts | 166 +++++- .../durable-remote-recovery.ts | 193 +++++++ .../runtime-recovery/invocation-recovery.ts | 33 +- .../runtime-startup-recovery-service.ts | 402 ++++++++++++- src/services/shutdown-container-service.ts | 130 ++++- src/services/sprint-file-browser-service.ts | 4 + src/services/sprint-preview-docker-plan.ts | 2 + src/services/sprint-preview-service.ts | 5 +- src/services/sprint-task-dispatch-service.ts | 280 ++++++++- .../structured-agent-request-service.ts | 2 + src/shared/config/runtime-owner.ts | 30 + src/shared/logging/logger.ts | 15 +- src/shared/subprocess/command-runner.ts | 222 ++++++- src/shared/subprocess/command-spawner-host.ts | 16 +- .../subprocess/command-spawner-protocol.ts | 14 + src/sprint/steps/session-sync-step.ts | 95 ++- src/sprint/steps/start-ready-tasks-step.ts | 79 ++- .../dashboard-snapshot-cache.test.ts | 29 + tests/backend/ci/workflow-health.test.ts | 15 +- .../domain/sprint/ci/feature-pr-gate.test.ts | 14 + .../sprint/orchestrator/cycle-runner.test.ts | 202 ++++++- tests/backend/electron-builder-config.test.ts | 43 ++ tests/backend/electron/startup-smoke.test.ts | 52 ++ .../infrastructure/git/local-merge.test.ts | 49 +- .../cli/claude-code-log-parser.test.ts | 40 +- .../providers/cli/codex-log-parser.test.ts | 29 + .../providers/cli/docker-helper-pool.test.ts | 302 +++++++++- .../providers/cli/docker-runner.test.ts | 85 ++- .../cli/mock-provider-cli-shim.test.ts | 40 ++ .../cli/provider-execution-loop.test.ts | 36 ++ .../cli/provider-telemetry-watcher.test.ts | 77 +++ .../providers/cli/workspace-manager.test.ts | 406 +++++++------ .../cli/workspace-volume-helper.test.ts | 415 +++++++++++--- .../integrations/jules-api-client.test.ts | 99 +++- .../repositories/app-db-storage.test.ts | 2 + .../repositories/db/app-db-schema.test.ts | 2 + .../repositories/execution-repository.test.ts | 45 ++ .../repositories/guardrail-repository.test.ts | 32 ++ .../runtime-status-projection.test.ts | 30 + .../repositories/qa-review-repository.test.ts | 13 + .../session-tracking-repository.test.ts | 40 +- .../mockup-sprint-pentest-runner.test.ts | 330 +++++++++++ .../server/activity-cache-service.test.ts | 29 + ...ashboard-realtime-websocket-server.test.ts | 8 +- .../backend/server/jules-agent-server.test.ts | 23 +- .../sprint-preview-docker-plan.test.ts.snap | 2 + .../services/activity-write-coalescer.test.ts | 16 + .../services/cli-process-runner.test.ts | 26 +- .../services/cli-workflow-service.test.ts | 12 +- .../pipeline/pipeline-stages.test.ts | 4 + .../dashboard-realtime-service.test.ts | 42 +- .../database-maintenance-service.test.ts | 17 + .../docker-asset-prune-service.test.ts | 63 ++ .../services/git-status-service.test.ts | 49 ++ .../services/guardrail-service.test.ts | 12 + .../backend/services/jules-api-client.test.ts | 21 +- .../services/planning-agent-service.test.ts | 110 +++- .../quality-assurance-service.test.ts | 160 +++++- .../runtime-startup-recovery-service.test.ts | 519 ++++++++++++++++- .../shutdown-container-service.test.ts | 106 +++- .../sprint-preview-docker-plan.test.ts | 4 +- .../sprint-task-dispatch-service.test.ts | 293 ++++++++++ tests/backend/shared/logging/logger.test.ts | 14 + .../shared/subprocess/command-runner.test.ts | 534 +++++++++++++++++ .../subprocess/command-spawner-client.test.ts | 16 + .../backend/sprint/session-sync-step.test.ts | 31 +- .../steps/start-ready-tasks-step.test.ts | 80 +++ .../v2/sprint-menu-positioning.test.ts | 12 + .../v2/sprints-page-integration.test.tsx | 2 +- tests/dashboard/v2/ui-components.test.tsx | 59 ++ tests/e2e/agents/agent-avatar-scene.spec.ts | 25 +- 186 files changed, 10511 insertions(+), 1188 deletions(-) create mode 100644 scripts/smoke-installed-electron.mjs create mode 100644 src/electron/startup-smoke.ts create mode 100644 src/services/runtime-recovery/durable-remote-recovery.ts create mode 100644 src/shared/config/runtime-owner.ts create mode 100644 tests/backend/electron/startup-smoke.test.ts create mode 100644 tests/backend/infrastructure/providers/cli/mock-provider-cli-shim.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f18e9babb2..5f15ffe741 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -567,7 +567,7 @@ jobs: needs: package-smoke if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.base_ref == 'main' }} runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 strategy: fail-fast: false max-parallel: 3 @@ -625,7 +625,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y libopenjp2-tools + sudo apt-get install --no-install-recommends -y libopenjp2-tools xvfb - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts @@ -648,6 +648,9 @@ jobs: - name: Build unsigned desktop package run: pnpm exec electron-builder --config electron-builder.config.cjs ${{ matrix.electron-target }} --publish never + - name: Install and start release candidate + run: pnpm run electron:smoke-installed + - name: Upload release candidate artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 3cb74d4fae..f7f4daf469 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -19,6 +19,7 @@ jobs: build-desktop: name: Build ${{ matrix.name }} runs-on: ${{ matrix.os }} + timeout-minutes: 40 strategy: fail-fast: false @@ -81,7 +82,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y libopenjp2-tools + sudo apt-get install --no-install-recommends -y libopenjp2-tools xvfb - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts @@ -95,6 +96,9 @@ jobs: - name: Build desktop package run: pnpm run ${{ matrix.script }} + - name: Install and start desktop package + run: pnpm run electron:smoke-installed + - name: Upload workflow artifact uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/release-checks.yml b/.github/workflows/release-checks.yml index cd4ad96f6d..845930ac3d 100644 --- a/.github/workflows/release-checks.yml +++ b/.github/workflows/release-checks.yml @@ -14,6 +14,7 @@ jobs: release-checks: name: Release checks (${{ matrix.name }}) runs-on: ${{ matrix.os }} + timeout-minutes: 45 strategy: fail-fast: false @@ -73,7 +74,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y libopenjp2-tools + sudo apt-get install --no-install-recommends -y libopenjp2-tools xvfb - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts @@ -93,6 +94,9 @@ jobs: - name: Build desktop package run: pnpm run ${{ matrix.electron-script }} -- --publish never + - name: Install and start release candidate + run: pnpm run electron:smoke-installed + - name: Upload release-check artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38ec973d8f..3cd1308be5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -113,7 +113,7 @@ jobs: name: 03 Release / desktop package (${{ matrix.name }}) needs: release-preflight runs-on: ${{ matrix.os }} - timeout-minutes: 25 + timeout-minutes: 35 permissions: contents: write strategy: @@ -178,7 +178,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y libopenjp2-tools + sudo apt-get install --no-install-recommends -y libopenjp2-tools xvfb - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts @@ -192,6 +192,9 @@ jobs: - name: Build desktop package run: pnpm run build && pnpm run electron:prepare-deps && pnpm exec electron-builder --config electron-builder.config.cjs ${{ matrix.electron-target }} --publish never + - name: Install and start desktop package + run: pnpm run electron:smoke-installed + - name: Upload workflow artifact uses: actions/upload-artifact@v4 with: diff --git a/dashboard/src/v2/components/search/SearchOverlay.tsx b/dashboard/src/v2/components/search/SearchOverlay.tsx index a56f56fd2a..9580c6125b 100644 --- a/dashboard/src/v2/components/search/SearchOverlay.tsx +++ b/dashboard/src/v2/components/search/SearchOverlay.tsx @@ -111,6 +111,7 @@ export interface SearchResults { interface SearchOverlayProps { anchorRef?: preact.RefObject; + restoreFocusRef?: preact.RefObject; committedSearchQuery?: string; isLoading?: boolean; isOpen: boolean; @@ -121,7 +122,7 @@ interface SearchOverlayProps { hasProjectData?: boolean; } -export const SearchOverlay: FunctionComponent = ({ anchorRef, committedSearchQuery, isOpen, onClose, searchQuery, onSearchChange, results, isLoading, hasProjectData = true }) => { +export const SearchOverlay: FunctionComponent = ({ anchorRef, restoreFocusRef, committedSearchQuery, isOpen, onClose, searchQuery, onSearchChange, results, isLoading, hasProjectData = true }) => { const { translate, translatePlural } = useOptionalDashboardI18n(); const overlayRef = useRef(null); const inputRef = useRef(null); @@ -129,6 +130,7 @@ export const SearchOverlay: FunctionComponent = ({ anchorRef const containerRef = useFocusTrap(isOpen, { onClose, initialFocusRef: inputRef, + restoreFocusRef, restoreFocus: true }) as preact.RefObject; const [focusedIndex, setFocusedIndex] = useState(-1); diff --git a/dashboard/src/v2/components/top-nav/GlobalSearch.tsx b/dashboard/src/v2/components/top-nav/GlobalSearch.tsx index 6f83f127b8..351537973d 100644 --- a/dashboard/src/v2/components/top-nav/GlobalSearch.tsx +++ b/dashboard/src/v2/components/top-nav/GlobalSearch.tsx @@ -226,6 +226,7 @@ export const GlobalSearch: FunctionComponent = ({ projectId, setIsSearchOpen(false)} diff --git a/dashboard/src/v2/components/ui/DropdownMenu.tsx b/dashboard/src/v2/components/ui/DropdownMenu.tsx index bb8520d483..c2f4feeba5 100644 --- a/dashboard/src/v2/components/ui/DropdownMenu.tsx +++ b/dashboard/src/v2/components/ui/DropdownMenu.tsx @@ -37,6 +37,14 @@ interface DropdownMenuProps { menuAriaLabel?: string; } +interface DropdownCoordinates { + top: number; + left: number; + maxHeight?: number; +} + +const VIEWPORT_PADDING = 8; + type DropdownMenuItemProps = JSX.HTMLAttributes & { children?: ComponentChildren; disabled?: boolean; @@ -115,7 +123,7 @@ export const DropdownMenu = ({ const triggerRef = externalTriggerRef || localTriggerRef; const menuRef = useRef(null); const previousFocusRef = useRef(null); - const [coords, setCoords] = useState({ top: 0, left: 0 }); + const [coords, setCoords] = useState({ top: 0, left: 0 }); const [transformOrigin, setTransformOrigin] = useState("top center"); // Generate a unique ID for ARIA wiring if none exists @@ -156,33 +164,62 @@ export const DropdownMenu = ({ if (!triggerRef.current || !menuRef.current) return; const triggerRect = triggerRef.current.getBoundingClientRect(); - const menuRect = menuRef.current.getBoundingClientRect(); + const menuElement = menuRef.current; + const menuRect = menuElement.getBoundingClientRect(); + // Transforms and a previous max-height can make the visual rectangle smaller + // than the menu's contents. Position against the intrinsic dimensions so a + // growing action list still flips before it becomes unreachable. + const intrinsicRect = { + ...menuRect, + width: Math.max(menuRect.width, menuElement.scrollWidth), + height: Math.max(menuRect.height, menuElement.scrollHeight), + } as DOMRect; + const viewport = { + width: window.innerWidth, + height: window.innerHeight, + }; + let top: number; + let left: number; + let nextTransformOrigin = "top center"; + if (computePosition) { const custom = computePosition({ triggerRect, - menuRect, - viewport: { - width: window.innerWidth, - height: window.innerHeight, - }, + menuRect: intrinsicRect, + viewport, defaultPosition: position, defaultAlign: align, gap, }); - setCoords({ top: custom.top, left: custom.left }); - setTransformOrigin(custom.transformOrigin ?? "top center"); - return; + top = custom.top; + left = custom.left; + nextTransformOrigin = custom.transformOrigin ?? "top center"; + } else { + const calculated = calculatePosition({ + triggerRect, + contentRect: intrinsicRect, + position, + align, + gap, + padding: VIEWPORT_PADDING, + viewportWidth: viewport.width, + viewportHeight: viewport.height, + }); + top = calculated.top; + left = calculated.left; } - const { top, left } = calculatePosition({ - triggerRect, - contentRect: menuRect, - position, - align, - gap, - padding: 8, + + const maxHeight = Math.max( + 0, + viewport.height - Math.max(VIEWPORT_PADDING, top) - VIEWPORT_PADDING, + ); + setCoords((current) => { + if (current.top === top && current.left === left && current.maxHeight === maxHeight) { + return current; + } + return { top, left, maxHeight }; }); - setCoords({ top, left }); - setTransformOrigin("top center"); + setTransformOrigin(nextTransformOrigin); }, [align, computePosition, gap, position, triggerRef]); useEffect(() => { @@ -198,6 +235,16 @@ export const DropdownMenu = ({ if (isOpen && isRendered) updatePosition(); }, [isOpen, isRendered, updatePosition]); + useLayoutEffect(() => { + if (!isOpen || !isRendered || typeof ResizeObserver === "undefined") return undefined; + + const observer = new ResizeObserver(() => updatePosition()); + if (triggerRef.current) observer.observe(triggerRef.current); + if (menuRef.current) observer.observe(menuRef.current); + + return () => observer.disconnect(); + }, [isOpen, isRendered, triggerRef, updatePosition]); + useEffect(() => { if (!isOpen) return undefined; @@ -426,8 +473,8 @@ export const DropdownMenu = ({ role="menu" aria-label={menuAriaLabel} aria-labelledby={menuAriaLabel ? undefined : (isValidElement(children) && (children.props as any).id ? (children.props as any).id : triggerId)} - className={`fixed z-[100] bg-white dark:bg-void-800 border border-black/[0.08] dark:border-white/[0.08] shadow-[0_16px_36px_rgba(15,23,42,0.14)] dark:shadow-[0_16px_36px_rgba(0,0,0,0.4)] rounded-2xl p-2 ${!isOpen ? "pointer-events-none" : ""} ${className}`} - style={{ top: coords.top, left: coords.left, transformOrigin }} + className={`fixed z-[100] max-h-[calc(100dvh-1rem)] overflow-y-auto bg-white dark:bg-void-800 border border-black/[0.08] dark:border-white/[0.08] shadow-[0_16px_36px_rgba(15,23,42,0.14)] dark:shadow-[0_16px_36px_rgba(0,0,0,0.4)] rounded-2xl p-2 ${!isOpen ? "pointer-events-none" : ""} ${className}`} + style={{ top: coords.top, left: coords.left, maxHeight: coords.maxHeight, transformOrigin }} onClick={(e) => e.stopPropagation()} > {enhancedContent} diff --git a/dashboard/src/v2/hooks/__tests__/use-focus-trap.test.tsx b/dashboard/src/v2/hooks/__tests__/use-focus-trap.test.tsx index 43c3163a35..2050aedc52 100644 --- a/dashboard/src/v2/hooks/__tests__/use-focus-trap.test.tsx +++ b/dashboard/src/v2/hooks/__tests__/use-focus-trap.test.tsx @@ -10,8 +10,8 @@ describe("useFocusTrap", () => { cleanup(); }); - const TestComponent = ({ active, onClose, empty = false, initialFocusRef, restoreFocus = true }: any) => { - const trapRef = useFocusTrap(active, { onClose, initialFocusRef, restoreFocus }); + const TestComponent = ({ active, onClose, empty = false, initialFocusRef, restoreFocusRef, restoreFocus = true }: any) => { + const trapRef = useFocusTrap(active, { onClose, initialFocusRef, restoreFocusRef, restoreFocus }); return (
@@ -182,6 +182,25 @@ describe("useFocusTrap", () => { expect(focusSpy).toHaveBeenLastCalledWith({ preventScroll: true }); }); + test("prefers an explicit return target when the captured trigger is replaced", async () => { + const capturedTrigger = document.createElement("button"); + const replacementTrigger = document.createElement("button"); + replacementTrigger.id = "replacement-trigger"; + document.body.append(capturedTrigger, replacementTrigger); + capturedTrigger.focus(); + const restoreFocusRef = { current: replacementTrigger }; + + const { unmount } = render( + {}} restoreFocusRef={restoreFocusRef} />, + ); + await waitFor(() => expect(document.activeElement?.id).toBe("inside1")); + capturedTrigger.remove(); + unmount(); + + await waitFor(() => expect(document.activeElement).toBe(replacementTrigger)); + replacementTrigger.remove(); + }); + test("keeps focus trapped when the focused element is removed dynamically", async () => { const DynamicTrap = () => { const [showFirst, setShowFirst] = useState(true); diff --git a/dashboard/src/v2/hooks/use-focus-trap.ts b/dashboard/src/v2/hooks/use-focus-trap.ts index 4726f65d2e..140e902c39 100644 --- a/dashboard/src/v2/hooks/use-focus-trap.ts +++ b/dashboard/src/v2/hooks/use-focus-trap.ts @@ -63,6 +63,8 @@ export function restoreFocusSafely(...candidates: Array void; initialFocusRef?: { current: HTMLElement | null }; + /** Preferred return target when reactive rendering may replace activeElement. */ + restoreFocusRef?: { current: HTMLElement | null }; restoreFocus?: boolean; /** Temporarily yield keyboard handling without losing the original opener. */ paused?: boolean; @@ -79,7 +81,7 @@ export function useFocusTrap( ? { onClose: optionsOrOnClose } : (optionsOrOnClose || {}); - const { onClose, initialFocusRef, restoreFocus = true, paused = false } = options; + const { onClose, initialFocusRef, restoreFocusRef, restoreFocus = true, paused = false } = options; const onCloseRef = useRef(onClose); const pausedRef = useRef(paused); @@ -168,7 +170,7 @@ export function useFocusTrap( // Defer focus restoration to ensure element is re-enabled or DOM is updated const trigger = triggerRef.current; window.setTimeout(() => { - restoreFocusSafely(trigger); + restoreFocusSafely(restoreFocusRef?.current, trigger); }, 0); } }; diff --git a/dashboard/src/v2/lib/sprint-menu-positioning.ts b/dashboard/src/v2/lib/sprint-menu-positioning.ts index 889d322e8d..596b1679f4 100644 --- a/dashboard/src/v2/lib/sprint-menu-positioning.ts +++ b/dashboard/src/v2/lib/sprint-menu-positioning.ts @@ -40,15 +40,21 @@ export function computeSprintActionMenuPosition( const left = Math.max(VIEWPORT_PADDING, Math.min(rightAlignedLeft, maxLeft)); const belowTop = triggerRect.bottom + MENU_GAP; - const canFitBelow = belowTop + height <= viewport.height - VIEWPORT_PADDING; - const top = canFitBelow + const spaceBelow = Math.max(0, viewport.height - VIEWPORT_PADDING - belowTop); + const spaceAbove = Math.max(0, triggerRect.top - MENU_GAP - VIEWPORT_PADDING); + const canFitBelow = height <= spaceBelow; + const canFitAbove = height <= spaceAbove; + // When both sides fit, prefer the larger region. This stays stable if the + // first layout pass underestimates a long menu while fonts or actions settle. + const placeBelow = canFitBelow && (!canFitAbove || spaceBelow >= spaceAbove); + const top = placeBelow ? belowTop : Math.max(VIEWPORT_PADDING, triggerRect.top - height - MENU_GAP); return { top, left, - placement: canFitBelow ? "bottom" : "top", - transformOrigin: canFitBelow ? "top right" : "bottom right", + placement: placeBelow ? "bottom" : "top", + transformOrigin: placeBelow ? "top right" : "bottom right", }; } diff --git a/dashboard/src/v2/pages/sprints/SprintsPage.tsx b/dashboard/src/v2/pages/sprints/SprintsPage.tsx index b8533384b0..b91e8d941c 100644 --- a/dashboard/src/v2/pages/sprints/SprintsPage.tsx +++ b/dashboard/src/v2/pages/sprints/SprintsPage.tsx @@ -54,6 +54,10 @@ import type { Sprint, SprintLinkedIssueInput } from "../../types.js"; import type { SprintImportedTaskInput } from "../../types.js"; import { useDashboardI18n } from "../../i18n/index.js"; import { sprintsMessages } from "../../i18n/messages/sprints.js"; +import { + computeSprintActionMenuPosition, + type SprintMenuRect, +} from "../../lib/sprint-menu-positioning.js"; const ACCENT_CYCLE = ["text-signal-500", "text-ember-500", "text-status-green"] as const; const SPRINT_GALLERY_VISIBILITY_STORAGE_KEY = "code_ux_sprints_show_gallery"; @@ -202,11 +206,16 @@ export const SprintsPage: FunctionComponent = () => { clearError: clearImportedTaskFeedbackError, } = useActionFeedback(); const previousSelectedProjectIdRef = useRef(null); + const rowMenuRef = useRef(null); + const rowMenuTriggerRef = useRef(null); const [rowMenu, setRowMenu] = useState<{ sprintId: string; + triggerRect: SprintMenuRect; top: number; left: number; - openUp: boolean; + maxHeight: number; + transformOrigin: string; + positioned: boolean; } | null>(null); const { @@ -359,21 +368,104 @@ export const SprintsPage: FunctionComponent = () => { event.stopPropagation(); const trigger = event.currentTarget as HTMLElement; const rect = trigger.getBoundingClientRect(); - const estimatedMenuHeight = 228; - const openUp = rect.bottom + estimatedMenuHeight > window.innerHeight - 16; + rowMenuTriggerRef.current = trigger; + const triggerRect: SprintMenuRect = { + top: rect.top, + left: rect.left, + right: rect.right, + bottom: rect.bottom, + width: rect.width, + height: rect.height, + }; setRowMenu((current) => ( current?.sprintId === sprintId ? null : { sprintId, - top: openUp ? rect.top - 8 : rect.bottom + 8, - left: rect.right, - openUp, + triggerRect, + top: 8, + left: Math.max(8, Math.min(rect.right, window.innerWidth - 8)), + maxHeight: Math.max(0, window.innerHeight - 16), + transformOrigin: "top right", + positioned: false, } )); }, [setRowMenu]); + const updateRowMenuPosition = useCallback(() => { + const menuElement = rowMenuRef.current; + if (!menuElement) { + return; + } + + setRowMenu((current) => { + if (!current) { + return current; + } + + const liveTriggerRect = rowMenuTriggerRef.current?.getBoundingClientRect(); + const triggerRect = liveTriggerRect + ? { + top: liveTriggerRect.top, + left: liveTriggerRect.left, + right: liveTriggerRect.right, + bottom: liveTriggerRect.bottom, + width: liveTriggerRect.width, + height: liveTriggerRect.height, + } + : current.triggerRect; + const visualRect = menuElement.getBoundingClientRect(); + const position = computeSprintActionMenuPosition( + triggerRect, + { width: window.innerWidth, height: window.innerHeight }, + { + width: Math.max(visualRect.width, menuElement.scrollWidth), + height: Math.max(visualRect.height, menuElement.scrollHeight), + }, + ); + const maxHeight = Math.max(0, window.innerHeight - Math.max(8, position.top) - 8); + + if ( + current.positioned + && current.top === position.top + && current.left === position.left + && current.maxHeight === maxHeight + && current.transformOrigin === position.transformOrigin + ) { + return current; + } + + return { + ...current, + triggerRect, + top: position.top, + left: position.left, + maxHeight, + transformOrigin: position.transformOrigin, + positioned: true, + }; + }); + }, []); + + useLayoutEffect(() => { + if (!rowMenu?.sprintId) { + return undefined; + } + + updateRowMenuPosition(); + if (typeof ResizeObserver === "undefined" || !rowMenuRef.current) { + return undefined; + } + + const observer = new ResizeObserver(updateRowMenuPosition); + observer.observe(rowMenuRef.current); + if (rowMenuTriggerRef.current) { + observer.observe(rowMenuTriggerRef.current); + } + return () => observer.disconnect(); + }, [rowMenu?.sprintId, updateRowMenuPosition]); + const [listWindow, setListWindow] = useState(DEFAULT_LIST_WINDOW); useLayoutEffect(() => { @@ -454,25 +546,29 @@ export const SprintsPage: FunctionComponent = () => { if (!rowMenu) { return; } - const closeMenu = () => setRowMenu(null); + const closeMenu = () => { + rowMenuTriggerRef.current = null; + setRowMenu(null); + }; const handleEscape = (event: KeyboardEvent) => { if (event.key === "Escape") { closeMenu(); } }; + const handleViewportChange = () => updateRowMenuPosition(); document.addEventListener("click", closeMenu); document.addEventListener("keydown", handleEscape); - window.addEventListener("resize", closeMenu); - window.addEventListener("scroll", closeMenu, true); + window.addEventListener("resize", handleViewportChange); + window.addEventListener("scroll", handleViewportChange, true); return () => { document.removeEventListener("click", closeMenu); document.removeEventListener("keydown", handleEscape); - window.removeEventListener("resize", closeMenu); - window.removeEventListener("scroll", closeMenu, true); + window.removeEventListener("resize", handleViewportChange); + window.removeEventListener("scroll", handleViewportChange, true); }; - }, [rowMenu, setRowMenu]); + }, [rowMenu, setRowMenu, updateRowMenuPosition]); const animateLatestCell = useCallback(() => { requestAnimationFrame(() => { @@ -1144,11 +1240,16 @@ export const SprintsPage: FunctionComponent = () => { {rowMenu && activeRowMenuSprint && createPortal(
event.stopPropagation()} > @@ -1199,6 +1300,7 @@ export const SprintsPage: FunctionComponent = () => { }} onClose={() => setRowMenu(null)} markCompletedIcon="square" + role="menuitem" buttonClassName="flex w-full min-w-0 items-center gap-2 rounded-[0.9rem] px-3 py-2 text-left text-xs font-medium leading-snug text-slate-600 transition-colors hover:bg-black/[0.04] hover:text-slate-900 focus-visible:ring-2 focus-visible:ring-signal-500/30 focus-visible:ring-offset-2 dark:text-slate-300 dark:hover:bg-white/[0.05] dark:hover:text-white focus:outline-none" />
diff --git a/docs-web/architecture/execution-invocation-tracking.md b/docs-web/architecture/execution-invocation-tracking.md index 191d9ac418..e0f6bc380b 100644 --- a/docs-web/architecture/execution-invocation-tracking.md +++ b/docs-web/architecture/execution-invocation-tracking.md @@ -65,6 +65,12 @@ Live provider telemetry is metadata-first. `provider-telemetry-watcher.ts` check Final post-process usage collection remains authoritative. Live telemetry is best effort for dashboard freshness; final collection reconciles the persisted provider usage row when the provider finishes. +Claude Code JSONL polling is append-only. The watcher retains parser state and an incomplete-line +tail, feeds each appended record through `ClaudeCodeLogAccumulator` once, and persists only the +changed conversation suffix. It does not concatenate and reparse the full growing session on every +poll. Codex similarly ignores fallback `event_msg` user/assistant rows after canonical item records +exist, preventing the same turn from being retained twice. + Jules remains outside this local CLI parser and watcher path. Its remote session synchronizer records its transcript separately and derives estimated usage from accumulated input/output characters; Code UX does not describe those estimates as provider-native token telemetry. ## Dashboard and recovery behavior @@ -75,7 +81,7 @@ The cinematic feedback model is separate from whichever invocation is selected i Logical tool activity is deduplicated by normalized `metadata.toolCallId`; a stable message id is the fallback only when no call id exists. The frontend refreshes this projection when the active invocation or its `messageCount`, `lastMessageAt`, or `updatedAt` changes, preserves same-invocation feedback during refresh, and aborts or generation-invalidates stale work after project/invocation changes. Terminal or missing invocations clear the feedback. A transcript request failure remains a local, non-fatal state and does not replace the normal chat transcript or make unrelated work foreground activity. -Startup recovery reconciles stale workflow and provider rows from durable task-run, sprint-run, dispatch, process, and Docker-container evidence. Preparation-only rows can fail without provider linkage, terminal provider rows are reconciled without extending their usage window, and a recovered completed provider attempt may continue from its preserved workspace without a duplicate provider run. +Startup recovery reconciles stale workflow and provider rows from durable task-run, sprint-run, dispatch, process, and Docker-container evidence. Preparation-only rows can fail without provider linkage, terminal provider rows are reconciled without extending their usage window, and a recovered completed provider attempt may continue from its preserved workspace without a duplicate provider run. Interrupted sprint-planning requests preserve their complete durable options and continue the exact recorded provider conversation in the stable planning workspace; a missing recorded conversation fails closed instead of becoming a fresh session. Only pre-provider interruptions are reissued from durable input because no provider conversation existed yet. ## Focused verification diff --git a/docs-web/architecture/high-concurrency-orchestration.md b/docs-web/architecture/high-concurrency-orchestration.md index e90c9f0585..39767404c1 100644 --- a/docs-web/architecture/high-concurrency-orchestration.md +++ b/docs-web/architecture/high-concurrency-orchestration.md @@ -19,8 +19,50 @@ the dashboard, and interactive replies. - Passive idle-time WAL checkpoints and a 256-page incremental-vacuum cap avoid full-file barriers. - Runtime and startup asset cleanup are single-flight; stale-path filesystem work is asynchronous, while Docker inspection/removal uses bounded parallel batches. -- Unchanged provider-cap diagnostics are coalesced per provider, avoiding per-cycle log writes while - a wide ready queue waits for one of the running slots. +- Managed Docker containers and volumes carry a state-home-derived runtime-owner label. Cleanup, + shutdown, preview/file-browser reconciliation, and warm-helper names are owner-scoped, so local + stress tests can share the daemon with a live runtime. A stopped helper is retried by generation + and falls back to one-shot execution if its replacement also disappears. +- An active sprint lazily starts one host-backed Git helper per project/runtime owner. Concurrent + sprints in that project share it with at most four commands in flight; the final sprint drains and + removes it, while inactive projects use one-shot helpers. Credentials and stdin remain scoped to + each exec. Docker-volume work uses a network-disabled, `no-new-privileges` sidecar per active + workspace/runtime-volume pair. Its Git home is a bounded tmpfs; coding and QA reserve the exact + sidecar for the complete workflow, release drains commands, and restartable volumes remain. The + workspace pool is capped at 16; new work evicts only an unreserved idle sidecar or waits. Helper + create/remove operations share a four-operation Docker control-plane limit. Fresh + helper creation avoids speculative removal and reclaims its deterministic name only after an + explicit Docker name conflict. Network Git remains one-shot. Once shutdown begins, late + host-backed Git work also stays one-shot so it cannot recreate a persistent helper generation + after the warm pool drain. +- Wide LOCAL merge drains inspect worker/feature ancestry once and reuse the last published target + SHA. Each serial publication retains compare-and-swap protection; only a concurrent target change + triggers a ref refresh and retry. +- Startup removes stale owner-scoped provider containers in every Docker state before recovery, + including never-started `created` generations. Shutdown also inspects every state and treats + concurrent disappearance as successful idempotent cleanup. It signals active dispatches before + draining helper leases, then removes the remaining owner-scoped containers in bounded batches so + restart latency does not wait for uncancelled workspace commands or one oversized Docker call. + Initial background-loop callbacks use the server's tracked startup timers; shutdown cancels them + before SQLite closes, and periodic callbacks reject new repository work after closing begins. +- Provider-cap diagnostics are limited to one write per sprint run and provider every ten seconds, + even when the blocked queue changes. Long-lived bounded throttle state prevents per-cycle child + loggers from defeating the limit or retaining unbounded history. +- Jules admission reads a fresh, coalesced first page from the API and counts executing `QUEUED`, + `PLANNING`, and `IN_PROGRESS` sessions missing from local accounting. Waiting/paused history does + not consume the execution count. An unavailable preflight fails closed with the task queued. Since + Jules exposes no state filter or subscription-slot endpoint and old running work may be paginated, + capacity `400`/`409`/exhausted `429` responses plus the generic capacity + `400 FAILED_PRECONDITION` remain authoritative retryable deferrals that release the provisional + claim and apply a 30-second learned-cap backoff. +- Startup preserves persisted Jules sessions and repairs false local terminal projections from a + fresh remote-active snapshot before ordinary reconciliation; completed/cancelled sprints, merged + tasks, and human QA handoffs remain terminal. The snapshot repair is bounded to five seconds so a + slow hosted API cannot hold runtime readiness. On timeout, local durable session/runtime evidence + keeps monitoring alive until the late snapshot or ordinary sync verifies the provider state. +- Scheduler starts consume current purpose-aware provider capacity, including adaptive reply + reservations. Task QA runs in waves of at most four so merges and newly unblocked coding progress + between review waves instead of waiting behind a full-DAG QA backlog. The published architecture page is available at [`/docs/architecture-high-concurrency-orchestration`](/docs/architecture-high-concurrency-orchestration). diff --git a/docs-web/content/docs/architecture-execution-invocation-tracking.mdx b/docs-web/content/docs/architecture-execution-invocation-tracking.mdx index 191d9ac418..e0f6bc380b 100644 --- a/docs-web/content/docs/architecture-execution-invocation-tracking.mdx +++ b/docs-web/content/docs/architecture-execution-invocation-tracking.mdx @@ -65,6 +65,12 @@ Live provider telemetry is metadata-first. `provider-telemetry-watcher.ts` check Final post-process usage collection remains authoritative. Live telemetry is best effort for dashboard freshness; final collection reconciles the persisted provider usage row when the provider finishes. +Claude Code JSONL polling is append-only. The watcher retains parser state and an incomplete-line +tail, feeds each appended record through `ClaudeCodeLogAccumulator` once, and persists only the +changed conversation suffix. It does not concatenate and reparse the full growing session on every +poll. Codex similarly ignores fallback `event_msg` user/assistant rows after canonical item records +exist, preventing the same turn from being retained twice. + Jules remains outside this local CLI parser and watcher path. Its remote session synchronizer records its transcript separately and derives estimated usage from accumulated input/output characters; Code UX does not describe those estimates as provider-native token telemetry. ## Dashboard and recovery behavior @@ -75,7 +81,7 @@ The cinematic feedback model is separate from whichever invocation is selected i Logical tool activity is deduplicated by normalized `metadata.toolCallId`; a stable message id is the fallback only when no call id exists. The frontend refreshes this projection when the active invocation or its `messageCount`, `lastMessageAt`, or `updatedAt` changes, preserves same-invocation feedback during refresh, and aborts or generation-invalidates stale work after project/invocation changes. Terminal or missing invocations clear the feedback. A transcript request failure remains a local, non-fatal state and does not replace the normal chat transcript or make unrelated work foreground activity. -Startup recovery reconciles stale workflow and provider rows from durable task-run, sprint-run, dispatch, process, and Docker-container evidence. Preparation-only rows can fail without provider linkage, terminal provider rows are reconciled without extending their usage window, and a recovered completed provider attempt may continue from its preserved workspace without a duplicate provider run. +Startup recovery reconciles stale workflow and provider rows from durable task-run, sprint-run, dispatch, process, and Docker-container evidence. Preparation-only rows can fail without provider linkage, terminal provider rows are reconciled without extending their usage window, and a recovered completed provider attempt may continue from its preserved workspace without a duplicate provider run. Interrupted sprint-planning requests preserve their complete durable options and continue the exact recorded provider conversation in the stable planning workspace; a missing recorded conversation fails closed instead of becoming a fresh session. Only pre-provider interruptions are reissued from durable input because no provider conversation existed yet. ## Focused verification diff --git a/docs-web/content/docs/architecture-high-concurrency-orchestration.mdx b/docs-web/content/docs/architecture-high-concurrency-orchestration.mdx index e90c9f0585..39767404c1 100644 --- a/docs-web/content/docs/architecture-high-concurrency-orchestration.mdx +++ b/docs-web/content/docs/architecture-high-concurrency-orchestration.mdx @@ -19,8 +19,50 @@ the dashboard, and interactive replies. - Passive idle-time WAL checkpoints and a 256-page incremental-vacuum cap avoid full-file barriers. - Runtime and startup asset cleanup are single-flight; stale-path filesystem work is asynchronous, while Docker inspection/removal uses bounded parallel batches. -- Unchanged provider-cap diagnostics are coalesced per provider, avoiding per-cycle log writes while - a wide ready queue waits for one of the running slots. +- Managed Docker containers and volumes carry a state-home-derived runtime-owner label. Cleanup, + shutdown, preview/file-browser reconciliation, and warm-helper names are owner-scoped, so local + stress tests can share the daemon with a live runtime. A stopped helper is retried by generation + and falls back to one-shot execution if its replacement also disappears. +- An active sprint lazily starts one host-backed Git helper per project/runtime owner. Concurrent + sprints in that project share it with at most four commands in flight; the final sprint drains and + removes it, while inactive projects use one-shot helpers. Credentials and stdin remain scoped to + each exec. Docker-volume work uses a network-disabled, `no-new-privileges` sidecar per active + workspace/runtime-volume pair. Its Git home is a bounded tmpfs; coding and QA reserve the exact + sidecar for the complete workflow, release drains commands, and restartable volumes remain. The + workspace pool is capped at 16; new work evicts only an unreserved idle sidecar or waits. Helper + create/remove operations share a four-operation Docker control-plane limit. Fresh + helper creation avoids speculative removal and reclaims its deterministic name only after an + explicit Docker name conflict. Network Git remains one-shot. Once shutdown begins, late + host-backed Git work also stays one-shot so it cannot recreate a persistent helper generation + after the warm pool drain. +- Wide LOCAL merge drains inspect worker/feature ancestry once and reuse the last published target + SHA. Each serial publication retains compare-and-swap protection; only a concurrent target change + triggers a ref refresh and retry. +- Startup removes stale owner-scoped provider containers in every Docker state before recovery, + including never-started `created` generations. Shutdown also inspects every state and treats + concurrent disappearance as successful idempotent cleanup. It signals active dispatches before + draining helper leases, then removes the remaining owner-scoped containers in bounded batches so + restart latency does not wait for uncancelled workspace commands or one oversized Docker call. + Initial background-loop callbacks use the server's tracked startup timers; shutdown cancels them + before SQLite closes, and periodic callbacks reject new repository work after closing begins. +- Provider-cap diagnostics are limited to one write per sprint run and provider every ten seconds, + even when the blocked queue changes. Long-lived bounded throttle state prevents per-cycle child + loggers from defeating the limit or retaining unbounded history. +- Jules admission reads a fresh, coalesced first page from the API and counts executing `QUEUED`, + `PLANNING`, and `IN_PROGRESS` sessions missing from local accounting. Waiting/paused history does + not consume the execution count. An unavailable preflight fails closed with the task queued. Since + Jules exposes no state filter or subscription-slot endpoint and old running work may be paginated, + capacity `400`/`409`/exhausted `429` responses plus the generic capacity + `400 FAILED_PRECONDITION` remain authoritative retryable deferrals that release the provisional + claim and apply a 30-second learned-cap backoff. +- Startup preserves persisted Jules sessions and repairs false local terminal projections from a + fresh remote-active snapshot before ordinary reconciliation; completed/cancelled sprints, merged + tasks, and human QA handoffs remain terminal. The snapshot repair is bounded to five seconds so a + slow hosted API cannot hold runtime readiness. On timeout, local durable session/runtime evidence + keeps monitoring alive until the late snapshot or ordinary sync verifies the provider state. +- Scheduler starts consume current purpose-aware provider capacity, including adaptive reply + reservations. Task QA runs in waves of at most four so merges and newly unblocked coding progress + between review waves instead of waiting behind a full-DAG QA backlog. The published architecture page is available at [`/docs/architecture-high-concurrency-orchestration`](/docs/architecture-high-concurrency-orchestration). diff --git a/docs-web/content/docs/developer-building-from-source.mdx b/docs-web/content/docs/developer-building-from-source.mdx index fd0754aedd..1b15809f8e 100644 --- a/docs-web/content/docs/developer-building-from-source.mdx +++ b/docs-web/content/docs/developer-building-from-source.mdx @@ -161,12 +161,12 @@ pnpm run audit # pnpm audit --audit-level=high pnpm run smoke-test # node dist/index.js --help pnpm run dev:server-only # boot just the server from source # Electron helper scripts: -# electron:generate-icons, electron:prepare-deps, electron:dev, electron:pack, electron:dist, electron:dist:linux, electron:dist:mac, electron:dist:win, electron:benchmark:runtime, electron:benchmark:win, electron:install-deps +# electron:generate-icons, electron:prepare-deps, electron:dev, electron:pack, electron:dist, electron:dist:linux, electron:dist:mac, electron:dist:win, electron:benchmark:runtime, electron:benchmark:win, electron:smoke-installed, electron:install-deps ``` Electron and npm package builds must include the `docs-web` runtime catalog. The dashboard Docs page fetches its collection and markdown through `/api/docs-web`, so installed desktop builds and npm-installed CLI/server runs need the same `docs-web` directory beside the compiled runtime root. -Electron runtime dependency preparation runs a production-only pnpm 11 install. The workspace `allowBuilds` policy approves only `onnxruntime-node`; preparation sets `ONNXRUNTIME_NODE_INSTALL=skip` because the CPU bindings used by Code UX are bundled and the upstream Linux default fetches optional CUDA/TensorRT binaries from NuGet. This keeps desktop packaging deterministic without suppressing the dependency postinstall or pnpm's build-policy check. Keep that allowlist narrow and review any addition as release-executed code. +Electron runtime dependency preparation runs a production-only pnpm 11 install with `--config.node-linker=hoisted` passed on the command line. A nested runtime `.npmrc` is not enough once pnpm discovers the enclosing workspace. Preparation rejects symbolic links, missing direct production packages, and failed MCP SDK/`zod` imports so Electron Builder cannot silently copy a broken peer-dependency layout. The workspace `allowBuilds` policy approves only `onnxruntime-node`; preparation sets `ONNXRUNTIME_NODE_INSTALL=skip` because the CPU bindings used by Code UX are bundled and the upstream Linux default fetches optional CUDA/TensorRT binaries from NuGet. This keeps desktop packaging deterministic without suppressing the dependency postinstall or pnpm's build-policy check. Keep that allowlist narrow and review any addition as release-executed code. They must also include `assets/models-dev/catalog.json`. The automatic token-pricing path reads this snapshot beside the compiled runtime; without it, known models can appear unpriced only in the desktop build. Electron packaging tests pin both runtime assets and a representative GPT-5.5 catalogue rate. diff --git a/docs-web/content/docs/developer-orchestration-debugging.mdx b/docs-web/content/docs/developer-orchestration-debugging.mdx index 94e629ba24..a77ee08410 100644 --- a/docs-web/content/docs/developer-orchestration-debugging.mdx +++ b/docs-web/content/docs/developer-orchestration-debugging.mdx @@ -13,6 +13,7 @@ Use this suite when a sprint stalls, local merges fail, worker-owned attention i | Full mockup pentest | `pnpm run test:orchestration:full` | Manual escalation for all deterministic mockup scenarios: smoke, CI repair, merge conflict, parallel DAG, dirty checkout, multi-project overrides. | | Large DAG stress | `pnpm run test:orchestration:large-dag` | Heavy 129-task mockup DAG with wide fan-out and layered joins. | | Full heavy pentest | `pnpm run test:orchestration:pentest` | Default mockup catalog plus heavy stress scenarios. | +| Extreme DAG recovery | `pnpm run test:orchestration:extreme-dag` | Local-only 400-task adversarial DAG, eight runtime restarts, QA context-cap validation, and resource ceilings; never runs in CI. | | Backend broadening | `pnpm run test:backend` | Full backend suite after focused fixes. | | Release validation | `pnpm run lint && pnpm run build` | Type safety and compiled server/dashboard output. | @@ -24,10 +25,12 @@ Run `pnpm run test:orchestration:ci-dag` for the Linux no-secret CI lane. It bui Run `pnpm run test:orchestration:ci-dag:electron` for the native macOS and Windows Electron lane. It launches `dist/electron/main.js`, waits for the embedded Code UX server, and runs the same QA DAG shape through a host-execution mockup fixture. -In GitHub Actions, `08 Orchestration` is one OS matrix: Linux runs the Docker-backed compiled-runtime DAG, and macOS/Windows run the Electron DAG with Electron binary install and native dependency rebuild exposed as separate steps. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. During orchestration, it streams redacted runtime stdout/stderr, emits `mockup_pentest_progress` records on sprint/task changes plus 15-second heartbeats, fails after `--stall-timeout-ms 180000` when no sprint, task status, merge, or expected-output state changes after polling starts, and writes a final Markdown table to `GITHUB_STEP_SUMMARY`. Progress records contain status counts and at most 32 changed tasks rather than repeating a wide DAG on every heartbeat; failures retain the full snapshot. Successful status reads are retained in a bounded trace without resetting the no-state-progress timer. Claimed tasks waiting on adaptive CPU/memory admission persist low-frequency wait events and dispatch heartbeats; the runner emits `mockup_pentest_provider_admission_wait` for diagnosis, but those heartbeats do not reset the no-state-progress stall timer. +In GitHub Actions, `08 Orchestration` is one OS matrix: Linux runs the Docker-backed compiled-runtime DAG, and macOS/Windows run the Electron DAG with Electron binary install and native dependency rebuild exposed as separate steps. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. During orchestration, it streams redacted runtime stdout/stderr, emits `mockup_pentest_progress` records on sprint/task changes plus 15-second heartbeats, fails after `--stall-timeout-ms 180000` when no sprint, task status, merge, expected-output state, or explicitly requested completed runtime restart changes after polling starts, and writes a final Markdown table to `GITHUB_STEP_SUMMARY`. Progress records contain status counts and at most 32 changed tasks rather than repeating a wide DAG on every heartbeat; failures retain the full snapshot. Successful status reads are retained in a bounded trace without resetting the no-state-progress timer. Finite completed restart events reset the watchdog so the configured interval measures post-recovery progress; admission heartbeats and failed restart attempts do not. Claimed tasks waiting on adaptive CPU/memory admission persist low-frequency wait events and dispatch heartbeats; the runner emits `mockup_pentest_provider_admission_wait` for diagnosis, but those heartbeats do not reset the no-state-progress stall timer. ## Local Branch Merge Drain +The local session snapshot used for watch-loop state matching excludes stored CLI prompts. Full prompts remain available through direct session and invocation reads, while large QA or planning payloads are not decoded into the server heap on every watch cycle. + The orchestrator performs a final branch-only merge drain immediately before rendering merge protocol instructions during an `orchestrate` cycle. This handles fast local CLI tasks that finish and push a worker branch after the earlier merge-gate snapshot but before protocol handling. If the CI DAG artifact shows a task stuck at `coding_completed` with `mergeIndicator: null`, a `cli_git_pushed` event, and a later `protocol_merge_required` event, inspect this final drain before increasing stall timeouts or rerunning blindly. The compact DAG's final validation command runs as a scenario-level assertion after all task branches have merged, not inside the final worker worktree. During polling, the runner also enforces the declared DAG: a task with dependencies may not leave `pending` until each dependency is marked merged. If a future task starts early, the runner emits `mockup_pentest_dependency_merge_violation` and fails the test run immediately. The runner does not treat a completed sprint as terminal for these scenarios until expected repository files are visible in the project checkout; while it waits, it emits `mockup_pentest_waiting_for_expected_output`, but only an actual expected-output readiness change refreshes the stall watchdog. This keeps native Electron runners from validating against a dependency branch before Windows has made the parent merge visible, while still failing within the configured stall timeout if a completed sprint never exposes the merged files. @@ -50,8 +53,23 @@ Run `pnpm run test:orchestration:full` manually when a scheduler, provider, CI, Run `pnpm run test:orchestration:large-dag` for a heavy 129-task DAG with 96 leaf tasks, 24 batch joins, 6 group joins, one final manifest, and one validation task. Use `pnpm run test:orchestration:pentest` for the default catalog plus heavy stress scenarios. +Run `pnpm run test:orchestration:extreme-dag` only as an explicit local pentest. It creates 400 deterministic tasks with wide, distant, diagonal, no-change, and long-tail dependency cases; runs task QA across every output-producing task including one changes-requested/follow-up/pass cycle; requires sprint QA and routed sprint-level CI repair; restarts the complete isolated runtime eight times; and enforces exact graph, merge, final-output, RSS, WAL, task-run, event, invocation, and 16-active-reservation scheduler invariants. The helper contract permits one project Git helper and at most 16 concurrent workspace sidecars; the restart contract caps failed task runs at 160 and final task-attempt amplification at 1.45, while the Docker contract requires zero non-running owner-scoped containers in the final sample. The sampler uses `docker ps -a`, so `created`, exited, and dead leaks are visible. p50 ceilings are 20 seconds for preparation, 10 seconds for Git finalization, and 35 seconds for the complete CLI workflow. Task QA contains full details only for the current task plus title-only completed siblings. Inert instruction padding pushes the unshortened sprint-QA context above 100,000 estimated tokens, every executable directive remains in the retained first half, and separate task/sprint prompt-size assertions prove both policies engaged. Restart-interrupted coding charges are refunded idempotently per task run, so the restart storm cannot exhaust a healthy task's coding guardrail. It is marked `localOnly`, so `all`, `pentest`, and CI never select it. Resource samples and final phase percentiles are retained in the project-run `resource-samples.json` artifact. + ## Local merge checklist +If a live provider invocation reports `container ... is not running` while an isolated restart test +is active, compare its timestamps with the test's restart events. Current builds label every managed +Docker asset with a state-home-derived runtime owner; startup cleanup and shutdown select only that +owner. Warm-helper retries invalidate only the failed generation and fall back to one-shot execution +if the replacement also stops. + +If Jules session creation returns HTTP `400` for several ready tasks, inspect the bounded provider +message on the execution invocation. Active-session/concurrency responses must leave the dispatch +queued and enter learned-cap backoff; source or branch validation responses remain failures with +their provider explanation. If restart marks a Jules task failed while Jules still reports it +active, compare the linked task run and invocation timestamps: startup recovery must restore remote +active truth before terminal local reconciliation. + 1. Confirm `/ready` is healthy. 2. Inspect `sprint_runs`, run events, and active `project_attention_items` in `~/.code-ux/app.db`. 3. Inspect the approved local test repository with `git status --short --branch` and `git log --oneline --decorate`. @@ -85,6 +103,8 @@ When a Docker-backed sprint appears capped but no matching provider containers a ## Memory profile +Wide LOCAL DAG cycles batch latest task-run lookup for Git-finalization evidence. When profiling, repeated status-derivation or merge-protocol passes should not issue one latest-run query for every task, including tasks that have not started. + After deterministic lanes pass, run a long profile against compiled-runtime mockup scenarios: ```bash diff --git a/docs-web/content/docs/developer-testing.mdx b/docs-web/content/docs/developer-testing.mdx index a9587c8c13..2956ff11f5 100644 --- a/docs-web/content/docs/developer-testing.mdx +++ b/docs-web/content/docs/developer-testing.mdx @@ -41,6 +41,8 @@ The isolated loopback runtime uses normal automatic local-file credential custod The release-install verifier installs the packed tarball with the upstream optional ONNX CUDA/TensorRT download disabled, then imports `onnxruntime-node` from the clean install to prove its bundled CPU native runtime loads. The npm package smoke therefore remains independent of NuGet availability while retaining native-runtime validation. +The explicit local-only `pnpm run test:orchestration:extreme-dag` lane runs a deterministic 400-task DAG with task and sprint QA, a QA coding follow-up, routed CI repair, eight full runtime restarts, and a hard 16-worker ceiling. It is intentionally excluded from CI and the normal pentest catalog. Its resource artifact enforces runtime RSS, application/session database and WAL growth, bounded failed-attempt amplification, zero final leaked non-running containers across all Docker states, separate task/sprint QA prompt policies, task-run/event/invocation growth, and exact terminal ordering. + `playwright.config.ts` keeps `testDir: './tests/e2e'` and defines purpose projects selected by directory glob: `navigation`, `settings`, `projects`, `tasks`, `agents`, and `config`. Add new E2E specs under `tests/e2e//` so suites can grow without editing the config. Use `pnpm exec playwright test --list` to confirm discovery, or `pnpm exec playwright test --project=tasks` to run one group. The `navigation` project includes Docs page smoke coverage for exactly five routes: `/docs`, the docs overview, and three representative user/developer/architecture pages. In GitHub Actions, `.github/workflows/playwright.yml` builds once per OS, uploads `dist/`, `dashboard/dist/`, and `.cache/tsc/` together as an OS-scoped artifact, then runs each purpose project in parallel against the restored build. A separate npm-package job packs the package, verifies its explicit bundled `.code-ux` allowlist excludes local logs, databases, and environment files, installs the tarball into a clean project, and runs the installed CLI help command independently of the source checkout. @@ -125,7 +127,7 @@ It is staged as: 3. `05 Backend`, `06 Dashboard`, `07 Package`, and `08 Orchestration`: run in parallel after those prerequisites. The orchestration matrix includes Linux Docker and macOS/Windows Electron on feature branches, `dev`, and `main`. 4. `09 Docs / five-page smoke`: loads the Docs index, its overview route, and three representative pages on Linux for every target branch. It fails on HTTP, console, or page errors without crawling all subpages. 5. `09 E2E`: full Playwright on Linux, macOS, and Windows only for `main` validation and manual dispatches. -6. `10 Release Candidate`: unsigned desktop release-candidate packages with `--publish never`, only for `main` validation and manual dispatches. +6. `10 Release Candidate`: unsigned desktop release-candidate packages with `--publish never`, only for `main` validation and manual dispatches. Every native row installs its finished `.deb`, NSIS `.exe`, or DMG app, starts the installed copy with isolated state, and requires packaged backend and renderer readiness plus a clean exit before upload. The main branch ruleset still includes historical context names from older CI numbering and matrix definitions. Compatibility aggregate jobs emit those names only after the corresponding current backend, dashboard, audit, package, orchestration, 18-shard E2E, or desktop release-candidate gate succeeds. They preserve branch-protection compatibility without replacing any current validation job and can be removed once a repository administrator cleans up the obsolete ruleset entries. diff --git a/docs-web/content/docs/developer-websocket-realtime.mdx b/docs-web/content/docs/developer-websocket-realtime.mdx index ab2b5f09d2..33841a8ca4 100644 --- a/docs-web/content/docs/developer-websocket-realtime.mdx +++ b/docs-web/content/docs/developer-websocket-realtime.mdx @@ -90,6 +90,19 @@ If the WebSocket connection cannot be established, consumers continue using thei - An invalid client frame produces `snapshot_required` with reason `invalid_client_message`; reconnect with a valid `set_subscriptions` payload. - If a requested resource no longer exists, its REST recovery request reports that condition and the consumer should remove the corresponding scope. +## Flow control and idle work + +The server checks subscription interest before assembling project execution, runtime, structure, +live, Git, overview, and project-collection payloads. With no interested client, those background +refreshes do not run their database loaders or assemble heavy frames. Lightweight in-memory +watermarks still advance so a disconnected client can detect missed invalidations on reconnect. + +All outbound paths—including replay, `subscribed`, `snapshot_required`, and ordinary event +frames—check the projected socket queue size. A client that cannot drain the bounded queue is +disconnected and should recover through the normal reconnect/snapshot flow. TCP keepalive also +helps remove abandoned peers; clients must treat disconnect as recoverable rather than relying on +an indefinitely buffered stream. + ## Sample session ```text diff --git a/docs-web/content/docs/operations-security-hardening.mdx b/docs-web/content/docs/operations-security-hardening.mdx index 54198f46c3..9ac70334ee 100644 --- a/docs-web/content/docs/operations-security-hardening.mdx +++ b/docs-web/content/docs/operations-security-hardening.mdx @@ -6,7 +6,9 @@ The proxy must remove client-supplied identity headers and inject its own princi Encrypted credential data has no plaintext fallback. Missing mounted/KMS/Vault key versions fail startup and `/ready`; `/health` remains useful for liveness. Backups are incomplete without the referenced key versions. Audit export is recursively secret-redacted but still restricted operational data. -Shared subprocess execution validates command names, arguments, stdin files, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, so stale ancestor markers cannot widen host bind mounts. `shell: false` prevents argument values from being reinterpreted as shell syntax. +Shared subprocess execution validates command names, arguments, stdin files, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, so stale ancestor markers cannot widen host bind mounts. Poolable Git commands use one runtime-owned warm helper per repository: repo-local worktrees share it, separate repositories remain isolated, stdin files stream through `docker exec -i`, and Git/auth environment applies only to each exec. Commands needing another host bind mount use a one-shot helper. `shell: false` prevents argument values from being reinterpreted as shell syntax. + +Docker provider launches stage selected environment values, provider argv, and generated provider configuration in restrictive temporary files. Oversized prompts for stdin-capable CLIs are streamed from a separate restrictive file through `docker run -i`, keeping the prompt out of both the host Docker command line and the container's final `execve` arguments while avoiding operating-system argument-size failures. See [Authenticated Headless Server Mode](/docs/operations-server-mode) for deployment, recovery, rotation, SLOs, and limitations. diff --git a/docs-web/content/docs/settings-database-settings.mdx b/docs-web/content/docs/settings-database-settings.mdx index b0384fc8f2..1c54912a82 100644 --- a/docs-web/content/docs/settings-database-settings.mdx +++ b/docs-web/content/docs/settings-database-settings.mdx @@ -25,8 +25,16 @@ and optional startup page reclaim releases a bounded amount of free SQLite space ## Recommended Configuration Keep pruning enabled. Leave startup page reclaim disabled unless bounded free-page reclamation is -useful for the local database. Provider work always takes priority: pruning, reclaim, and WAL -checkpointing are deferred while an invocation is running. +useful for the local database. Provider work always takes priority over pruning and page reclaim. +Passive WAL checkpoints still run during active provider work because they do not wait for readers +or writers; this bounds disk growth during continuously busy DAGs. Graceful shutdown performs a +final checkpoint and explicitly closes all runtime SQLite connections. + +The legacy `session-tracking.db` keeps provider lifecycle, branch, and activity projections. It +retains Jules prompts for hosted usage estimation, but does not copy local CLI prompts because the +durable invocation message history already stores those in `app.db`. This avoids a second large +prompt copy for wide DAG, QA, and CI-repair sessions. The schema upgrade clears legacy local prompt +copies once while preserving Jules prompts. A practical review flow is: diff --git a/docs-web/content/docs/settings-quality-assurance.mdx b/docs-web/content/docs/settings-quality-assurance.mdx index 8b8b1cd591..f9f9f77e20 100644 --- a/docs-web/content/docs/settings-quality-assurance.mdx +++ b/docs-web/content/docs/settings-quality-assurance.mdx @@ -43,6 +43,12 @@ Before applying changes, check: - Whether a project override is masking the system value you expected to change. - Whether a running sprint needs to be paused, restarted, or allowed to finish before the new value can be observed. +Task-level QA prompts contain full details only for the task under review: title, status, provider, worker branch, PR, dependencies, the complete unshortened prompt, and the latest eight activity entries without content truncation. Other sprint tasks appear only after they reach `completed`, and then only their titles are listed; unfinished siblings and all sibling instructions, metadata, and activity are omitted. Sprint-completion QA still receives every task because it reviews cross-task integration. When that full sprint context exceeds 100,000 estimated tokens (using the runtime's four-characters-per-token estimate), sprint QA receives the first half of every task instruction with an explicit notice while task metadata, ordering, and recent activity remain intact. + +During orchestration, QA reconciliation and initial merge-gate evaluation load the whole DAG's latest review cycles and attempt counts in a chunked batch. Review decisions, retry budgets, and fail-closed behavior are unchanged. + +Task QA runs in waves of at most four reviews per orchestration cycle, or a lower positive capacity when the providers routed to `qa_review` are configured more conservatively. The cycle settles that wave, merges ready branches, and starts newly unblocked coding before scheduling more reviews. Provider admission remains authoritative and may reduce effective concurrency further under host pressure. + ## Troubleshooting If the saved setting does not appear to take effect: diff --git a/docs-web/content/docs/settings-restart-behavior.mdx b/docs-web/content/docs/settings-restart-behavior.mdx index f3488c599a..9856efd444 100644 --- a/docs-web/content/docs/settings-restart-behavior.mdx +++ b/docs-web/content/docs/settings-restart-behavior.mdx @@ -15,7 +15,7 @@ Use it when you are configuring a new project, auditing inherited settings, or d Sprint policy continues, pauses, or cancels active sprints; invocation policy continues, cancels, or restarts interrupted work. -The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. +The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes sprint planning, task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. | Control Surface | Runtime Effect | Review Before Saving | | --- | --- | --- | @@ -28,12 +28,14 @@ The invocation policy applies to every provider-backed orchestration stage, not When `restartSprintPolicy = continue` and `restartInvocationPolicy = continue`, startup recovery: - resumes the existing sprint run and watch loop instead of creating a replacement sprint run +- preserves the complete sprint-planning request and its routing options, closes the process-bound invocation interrupted by shutdown, and continues the exact provider-native planning session in the preserved planning workspace. A missing recorded provider conversation fails closed rather than silently creating a new conversation. Only a request interrupted before provider linkage is reissued from its durable full prompt, because no provider session existed yet. - correlates each interrupted QA reviewer with its exact execution invocation, reviewer preset, logical provider session, and isolated review workspace - reuses the QA review workspace and provider conversation only while completing the same interrupted review cycle; verification after a decisive verdict starts from a fresh branch snapshot so it sees any coding follow-up - checkpoints every configured reviewer in a multi-reviewer cycle before invoking the first reviewer; recovery keeps completed verdicts, resumes only interrupted reviewers, and fills any reviewer row missing from a legacy partial cycle without spending another QA cycle - preserves task-level and sprint-completion `changes_requested` verdicts before starting their coding handoffs; if restart occurs between the verdict and the follow-up invocation, the next cycle resumes that pending handoff instead of leaving QA indefinitely blocked - returns an abruptly failed QA coding handoff to `CODING_COMPLETED`/`QA_PENDING` and retries it from the recorded coding session and workspace. A successful or reconciled handoff remains in that verification-ready state until the next QA review starts, preventing the restart window from launching unrelated coding work. Provider failures are bounded to three continuation attempts, while resuming a `running` checkpoint after a runtime restart does not consume another failure allowance; exhaustion then follows the configured QA exhaustion policy instead of redispatching the task as unrelated coding or heartbeating forever. - records the original worker-branch baseline before invoking a QA coding follow-up and reuses it after restart, so provider commits made before host-branch publication are still exported and published instead of being mistaken for an empty follow-up +- treats coding-provider completion as an intermediate checkpoint until Git finalization records a pushed branch or a verified no-change result. Task QA waits for that evidence. If restart interrupts this window, startup uses recovered-session membership (including hard kills with no shutdown event), preserves the workspace, requeues the premature terminal projection, and continues at Git finalization without calling the coding provider again. - reconciles the recovered coding task-run and dispatch after a successful handoff, preventing an earlier transient failure marker from incorrectly failing the sprint during terminal evaluation - requeues interrupted worker-owned CI-fix and merge-conflict attention, clearing ownership left by the stopped virtual worker - closes the stopped repair attempt's provider-usage row before requeueing it, so a hard restart cannot leave a stale invocation occupying the provider concurrency limit. A durable `workspace_finalized`, `host_publishing`, or `host_published` checkpoint proves that the provider returned successfully, so recovery records that attempt as completed; an attempt interrupted before that boundary is recorded as cancelled. diff --git a/docs-web/content/docs/user-dashboard-sprints.mdx b/docs-web/content/docs/user-dashboard-sprints.mdx index e046b2e7fb..5824ad2e3c 100644 --- a/docs-web/content/docs/user-dashboard-sprints.mdx +++ b/docs-web/content/docs/user-dashboard-sprints.mdx @@ -82,6 +82,8 @@ You can create a sprint from the Sprints page or directly from the top-bar sprin From the top bar, open the sprint selector and click **Add Sprint**. This compact flow asks only for a name and goal, creates an idle sprint, refreshes the sprint collection, and selects the new sprint. The selector stays enabled even when the project has no sprints so this action remains available for first-sprint creation. Click **Manage Sprints** in the same selector to open the full Sprints page. +Sprint action menus automatically flip at viewport edges and scroll internally when their full action list is taller than the available screen space, so every action remains reachable on short windows. + From the Sprints page: 1. Click **+ New sprint**. diff --git a/docs-web/content/docs/user-sprint-orchestration.mdx b/docs-web/content/docs/user-sprint-orchestration.mdx index f36cf9e622..5b11c3f931 100644 --- a/docs-web/content/docs/user-sprint-orchestration.mdx +++ b/docs-web/content/docs/user-sprint-orchestration.mdx @@ -116,12 +116,21 @@ Docker capacity. For CLI/Docker providers, Code UX counts both running provider invocations and running task runs when enforcing provider capacity. A task run can reserve orchestration capacity before its provider invocation row starts, so this prevents wide DAGs from creating hidden running backlogs while provider calls appear idle. +Before starting hosted work, Code UX also reads a fresh, coalesced Jules API preflight. Remote +`QUEUED`, `PLANNING`, and `IN_PROGRESS` work counts against execution capacity; sessions waiting for +approval/feedback or paused do not. The remaining local slots are claimed atomically, and an +unavailable preflight leaves the task queued. Because Jules has no dedicated slot-count endpoint and +its history is paginated, a provider capacity rejection remains authoritative and is retried after a +short backoff rather than failing the task. + Docker-backed task workspaces prepare independently. Code UX locks only the workspace being created or resumed, deduplicates exact remote-branch fetches, reuses short-lived targeted seed bundles for concurrent workspaces with identical ref tips, checks out the worker branch during the seed container, and caches the public helper image readiness check per process. Completed task patch export and host-side patch materialization are collapsed into single Git shell phases to avoid repeated helper-container startup and large argv transfers. On restart, interrupted local CLI task runs may be cancelled and redispatched, but their workspace volumes are preserved. If the coding provider had already finished, the resumed run continues with Git finalization from that workspace instead of invoking the coding agent again. Session sync treats finished local CLI task runs as terminal even if a stale cached session snapshot still reports the old session as running. With restart invocation policy `continue`, the same continuity applies to QA reviewers, CI-fix workers, and merge-conflict workers. Each resumed invocation keeps its logical session and preserved workspace, and continues the provider-native conversation when supported. QA reuses that review workspace only to finish the same interrupted review cycle; verification after a saved verdict starts from a fresh branch snapshot so it sees any coding follow-up. Code UX releases repair attention left claimed by the stopped virtual worker and returns it to the queue without consuming another repair attempt. +Interrupted sprint planning also resumes automatically under `continue`. Code UX preserves the complete request and options and continues the exact provider-native conversation from the stable planning workspace. It fails closed if that recorded conversation cannot be resumed instead of silently starting a fresh one. A request stopped before provider linkage can be reissued from the durable full prompt because no provider session existed yet. + Before a Docker snapshot is reused, Code UX verifies that it has a valid Git `HEAD`. If restart interrupted snapshot initialization after the volume was created, the runtime rebuilds that snapshot from the requested branch so resumed QA and provider work never starts from an empty workspace. QA fix handoffs are durable as well. Code UX records a requested-fix handoff before invoking the task's coding session. If restart occurs in that gap, the watch loop resumes the pending handoff; if the coding follow-up finished before its final QA update was written, the loop recognizes that execution and proceeds to verification instead of repeating the fix or leaving the task at `QA_PENDING`. diff --git a/docs-web/content/docs/user-troubleshooting.mdx b/docs-web/content/docs/user-troubleshooting.mdx index a1f754a423..1084b7e9f9 100644 --- a/docs-web/content/docs/user-troubleshooting.mdx +++ b/docs-web/content/docs/user-troubleshooting.mdx @@ -122,6 +122,21 @@ CLI-backed tasks refresh the remote branch before preparing the worker branch. T **Fix:** verify `git fetch origin ` works in the project repository and that the dashboard GitHub/GitLab token or local SSH setup can read the remote. Slow GitHub/GitLab smart HTTP connections may exceed short local timeouts; Code UX waits 120 seconds by default, and operators can raise it with `CODE_UX_GIT_FETCH_TIMEOUT_MS`. +### Jules session creation returns HTTP 400 + +Read the provider explanation stored on the failed/deferred invocation. If it reports an active +session or concurrency ceiling, current builds keep the task queued, release the provisional slot, +and retry after a short learned-cap backoff. If it reports an invalid source or starting branch, +repair that request instead. A runtime restart must preserve persisted Jules sessions that remain +active remotely; completed/cancelled sprints and merged tasks are not reopened. New dispatches first +count executing `QUEUED`, `PLANNING`, and `IN_PROGRESS` sessions from a fresh, coalesced Jules API +preflight and reserve visible remote capacity absent from local accounting. Waiting and paused history +does not consume that execution count. If the preflight is unavailable, the task remains queued. Since +Jules exposes no slot-count/state-filter endpoint and history is paginated, a create-time +`FAILED_PRECONDITION` is also treated as authoritative capacity and retried instead of failing the task. +If the bounded startup snapshot times out, locally durable Jules rows remain monitored until a late +snapshot or normal session sync confirms their state; the timeout alone must not fail the sprint. + ### CI autofix loops A `VirtualWorkerService` doing `ci_fix` tasks keeps trying and failing. @@ -160,7 +175,9 @@ For packaged Windows builds, Docker errors that show `C:\...` as a container `-- Current preview routing fixes **Forbidden: Untrusted host** without weakening application allowlists by presenting one coherent local upstream host boundary. Exit-code-137 previews report the actual termination and previously healthy previews receive one bounded recovery attempt. Docker commands require the explicit Docker Access setting and grant effective host-level control. -For packaged Windows builds, `spawn ENAMETOOLONG` during Docker provider launch indicates an outdated build or a launch path still passing a large prompt through the host command line. Current Docker provider runs mount provider arguments from a generated file so large prompts do not become `docker run` arguments. +For packaged Windows builds, `spawn ENAMETOOLONG` during Docker provider launch indicates an outdated build or a launch path still passing a large prompt through the host command line. A Linux container error such as `Argument list too long` has the same root cause at the container `execve` boundary. Current Docker provider runs mount provider arguments from a generated file and stream oversized prompts through stdin for supported CLIs, so large prompts become neither host `docker run` arguments nor reconstructed container arguments. + +A provider error containing `container ... is not running` can come from an older build whose startup or shutdown cleanup removed a warm Git/workspace helper owned by another Code UX runtime on the same Docker daemon. Current builds owner-scope managed Docker assets and retry a stopped helper with a one-shot fallback. Stop parallel older runtimes, update Code UX, restart once, and rerun the failed task; the original provider may never have been contacted. Chromium `tile_manager.cc` warnings about tile memory limits indicate renderer pressure. Current builds use an opaque desktop shell and GPU memory hints; hidden dashboard tabs also release animated WebGL and realtime resources, and the Nodes canvas uses a static background automatically. Returning to a tab reconnects realtime data and performs a fallback refresh. If warnings persist on older builds, switch to a lighter animation or set background mode to Static in Settings > Appearance. diff --git a/docs-web/developer/building-from-source.md b/docs-web/developer/building-from-source.md index b75e680c05..e1eebb106c 100644 --- a/docs-web/developer/building-from-source.md +++ b/docs-web/developer/building-from-source.md @@ -161,12 +161,12 @@ pnpm run audit # pnpm audit --audit-level=high pnpm run smoke-test # node dist/index.js --help pnpm run dev:server-only # boot just the server from source # Electron helper scripts: -# electron:generate-icons, electron:prepare-deps, electron:dev, electron:pack, electron:dist, electron:dist:linux, electron:dist:mac, electron:dist:win, electron:benchmark:runtime, electron:benchmark:win, electron:install-deps +# electron:generate-icons, electron:prepare-deps, electron:dev, electron:pack, electron:dist, electron:dist:linux, electron:dist:mac, electron:dist:win, electron:benchmark:runtime, electron:benchmark:win, electron:smoke-installed, electron:install-deps ``` Electron and npm package builds must include the `docs-web` runtime catalog. The dashboard Docs page fetches its collection and markdown through `/api/docs-web`, so installed desktop builds and npm-installed CLI/server runs need the same `docs-web` directory beside the compiled runtime root. -Electron runtime dependency preparation runs a production-only pnpm 11 install. The workspace `allowBuilds` policy approves only `onnxruntime-node`; preparation sets `ONNXRUNTIME_NODE_INSTALL=skip` because the CPU bindings used by Code UX are bundled and the upstream Linux default fetches optional CUDA/TensorRT binaries from NuGet. This keeps desktop packaging deterministic without suppressing the dependency postinstall or pnpm's build-policy check. Keep that allowlist narrow and review any addition as release-executed code. +Electron runtime dependency preparation runs a production-only pnpm 11 install with `--config.node-linker=hoisted` passed on the command line. A nested runtime `.npmrc` is not enough once pnpm discovers the enclosing workspace. Preparation rejects symbolic links, missing direct production packages, and failed MCP SDK/`zod` imports so Electron Builder cannot silently copy a broken peer-dependency layout. The workspace `allowBuilds` policy approves only `onnxruntime-node`; preparation sets `ONNXRUNTIME_NODE_INSTALL=skip` because the CPU bindings used by Code UX are bundled and the upstream Linux default fetches optional CUDA/TensorRT binaries from NuGet. This keeps desktop packaging deterministic without suppressing the dependency postinstall or pnpm's build-policy check. Keep that allowlist narrow and review any addition as release-executed code. They must also include `assets/models-dev/catalog.json`. The automatic token-pricing path reads this snapshot beside the compiled runtime; without it, known models can appear unpriced only in the desktop build. Electron packaging tests pin both runtime assets and a representative GPT-5.5 catalogue rate. diff --git a/docs-web/developer/orchestration-debugging.md b/docs-web/developer/orchestration-debugging.md index 94e629ba24..a77ee08410 100644 --- a/docs-web/developer/orchestration-debugging.md +++ b/docs-web/developer/orchestration-debugging.md @@ -13,6 +13,7 @@ Use this suite when a sprint stalls, local merges fail, worker-owned attention i | Full mockup pentest | `pnpm run test:orchestration:full` | Manual escalation for all deterministic mockup scenarios: smoke, CI repair, merge conflict, parallel DAG, dirty checkout, multi-project overrides. | | Large DAG stress | `pnpm run test:orchestration:large-dag` | Heavy 129-task mockup DAG with wide fan-out and layered joins. | | Full heavy pentest | `pnpm run test:orchestration:pentest` | Default mockup catalog plus heavy stress scenarios. | +| Extreme DAG recovery | `pnpm run test:orchestration:extreme-dag` | Local-only 400-task adversarial DAG, eight runtime restarts, QA context-cap validation, and resource ceilings; never runs in CI. | | Backend broadening | `pnpm run test:backend` | Full backend suite after focused fixes. | | Release validation | `pnpm run lint && pnpm run build` | Type safety and compiled server/dashboard output. | @@ -24,10 +25,12 @@ Run `pnpm run test:orchestration:ci-dag` for the Linux no-secret CI lane. It bui Run `pnpm run test:orchestration:ci-dag:electron` for the native macOS and Windows Electron lane. It launches `dist/electron/main.js`, waits for the embedded Code UX server, and runs the same QA DAG shape through a host-execution mockup fixture. -In GitHub Actions, `08 Orchestration` is one OS matrix: Linux runs the Docker-backed compiled-runtime DAG, and macOS/Windows run the Electron DAG with Electron binary install and native dependency rebuild exposed as separate steps. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. During orchestration, it streams redacted runtime stdout/stderr, emits `mockup_pentest_progress` records on sprint/task changes plus 15-second heartbeats, fails after `--stall-timeout-ms 180000` when no sprint, task status, merge, or expected-output state changes after polling starts, and writes a final Markdown table to `GITHUB_STEP_SUMMARY`. Progress records contain status counts and at most 32 changed tasks rather than repeating a wide DAG on every heartbeat; failures retain the full snapshot. Successful status reads are retained in a bounded trace without resetting the no-state-progress timer. Claimed tasks waiting on adaptive CPU/memory admission persist low-frequency wait events and dispatch heartbeats; the runner emits `mockup_pentest_provider_admission_wait` for diagnosis, but those heartbeats do not reset the no-state-progress stall timer. +In GitHub Actions, `08 Orchestration` is one OS matrix: Linux runs the Docker-backed compiled-runtime DAG, and macOS/Windows run the Electron DAG with Electron binary install and native dependency rebuild exposed as separate steps. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. During orchestration, it streams redacted runtime stdout/stderr, emits `mockup_pentest_progress` records on sprint/task changes plus 15-second heartbeats, fails after `--stall-timeout-ms 180000` when no sprint, task status, merge, expected-output state, or explicitly requested completed runtime restart changes after polling starts, and writes a final Markdown table to `GITHUB_STEP_SUMMARY`. Progress records contain status counts and at most 32 changed tasks rather than repeating a wide DAG on every heartbeat; failures retain the full snapshot. Successful status reads are retained in a bounded trace without resetting the no-state-progress timer. Finite completed restart events reset the watchdog so the configured interval measures post-recovery progress; admission heartbeats and failed restart attempts do not. Claimed tasks waiting on adaptive CPU/memory admission persist low-frequency wait events and dispatch heartbeats; the runner emits `mockup_pentest_provider_admission_wait` for diagnosis, but those heartbeats do not reset the no-state-progress stall timer. ## Local Branch Merge Drain +The local session snapshot used for watch-loop state matching excludes stored CLI prompts. Full prompts remain available through direct session and invocation reads, while large QA or planning payloads are not decoded into the server heap on every watch cycle. + The orchestrator performs a final branch-only merge drain immediately before rendering merge protocol instructions during an `orchestrate` cycle. This handles fast local CLI tasks that finish and push a worker branch after the earlier merge-gate snapshot but before protocol handling. If the CI DAG artifact shows a task stuck at `coding_completed` with `mergeIndicator: null`, a `cli_git_pushed` event, and a later `protocol_merge_required` event, inspect this final drain before increasing stall timeouts or rerunning blindly. The compact DAG's final validation command runs as a scenario-level assertion after all task branches have merged, not inside the final worker worktree. During polling, the runner also enforces the declared DAG: a task with dependencies may not leave `pending` until each dependency is marked merged. If a future task starts early, the runner emits `mockup_pentest_dependency_merge_violation` and fails the test run immediately. The runner does not treat a completed sprint as terminal for these scenarios until expected repository files are visible in the project checkout; while it waits, it emits `mockup_pentest_waiting_for_expected_output`, but only an actual expected-output readiness change refreshes the stall watchdog. This keeps native Electron runners from validating against a dependency branch before Windows has made the parent merge visible, while still failing within the configured stall timeout if a completed sprint never exposes the merged files. @@ -50,8 +53,23 @@ Run `pnpm run test:orchestration:full` manually when a scheduler, provider, CI, Run `pnpm run test:orchestration:large-dag` for a heavy 129-task DAG with 96 leaf tasks, 24 batch joins, 6 group joins, one final manifest, and one validation task. Use `pnpm run test:orchestration:pentest` for the default catalog plus heavy stress scenarios. +Run `pnpm run test:orchestration:extreme-dag` only as an explicit local pentest. It creates 400 deterministic tasks with wide, distant, diagonal, no-change, and long-tail dependency cases; runs task QA across every output-producing task including one changes-requested/follow-up/pass cycle; requires sprint QA and routed sprint-level CI repair; restarts the complete isolated runtime eight times; and enforces exact graph, merge, final-output, RSS, WAL, task-run, event, invocation, and 16-active-reservation scheduler invariants. The helper contract permits one project Git helper and at most 16 concurrent workspace sidecars; the restart contract caps failed task runs at 160 and final task-attempt amplification at 1.45, while the Docker contract requires zero non-running owner-scoped containers in the final sample. The sampler uses `docker ps -a`, so `created`, exited, and dead leaks are visible. p50 ceilings are 20 seconds for preparation, 10 seconds for Git finalization, and 35 seconds for the complete CLI workflow. Task QA contains full details only for the current task plus title-only completed siblings. Inert instruction padding pushes the unshortened sprint-QA context above 100,000 estimated tokens, every executable directive remains in the retained first half, and separate task/sprint prompt-size assertions prove both policies engaged. Restart-interrupted coding charges are refunded idempotently per task run, so the restart storm cannot exhaust a healthy task's coding guardrail. It is marked `localOnly`, so `all`, `pentest`, and CI never select it. Resource samples and final phase percentiles are retained in the project-run `resource-samples.json` artifact. + ## Local merge checklist +If a live provider invocation reports `container ... is not running` while an isolated restart test +is active, compare its timestamps with the test's restart events. Current builds label every managed +Docker asset with a state-home-derived runtime owner; startup cleanup and shutdown select only that +owner. Warm-helper retries invalidate only the failed generation and fall back to one-shot execution +if the replacement also stops. + +If Jules session creation returns HTTP `400` for several ready tasks, inspect the bounded provider +message on the execution invocation. Active-session/concurrency responses must leave the dispatch +queued and enter learned-cap backoff; source or branch validation responses remain failures with +their provider explanation. If restart marks a Jules task failed while Jules still reports it +active, compare the linked task run and invocation timestamps: startup recovery must restore remote +active truth before terminal local reconciliation. + 1. Confirm `/ready` is healthy. 2. Inspect `sprint_runs`, run events, and active `project_attention_items` in `~/.code-ux/app.db`. 3. Inspect the approved local test repository with `git status --short --branch` and `git log --oneline --decorate`. @@ -85,6 +103,8 @@ When a Docker-backed sprint appears capped but no matching provider containers a ## Memory profile +Wide LOCAL DAG cycles batch latest task-run lookup for Git-finalization evidence. When profiling, repeated status-derivation or merge-protocol passes should not issue one latest-run query for every task, including tasks that have not started. + After deterministic lanes pass, run a long profile against compiled-runtime mockup scenarios: ```bash diff --git a/docs-web/developer/testing.md b/docs-web/developer/testing.md index a9587c8c13..2956ff11f5 100644 --- a/docs-web/developer/testing.md +++ b/docs-web/developer/testing.md @@ -41,6 +41,8 @@ The isolated loopback runtime uses normal automatic local-file credential custod The release-install verifier installs the packed tarball with the upstream optional ONNX CUDA/TensorRT download disabled, then imports `onnxruntime-node` from the clean install to prove its bundled CPU native runtime loads. The npm package smoke therefore remains independent of NuGet availability while retaining native-runtime validation. +The explicit local-only `pnpm run test:orchestration:extreme-dag` lane runs a deterministic 400-task DAG with task and sprint QA, a QA coding follow-up, routed CI repair, eight full runtime restarts, and a hard 16-worker ceiling. It is intentionally excluded from CI and the normal pentest catalog. Its resource artifact enforces runtime RSS, application/session database and WAL growth, bounded failed-attempt amplification, zero final leaked non-running containers across all Docker states, separate task/sprint QA prompt policies, task-run/event/invocation growth, and exact terminal ordering. + `playwright.config.ts` keeps `testDir: './tests/e2e'` and defines purpose projects selected by directory glob: `navigation`, `settings`, `projects`, `tasks`, `agents`, and `config`. Add new E2E specs under `tests/e2e//` so suites can grow without editing the config. Use `pnpm exec playwright test --list` to confirm discovery, or `pnpm exec playwright test --project=tasks` to run one group. The `navigation` project includes Docs page smoke coverage for exactly five routes: `/docs`, the docs overview, and three representative user/developer/architecture pages. In GitHub Actions, `.github/workflows/playwright.yml` builds once per OS, uploads `dist/`, `dashboard/dist/`, and `.cache/tsc/` together as an OS-scoped artifact, then runs each purpose project in parallel against the restored build. A separate npm-package job packs the package, verifies its explicit bundled `.code-ux` allowlist excludes local logs, databases, and environment files, installs the tarball into a clean project, and runs the installed CLI help command independently of the source checkout. @@ -125,7 +127,7 @@ It is staged as: 3. `05 Backend`, `06 Dashboard`, `07 Package`, and `08 Orchestration`: run in parallel after those prerequisites. The orchestration matrix includes Linux Docker and macOS/Windows Electron on feature branches, `dev`, and `main`. 4. `09 Docs / five-page smoke`: loads the Docs index, its overview route, and three representative pages on Linux for every target branch. It fails on HTTP, console, or page errors without crawling all subpages. 5. `09 E2E`: full Playwright on Linux, macOS, and Windows only for `main` validation and manual dispatches. -6. `10 Release Candidate`: unsigned desktop release-candidate packages with `--publish never`, only for `main` validation and manual dispatches. +6. `10 Release Candidate`: unsigned desktop release-candidate packages with `--publish never`, only for `main` validation and manual dispatches. Every native row installs its finished `.deb`, NSIS `.exe`, or DMG app, starts the installed copy with isolated state, and requires packaged backend and renderer readiness plus a clean exit before upload. The main branch ruleset still includes historical context names from older CI numbering and matrix definitions. Compatibility aggregate jobs emit those names only after the corresponding current backend, dashboard, audit, package, orchestration, 18-shard E2E, or desktop release-candidate gate succeeds. They preserve branch-protection compatibility without replacing any current validation job and can be removed once a repository administrator cleans up the obsolete ruleset entries. diff --git a/docs-web/developer/websocket-realtime.md b/docs-web/developer/websocket-realtime.md index ab2b5f09d2..33841a8ca4 100644 --- a/docs-web/developer/websocket-realtime.md +++ b/docs-web/developer/websocket-realtime.md @@ -90,6 +90,19 @@ If the WebSocket connection cannot be established, consumers continue using thei - An invalid client frame produces `snapshot_required` with reason `invalid_client_message`; reconnect with a valid `set_subscriptions` payload. - If a requested resource no longer exists, its REST recovery request reports that condition and the consumer should remove the corresponding scope. +## Flow control and idle work + +The server checks subscription interest before assembling project execution, runtime, structure, +live, Git, overview, and project-collection payloads. With no interested client, those background +refreshes do not run their database loaders or assemble heavy frames. Lightweight in-memory +watermarks still advance so a disconnected client can detect missed invalidations on reconnect. + +All outbound paths—including replay, `subscribed`, `snapshot_required`, and ordinary event +frames—check the projected socket queue size. A client that cannot drain the bounded queue is +disconnected and should recover through the normal reconnect/snapshot flow. TCP keepalive also +helps remove abandoned peers; clients must treat disconnect as recoverable rather than relying on +an indefinitely buffered stream. + ## Sample session ```text diff --git a/docs-web/operations/security-hardening.md b/docs-web/operations/security-hardening.md index 0a38ce15a2..cfb7d2da90 100644 --- a/docs-web/operations/security-hardening.md +++ b/docs-web/operations/security-hardening.md @@ -6,7 +6,9 @@ The proxy must remove client-supplied identity headers and inject its own princi Encrypted credential data has no plaintext fallback. Missing mounted/KMS/Vault key versions fail startup and `/ready`; `/health` remains useful for liveness. Backups are incomplete without the referenced key versions. Audit export is recursively secret-redacted but still restricted operational data. -Shared subprocess execution validates command names, arguments, stdin files, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, so stale ancestor markers cannot widen host bind mounts. `shell: false` prevents argument values from being reinterpreted as shell syntax. +Shared subprocess execution validates command names, arguments, stdin files, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, so stale ancestor markers cannot widen host bind mounts. Poolable Git commands use one runtime-owned warm helper per repository: repo-local worktrees share it, separate repositories remain isolated, stdin files stream through `docker exec -i`, and Git/auth environment applies only to each exec. Commands needing another host bind mount use a one-shot helper. `shell: false` prevents argument values from being reinterpreted as shell syntax. + +Docker provider launches stage selected environment values, provider argv, and generated provider configuration in restrictive temporary files. Oversized prompts for stdin-capable CLIs are streamed from a separate restrictive file through `docker run -i`, keeping the prompt out of both the host Docker command line and the container's final `execve` arguments while avoiding operating-system argument-size failures. See [Authenticated Headless Server Mode](./server-mode.md) for deployment, recovery, rotation, SLOs, and limitations. diff --git a/docs-web/settings/database-settings.md b/docs-web/settings/database-settings.md index b0384fc8f2..1c54912a82 100644 --- a/docs-web/settings/database-settings.md +++ b/docs-web/settings/database-settings.md @@ -25,8 +25,16 @@ and optional startup page reclaim releases a bounded amount of free SQLite space ## Recommended Configuration Keep pruning enabled. Leave startup page reclaim disabled unless bounded free-page reclamation is -useful for the local database. Provider work always takes priority: pruning, reclaim, and WAL -checkpointing are deferred while an invocation is running. +useful for the local database. Provider work always takes priority over pruning and page reclaim. +Passive WAL checkpoints still run during active provider work because they do not wait for readers +or writers; this bounds disk growth during continuously busy DAGs. Graceful shutdown performs a +final checkpoint and explicitly closes all runtime SQLite connections. + +The legacy `session-tracking.db` keeps provider lifecycle, branch, and activity projections. It +retains Jules prompts for hosted usage estimation, but does not copy local CLI prompts because the +durable invocation message history already stores those in `app.db`. This avoids a second large +prompt copy for wide DAG, QA, and CI-repair sessions. The schema upgrade clears legacy local prompt +copies once while preserving Jules prompts. A practical review flow is: diff --git a/docs-web/settings/quality-assurance.md b/docs-web/settings/quality-assurance.md index 8b8b1cd591..f9f9f77e20 100644 --- a/docs-web/settings/quality-assurance.md +++ b/docs-web/settings/quality-assurance.md @@ -43,6 +43,12 @@ Before applying changes, check: - Whether a project override is masking the system value you expected to change. - Whether a running sprint needs to be paused, restarted, or allowed to finish before the new value can be observed. +Task-level QA prompts contain full details only for the task under review: title, status, provider, worker branch, PR, dependencies, the complete unshortened prompt, and the latest eight activity entries without content truncation. Other sprint tasks appear only after they reach `completed`, and then only their titles are listed; unfinished siblings and all sibling instructions, metadata, and activity are omitted. Sprint-completion QA still receives every task because it reviews cross-task integration. When that full sprint context exceeds 100,000 estimated tokens (using the runtime's four-characters-per-token estimate), sprint QA receives the first half of every task instruction with an explicit notice while task metadata, ordering, and recent activity remain intact. + +During orchestration, QA reconciliation and initial merge-gate evaluation load the whole DAG's latest review cycles and attempt counts in a chunked batch. Review decisions, retry budgets, and fail-closed behavior are unchanged. + +Task QA runs in waves of at most four reviews per orchestration cycle, or a lower positive capacity when the providers routed to `qa_review` are configured more conservatively. The cycle settles that wave, merges ready branches, and starts newly unblocked coding before scheduling more reviews. Provider admission remains authoritative and may reduce effective concurrency further under host pressure. + ## Troubleshooting If the saved setting does not appear to take effect: diff --git a/docs-web/settings/restart-behavior.md b/docs-web/settings/restart-behavior.md index f3488c599a..9856efd444 100644 --- a/docs-web/settings/restart-behavior.md +++ b/docs-web/settings/restart-behavior.md @@ -15,7 +15,7 @@ Use it when you are configuring a new project, auditing inherited settings, or d Sprint policy continues, pauses, or cancels active sprints; invocation policy continues, cancels, or restarts interrupted work. -The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. +The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes sprint planning, task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. | Control Surface | Runtime Effect | Review Before Saving | | --- | --- | --- | @@ -28,12 +28,14 @@ The invocation policy applies to every provider-backed orchestration stage, not When `restartSprintPolicy = continue` and `restartInvocationPolicy = continue`, startup recovery: - resumes the existing sprint run and watch loop instead of creating a replacement sprint run +- preserves the complete sprint-planning request and its routing options, closes the process-bound invocation interrupted by shutdown, and continues the exact provider-native planning session in the preserved planning workspace. A missing recorded provider conversation fails closed rather than silently creating a new conversation. Only a request interrupted before provider linkage is reissued from its durable full prompt, because no provider session existed yet. - correlates each interrupted QA reviewer with its exact execution invocation, reviewer preset, logical provider session, and isolated review workspace - reuses the QA review workspace and provider conversation only while completing the same interrupted review cycle; verification after a decisive verdict starts from a fresh branch snapshot so it sees any coding follow-up - checkpoints every configured reviewer in a multi-reviewer cycle before invoking the first reviewer; recovery keeps completed verdicts, resumes only interrupted reviewers, and fills any reviewer row missing from a legacy partial cycle without spending another QA cycle - preserves task-level and sprint-completion `changes_requested` verdicts before starting their coding handoffs; if restart occurs between the verdict and the follow-up invocation, the next cycle resumes that pending handoff instead of leaving QA indefinitely blocked - returns an abruptly failed QA coding handoff to `CODING_COMPLETED`/`QA_PENDING` and retries it from the recorded coding session and workspace. A successful or reconciled handoff remains in that verification-ready state until the next QA review starts, preventing the restart window from launching unrelated coding work. Provider failures are bounded to three continuation attempts, while resuming a `running` checkpoint after a runtime restart does not consume another failure allowance; exhaustion then follows the configured QA exhaustion policy instead of redispatching the task as unrelated coding or heartbeating forever. - records the original worker-branch baseline before invoking a QA coding follow-up and reuses it after restart, so provider commits made before host-branch publication are still exported and published instead of being mistaken for an empty follow-up +- treats coding-provider completion as an intermediate checkpoint until Git finalization records a pushed branch or a verified no-change result. Task QA waits for that evidence. If restart interrupts this window, startup uses recovered-session membership (including hard kills with no shutdown event), preserves the workspace, requeues the premature terminal projection, and continues at Git finalization without calling the coding provider again. - reconciles the recovered coding task-run and dispatch after a successful handoff, preventing an earlier transient failure marker from incorrectly failing the sprint during terminal evaluation - requeues interrupted worker-owned CI-fix and merge-conflict attention, clearing ownership left by the stopped virtual worker - closes the stopped repair attempt's provider-usage row before requeueing it, so a hard restart cannot leave a stale invocation occupying the provider concurrency limit. A durable `workspace_finalized`, `host_publishing`, or `host_published` checkpoint proves that the provider returned successfully, so recovery records that attempt as completed; an attempt interrupted before that boundary is recorded as cancelled. diff --git a/docs-web/user/dashboard/sprints.md b/docs-web/user/dashboard/sprints.md index 58846e47d5..5171245c8b 100644 --- a/docs-web/user/dashboard/sprints.md +++ b/docs-web/user/dashboard/sprints.md @@ -82,6 +82,8 @@ You can create a sprint from the Sprints page or directly from the top-bar sprin From the top bar, open the sprint selector and click **Add Sprint**. This compact flow asks only for a name and goal, creates an idle sprint, refreshes the sprint collection, and selects the new sprint. The selector stays enabled even when the project has no sprints so this action remains available for first-sprint creation. Click **Manage Sprints** in the same selector to open the full Sprints page. +Sprint action menus automatically flip at viewport edges and scroll internally when their full action list is taller than the available screen space, so every action remains reachable on short windows. + From the Sprints page: 1. Click **+ New sprint**. diff --git a/docs-web/user/sprint-orchestration.md b/docs-web/user/sprint-orchestration.md index 841346cd78..a39a1c346c 100644 --- a/docs-web/user/sprint-orchestration.md +++ b/docs-web/user/sprint-orchestration.md @@ -116,12 +116,21 @@ Docker capacity. For CLI/Docker providers, Code UX counts both running provider invocations and running task runs when enforcing provider capacity. A task run can reserve orchestration capacity before its provider invocation row starts, so this prevents wide DAGs from creating hidden running backlogs while provider calls appear idle. +Before starting hosted work, Code UX also reads a fresh, coalesced Jules API preflight. Remote +`QUEUED`, `PLANNING`, and `IN_PROGRESS` work counts against execution capacity; sessions waiting for +approval/feedback or paused do not. The remaining local slots are claimed atomically, and an +unavailable preflight leaves the task queued. Because Jules has no dedicated slot-count endpoint and +its history is paginated, a provider capacity rejection remains authoritative and is retried after a +short backoff rather than failing the task. + Docker-backed task workspaces prepare independently. Code UX locks only the workspace being created or resumed, deduplicates exact remote-branch fetches, reuses short-lived targeted seed bundles for concurrent workspaces with identical ref tips, checks out the worker branch during the seed container, and caches the public helper image readiness check per process. Completed task patch export and host-side patch materialization are collapsed into single Git shell phases to avoid repeated helper-container startup and large argv transfers. On restart, interrupted local CLI task runs may be cancelled and redispatched, but their workspace volumes are preserved. If the coding provider had already finished, the resumed run continues with Git finalization from that workspace instead of invoking the coding agent again. Session sync treats finished local CLI task runs as terminal even if a stale cached session snapshot still reports the old session as running. With restart invocation policy `continue`, the same continuity applies to QA reviewers, CI-fix workers, and merge-conflict workers. Each resumed invocation keeps its logical session and preserved workspace, and continues the provider-native conversation when supported. QA reuses that review workspace only to finish the same interrupted review cycle; verification after a saved verdict starts from a fresh branch snapshot so it sees any coding follow-up. Code UX releases repair attention left claimed by the stopped virtual worker and returns it to the queue without consuming another repair attempt. +Interrupted sprint planning also resumes automatically under `continue`. Code UX preserves the complete request and options and continues the exact provider-native conversation from the stable planning workspace. It fails closed if that recorded conversation cannot be resumed instead of silently starting a fresh one. A request stopped before provider linkage can be reissued from the durable full prompt because no provider session existed yet. + Before a Docker snapshot is reused, Code UX verifies that it has a valid Git `HEAD`. If restart interrupted snapshot initialization after the volume was created, the runtime rebuilds that snapshot from the requested branch so resumed QA and provider work never starts from an empty workspace. QA fix handoffs are durable as well. Code UX records a requested-fix handoff before invoking the task's coding session. If restart occurs in that gap, the watch loop resumes the pending handoff; if the coding follow-up finished before its final QA update was written, the loop recognizes that execution and proceeds to verification instead of repeating the fix or leaving the task at `QA_PENDING`. diff --git a/docs-web/user/troubleshooting.md b/docs-web/user/troubleshooting.md index bb00b92bbd..04134585d7 100644 --- a/docs-web/user/troubleshooting.md +++ b/docs-web/user/troubleshooting.md @@ -122,6 +122,21 @@ CLI-backed tasks refresh the remote branch before preparing the worker branch. T **Fix:** verify `git fetch origin ` works in the project repository and that the dashboard GitHub/GitLab token or local SSH setup can read the remote. Slow GitHub/GitLab smart HTTP connections may exceed short local timeouts; Code UX waits 120 seconds by default, and operators can raise it with `CODE_UX_GIT_FETCH_TIMEOUT_MS`. +### Jules session creation returns HTTP 400 + +Read the provider explanation stored on the failed/deferred invocation. If it reports an active +session or concurrency ceiling, current builds keep the task queued, release the provisional slot, +and retry after a short learned-cap backoff. If it reports an invalid source or starting branch, +repair that request instead. A runtime restart must preserve persisted Jules sessions that remain +active remotely; completed/cancelled sprints and merged tasks are not reopened. New dispatches first +count executing `QUEUED`, `PLANNING`, and `IN_PROGRESS` sessions from a fresh, coalesced Jules API +preflight and reserve visible remote capacity absent from local accounting. Waiting and paused history +does not consume that execution count. If the preflight is unavailable, the task remains queued. Since +Jules exposes no slot-count/state-filter endpoint and history is paginated, a create-time +`FAILED_PRECONDITION` is also treated as authoritative capacity and retried instead of failing the task. +If the bounded startup snapshot times out, locally durable Jules rows remain monitored until a late +snapshot or normal session sync confirms their state; the timeout alone must not fail the sprint. + ### CI autofix loops A `VirtualWorkerService` doing `ci_fix` tasks keeps trying and failing. @@ -160,7 +175,9 @@ For packaged Windows builds, Docker errors that show `C:\...` as a container `-- Current preview routing fixes **Forbidden: Untrusted host** without weakening application allowlists by presenting one coherent local upstream host boundary. Exit-code-137 previews report the actual termination and previously healthy previews receive one bounded recovery attempt. Docker commands require the explicit Docker Access setting and grant effective host-level control. -For packaged Windows builds, `spawn ENAMETOOLONG` during Docker provider launch indicates an outdated build or a launch path still passing a large prompt through the host command line. Current Docker provider runs mount provider arguments from a generated file so large prompts do not become `docker run` arguments. +For packaged Windows builds, `spawn ENAMETOOLONG` during Docker provider launch indicates an outdated build or a launch path still passing a large prompt through the host command line. A Linux container error such as `Argument list too long` has the same root cause at the container `execve` boundary. Current Docker provider runs mount provider arguments from a generated file and stream oversized prompts through stdin for supported CLIs, so large prompts become neither host `docker run` arguments nor reconstructed container arguments. + +A provider error containing `container ... is not running` can come from an older build whose startup or shutdown cleanup removed a warm Git/workspace helper owned by another Code UX runtime on the same Docker daemon. Current builds owner-scope managed Docker assets and retry a stopped helper with a one-shot fallback. Stop parallel older runtimes, update Code UX, restart once, and rerun the failed task; the original provider may never have been contacted. Chromium `tile_manager.cc` warnings about tile memory limits indicate renderer pressure. Current builds use an opaque desktop shell and GPU memory hints; hidden dashboard tabs also release animated WebGL and realtime resources, and the Nodes canvas uses a static background automatically. Returning to a tab reconnects realtime data and performs a fallback refresh. If warnings persist on older builds, switch to a lighter animation or set background mode to Static in Settings > Appearance. diff --git a/docs/architecture/code-quality-performance-contracts.md b/docs/architecture/code-quality-performance-contracts.md index 600d939092..b54c99ba1e 100644 --- a/docs/architecture/code-quality-performance-contracts.md +++ b/docs/architecture/code-quality-performance-contracts.md @@ -8,11 +8,12 @@ Use this page when changing hot paths that affect `/api/live`, `/api/execution`, | Contract | Keep this true | Primary owner files | Focused verification | | --- | --- | --- | --- | -| Bounded live snapshots | Live payload assembly must remain server-owned, semantically deduplicated, cached by explicit policy, and scoped to subscribed realtime channels. Do not add browser reconciliation or raw timestamp churn as a change signal. | `src/app/live/project-live-snapshot.ts`, `src/app/lifecycle/dashboard-snapshot-cache.ts`, `src/app/lifecycle/dashboard-snapshot-cache-policy.ts`, `src/services/dashboard-realtime-service.ts`, `src/repositories/dashboard-realtime-event-repository.ts`, `dashboard/src/lib/runtime-snapshot-stability.ts`, `dashboard/src/hooks/use-dashboard-runtime-data.ts` | `pnpm exec vitest run tests/backend/app/live/project-live-snapshot.test.ts tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts tests/dashboard/v2/lib/live-session-view-model.test.ts` | +| Bounded live snapshots | Live payload assembly must remain server-owned, semantically deduplicated, cached by explicit TTL and entry-count policy, and scoped to subscribed realtime channels. Expired parameterized snapshots must not remain reachable indefinitely. Do not add browser reconciliation or raw timestamp churn as a change signal. | `src/app/live/project-live-snapshot.ts`, `src/app/lifecycle/dashboard-snapshot-cache.ts`, `src/app/lifecycle/dashboard-snapshot-cache-policy.ts`, `src/services/dashboard-realtime-service.ts`, `src/repositories/dashboard-realtime-event-repository.ts`, `dashboard/src/lib/runtime-snapshot-stability.ts`, `dashboard/src/hooks/use-dashboard-runtime-data.ts` | `pnpm exec vitest run tests/backend/app/live/project-live-snapshot.test.ts tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts tests/dashboard/v2/lib/live-session-view-model.test.ts` | | Scoped execution caches | Execution snapshot enrichment must deduplicate task, sprint-run, dispatch, and invocation IDs before rollups and must not introduce per-task follow-up queries. Cache scope and TTL changes belong in the lifecycle cache policy. | `src/repositories/execution/project-execution-snapshot-query.ts`, `src/repositories/execution/execution-usage-query.ts`, `src/repositories/execution/execution-wall-time-query.ts`, `src/repositories/execution/execution-invocations-query.ts`, `src/app/lifecycle/dashboard-snapshot-cache-policy.ts` | `pnpm exec vitest run tests/backend/repositories/execution/project-execution-snapshot-query.test.ts tests/backend/repositories/execution/execution-usage-query.test.ts tests/backend/repositories/execution/execution-wall-time-query.test.ts` | -| Indexed projection slices | Execution snapshots must load bounded SQL slices for sprint runs, dispatches, runtime events, invocations, attention, usage, and wall time, then merge by stable identifiers in memory. Avoid all-history scans, unbounded `ORDER BY`, JSON-expression indexes, and repeated array filters in task loops. | `src/repositories/execution/execution-sprint-runs-query.ts`, `src/repositories/execution/execution-task-dispatches-query.ts`, `src/repositories/execution/execution-runtime-events-query.ts`, `src/repositories/execution/execution-invocations-query.ts`, `src/repositories/execution/execution-human-intervention-query.ts`, `src/repositories/db/app-db-schema.ts`, `src/repositories/db/app-db-migrations.ts` | `pnpm exec vitest run tests/backend/repositories/execution/execution-snapshot-slice-queries.test.ts tests/backend/repositories/execution/execution-task-dispatches-query.test.ts tests/backend/repositories/execution/project-execution-snapshot-query.test.ts` | +| Indexed projection slices | Execution snapshots and orchestration cycles must load bounded SQL slices, then merge by stable identifiers in memory. Wide-DAG event evidence uses one event-type-filtered, per-run-limited batch query and its covering index; never scan the same task-run history once per task or once per decision. Avoid all-history scans, unbounded `ORDER BY`, JSON-expression indexes, and repeated array filters in task loops. | `src/repositories/execution/execution-sprint-runs-query.ts`, `src/repositories/execution/execution-task-dispatches-query.ts`, `src/repositories/execution/execution-runtime-events-query.ts`, `src/repositories/execution-repository.ts`, `src/domain/sprint/orchestrator/cycle-runner.ts`, `src/domain/sprint/orchestrator/watch-loop-runner.ts`, `src/repositories/db/app-db-schema.ts`, `src/repositories/db/app-db-migrations.ts` | `pnpm exec vitest run tests/backend/repositories/execution/execution-snapshot-slice-queries.test.ts tests/backend/repositories/execution-repository.test.ts tests/backend/domain/sprint/orchestrator/cycle-runner.test.ts tests/backend/domain/sprint/orchestrator/watch-loop-runner.test.ts` | | Delta provider telemetry | Live telemetry must use cheap metadata before content, bounded byte-range transport where the source supports it, append-efficient process buffers, and suffix-only message reconciliation. Final post-process usage remains authoritative, and cumulative-session providers subtract baselines so resumed runs do not re-report earlier usage. | `src/infrastructure/providers/cli/provider-telemetry-watcher.ts`, `src/infrastructure/providers/cli/provider-transcript-chunks.ts`, `src/infrastructure/providers/cli/provider-runner.ts`, `src/infrastructure/providers/cli/provider-usage.ts`, `src/infrastructure/providers/cli/provider-logs/codex-log-parser.ts`, `src/shared/subprocess/bounded-text-buffer.ts`, `src/services/provider-execution-service.ts`, `src/repositories/execution/execution-invocation-writes.ts` | `pnpm exec vitest run tests/backend/infrastructure/providers/cli/provider-transcript-chunks.test.ts tests/backend/infrastructure/providers/cli/provider-telemetry-watcher.test.ts tests/backend/infrastructure/providers/cli/codex-log-parser.test.ts tests/backend/services/provider-execution-service.test.ts tests/backend/repositories/execution-repository.test.ts tests/backend/shared/subprocess/bounded-text-buffer.test.ts` | -| Bounded activity fetches | Session sync must plan the smallest set of active sessions, skip foreign or locally terminal work where possible, cap per-session activity pages, limit concurrency, and timeout provider reads without failing the whole sync loop. | `src/domain/sprint/session-sync/activity-fetch-plan.ts`, `src/domain/sprint/session-sync/activity-fetch-utils.ts`, `src/domain/sprint/session-sync/bounded-activity-fetch.ts`, `src/sprint/steps/session-sync-step.ts`, `src/server/activity-cache-service.ts` | `pnpm exec vitest run tests/backend/domain/sprint/session-sync/activity-fetch-plan.test.ts tests/backend/domain/sprint/session-sync/bounded-activity-fetch.test.ts tests/backend/sprint/session-sync-step.test.ts tests/backend/server/activity-cache-service.test.ts` | +| Bounded activity fetches | Session sync must plan the smallest set of active sessions, skip foreign or locally terminal work where possible, cap per-session activity pages, limit concurrency, and timeout provider reads without failing the whole sync loop. Persisted and projected activity text, plans, completion payloads, and caches must also have byte/character and entry bounds so legacy oversized rows cannot be retained across live snapshots. | `src/domain/sprint/session-sync/activity-fetch-plan.ts`, `src/domain/sprint/session-sync/activity-fetch-utils.ts`, `src/domain/sprint/session-sync/bounded-activity-fetch.ts`, `src/sprint/steps/session-sync-step.ts`, `src/server/activity-cache-service.ts`, `src/repositories/project-runtime/runtime-status-projection.ts` | `pnpm exec vitest run tests/backend/domain/sprint/session-sync/activity-fetch-plan.test.ts tests/backend/domain/sprint/session-sync/bounded-activity-fetch.test.ts tests/backend/sprint/session-sync-step.test.ts tests/backend/server/activity-cache-service.test.ts tests/backend/repositories/project-runtime/runtime-status-projection.test.ts` | +| Bounded runtime queues and WAL | Slow stderr/file consumers, subprocess stream lines, WebSocket clients, and continuously busy WAL files must remain bounded. Passive WAL checkpoints continue during provider work; pruning remains idle-only; graceful shutdown checkpoints and closes every SQLite connection. | `src/shared/logging/logger.ts`, `src/shared/subprocess/command-spawner-host.ts`, `src/server/dashboard-realtime-websocket-server.ts`, `src/services/database-maintenance-service.ts`, `src/server/code-ux-server.ts` | `pnpm exec vitest run tests/backend/shared/logging/logger.test.ts tests/backend/shared/subprocess/command-spawner-client.test.ts tests/backend/server/dashboard-realtime-websocket-server.test.ts tests/backend/services/database-maintenance-service.test.ts` | | Pure dashboard view models | v2 dashboard pages must build task, live runtime, and stats render models through pure helpers before JSX composition. Components should memoize scoped inputs and avoid rebuilding indexes, filter counts, task-card invocation feeds, or board columns inline during render. | `dashboard/src/v2/lib/live-session-view-model.ts`, `dashboard/src/v2/lib/tasks/task-board-view-model.ts`, `dashboard/src/v2/lib/task-board-state.ts`, `dashboard/src/v2/pages/stats/use-stats-page-data.ts`, `dashboard/src/v2/pages/stats/chart-view-models.ts`, `dashboard/src/v2/LiveSessionPage.tsx`, `dashboard/src/v2/TasksPage.tsx` | `pnpm exec vitest run tests/dashboard/v2/lib/live-session-view-model.test.ts tests/dashboard/lib/task-board-view-model.test.ts tests/dashboard/lib/task-board-state.test.ts tests/dashboard/v2/use-stats-page-data.test.tsx` | | Guardrail-backed regressions | Hot-path regressions must be enforced by typed tests and repository guardrails, not reviewer memory. Keep guardrail checks focused on durable risks: stale artifacts, broad `any`, unsafe dependency placeholders, realtime snapshot persistence, bounded execution runtime-event reads, duplicate optimistic insertion, and large duplicate implementation blocks. | `scripts/check-quality-guardrails.mjs`, `tests/backend/scripts/quality-guardrails.test.ts`, `src/shared/late-bound-dependency.ts`, `src/app/dependency-factory/dashboard-factory.ts`, `src/repositories/execution/execution-runtime-events-query.ts` | `pnpm run quality:guardrails`, `pnpm exec vitest run tests/backend/scripts/quality-guardrails.test.ts` | @@ -50,9 +51,10 @@ Related docs: [Execution Dashboard Projection](./execution-dashboard-projection. - Codex live and final transcript transport must retain its byte cursor, cap a chunk at 2 MiB, drain at most four chunks per poll, preserve split UTF-8/JSONL records, and reset safely on rotation or truncation. Do not replace the seek-based large-block `dd` transport with byte-at-a-time reads. -- Claude transport reads appended bytes, but its parser currently rebuilds the accumulated changed - transcript. Qwen mutable JSON records and the Antigravity SQLite source require coherent full reads - only after metadata changes. Unchanged polls for all three must remain metadata-only. +- Claude transport and parsing are append-only: retain the partial-line tail and parser state, process + each appended JSONL record once, and emit only the changed conversation suffix. Qwen mutable JSON + records and the Antigravity SQLite source require coherent full reads only after metadata changes. + Unchanged polls for all three must remain metadata-only. - Raw stdout/stderr capture must remain append-efficient and bounded. Snapshot at observation boundaries, and release chunk backing after finalization instead of repeatedly concatenating or hashing the full process lifetime output. @@ -85,6 +87,9 @@ Related docs: [Usage Telemetry And Stats](./usage-telemetry-and-stats.md), [Exec - Keep generic timeout, error metadata normalization, and ordered bounded mapping semantics in `activity-fetch-utils.ts`; Jules-specific activity shaping belongs in caller wrappers such as `bounded-activity-fetch.ts`. - Activity sync must preserve ordering after bounded concurrent fetches so task updates stay deterministic. - Dashboard activity cache changes must consume the shared bounded activity fetch utilities, preserve stale positive cache fallbacks on per-session fetch failures, keep short negative caching for real empty reads, and log structured timeout/error metadata without rejecting the whole live activity snapshot. +- Activity payload bounds apply on write and again on legacy read projection. The recent-activity + projection cache has both an entry cap and an estimated-character budget; do not replace either + with TTL-only retention. Related docs: [System Overview](./system-overview.md), [Project Runtime Integration](./project-runtime-integration.md), [Operations Runbook](../operations/runbook.md). diff --git a/docs/architecture/dashboard-realtime-foundation.md b/docs/architecture/dashboard-realtime-foundation.md index 0cccedb4fd..2345fa4150 100644 --- a/docs/architecture/dashboard-realtime-foundation.md +++ b/docs/architecture/dashboard-realtime-foundation.md @@ -187,7 +187,8 @@ Production refinement shipped on March 15, 2026: - project execution, runtime-status, and structure refresh scheduling now also fan into `project.live.updated`, so the Live page always receives a fresh combined snapshot after any committed runtime mutation - the server now performs a periodic background live-snapshot refresh for the selected project so git status and other slower-changing runtime metadata continue to stream even when no new task event is being written -- large live and git snapshot publishers check websocket subscription demand before running their loaders, so task churn does not assemble or serialize heavy frames when no tab is subscribed to `project::live` or `project::git` +- live, git, project execution, runtime-status, structure, overview, and project-collection publishers check websocket subscription demand before running their loaders, so task churn does not assemble or serialize heavy frames when no tab owns the corresponding scope; lightweight non-replayable watermarks still advance so disconnected clients can detect missed invalidations +- every outbound frame path, including replay, subscription acknowledgements, and recovery control frames, checks the projected socket queue size; clients that cross the queue ceiling are disconnected before Node retains another frame, and TCP keepalive detects abandoned peers ## What This Improves @@ -269,7 +270,7 @@ The live dashboard transport is designed to send updates as fast as mutations oc To measure current latency and payload sizes against a representative active-project fixture, run the benchmark harness: ```bash -node --loader ts-node/esm scripts/measure-live-snapshot.ts +node --import ./scripts/tsnode-register.mjs scripts/measure-live-snapshot.ts ``` This harness tracks: diff --git a/docs/architecture/execution-dashboard-controls.md b/docs/architecture/execution-dashboard-controls.md index 7b9ea99a90..2290740730 100644 --- a/docs/architecture/execution-dashboard-controls.md +++ b/docs/architecture/execution-dashboard-controls.md @@ -100,7 +100,7 @@ That startup pass is intentionally different from stale-runtime cleanup: - `queued` and `running` sprint runs are resumed in place (provided their associated sprint in the `sprints` table is still in the `running` status; if the sprint is no longer active, the run is finalized as failed), keeping the original `sprint_run` id instead of creating a fresh restart run - Code UX releases the orphaned in-process sprint lease from the old server process before reacquiring a fresh lease for the resumed watch loop - if corrupted state left more than one active `queued` or `running` run for the same sprint, Code UX resumes only the newest run and fails older duplicates as superseded -- interrupted local CLI task dispatches (`docker_cli`) are not treated as still running after process restart; they are rewritten to failed/retryable state so the resumed sprint loop can launch them again safely +- interrupted local CLI task dispatches (`docker_cli`) are not treated as still running after process restart; they are rewritten to failed/retryable state so the resumed sprint loop can launch them again safely. When a persisted session proves the coding attempt had already been charged, recovery applies one durable, task-run-keyed guardrail refund. Reprocessing the same restart is idempotent, and non-restart provider failures still consume their normal budget. - Docker workspace/runtime volumes for those tracked CLI sessions are preserved even after the session is marked `FAILED`, so the retry can bind to the old workspace when same-workspace retry is enabled - rerun recovery resolves the resume target from the latest `cli_workspace_bound` task event before falling back to older task-run metadata. This keeps the reusable workspace volume tied to the real workspace session id even when a restarted provider invocation recorded a newer local session id before being interrupted. - missing recorded-session recovery is provider-scoped: only Jules task sessions are checked against the Jules API and failed as missing remote sessions. Local CLI session ids such as `cli-codex-*` are not queried through Jules and are left to the CLI runtime/session-tracking recovery paths. diff --git a/docs/architecture/execution-invocation-tracking.md b/docs/architecture/execution-invocation-tracking.md index 53ca056167..f563bfe060 100644 --- a/docs/architecture/execution-invocation-tracking.md +++ b/docs/architecture/execution-invocation-tracking.md @@ -106,7 +106,9 @@ The 3D Chat cinematic feedback model is independent from the invocation selected `useCinematicInvocationFeedback` refreshes when the foreground invocation changes or its `messageCount`, `lastMessageAt`, or `updatedAt` summary changes. Same-invocation copy remains visible during a refresh, while project/invocation changes abort and generation-invalidate older requests. Terminal or missing invocations clear the projection immediately. Transcript fetch errors stay local and non-fatal; they do not replace the normal chat transcript or activate unrelated project work. -The Chat -> Invocations detail view exposes same-session recovery actions for failed or cancelled planning invocations. **Restart** preserves the original terminal transcript, creates a new invocation row, and resends the full planning prompt while passing the terminal provider row's native session id as `continueSessionId` (Claude Code uses `--resume `). **Continue** uses the same native-session resume path and asks the provider to finish the previous planning attempt, but the continuation prompt also embeds the original planning instructions so a provider fallback to a fresh session still has the full schema, sprint goal, and task-generation context. Docker-backed planning runs use a stable project/sprint snapshot workspace and preserve its paired provider runtime volume while the run is failed, cancelled, or incomplete. Restart and Continue reuse that workspace so provider-local session files remain available; fresh planning invocations in `REMOTE` git mode still refresh `origin` and build a new snapshot from `origin/`, using the explicit sprint feature branch when present or the effective runtime git default branch otherwise. Successful planning cleans up that workspace and paired runtime volume. The replacement invocation has its own provider usage trail; the terminal row remains immutable evidence of the quota/error/cancellation history. If Claude Code reports "No conversation found" during resume, Code UX retries once with a fresh Claude session and persists that fresh native session id rather than the rejected id. +The Chat -> Invocations detail view exposes same-session recovery actions for failed or cancelled planning invocations. **Restart** preserves the original terminal transcript, creates a new invocation row, and resends the full planning prompt while passing the terminal provider row's native session id as `continueSessionId` (Claude Code uses `--resume `). **Continue** uses the same native-session resume path and asks the provider to finish the previous planning attempt; the continuation prompt also embeds the complete original planning instructions. Docker-backed planning runs use a stable project/sprint snapshot workspace and preserve its paired provider runtime volume while the run is failed, cancelled, or incomplete. Restart and Continue reuse that workspace so provider-local session files remain available; they fail closed if the recorded provider conversation is missing and never silently replace it with a fresh planning conversation. Fresh planning invocations in `REMOTE` git mode still refresh `origin` and build a new snapshot from `origin/`, using the explicit sprint feature branch when present or the effective runtime git default branch otherwise. Successful planning cleans up that workspace and paired runtime volume. The replacement invocation has its own provider usage trail; the terminal row remains immutable evidence of the quota/error/cancellation history. + +Startup applies the same contract automatically when a sprint-planning process is interrupted. The original invocation is preserved, the complete request/options are loaded from its durable user-message metadata, and a correlated replacement continues the same logical/native session. A planning request interrupted before any provider row was linked may be reissued from that metadata because there was no previous provider conversation to replace. Running invocations can also be cancelled from the same detail header. Cancellation is available for every running invocation type, not just planning. The dashboard posts to `/api/execution/invocations/:invocationId/cancel`; the server requests any registered active dispatch to stop, finds Docker containers by the existing `code-ux.session-id` label from the linked provider/task runtime, kills those containers, marks the provider usage row `cancelled`, and appends a system cancellation message to the invocation transcript. Provider finalizers check the current invocation state before writing terminal status so a cancelled row is not overwritten by a late provider failure while the process unwinds. diff --git a/docs/architecture/high-concurrency-orchestration.md b/docs/architecture/high-concurrency-orchestration.md index 5ff2acf4db..f5b0e1db74 100644 --- a/docs/architecture/high-concurrency-orchestration.md +++ b/docs/architecture/high-concurrency-orchestration.md @@ -35,8 +35,36 @@ work is never killed; admission resumes as pressure falls. The policy does not call Docker. A rejected bounded claim may invoke stale-runtime reconciliation, but a claim with available capacity reaches the atomic SQLite boundary first. -Unchanged provider-cap deferrals are coalesced to one structured diagnostic per provider every ten -seconds, so a wide ready queue does not turn a one-second orchestration loop into a log-write loop. +Provider-cap deferrals are limited to one structured diagnostic per sprint run and provider every +ten seconds, even when the blocked queue changes. The throttle state lives on the long-lived cycle +runner and is bounded, so per-cycle child loggers and wide ready queues cannot create a log-write +loop or an unbounded diagnostics cache. + +Before creating a Jules session, Code UX reads a fresh, coalesced first page from `sessions.list` and +counts remote states that consume concurrent execution (`QUEUED`, `PLANNING`, and `IN_PROGRESS`). +Waiting-for-approval, waiting-for-feedback, and paused sessions do not consume this execution count; +established accounts can retain more of those historical waiting sessions than their concurrent-task +plan limit. Executing sessions visible to the API but absent from local runtime accounting reserve +part of the configured cap, and the remaining local slots are claimed atomically. Capacity +verification fails closed if the fresh preflight cannot be read. + +Jules does not expose a state-filtered list, subscription-slot counter, or atomic slot-reservation +endpoint. Its history is paginated and old queued work can occur beyond the bounded preflight page, +so the provider's create response remains authoritative for the unavoidable list/create and +pagination races. Explicit capacity `400`/`409`/exhausted `429` responses and the generic +`400 FAILED_PRECONDITION` currently emitted for a full subscription are retryable deferrals: Code UX +releases the provisional claim and applies a 30-second learned-cap backoff. `INVALID_ARGUMENT` and +other validation failures remain terminal with bounded provider detail. + +Persisted Jules sessions are durable across runtime restarts. Startup recovery preserves running +hosted invocations even when a stale local sprint projection is terminal, then compares the cached +Jules session snapshot and reactivates only remotely active, unmerged work. Completed or cancelled +sprints and human-owned QA failures are never reopened by this repair. The startup snapshot repair +has a five-second bound; a slow hosted API cannot hold readiness, and normal session sync completes +the work later. If that bound expires, locally persisted Jules session ids plus running external +provider rows are fail-safe evidence: startup keeps those rows and their sprint monitor alive until +the late snapshot or ordinary session sync verifies the remote state. A timeout therefore cannot +terminalize hosted work merely because the local sprint summary was already failed. ## Docker Inventory @@ -94,6 +122,47 @@ Runtime volume ownership uses a durable `.codeux-owner` marker and the actual vo The recursive ownership repair runs only for a new volume, an owner mismatch, or recovery from an externally recreated volume. Ordinary launches do not run `chown -R`. +Git control-plane work uses two bounded helper tiers. An active sprint holds a reference-counted +project lease keyed by the repository's Git common directory and runtime owner. The first eligible +Git command lazily starts one warm helper; concurrent sprints and worktrees for that project share +it, with at most four commands executing in parallel. The last active sprint drains pending and +in-flight commands and removes the helper. Projects without an active sprint use one-shot helpers, +so idle projects retain no container. Different repositories remain isolated, which also keeps +project-specific credentials separate. Credentials, Git environment, and stdin are attached only +to the individual `docker exec`; they are never retained in the helper. Commands requiring an extra +host bind mount remain one-shot. After runtime shutdown begins, late Git commands also use the +containerized one-shot path so they cannot recreate a persistent generation after the warm pool has +been drained. + +Docker-volume workspaces reuse one short-lived sidecar per active workspace/runtime-volume pair for +local checkout, inspection, export, and bundle bootstrap commands. The sidecar is Git-capable, +network-disabled, `no-new-privileges`, and receives only the explicit Git environment allowlist. +Its transient Git home is a 1 MiB tmpfs. Coding and QA workflows reserve the exact +workspace/runtime-volume pair for their complete prepare-through-finalize lifetime, so the pool +cannot evict a sidecar merely because that workflow is temporarily between commands. Release drains +in-flight work before removing the sidecar without deleting restartable volumes. +Network Git commands use an isolated one-shot container. The workspace pool admits at most 16 +sidecars: a new workspace evicts the least-recently-used unreserved idle generation, or waits when +every slot is executing or reserved. Helper creation and removal share a four-operation Docker +control-plane limit. This keeps overlapping task-QA and coding waves inside the same resource bound +without creating `created`/`dead` container storms. Otherwise-idle, unreserved sidecars expire after +30 seconds, and shutdown drains both helper pools before owner-scoped Docker cleanup. Fresh helper +creation runs first without a speculative remove; only an explicit Docker container-name conflict +reclaims the deterministic name and retries once. +The shutdown sequence signals every active dispatch before beginning that drain, so provider and +workspace commands can release their helper leases concurrently instead of making restart latency +depend on their natural completion. Small bounded-parallel removal batches then reconcile every +remaining owner-scoped container state without exceeding the Docker command deadline during a +full admission wave. Initial cleanup, preview-reconciliation, and live-snapshot callbacks share the +server's tracked startup-timer set; shutdown clears that set before SQLite checkpoint/close, and +periodic callbacks refuse new repository work once closing begins. Fast restarts therefore cannot +leave a delayed loop callback querying closed storage. + +Wide LOCAL merge drains resolve each worker/feature relationship once, then reuse the detached +merger's last published target SHA. Every publication remains a compare-and-swap update; only a CAS +failure rereads and resets to the concurrently advanced target. Merge history stays serial and +conflict attribution remains per task without repeating ref-existence scans for every branch. + Provider container names are reclaimed only after Docker reports a real name conflict. The normal launch path no longer runs a speculative `docker rm`. @@ -134,11 +203,15 @@ filesystem traversal. Runtime directory listing, age checks, and recursive remov filesystem operations with an eight-operation bound, so stale-path cleanup does not synchronously block container launch, telemetry, or dashboard work on the Node.js event loop. -Startup Docker asset cleanup is also single-flight. Helper and login containers are removed before -workspace volumes so mounted volumes retain the existing safety ordering. After that prerequisite, -workspace, provider-tool, and browser-volume pipelines run independently. Docker inspection and -removal use batches of at most 50 with at most four cleanup commands active at once; a failed batch -falls back to bounded per-item work instead of a serial control-plane loop. +Startup Docker asset cleanup is also single-flight. Helper containers and owner-scoped provider +containers are removed before recovery and workspace-volume pruning. Provider cleanup includes +running, exited, dead, and never-started `created` generations because a local Docker client cannot +be reattached after process loss and `docker run --rm` cannot remove a container that never started. +Shutdown likewise lists all states and force-removes owner-scoped containers; concurrent +disappearance is an idempotent success. After that prerequisite, workspace, provider-tool, and +browser-volume pipelines run independently. Docker inspection and removal use batches of at most 50 +with at most four cleanup commands active at once; a failed batch falls back to bounded per-item +work instead of a serial control-plane loop. Tracked-session snapshots, the ten-minute new-workspace grace period, active managed-volume state, the newest-two cache generations, and the 30-day managed-volume retention window are unchanged. @@ -166,6 +239,10 @@ while older files that predate that mode may safely no-op until an explicit offl - Independent queued dispatches within one project fan out up to both `workers.maxConcurrency` and effective provider capacity. +- Each cycle starts no more work than the provider admission service's current purpose-aware + capacity, including adaptive reply reservations; configured capacity is only an upper bound. +- Task QA runs in waves of at most four reviews. The cycle merges settled work and starts newly + unblocked coding before scheduling another QA wave. - Provider claims remain atomic across every project and runtime process. - Interactive work cannot exceed an explicit positive provider cap. - Existing providers are not killed in response to pressure. @@ -176,6 +253,12 @@ while older files that predate that mode may safely no-op until an explicit offl overlap their own previous interval. - Periodic runtime and startup Docker cleanup cannot overlap themselves, block the event loop with synchronous recursive filesystem work, or issue unbounded Docker commands. +- Every managed Docker container and volume carries a state-home-derived `code-ux.runtime-owner` + label. Startup pruning, preview/file-browser reconciliation, login cleanup, and shutdown select + that owner before removing assets, so an isolated stress-test runtime sharing the Docker daemon + cannot stop a live runtime. Warm Git/workspace helper names include the same owner identity; + generation-aware invalidation preserves a concurrent replacement and a second stopped-helper + result falls back to a one-shot command instead of failing the provider invocation. ## Focused Verification diff --git a/docs/architecture/usage-telemetry-and-stats.md b/docs/architecture/usage-telemetry-and-stats.md index 74964e0c1a..e7a97812e3 100644 --- a/docs/architecture/usage-telemetry-and-stats.md +++ b/docs/architecture/usage-telemetry-and-stats.md @@ -146,6 +146,12 @@ If usage is absent or totals are zero, Code UX falls back to token estimation us Host runs read the active session under `~/.claude/projects`; Docker-backed runs read the same JSONL contract from the paired provider runtime volume mounted at `/code-ux-runtime-home` before the Docker workspace and runtime volumes are cleaned up. +Live Claude polling uses `ClaudeCodeLogAccumulator`: it preserves the incomplete trailing line, +parses only bytes appended since the previous poll, replaces duplicate message-id snapshots in +place, and reports a stable conversation revision plus the changed turn suffix. It does not join +all prior chunks and reparse the entire session on every poll. Final collection uses the same parser +contract, while reported usage remains authoritative over estimates. + ### Antigravity Antigravity runs with `agy` CLI commands. diff --git a/docs/dashboard/design-system-sprints.md b/docs/dashboard/design-system-sprints.md index 20b8d65038..e192218df6 100644 --- a/docs/dashboard/design-system-sprints.md +++ b/docs/dashboard/design-system-sprints.md @@ -23,7 +23,7 @@ This document outlines the design system for the Sprints page and related planni * **Localization boundary:** Ledger headings, filters, sort descriptions, selection feedback, bulk actions, pending reasons, status labels, mobile cell labels, and live announcements come from the Sprints message catalog. Names, goals, issue data, Git metadata, review results, and runtime content must never be translated. Counts, dates, times, percentages, and affected-sprint lists use the dashboard `Intl` formatters while sorting continues to use the underlying values. -* **Responsive Ledger Pattern:** Uses the shared `Table` contract with `mobileLabel` mapping for narrow screens. Rows stack gracefully, ensuring that sprint name, status, completion, dates, selection, pin state, and row controls remain discoverable without horizontal scrolling. Row controls and bulk action bars use flexible, touch-friendly layouts that wrap cleanly to prevent text clipping. Fixed-position inline menus (like row actions) use viewport-clamping to remain usable near screen edges. +* **Responsive Ledger Pattern:** Uses the shared `Table` contract with `mobileLabel` mapping for narrow screens. Rows stack gracefully, ensuring that sprint name, status, completion, dates, selection, pin state, and row controls remain discoverable without horizontal scrolling. Row controls and bulk action bars use flexible, touch-friendly layouts that wrap cleanly to prevent text clipping. Fixed-position inline menus (like row actions) position against their intrinsic content dimensions, update after trigger or menu resizing, prefer the larger viewport region when both sides fit, flip at viewport edges, and scroll internally when the full action list exceeds the available height. * **Rows & Headers:** Refined row heights, consistent padding, and clear separators. Column headers must align perfectly with their corresponding data. * **Metadata Hierarchy:** Prioritize sprint names and status. Secondary metadata (dates, task counts) should be styled as supporting information (e.g., smaller text, muted colors). * **Interactive Elements:** Row action menus and bulk actions should have clear active/hover states, unified menu padding, and consistent icon scaling. diff --git a/docs/deployment/electron-desktop.md b/docs/deployment/electron-desktop.md index 26a2748cc0..fe06d6603a 100644 --- a/docs/deployment/electron-desktop.md +++ b/docs/deployment/electron-desktop.md @@ -67,11 +67,12 @@ macOS DMG builds include the MIT license resource through `build/license_en.txt` - `pnpm run electron:dist:win`: build Windows targets. - `pnpm run electron:benchmark:runtime`: launch Electron with an isolated temporary user profile, navigate dashboard routes, probe backend endpoints, and write route/API/renderer/runtime metrics under `.cache/electron-runtime-benchmark/`. - `pnpm run electron:benchmark:win`: build Windows installers with `normal` and `store` compression and write timing/size data to `release/electron-benchmark/summary.json`. +- `pnpm run electron:smoke-installed`: select the native package in `release/electron/` whose artifact name matches the current `package.json` version, install it, start that installed app with an isolated profile, wait until its backend and renderer are ready, and require a clean exit. Linux requires passwordless `sudo` and `xvfb-run`, as provided by the release runners. - `pnpm run electron:install-deps`: rebuild native app dependencies for Electron. The release output is written to `release/electron/`. -Electron package builds run `pnpm run electron:prepare-deps` before Electron Builder. That script creates a production-only, hoisted runtime dependency tree in `.cache/electron-runtime/node_modules`, prunes non-runtime package files, generates deterministic PNG/ICO/BMP desktop artwork, and Electron Builder copies it to `resources/node_modules` so ASAR-packaged builds can resolve pnpm transitive dependencies at runtime. +Electron package builds run `pnpm run electron:prepare-deps` before Electron Builder. That script creates a production-only, hoisted runtime dependency tree in `.cache/electron-runtime/node_modules`, prunes non-runtime package files, generates deterministic PNG/ICO/BMP desktop artwork, and Electron Builder copies it to `resources/node_modules` so ASAR-packaged builds can resolve pnpm transitive dependencies at runtime. The pnpm invocation passes `--config.node-linker=hoisted` directly; a nested runtime `.npmrc` is not sufficient when pnpm 11 discovers the enclosing workspace. Preparation rejects symbolic-link layouts, missing direct production packages, and failed MCP SDK, `dotenv`, or `zod` imports before accepting or fingerprinting the tree. `zod` remains a direct production dependency because the MCP SDK consumes it as a runtime peer as well as a transitive dependency. pnpm 11 treats ignored dependency build scripts as an error. The workspace `allowBuilds` policy therefore approves only `onnxruntime-node`. Electron runtime preparation explicitly sets `ONNXRUNTIME_NODE_INSTALL=skip`: the CPU native bindings used by Code UX are already bundled, while the upstream default on Linux downloads optional CUDA/TensorRT binaries from NuGet. This keeps desktop packaging deterministic and offline from that optional feed without suppressing the dependency postinstall or its pnpm policy check. Keep the allowlist narrow; adding another package requires confirming that its build script is necessary for the packaged runtime and safe on every release runner. @@ -87,7 +88,7 @@ Speech transcription uses the same packaged backend route as the npm-served dash Local speech models are user-cache data, not application bundle data. The service resolves them under `~/.code-ux/models/speech//`, where the default `onnx-community/whisper-base.en` becomes `onnx-community--whisper-base.en` and contains an encoder, merged decoder, tokenizer, preprocessing metadata, and generation metadata. Whisper Tiny uses the same bundle layout as a faster, lower-footprint alternative. Missing model files produce a structured `missing_local_model` or setup `client_error`; the desktop package should not bundle model weights by default because they are large and user-replaceable. Local is the default provider mode and never sends audio externally. An OpenAI-compatible endpoint is used only when API mode is selected and its base URL, API key, and model are configured. -The runtime dependency tree is fingerprinted from production dependencies and the lockfile. If the fingerprint matches a previous run, `electron:prepare-deps` reuses the existing tree instead of deleting and reinstalling it. +The runtime dependency tree is fingerprinted from production dependencies and the lockfile. If the fingerprint matches a previous run, `electron:prepare-deps` revalidates the copy-safe layout and runtime imports before reusing the existing tree instead of deleting and reinstalling it. Dashboard-only libraries belong in `devDependencies` because Vite bundles them into `dashboard/dist/`; keeping them out of production dependencies prevents Electron packages from copying unused source packages into `resources/node_modules`. @@ -128,7 +129,7 @@ Use `.github/workflows/release.yml` for published desktop releases. It is the la The no-secret release-candidate package lane is part of `.github/workflows/ci.yml`, named `Code UX CI Pipeline`. It runs for `main` validation and manual dispatches after package smoke, keeping the full desktop package proof out of the routine `dev` lane. -The `10 Release Candidate / desktop package` matrix starts as soon as the package smoke job passes, so desktop packaging can run beside the E2E and orchestration matrices instead of waiting for them to finish. It downloads the shared `codeux-build-linux` artifact, installs the cached Electron binary, rebuilds Electron native dependencies, prepares runtime assets, and runs Electron Builder directly with `--linux`, `--mac`, or `--win` plus `--publish never`. The package smoke job that precedes it runs `node scripts/verify-release-install.mjs` with `CODE_UX_SKIP_RELEASE_INSTALL_BUILD=1`, so the npm tarball install check uses the same compiled artifact instead of rebuilding. +The `10 Release Candidate / desktop package` matrix starts as soon as the package smoke job passes, so desktop packaging can run beside the E2E and orchestration matrices instead of waiting for them to finish. It downloads the shared `codeux-build-linux` artifact, installs the cached Electron binary, rebuilds Electron native dependencies, prepares runtime assets, and runs Electron Builder directly with `--linux`, `--mac`, or `--win` plus `--publish never`. After compilation, every native runner installs its candidate—the Linux `.deb`, Windows NSIS `.exe`, or macOS app copied from the `.dmg`—and starts that installed copy with an isolated home and dashboard port. Success requires the packaged backend to start, the dashboard renderer to finish loading, an atomic readiness marker to be written, and the app to shut down cleanly. The package smoke job that precedes it runs `node scripts/verify-release-install.mjs` with `CODE_UX_SKIP_RELEASE_INSTALL_BUILD=1`, so the npm tarball install check uses the same compiled artifact instead of rebuilding. Release-candidate packaging sets `CSC_IDENTITY_AUTO_DISCOVERY=false` for unsigned Electron packaging and passes `--publish never` to Electron Builder. It does not require provider API keys, npm publishing credentials, Docker credentials, GitHub Release events, or real project state. When Electron output exists, the workflow uploads files from `release/electron/` as workflow artifacts only; it does not publish to npm or attach files to a GitHub Release. @@ -141,6 +142,7 @@ pnpm run build node scripts/verify-release-install.mjs pnpm run electron:install-deps pnpm run electron:dist -- --publish never +pnpm run electron:smoke-installed ``` Use `pnpm run electron:dist:linux -- --publish never`, `pnpm run electron:dist:mac -- --publish never`, or `pnpm run electron:dist:win -- --publish never` when matching a specific GitHub Actions matrix leg. diff --git a/docs/development/mockup-sprint-pentest-scenarios.md b/docs/development/mockup-sprint-pentest-scenarios.md index 0766ab7c20..b33e87fedd 100644 --- a/docs/development/mockup-sprint-pentest-scenarios.md +++ b/docs/development/mockup-sprint-pentest-scenarios.md @@ -6,7 +6,7 @@ The catalog exports: -- `SCENARIOS`: deterministic scenario definitions covering smoke completion, CI repair, merge-conflict DAG execution, parallel independent tasks, dirty-checkout finalization, sprint-completion merge conflict repair, multi-project settings overrides, the CI-sized QA DAG, and the heavy 129-task large-DAG stress run. +- `SCENARIOS`: deterministic scenario definitions covering smoke completion, CI repair, merge-conflict DAG execution, parallel independent tasks, dirty-checkout finalization, sprint-completion merge conflict repair, multi-project settings overrides, the CI-sized QA DAG, the heavy 129-task large-DAG stress run, and an explicit local-only 400-task restart/recovery pentest with task/sprint QA and routed CI repair. - `PROJECT_FIXTURES`: generic local project fixtures with different `featureBranchPrefix`, watch-loop intervals, QA settings, provider concurrency caps, and host/Docker execution modes. - `TERMINAL_TASK_STATUSES`, `SUCCESS_TASK_STATUSES`, and `TERMINAL_SPRINT_STATUSES`: shared status sets for runners. - `getScenario()`, `listScenarioIds()`, `getProjectFixture()`, `listProjectFixtureIds()`, and `getScenarioProjectRuns()`: lookup helpers for runner scripts. @@ -31,11 +31,11 @@ Scenarios may also define orchestration hooks: - `beforeOrchestration.dirtyFiles` and `beforeOrchestration.commands` mutate the visible fixture checkout before the sprint starts. This is used by dirty-checkout finalization coverage. - `duringOrchestration.defaultBranchMutations` waits until the orchestrator has created a non-default branch, then commits deterministic changes to the default branch through a detached temporary worktree. This is used by sprint-completion merge conflict coverage without touching live projects. -Artifacts are written under `.cache/e2e-mockup-sprint-pentest//`, including server logs, per-project-run summaries, task status snapshots, and a top-level `summary.json`. The runner redacts token-like values before writing logs or summaries even though it does not require provider credentials. +Artifacts are written under `.cache/e2e-mockup-sprint-pentest//`, including server logs, per-project-run summaries, task status snapshots, optional `resource-samples.json`, and a top-level `summary.json`. The runner redacts token-like values before writing logs or summaries even though it does not require provider credentials. Resource-enabled scenarios sample compiled-runtime RSS, application and session SQLite database/WAL size, task-run rows, task-run events, running task runs, active dispatch-backed task-run reservations (`PENDING`, `RUNNING`, or `PAUSED`), provider invocations, and the largest task- and sprint-level QA provider prompts every five seconds, then fail when declared resource bounds are violated. The extreme DAG bounds compact task QA separately from the integration-wide sprint QA context. Pending status-projection rows without a dispatch are deliberately excluded because they reserve no runtime work. When `CODE_UX_E2E_SERVER_NODE_OPTIONS` is set, the runner forwards it as `NODE_OPTIONS` only to the isolated compiled-runtime server child. Use this to capture Node CPU or heap profiles for stress scenarios without profiling the parent runner or a live Code UX process. Restart stress runs should omit explicit `--cpu-prof-name` and `--heap-prof-name` values so every restarted server child writes separate profile files. -The runner supports restart stress with `--restart-every-ms --restart-count `. Restarts happen during active project runs, preserve the same temporary HOME and dashboard port, and append all server output to the same `server.log`. Polling retries transient API failures while the server is down and retains a bounded trace of successful reads without treating unchanged reads as stall progress. Status-enabled DAG fixtures require observations before and after every completed restart, a durable QA-start event and completed review for each QA task, ordered changes-requested-to-pass follow-up, merged terminal tasks, passing sprint QA, and a terminal completed sprint. +The runner supports restart stress with `--restart-every-ms --restart-count `. Restarts happen during active project runs, preserve the same temporary HOME and dashboard port, and append all server output to the same `server.log`. Polling retries transient API failures while the server is down and retains a bounded trace of successful reads without treating unchanged reads as stall progress. Each finite, explicitly requested completed restart refreshes the stall watchdog so a restart campaign does not consume the post-recovery task-progress budget; ordinary status reads, admission heartbeats, and failed restart attempts do not. Status-enabled DAG fixtures require observations before and after every completed restart, a durable QA-start event and completed review for each QA task, ordered changes-requested-to-pass follow-up, merged terminal tasks, passing sprint QA, and a terminal completed sprint. ## CI Coverage @@ -45,7 +45,7 @@ The macOS and Windows entries in the same `08 Orchestration` matrix run native E The legacy orchestration workflow is now `Mockup Sprint Diagnostics` and runs only through manual `workflow_dispatch` for focused reruns. -The fast `pnpm run test:orchestration:rapid` lane remains available for local unit-level regression checks. The full `pnpm run test:orchestration:full` catalog remains a manual escalation lane. Manual full and pentest runs write artifacts under `.cache/e2e-mockup-sprint-pentest/`. +The fast `pnpm run test:orchestration:rapid` lane remains available for local unit-level regression checks. The full `pnpm run test:orchestration:full` catalog remains a manual escalation lane. Manual full and pentest runs write artifacts under `.cache/e2e-mockup-sprint-pentest/`. The `extreme-dag-recovery` scenario is marked `localOnly`; it is excluded from `all`, `pentest`, and every CI workflow, and runs only when selected explicitly or through `pnpm run test:orchestration:extreme-dag`. Its inert instruction padding makes the unshortened sprint-QA context exceed 100,000 estimated tokens. The retained first half still contains every executable directive; separate task-QA bounds prove single-task review prompts contain only completed sibling titles plus full current-task details. The resource contract includes one warm project Git helper, at most 16 concurrent workspace sidecars, at most 160 failed restart attempts, no more than 1.45 task attempts per completed task run, zero final non-running owner-scoped containers, and p50 ceilings of 20 seconds for preparation, 10 seconds for Git finalization, and 35 seconds end to end. ## Local Validation @@ -97,3 +97,9 @@ Run the full pentest catalog, including heavy stress scenarios, as a manual loca ```bash pnpm run test:orchestration:pentest ``` + +Run the opt-in 400-task DAG with eight full-runtime restarts and resource ceilings: + +```bash +pnpm run test:orchestration:extreme-dag +``` diff --git a/docs/development/mockup-sprint-pentest.md b/docs/development/mockup-sprint-pentest.md index b3a7e57911..37e1c06b6c 100644 --- a/docs/development/mockup-sprint-pentest.md +++ b/docs/development/mockup-sprint-pentest.md @@ -45,6 +45,14 @@ Run only the 129-task DAG stress scenario: pnpm run test:orchestration:large-dag ``` +Run the local-only 400-task adversarial DAG with eight full-runtime restarts: + +```bash +pnpm run test:orchestration:extreme-dag +``` + +This opt-in lane is excluded from `all`, `pentest`, and every CI workflow. It checks exact terminal state for 400 tasks, dependency/merge ordering across wide and diagonal edges, 399 task-QA passes, one deterministic changes-requested/coding-follow-up/pass cycle, sprint QA, routed sprint-level CI repair, no-change recovery, final repository output, runtime RSS, application/session SQLite and WAL size, separate task- and sprint-QA prompt bounds, a hard 16-reservation scheduler ceiling across dispatch-backed `PENDING`, `RUNNING`, and `PAUSED` task runs, and bounded task-run/event/invocation growth. It also rejects more than 160 failed restart attempts, task-attempt amplification above 1.45, any final non-running owner-scoped container, or p50 regressions above 20 seconds for workspace preparation, 10 seconds for Git finalization, or 35 seconds for the complete CLI workflow. Docker sampling uses all container states so a never-started `created` provider container cannot escape the assertion. Task QA contains full details only for the current task plus title-only completed siblings. Inert instruction padding pushes the unshortened sprint-QA task context above 100,000 estimated tokens; every executable directive remains in the retained first half. Pending projection-only rows without a dispatch are not runtime reservations. Per-sample telemetry and final phase percentiles are written to the project-run `resource-samples.json` artifact. + Docker scenarios use `node:24-bookworm` as an explicit custom test image so pull-request orchestration remains independent from managed-runtime publication. Set `CODE_UX_E2E_CONTAINER_IMAGE` to exercise another prebuilt image locally; @@ -80,11 +88,11 @@ Useful runner options: - `--scenario smoke` aliases the `smoke-completion` scenario. - `--scenario ` runs a single catalog scenario, including heavy scenarios such as `large-dag-stress`. - `--scenario all` runs the default CI-sized catalog and is the default. -- `--scenario pentest` runs every scenario, including heavy stress scenarios. +- `--scenario pentest` runs the standard and heavy scenarios but excludes scenarios marked `localOnly`. - `--runtime electron` launches the compiled Electron shell instead of the Node entrypoint. - `--execution-mode fixture` enables the guarded E2E provider shim and lets fixture settings select host execution. The runner passes the parent Node executable through `CODEUX_E2E_NODE_EXECUTABLE` so mockup provider subprocesses do not accidentally execute through the Electron binary. - `--timeout-ms ` overrides the scenario timeout. -- `--restart-every-ms --restart-count ` restarts the isolated compiled-runtime server during each active project run while preserving the same temporary HOME and port. +- `--restart-every-ms --restart-count ` restarts the isolated compiled-runtime server during each active project run while preserving the same temporary HOME and port. Each completed requested restart refreshes the stall watchdog, leaving the configured stall interval available to detect a genuine post-recovery stall. - `--keep-artifacts` preserves temporary runtime files that are normally trimmed after the run. For CPU or heap profiling, set `CODE_UX_E2E_SERVER_NODE_OPTIONS`. The runner forwards this value only to the isolated compiled-runtime server child as `NODE_OPTIONS`, so profiling does not affect the parent runner or the user's live Code UX process: @@ -120,7 +128,7 @@ The current runner covers: - Sprint-completion merge conflict repair through `completion-merge-conflict`, which waits for a running `task_coding` provider invocation before mutating the default branch. This places the mutation after sprint preflight and workspace preparation but during the fixture task's built-in delay, so the final LOCAL sprint merge deterministically invokes the mockup merge-conflict worker instead of racing feature-branch synchronization or worker-branch publication. The fixture synchronizes a checked-out default worktree after its detached mutation and requires a completed `merge_conflict` invocation, preventing stale-index dirty-work false positives or silent clean-merge passes. - CI-sized QA DAG orchestration through `ci-qa-dag`, a deterministic graph that covers task QA pass, task QA decline, follow-up file creation on the recovered worker branch, a second task QA pass, sprint QA, and final repository assertions. It records a bounded status trace, proves observations survive each forced restart, and requires current completed/pass review summaries plus merged task and completed sprint status. This scenario is the default no-secret GitHub Actions orchestration lane. - Large-DAG orchestration through `large-dag-stress`, a heavy 129-task graph with 96 leaf tasks, 24 batch joins, 6 group joins, a final manifest, and a validation task. This scenario is excluded from default `all` runs and included by `--scenario pentest`. -- Runtime restart recovery by restarting the isolated compiled-runtime server during active project runs with `--restart-every-ms` and `--restart-count`; polling tolerates transient API failures while the server is down, requires a successful status observation on both sides of each completed restart, and then validates terminal sprint/task/review state after recovery. +- Runtime restart recovery by restarting the isolated compiled-runtime server during active project runs with `--restart-every-ms` and `--restart-count`; polling tolerates transient API failures while the server is down, requires a successful status observation on both sides of each completed restart, and then validates terminal sprint/task/review state after recovery. Restart-interrupted task-coding runs are refunded from the coding guardrail exactly once, preventing a restart storm from parking otherwise healthy tasks at the attempt cap. - Sprint completion by polling the compiled runtime until tasks and sprints reach terminal statuses. - Terminal failure behavior by treating unknown scenarios, missing `dist/index.js`, Docker startup failure, server readiness failure, timeout, non-terminal tasks, failed task statuses, failed child commands, or failed assertions as non-zero runner failures. diff --git a/docs/development/rapid-orchestration-debugging.md b/docs/development/rapid-orchestration-debugging.md index dcd0391f1b..1a4376675b 100644 --- a/docs/development/rapid-orchestration-debugging.md +++ b/docs/development/rapid-orchestration-debugging.md @@ -15,6 +15,7 @@ The suite is intentionally split into fast deterministic lanes and slower compil | Full mockup pentest | `pnpm run test:orchestration:full` | Manual escalation for all deterministic mockup scenarios: smoke, CI repair, merge conflict, parallel DAG, multi-project overrides. | Longer-running | | Large DAG stress | `pnpm run test:orchestration:large-dag` | Heavy 129-task mockup DAG with wide fan-out and layered joins. | Long-running | | Full heavy pentest | `pnpm run test:orchestration:pentest` | Default mockup catalog plus heavy stress scenarios. | Long-running | +| Extreme DAG recovery | `pnpm run test:orchestration:extreme-dag` | Local-only 400-task adversarial DAG, eight runtime restarts, and resource ceilings. Never runs in CI. | Very long-running | | Backend broadening | `pnpm run test:backend` | Full backend suite after focused fixes. | Medium | | Release validation | `pnpm run lint && pnpm run build` | Type safety and compiled server/dashboard output. | Medium | @@ -75,10 +76,12 @@ pnpm run test:orchestration:ci-dag:electron That lane launches the Electron app, waits for the embedded Code UX server, and runs the same QA DAG shape through a host-execution mockup fixture. It runs directly on the hosted OS because GitHub-hosted Windows and macOS runners do not provide Docker job containers. -In GitHub Actions, `08 Orchestration` runs build-artifact download, Electron binary install, native dependency rebuild, and orchestration as separate steps where applicable so a stall is visible at the step boundary. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. The runner streams redacted runtime stdout/stderr and emits `mockup_pentest_progress` records whenever sprint or task state changes, plus heartbeat progress every 15 seconds. Progress records contain status counts and at most 32 changed tasks instead of repeating the entire wide DAG; failure records retain the full diagnostic snapshot. CI passes `--stall-timeout-ms 180000`; if no sprint, task status, merge, or expected-output state changes for three minutes after polling starts, the runner fails early with the last sprint/task snapshot and writes the final table to `GITHUB_STEP_SUMMARY`. Successful status observations are retained in a bounded diagnostic trace but do not reset the stall watchdog unless the existing progress signature changes. A claimed task waiting for adaptive CPU/memory admission persists `provider_admission_waiting` / `provider_admission_wait_ended` task-run events and refreshes its dispatch heartbeat every ten seconds. The runner still reports this as `mockup_pentest_provider_admission_wait` for diagnosis, but a fresh admission heartbeat is liveness rather than task progress and cannot keep an unchanged scenario alive indefinitely. +In GitHub Actions, `08 Orchestration` runs build-artifact download, Electron binary install, native dependency rebuild, and orchestration as separate steps where applicable so a stall is visible at the step boundary. Each DAG job has a 25-minute workflow timeout, and the mockup runner bounds individual HTTP calls at 60 seconds plus the full project run at the configured `--timeout-ms`. The runner streams redacted runtime stdout/stderr and emits `mockup_pentest_progress` records whenever sprint or task state changes, plus heartbeat progress every 15 seconds. Progress records contain status counts and at most 32 changed tasks instead of repeating the entire wide DAG; failure records retain the full diagnostic snapshot. CI passes `--stall-timeout-ms 180000`; if no sprint, task status, merge, expected-output state, or explicitly requested completed runtime restart changes for three minutes after polling starts, the runner fails early with the last sprint/task snapshot and writes the final table to `GITHUB_STEP_SUMMARY`. Successful status observations are retained in a bounded diagnostic trace but do not reset the stall watchdog unless the existing progress signature changes. Finite completed restart events reset the watchdog so the configured interval measures post-recovery progress; admission heartbeats and failed restart attempts do not. A claimed task waiting for adaptive CPU/memory admission persists `provider_admission_waiting` / `provider_admission_wait_ended` task-run events and refreshes its dispatch heartbeat every ten seconds. The runner still reports this as `mockup_pentest_provider_admission_wait` for diagnosis, but a fresh admission heartbeat is liveness rather than task progress and cannot keep an unchanged scenario alive indefinitely. For high-throughput diagnosis, distinguish a real state transition from a steady poll. Session sync writes task-run and dispatch rows only when provider-derived state changes; an otherwise unchanged active dispatch emits a liveness heartbeat no more than once per minute. The cycle runner owns the LOCAL Git-finalization evidence snapshot for the cycle and passes it into watch-loop terminal evaluation, while the feature gate reads each task's event history once per evaluation. Repeated source-keyed events that SQLite ignores do not invalidate wall-time caches or publish a dashboard refresh. +The local session snapshot used for state matching excludes stored CLI prompts. Full prompts remain available through direct session and invocation reads, while large QA or planning payloads are not decoded into the server heap on every watch cycle. + The default watch-loop interval is one second. This improves local task completion, merge-drain, and dependency-unlock latency; remote Git/CI refreshes retain their ten-second cache floor and provider-specific session clients keep their own throttling. Existing explicit project or sprint interval overrides are not migrated automatically. The compact DAG's final validation command runs as a scenario-level assertion after all task branches have merged, not inside the final worker worktree. During polling, the runner also enforces the declared DAG: a task with dependencies may not leave `pending` until each dependency is marked merged. If a future task starts early, the runner emits `mockup_pentest_dependency_merge_violation` and fails the test run immediately. The runner does not treat a completed sprint as terminal for these scenarios until expected repository files are visible in the project checkout; while it waits, it emits `mockup_pentest_waiting_for_expected_output`, but only an actual expected-output readiness change refreshes the stall watchdog. This keeps native Electron runners from validating against a dependency branch before Windows has made the parent merge visible, while still failing within the configured stall timeout if a completed sprint never exposes the merged files. @@ -212,6 +215,22 @@ Run the full heavy catalog with: pnpm run test:orchestration:pentest ``` +## Extreme DAG Restart/Resource Lane + +Run: + +```bash +pnpm run test:orchestration:extreme-dag +``` + +This explicit local-only lane is not part of `all`, `pentest`, or any workflow. Its 400 tasks combine a 240-wide root fan-out, 80 three-way joins, distant half-graph joins, diagonal cross-layer dependencies, priority skew, asymmetric regional barriers, a no-change dependency gate, and a final recovery tail. Task QA is enabled for every output-producing task: 398 pass directly and one layer exercises changes requested, same-worker-branch coding follow-up, and a later pass. The sprint then requires sprint QA and a routed sprint-level CI-fix worker marker before final settlement. The package command restarts the entire isolated compiled runtime eight times after durable orchestration begins while retaining the same HOME, database, port, task workspaces, and Docker volumes. + +Passing requires all 400 task rows to be completed and merge-safe, no dependency to start before every parent is merged, all expected task/sprint QA outcomes and the CI repair to complete, the final validation command to pass from the integrated default branch, and all eight restarts to finish. The runner also samples resources every five seconds and enforces a 1 GiB compiled-runtime RSS ceiling, a 512 MiB WAL ceiling, at most 16 dispatch-backed active task-run reservations (`PENDING`, `RUNNING`, or `PAUSED`), at most 560 total task runs, at most 40,000 task-run events, and at most 1,100 provider invocations. Pending status-projection rows without a dispatch do not reserve work and are excluded. Startup recovery refunds each runtime-interrupted task-coding charge exactly once using the task-run id; without that invariant, repeatedly interrupting the same deterministic wave can exhaust its coding guardrail and deadlock the remaining DAG. The row ceilings cover 400 successful coding attempts, the QA follow-up, task/sprint QA, routed CI repair, and the worst-case eight interrupted 16-wide waves with margin; they are not steady-state concurrency targets. Inspect `resource-samples.json` and the `runtimeResources` summary when a ceiling fails. + +Progress events contain only the current delta and aggregate task counts. Full restart history is +retained once in the final summary rather than copied into every polling event, keeping multi-hour +local stress-test output bounded while preserving the recovery audit trail. + ## Local Merge Incident Checklist Use this checklist while monitoring an approved local test project. @@ -296,6 +315,9 @@ Exercise these cases with an approved local test project or a temporary fixture: | Code-complete LOCAL task branches repeatedly become `MERGE_CONFLICT` but raw Git merges cleanly. | Fast branch-only gate logs, `local-merge.test.ts`, visible checkout status. | The visible checkout is blocking host `git checkout`; task branch settlement must use the temporary-worktree path instead of the visible worktree. | | Temporary-worktree merges fail with `fatal: not a git repository: /workspace/.git/worktrees/...`. | `local-merge.test.ts`, helper-container Git logs, the temp worktree `.git` file. | Containerized `git worktree add` left an absolute container gitdir pointer behind. Code UX must normalize that pointer to a relative gitdir before the next helper-container Git command. | | Mockup merge E2E passes but live provider fails. | Provider invocation row, Docker logs, provider transcript metadata. | Provider-specific output, workspace, or session-sync issue rather than orchestration policy. | +| Live provider fails with `container ... is not running` while an isolated restart test is active. | Compare the invocation timestamps with `runtime_restart_started` / `runtime_restart_completed`; inspect the container's `code-ux.runtime-owner` label. | An unowned or older runtime allowed its shutdown/startup cleanup to remove another runtime's warm helper. Current builds owner-scope every cleanup path; stopped-helper retries are generation-aware and fall back to one-shot execution. | +| Jules session creation fails repeatedly with HTTP `400` across ready tasks. | Read the bounded provider detail on the execution invocation and compare local running Jules claims with remotely active sessions. | A hosted active-session ceiling was treated as a task failure, or the source/branch request is invalid. Capacity-specific responses must queue/defer and back off; other validation responses must retain their exact provider explanation. | +| Restart marks Jules task runs failed while the provider still shows them active. | Compare `task_runs`, linked provider/execution invocations, and the Jules session state at the restart timestamp. | Local terminal sprint state overrode durable remote truth. Startup must restore remotely active, unmerged sessions before terminal dispatch/invocation reconciliation. | | Mockup merge E2E selects a credentialed provider. | `provider_invocations`, mockup runner `server.log`, virtual-worker provider pool. | The credential-free mockup route is being filtered before virtual-worker conflict or CI repair. | | Merge-conflict attention resolves but reopens until the guardrail escalates. | Task `merge_indicator`, project attention payload, virtual-worker resolution logs. | The worker resolved the branch but did not clear the stale task `MERGE_CONFLICT` marker, so the next protocol pass recreated the same conflict. | | Merge-conflict attention resolves and the sprint pauses as generic manual attention. | Task `worker_branch`, `task_runs.worker_branch`, `git rev-list feature..worker`. | Resolved-conflict clear history is being reused as merge-required suppression. Suppression must apply only after the source branch has no commits ahead of the feature branch. | diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 554d97daf3..c899d36d49 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -178,7 +178,7 @@ Purpose-grouped E2E specs should prepare normal app state through `tests/e2e/hel The focused `09 Docs / five-page smoke` gate is part of `.github/workflows/ci.yml` for every `dev` and `main` push or pull request. It restores the shared compiled artifact and runs only `tests/e2e/navigation/docs-page.spec.ts` on Linux. The test fails on HTTP 4xx/5xx responses, browser console errors, page errors, missing route landmarks, or missing markdown content without crawling every documentation page. -The full automatic Playwright matrix also lives in `.github/workflows/ci.yml`. It runs only for `main` validation and manual dispatches from the shared build artifact, so broad E2E coverage validates the same compiled app used by package smoke, orchestration, and release-candidate packaging. +The full automatic Playwright matrix also lives in `.github/workflows/ci.yml`. It runs only for `main` validation and manual dispatches from the shared build artifact, so broad E2E coverage validates the same compiled app used by package smoke, orchestration, and release-candidate packaging. Each release-candidate OS row then installs and starts its completed native package and waits for packaged backend plus renderer readiness before uploading artifacts. The automatic E2E stage has one shared job template. `09 E2E / full` fans out across Linux, macOS, and Windows, and each OS runs all purpose projects (`navigation`, `settings`, `projects`, `tasks`, `agents`, and `config`) with `max-parallel: 10`. Every shard downloads `codeux-build-linux`, installs Chromium with browser binaries cached under `.cache/ms-playwright`, installs Linux Chromium system dependencies only on Linux runners with `pnpm exec playwright install-deps chromium`, runs `pnpm exec playwright test --project=` directly, and uploads `test-results/` plus `playwright-report/` for seven days. Artifact names use `playwright--` for every OS. diff --git a/docs/operations/logging-and-correlation.md b/docs/operations/logging-and-correlation.md index ec731d5818..6fde863ee7 100644 --- a/docs/operations/logging-and-correlation.md +++ b/docs/operations/logging-and-correlation.md @@ -21,6 +21,9 @@ This project now uses a shared structured logger and request correlation context - `standard` is the default and keeps important lifecycle, orchestration, invocation, MCP, warning, and error logs visible. - `full` also prints routine dashboard HTTP request-completion logs. - File output uses its own severity threshold and is not hidden by console visibility filtering. + - Each stderr and debug-file stream has an 8 MiB pending-write ceiling. When a downstream + consumer is slower than the runtime, additional records are dropped before `write()` rather + than retained in the Node heap; application work and provider execution continue. - `src/shared/logging/correlation-id.ts` - Correlation ID context backed by `AsyncLocalStorage`. @@ -107,7 +110,7 @@ Expected provider telemetry event types: - `provider_telemetry_poll_failed`: A watcher tick failed to read or parse provider telemetry; the warning logs invocation context, `failureCount`, and `errorName` without provider transcript or usage payload text. - `provider_invocation_usage_updated`: A provider invocation usage row was updated; logs include the update shape and summary counters, not raw usage payloads. -Docker-backed provider launches pass secret-bearing provider environment values through a temporary `0600` env-file supplied with `--env-file`. Long prompts and provider argv are mounted from a generated argv file. Host `docker run` arguments and provider activity logs should show env-file or mount paths only, never API key values, provider env assignments, raw prompts, or usage JSON. +Docker-backed provider launches pass secret-bearing provider environment values through a temporary `0600` env-file supplied with `--env-file`. Provider argv is mounted from a generated file, and oversized prompts are streamed from a separate restrictive file through `docker run -i` when the provider supports stdin input. This keeps raw prompts out of both the host Docker argv and the container's final `execve` argument array. Host process arguments and provider activity logs should show temporary paths or bounded metadata only, never API key values, provider env assignments, raw prompts, or usage JSON. Focused verification: @@ -129,6 +132,9 @@ pnpm run test:dashboard -- tests/dashboard/lib/dashboard-realtime-client.test.ts - The CLI entrypoint installs a bootstrap warning filter before server modules load, suppressing Node's SQLite experimental warning. Dotenv is loaded in quiet mode so startup output is owned by the structured logger. - Treat `logPurpose` as required for new server, tool, provider, realtime, request, and security logs. The stable labels (`HTTP`, `INVK`, `LIVE`, `SEC`, and the other labels in the table above) are used by operators and tests to separate workflow concerns without string-matching free-form messages. - Keep provider invocation records and realtime event logs metadata-only. Persisted provider usage rows may retain raw usage JSON for diagnostics, but structured logs, realtime event metadata, and debug-file output must expose only bounded counters, identifiers, event types, sizes, `rawUsageJsonPresent`, `errorName`, and `correlationId`. +- Never remove the pending-stream ceiling or bypass it from a child logger. A blocked terminal, + redirected pipe, or slow debug-log filesystem must not turn routine invocation telemetry into an + unbounded JavaScript write queue. - Security validation failures should log through `logPurpose: "security"` with sanitized reason metadata. Do not include request bodies, authorization headers, API keys, token values, raw websocket frames, provider prompts, subprocess argv, or transcript text. - `DEBUG_LOG_FILE_LEVEL` increases `.code-ux/debug.log` detail only; it does not relax redaction or metadata-only provider logging. diff --git a/docs/operations/runbook.md b/docs/operations/runbook.md index 1257bfd924..3004994b36 100644 --- a/docs/operations/runbook.md +++ b/docs/operations/runbook.md @@ -94,7 +94,7 @@ Provider concurrency is enforced globally across all projects using `ProviderSet ### Provider invocation observability - Provider usage rows are the source of truth for runtime diagnostics. Confirm `provider_invocations` keeps the Code UX provider invocation id, Code UX session id, native provider session id, provider, purpose, status, model, execution mode, lifecycle timestamps, duration, token counters, transcript character count, tool-call count, usage source, invocation source, and raw-usage presence. Linked `execution_invocations` preserve the provider invocation id for cross-querying. - Structured invocation logs are metadata-only. They may include identifiers, lifecycle fields, counters, `failureCount`, `errorName`, and `correlationId`, but must not include raw transcripts, API keys, provider environment values, raw usage JSON, or full prompts. -- Docker provider launches should expose only env-file and controlled mount paths in the host process arguments. Provider API keys, Git tokens, custom provider env values, and long prompts are written to temporary files or controlled mounts and should not appear in `docker run` argv or activity log metadata. +- Docker provider launches should expose only env-file and controlled mount paths in the host process arguments. Provider API keys, Git tokens, custom provider env values, and long prompts are written to restrictive temporary files or controlled mounts and should not appear in `docker run` argv or activity log metadata. Oversized prompts are also removed from the reconstructed container argv: Code UX streams them through `docker run -i` to providers with a stdin prompt contract, preventing the container-side Linux `MAX_ARG_STRLEN` failure that an argv mount alone cannot prevent. - File logging has its own threshold. `DEBUG_LOG_FILE_LEVEL=debug` can persist debug-level provider diagnostics to `.code-ux/debug.log` even when console logging is filtered to `error`; use this only for focused diagnostics and keep the metadata-only rule in place. - New runtime logs should set a structured `logPurpose` label so request (`HTTP`), invocation (`INVK`), realtime (`LIVE`), security (`SEC`), orchestration (`ORCH`), storage (`DATA`), and lifecycle (`LIFE`) traffic stays separable in console and debug-file output. - Realtime event logs are operational metadata, not payload dumps. They may include event type, sequence, scope, bounded byte sizes, replay/recovery reason, and `correlationId`; they must not include full websocket frames, dashboard payloads, provider transcripts, request bodies, API keys, or authorization headers. @@ -167,6 +167,8 @@ Checks: - HTTPS remotes should fail fast when credentials are missing. Code UX disables interactive Git credential prompts and bounds branch-preflight/fetch checks so orchestration settles instead of waiting indefinitely on a local credential helper. CLI-backed task dispatch still requires a remote refresh before branch preparation; when the starting branch is known, Code UX fetches that branch's remote-tracking ref instead of every branch on `origin`. Slow GitHub/GitLab smart HTTP connections can still take longer than 30 seconds, so the default fetch timeout is 120 seconds. Set `CODE_UX_GIT_FETCH_TIMEOUT_MS` higher when the network or remote regularly needs more time. Backend Git commands run inside the helper container unless host mode is explicitly enabled for diagnostics. - For Jules dispatch, that local refresh is best-effort and a refresh failure should be logged without blocking Jules session creation. - After a restart, active Jules dispatches that never reached `session_created` are treated as interrupted pre-session dispatches and moved back to a retryable task state. Jules dispatches with a persisted session remain attached for normal sprint recovery. +- Before session creation, Code UX refreshes a coalesced, bounded Jules API preflight and counts `QUEUED`, `PLANNING`, and `IN_PROGRESS` sessions missing from local accounting; waiting/paused history does not consume the execution count. If the preflight cannot be verified, dispatch fails closed and leaves the task queued. Jules has no slot-count or state-filter endpoint, so explicit capacity responses and its generic `400 FAILED_PRECONDITION` create response are authoritative retryable deferrals with a short learned-cap backoff. `INVALID_ARGUMENT` and other validation responses still fail with their bounded provider explanation. +- On restart, a remotely active persisted Jules session is authoritative over stale local `failed` projections. Startup recovery restores its task run, dispatch, provider/execution invocation, and recoverable failed sprint run before the watch loop resumes. It never reopens completed/cancelled sprints, merged tasks, or `QA_REVIEW_FAILED` human handoffs. If the five-second startup snapshot bound expires, local session ids and running external-provider rows preserve the affected monitors until the late snapshot or normal sync verifies them; do not interpret the timeout warning as provider termination. - If the dispatch fails with an auth error, fix the dashboard GitHub token or remote URL, then rerun the task. ### 4. Local provider task sessions fail immediately @@ -195,7 +197,7 @@ Checks: - If auth is expected from host login state, is the relevant Docker auth mount enabled and is its mount path valid? Docker uses dedicated, isolated credential mounts per provider to keep raw tokens and key paths out of the broader workspace and process arguments. - Docker mode requires daemon-visible workspace paths. Runtime now prefers repo-scoped worktree paths for Docker sessions and mounts them as dedicated volumes alongside runtime volumes that hold provider home paths and package manager caches (`code-ux.workspace-runtime=true`). - Docker runtime state is stored under `~/.code-ux/runtime/docker//` by default (override with `JULES_DOCKER_RUNTIME_ROOT`). Cached setup image build contexts and build locks live under that root so setup-cache images survive dashboard restarts and concurrent post-restart jobs wait on the same build instead of starting duplicate builds. - - Startup pruning clears orphaned helper containers, login containers, temp credential dirs, and stale workspace/runtime volumes that are no longer referenced by active tracking. + - Startup pruning clears orphaned helper containers, login containers, temp credential dirs, and stale workspace/runtime volumes that are no longer referenced by active tracking. Docker assets are filtered by the state-home-derived `code-ux.runtime-owner` label, so a local pentest or second isolated runtime sharing the daemon cannot prune the live runtime. - During normal Code UX shutdown (`SIGINT`, `SIGTERM`, `SIGHUP`, or Electron quit), the server requests active dispatch aborts, drains persistent Git/workspace helper pools (including helpers that were still starting), and then kills any still-running Docker containers with `code-ux.*` labels or deterministic `code-ux-*` runtime names. It does not remove Docker workspace/runtime volumes. On the next start, recovery follows `Settings -> General -> Restart Behavior`: continue resumes active sprint runs by default, pause/cancel applies sprint-level policy before watch-loop recovery, and invocation restart/cancel removes labelled active containers without deleting preserved volumes. - Pausing a sprint run also pauses or stops active task dispatch rows, cancels linked provider and QA runtime rows, releases task and sprint leases, and resets affected project tasks to `pending`. Resuming that run uses existing-run recovery and will not create a second sprint run. - Dashboard and MCP HTTP listeners track and destroy open sockets during shutdown, including upgraded dashboard WebSocket sockets, so open browser tabs do not delay process exit or leave ports bound during rapid restarts. @@ -203,11 +205,11 @@ Checks: - Rerun resume uses the latest `cli_workspace_bound` event as the source of truth for the workspace session id. If the latest interrupted provider invocation has a different `session_id`, Code UX still resumes the Docker volume named by the recorded workspace binding. - Codex uses per-session container home directories under that runtime root to prevent stale state from previous Codex runs. - `RuntimeCleanupService` performs a periodic sweep for stale/offline connections, expired leases, terminal dispatch reconciliation, stale sprint runs, and runtime artifacts. -- During shutdown, Code UX disposes the command-spawner host before Docker cleanup (`DockerRuntimePruneService` and `DockerAssetPruneService`). `DockerRuntimePruneService` safely prunes stale per-runtime paths and shared temp paths after their age threshold while preserving active roots/Codex homes; its periodic traversal is single-flight and uses bounded asynchronous filesystem operations. `DockerAssetPruneService` cleans up orphaned workspace volumes, login containers, helper containers, and temporary credential directories on startup. Startup cleanup is single-flight, removes helper containers before workspace volumes, then uses bounded parallel Docker inspection/removal batches without changing active-session, grace-period, newest-version, or age-retention protections. Workspace volume helpers use `code-ux.managed=true` and `code-ux.helper=volume` on both persistent helpers and `docker run --rm` fallback helpers. Do not instruct operators to run broad manual `docker system prune` commands. -- Docker provider launches use readable container names such as `code-ux-codex-` and mount provider arguments through a generated argv file instead of passing the full prompt through the host `docker run` command line. Secret-bearing provider environment variables are written to temporary `0600` env-files and supplied with `--env-file`, so `ps`/process-list inspection should show only the env-file path and not API key values. If Docker reports that the deterministic provider container name is already in use, Code UX force-removes that named container with volumes and retries the launch once; repeated conflicts usually mean an external Docker daemon or another runtime is recreating the same session container. Packaged Windows Electron builds that fail with `spawn ENAMETOOLONG` during provider launch are using an older build or a non-provider launch path that still embeds a large payload in command arguments. +- During shutdown, Code UX disposes the command-spawner host before Docker cleanup (`DockerRuntimePruneService` and `DockerAssetPruneService`). `DockerRuntimePruneService` safely prunes stale per-runtime paths and shared temp paths after their age threshold while preserving active roots/Codex homes; its periodic traversal is single-flight and uses bounded asynchronous filesystem operations. `DockerAssetPruneService` cleans up orphaned workspace volumes, login containers, helper containers, and temporary credential directories on startup. Startup cleanup is single-flight, owner-scoped, removes this runtime's helper containers before its workspace volumes, then uses bounded parallel Docker inspection/removal batches without changing active-session, grace-period, newest-version, or age-retention protections. Workspace volume helpers use `code-ux.managed=true`, `code-ux.helper=volume`, and `code-ux.runtime-owner=` on both persistent helpers and `docker run --rm` fallback helpers. Shutdown applies the same owner boundary. Do not instruct operators to run broad manual `docker system prune` commands. +- Docker provider launches use readable container names such as `code-ux-codex-` and mount provider arguments through a generated argv file instead of passing the full prompt through the host `docker run` command line. When a prompt exceeds the safe per-argument size, Codex, Claude Code, Gemini, Qwen Code, OpenCode, and the deterministic test provider receive it through a restrictive temporary stdin file; Codex receives `-` as its stdin marker, while the other supported CLIs run without the prompt value in argv. Antigravity retains one positional prompt because its published `-p` / `--print` contract requires one argument and does not expose file/stdin prompt input. Code UX accepts that argument through 120 KiB and rejects a larger Antigravity Docker prompt with an explicit scoped error before `execve`, instead of splitting or silently truncating it. Secret-bearing provider environment variables are written to temporary `0600` env-files and supplied with `--env-file`, so `ps`/process-list inspection should show only temporary file paths and not API key or prompt values. If Docker reports that the deterministic provider container name is already in use, Code UX force-removes that named container with volumes and retries the launch once; repeated conflicts usually mean an external Docker daemon or another runtime is recreating the same session container. Packaged Windows Electron builds that fail with `spawn ENAMETOOLONG`, or Linux containers that report `Argument list too long`, are using an older build or a launch path that still reconstructs a large prompt argument. - In custom-image mode, setup-image caching may spend several minutes building a content-addressed `code-ux-setup-cache-*` image for the first base-image/setup-script combination. Activity logs call out the cache miss, stream Docker build steps, and report bounded progress; later runs reuse it until the base image, setup script, Dockerfile template, or Playwright setting changes. Managed mode performs no setup build: it preloads the Playwright-matched browser into a versioned local Docker volume and mounts `/ms-playwright` read-only. If a custom setup build fails, Code UX logs the fallback and runs the explicit setup script at container runtime instead. - Provider login uses a separate content-addressed `code-ux-login-base-node-24-bookworm-slim:*` image with curl and keyring prerequisites baked in. The image is prewarmed after dashboard logging is available, but this is best-effort: failures should be treated as startup warnings, not as a reason to block the dashboard or provider login. -- Backend Git commands and snapshot workspace bootstrap use public helper images such as `alpine/git`. Snapshot bootstrap verifies or pulls these helpers automatically, and if Docker reports a broken host credential helper while pulling a public helper image, Code UX retries that helper pull with an isolated empty Docker client config; provider/container images still use the normal Docker configuration. Local-mode temporary-worktree merges also use the shared Git helper: Code UX normalizes the temporary worktree's Git metadata after creation so later containerized commands can safely update host refs and checked-out worktrees. Persistent helper containers are removed with `docker rm -f -v` so image-declared anonymous volumes are cleaned with the container. Startup Docker pruning is scheduled in the background and only queries Code UX labels, so a large Docker daemon does not delay dashboard boot with full volume/container scans. +- Backend Git commands and snapshot workspace bootstrap use public helper images such as `alpine/git`. Snapshot bootstrap verifies or pulls these helpers automatically, and if Docker reports a broken host credential helper while pulling a public helper image, Code UX retries that helper pull with an isolated empty Docker client config; provider/container images still use the normal Docker configuration. Poolable commands reuse one warm helper per repository and runtime owner: repo-local worktrees share it, separate repositories remain isolated, stdin-backed Git operations stream through `docker exec -i`, and Git/auth environment is injected per exec so credentials are not retained between commands. Commands that need an additional host bind mount keep the one-shot path. Local-mode temporary-worktree merges also use the shared Git helper: Code UX normalizes the temporary worktree's Git metadata after creation so later containerized commands can safely update host refs and checked-out worktrees. Persistent helper containers are removed with `docker rm -f -v` so image-declared anonymous volumes are cleaned with the container. Helper names and cleanup filters include the runtime owner. If Docker reports that a pooled helper stopped, retries invalidate only the failed container generation; a concurrent replacement remains registered, and a second stopped replacement uses a one-shot `docker run --rm` command. Startup Docker pruning is scheduled in the background and only queries owner-scoped Code UX labels, so a large Docker daemon does not delay dashboard boot with full volume/container scans. - Snapshot workspace bootstrap creates the temporary Git bundle through the containerized Git helper using a portable `/mnt/code-ux/git-paths/*` target, then streams the bundle directly into `docker run` stdin. Packaged Windows Electron builds should not route `C:\...AppData\Local\Temp\code-ux-bundle-*` paths through `bash -lc` or use those paths as Docker mount targets; seeing `cat: 'C:\...\repo.bundle': No such file or directory` or `invalid mount path: 'C:/Users/.../code-ux-bundle-*'` indicates an older build. - Dashboard reply and clarification worker invocations resolve Git policy from the project-scoped dashboard settings before creating Docker snapshot workspaces. In local Git mode, those snapshots seed from local refs and should not require `origin/` to exist. - Packaged Windows Electron runs use an opaque desktop window to avoid Chromium tile-memory exhaustion (`tile_manager.cc:1012 WARNING: tile memory limits exceeded`). Visible non-canvas routes render the selected animated background normally. Hidden tabs release their WebGL background and realtime socket, pause fallback polling, and reconnect/refetch when visible; `/nodes` uses a context-free static fallback while its large draggable canvas is mounted. Additional mitigations remain at the WebGL layer (`powerPreference: "low-power"`, reduced render scale, `contain: strict`, and the Chromium `--force-gpu-mem-available-mb` flag). @@ -229,7 +231,7 @@ Checks: - Dashboard **Resume** for a paused sprint run reactivates that same run and starts the recovery/watch-loop path in place. It should not create a replacement sprint run or a second watch loop. If the old loop is still draining, resume schedules a short follow-up recovery attempt after the registry clears so a run is not left `running` without a heartbeat. - Sprint deletion is rejected while the sprint has any queued/running/cancel-pending sprint run, active task dispatch, running provider/execution invocation, preserved invocation transcript, or a sprint run that finished in the last 30 seconds. Cancel, pause, or let runtime cleanup (`RuntimeCleanupService`) settle first; this prevents database cascades from deleting rows while an in-memory watch loop or provider callback is still unwinding. - To clean up stale workspace branches that were merged or closed on origin, use `BranchReaperService` logic via the dashboard. -- `RuntimeStartupRecoveryService` closes active dispatch/task-run rows whose linked provider invocation already reached a terminal state. It reconciles persisted/runtime state after restart and cleans or marks stale execution artifacts according to service behavior. If the project task is already code-complete, the dispatch mirrors completion; otherwise the task is reset to pending for a clean retry instead of staying in a stale running state. +- `RuntimeStartupRecoveryService` closes active dispatch/task-run rows whose linked provider invocation already reached a terminal state. Persisted hosted Jules sessions are the exception: running remote work is preserved, and a fresh remote snapshot repairs false local terminal projections before ordinary reconciliation. If the project task is already code-complete, the dispatch mirrors completion; otherwise the task is reset to pending for a clean retry instead of staying in a stale running state. - Live provider telemetry refreshes the linked task-dispatch heartbeat (`HeartbeatService`) while the provider invocation is running. `HeartbeatService` acts to renew sprint-run heartbeat/lease on an interval and stops tracking when renewal fails. It is for liveness and lease maintenance, not a cleanup command. A dispatch heartbeat should not go stale when provider usage rows are still updating. - In local-git mode, an existing worker-owned main-merge conflict attention item suppresses additional `feature -> default` merge attempts while the worker is resolving the conflict. Human-escalated main-merge attention pauses the sprint with local conflict instructions. @@ -450,7 +452,7 @@ curl http://localhost:4444/api/git-status GitHub validation is split by signal: - `Code UX CI Pipeline` is the canonical automatic lane. It runs on pushes to every branch, pull requests targeting `dev` or `main`, and manual dispatches. Feature-branch and `dev` pushes run the core numbered jobs, including all three orchestration DAG rows; `main` pushes, `main` pull requests, and manual dispatches additionally run full Playwright and release-candidate matrices. - Static, build, and security are the prerequisite stage. The build job uploads `codeux-build-linux` for all downstream jobs. -- Backend coverage, dashboard tests, npm install smoke, and the cross-OS orchestration DAG matrix reuse that build artifact and run in parallel after the prerequisite stage. Release-candidate packaging starts after package smoke and can run beside the main-only E2E matrix. Matrix bounds are `08 Orchestration` at three shards, `09 E2E` at ten shards, and `10 Release Candidate` at three shards; GitHub's runner quota queues any excess work across the parallel lanes. +- Backend coverage, dashboard tests, npm install smoke, and the cross-OS orchestration DAG matrix reuse that build artifact and run in parallel after the prerequisite stage. Release-candidate packaging starts after package smoke and can run beside the main-only E2E matrix. Each release-candidate row installs its completed native package and requires packaged backend and renderer readiness plus a clean exit before artifact upload. Matrix bounds are `08 Orchestration` at three shards, `09 E2E` at ten shards, and `10 Release Candidate` at three shards; GitHub's runner quota queues any excess work across the parallel lanes. - `Playwright Diagnostics`, `Release Candidate Diagnostics`, and `Mockup Sprint Diagnostics` are manual-only workflows for focused reruns. They no longer run automatically on every PR. - Superseded runs for the same branch or pull request are cancelled by workflow concurrency groups. - Security validation is intentionally separated from build and Playwright lanes. The `04 Security / dependency audit` job runs the standard `pnpm run audit`, which is `pnpm audit --audit-level=high`; high-severity dependency findings fail that job without preventing typecheck, tests, build, or Playwright artifacts from reporting their own status. The repository pins pnpm 11.13.0 so the native audit command uses npm's supported bulk-advisory API. diff --git a/docs/operations/security-hardening.md b/docs/operations/security-hardening.md index 2b6e31b420..9dbaf930ac 100644 --- a/docs/operations/security-hardening.md +++ b/docs/operations/security-hardening.md @@ -100,12 +100,12 @@ While Code UX trusts the developer and any connected systems, several specific p - **Provider Transcript Sanitization:** Provider stdout/stderr callbacks, returned command output, and parsed usage transcript/conversation strings pass through the invocation-output sanitizer before they are persisted or surfaced to the dashboard. Raw provider streams may still be held transiently in memory while provider parsers compute usage and session metadata. - **MCP Gateway Log Hygiene:** Unauthorized, invalid-header, rate-limit, inactive-session, session-cap, idle-cleanup, and startup gateway logs omit bearer values, supplied session ids, supplied agent ids, raw request bodies, provider credentials, and login tokens. They retain only bounded operational metadata such as method, path, host, port, auth-required state, active-session counts, configured limits, and timeout values. - **Settings Secret Inputs:** Dashboard settings fields that store provider API keys, Git host tokens, Jira API tokens, and external embedding API keys render as masked secret inputs by default. Operators must explicitly use the reveal control to inspect a value. -- **Docker Secret Transport:** Provider and preview Docker launches write selected host/provider environment variables to temporary `0600` env-files and pass those files via `--env-file`. Provider argv and generated provider MCP/config artifacts are also staged in restrictive temporary files and mounted into the container instead of being inlined into `docker run` arguments or labels. Provider credentials use isolated credential mounts rather than broad workspace root exposure, ensuring secrets are strictly bounded and not inadvertently captured in workspace logs. This keeps API keys, MCP bearer tokens, and Git tokens out of the host `docker run` argv visible through process listings while preserving the same container environment. +- **Docker Secret and Prompt Transport:** Provider and preview Docker launches write selected host/provider environment variables to temporary `0600` env-files and pass those files via `--env-file`. Provider argv and generated provider MCP/config artifacts are also staged in restrictive temporary files and mounted into the container instead of being inlined into `docker run` arguments or labels. Oversized provider prompts use a separate restrictive file as `docker run -i` input for stdin-capable CLIs, so they are absent from both the host Docker argv and the container's final `execve` argument array. Provider credentials use isolated credential mounts rather than broad workspace root exposure, ensuring secrets are strictly bounded and not inadvertently captured in workspace logs. This keeps API keys, MCP bearer tokens, Git tokens, and large prompt bodies out of process listings while preserving the same provider input. - **Preview Environment Boundaries:** Browser Preview user-defined environment variables are validated before they are written to the Docker env-file. Keys must be shell-style env names, values must be single-line env-file values, and runtime-owned routing names such as `HOST`, `PORT`, `HOME`, `DASHBOARD_PORT`, `SPRINT_PREVIEW_*`, and `CODE_UX_GIT_USER_*` are reserved so UI edits cannot override Code UX port routing or container identity. - **Dashboard Login Port Binding:** Interactive dashboard-login containers do not use Docker host networking by default. Codex and Claude Code OAuth callback ports are published only on host loopback as `127.0.0.1::`; other provider login containers publish no host ports unless a provider-specific flow explicitly requires it. Public dashboard binding does not change this callback-port rule. ### Subprocess & Settings Mutation Safety -- **Shell-Free Command Execution:** Shared subprocess execution validates command names, argument null bytes, stdin file paths, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, preventing a stale or transient ancestor marker from widening the host bind mount. Commands run with `shell: false` so arguments are not reinterpreted by a shell. +- **Shell-Free Command Execution:** Shared subprocess execution validates command names, argument null bytes, stdin file paths, and working directories immediately before spawning. Working directories must resolve to existing real directories inside the user home, application directory, OS temporary directory, or an explicit `CODE_UX_DIRECTORY_BROWSER_ROOTS` entry before either the inline or helper-process boundary. Git helper repository discovery accepts only `.git` directories and worktree targets with a valid `HEAD`, preventing a stale or transient ancestor marker from widening the host bind mount. Poolable Git commands use one runtime-owned warm helper per repository; repo-local worktrees share that helper, separate repositories remain isolated, stdin files stream through `docker exec -i`, and Git/auth environment is applied only to the individual exec. Commands needing another host bind mount retain the one-shot helper path. Commands run with `shell: false` so arguments are not reinterpreted by a shell. - **Prototype Pollution Guards:** Dotted settings paths are parsed through a safe-key validator before clone-on-write mutation. `__proto__`, `constructor`, `prototype`, and empty path segments are rejected before any assignment. ## Trust Model & Limitations diff --git a/docs/settings/configuration-and-storage.md b/docs/settings/configuration-and-storage.md index 8de052a3e3..57e8380f28 100644 --- a/docs/settings/configuration-and-storage.md +++ b/docs/settings/configuration-and-storage.md @@ -69,7 +69,7 @@ Storage: - `project_settings` - `sprint_settings` - `app_settings` is retained only as a one-time legacy migration source for development data that predates the scoped model -- provider session DB at `~/.code-ux/session-tracking.db` +- provider session DB at `~/.code-ux/session-tracking.db`; it stores provider lifecycle, branch, and activity projections. Jules prompts remain there for hosted usage estimation, while local CLI prompts are not duplicated because their durable invocation messages already live in `app.db`. The session schema upgrade clears pre-existing local prompt copies once and preserves Jules prompts. - Code UX app DB at `~/.code-ux/app.db` - includes project planning tables (sprints with `original_prompt` and `goal`) plus sprint-scoped runtime projection in `app_settings`, `task_runs`, and `task_run_events` - runtime context rows are keyed by sprint (`runtime_context::`); legacy unscoped project-level runtime rows are deprecated and are no longer used for explicit sprint reads or rerun context @@ -105,7 +105,7 @@ Runtime resolution: - When a Docker-backed provider run is cancelled, Code UX now kills the backing container directly on abort instead of relying on the local `docker run` client to tear it down. This keeps deterministic container names safe for retries while ensuring the daemon-side container stops promptly. - Interactive provider login containers use readable names such as `code-ux-login--` and run on a small cached prerequisite image named like `code-ux-login-base-node-24-bookworm-slim:`. - Packaged Windows Electron uses an opaque BrowserWindow and Chromium GPU memory hints to mitigate tile-memory pressure. All animated backgrounds render at full fidelity; WebGL backgrounds use `powerPreference: "low-power"` and 0.5× render scale, and all background layers apply CSS `contain: strict` to limit compositor tile scope. -- On startup, Code UX schedules Docker asset pruning in the background so dashboard boot is not blocked by Docker cleanup. The prune path uses label-filtered Docker queries for managed workspace/runtime volumes plus helper/login containers, removes containers and volumes in batches, and applies a short per-command timeout. Helper/login container cleanup uses `docker rm -f -v` so anonymous image-declared volumes are removed with the container. Cached setup-script images are content-addressed and are intentionally preserved across dashboard restarts so provider launches can reuse them until the base image, setup script content, or setup Dockerfile changes. +- On startup, Code UX schedules Docker asset pruning in the background so dashboard boot is not blocked by Docker cleanup. The prune path uses the state-home-derived `code-ux.runtime-owner` label together with asset labels for managed workspace/runtime volumes plus helper/login containers, removes only this runtime's containers and volumes in batches, and applies a short per-command timeout. This permits the live app and an isolated local test runtime to share one Docker daemon safely. Helper/login container cleanup uses `docker rm -f -v` so anonymous image-declared volumes are removed with the container. Cached setup-script images are content-addressed and are intentionally preserved across dashboard restarts so provider launches can reuse them until the base image, setup script content, or setup Dockerfile changes. - After startup, Code UX schedules automated database maintenance in the background. Each idle pass scans and mutates at most 500 rows per table while pruning old task runs, terminal execution/provider invocation trees, VM activities, attention items, realtime events, and released virtual-worker assignment history according to the configured retention policy. Parent records wait for their bounded child cleanup, and preserved invocation trees remain linked. Raw terminal provider activity has a one-day window because the durable execution transcript remains available. Automatic maintenance never issues full `VACUUM` or `TRUNCATE` operations; the optional startup reclaim requests at most 256 incremental-vacuum pages, and controlled WAL maintenance uses `PASSIVE` checkpoints only when no provider invocation is active. - restart recovery treats interrupted Docker sessions without a live backing container as cancelled/retryable, so app shutdowns and restarts do not inflate invocation failure statistics while abandoned runtime callbacks are still cleared. - restart behavior is controlled from `Settings -> General -> Restart Behavior`: @@ -118,7 +118,7 @@ Runtime resolution: - startup recovery also repairs parent sprint projection drift for paused runs: if the latest run is paused and no queued/running/cancel-pending run exists for the sprint, the parent sprint row is synced back to `paused` instead of allowing the dashboard to show a false running state. - restart recovery respects live sprint lease ownership: if an active run has an unexpired `sprint_orchestrator:` lease and that PID is still alive, the new process skips recovery instead of releasing the lease and starting a duplicate watch loop. - QA review keepalives refresh the sprint-run heartbeat only; the orchestrator heartbeat owns sprint lease renewal with its original lease token. -- On Code UX shutdown (`SIGINT`, `SIGTERM`, `SIGHUP`, or Electron quit), the server first requests registered active dispatches to abort and then kills any still-running Docker containers with `code-ux.*` labels or deterministic `code-ux-*` runtime names. This prevents provider, preview, browser, login, and workspace-helper containers from surviving a normal app stop. Persistent workspace helper containers and their one-shot fallback containers both carry `code-ux.managed=true` and `code-ux.helper=volume` labels so cleanup and inspection can find either path. Shutdown does not remove Docker workspace/runtime volumes, and startup recovery can continue from the same workspace volume when `Resume failed task in same workspace` is enabled. +- On Code UX shutdown (`SIGINT`, `SIGTERM`, `SIGHUP`, or Electron quit), the server first requests registered active dispatches to abort and then kills still-running Docker containers only when their `code-ux.runtime-owner` matches the current state home. This prevents provider, preview, browser, login, and workspace-helper containers from surviving a normal app stop without terminating containers belonging to an isolated runtime on the same daemon. Persistent workspace helper containers and their one-shot fallback containers carry `code-ux.managed=true`, `code-ux.helper=volume`, and the runtime-owner label so cleanup and inspection can find either path. Shutdown does not remove Docker workspace/runtime volumes, and startup recovery can continue from the same workspace volume when `Resume failed task in same workspace` is enabled. - Failed-task retry uses the latest `cli_workspace_bound` task-run event as the authoritative Docker workspace binding. This matters after restart recovery because the interrupted provider session id can differ from the workspace session id that actually names the preserved volume. - startup recovery now also requeues task-level CLI follow-up runs that were left in `in_progress` after QA/repair `Fix` work lost its backing container, so the orchestrator can start the container again instead of leaving the sprint stuck after a server restart. - startup recovery treats Jules task sessions as durable remote runtime. If Code UX restarts after a sprint run or task dispatch was incorrectly terminalized while the sprint itself is still active, recovery rehydrates one sprint run, reattaches active Jules task runs/dispatches/provider invocation rows to it, and resumes the watch loop instead of failing the sessions. diff --git a/docs/settings/quality-assurance.md b/docs/settings/quality-assurance.md index 91dce31897..400b2094ed 100644 --- a/docs/settings/quality-assurance.md +++ b/docs/settings/quality-assurance.md @@ -41,6 +41,12 @@ Before applying changes, check: - Whether a project override is masking the system value you expected to change. - Whether a running sprint needs to be paused, restarted, or allowed to finish before the new value can be observed. +Task-level QA prompts contain full details only for the task under review: title, status, provider, worker branch, PR, dependencies, the complete unshortened prompt, and the latest eight activity entries without content truncation. Other sprint tasks appear only after they reach `completed`, and then only their titles are listed; unfinished siblings and all sibling instructions, metadata, and activity are omitted. Sprint-completion QA still receives every task because it reviews cross-task integration. When that full sprint context exceeds 100,000 estimated tokens (using the runtime's four-characters-per-token estimate), sprint QA receives the first half of every task instruction with an explicit notice while task metadata, ordering, and recent activity remain intact. + +During orchestration, QA reconciliation and initial merge-gate evaluation load the whole DAG's latest review cycles and attempt counts in a chunked batch. Review decisions, retry budgets, and fail-closed behavior are unchanged. + +Task QA runs in waves of at most four reviews per orchestration cycle, or a lower positive capacity when the providers routed to `qa_review` are configured more conservatively. The cycle settles that wave, merges ready branches, and starts newly unblocked coding before scheduling more reviews. Provider admission remains authoritative and may reduce effective concurrency further under host pressure. + ## Troubleshooting If the saved setting does not appear to take effect: diff --git a/docs/settings/restart-behavior.md b/docs/settings/restart-behavior.md index 85b80a0110..741036202f 100644 --- a/docs/settings/restart-behavior.md +++ b/docs/settings/restart-behavior.md @@ -15,7 +15,7 @@ Use it when you are configuring a new project, auditing inherited settings, or d Sprint policy continues, pauses, or cancels active sprints; invocation policy continues, cancels, or restarts interrupted work. -The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. +The invocation policy applies to every provider-backed orchestration stage, not only task coding. Under `continue`, Code UX durably resumes sprint planning, task coding, QA review, QA-requested coding follow-up, CI-fix, and merge-conflict work from their recorded logical session and workspace. When the provider exposed a resumable native session, the replacement invocation continues that native conversation as well. | Control Surface | Runtime Effect | Review Before Saving | | --- | --- | --- | @@ -28,12 +28,14 @@ The invocation policy applies to every provider-backed orchestration stage, not When `restartSprintPolicy = continue` and `restartInvocationPolicy = continue`, startup recovery: - resumes the existing sprint run and watch loop instead of creating a replacement sprint run +- preserves the complete sprint-planning request and its routing options, closes the process-bound invocation interrupted by shutdown, and continues the exact provider-native planning session in the preserved planning workspace. A missing recorded provider conversation fails closed rather than silently creating a new conversation. Only a request interrupted before provider linkage is reissued from its durable full prompt, because no provider session existed yet. - correlates each interrupted QA reviewer with its exact execution invocation, reviewer preset, logical provider session, and isolated review workspace - reuses the QA review workspace and provider conversation only while completing the same interrupted review cycle; verification after a decisive verdict starts from a fresh branch snapshot so it sees any coding follow-up - checkpoints every configured reviewer in a multi-reviewer cycle before invoking the first reviewer; recovery keeps completed verdicts, resumes only interrupted reviewers, and fills any reviewer row missing from a legacy partial cycle without spending another QA cycle - preserves task-level and sprint-completion `changes_requested` verdicts before starting their coding handoffs; if restart occurs between the verdict and the follow-up invocation, the next cycle resumes that pending handoff instead of leaving QA indefinitely blocked - returns an abruptly failed QA coding handoff to `CODING_COMPLETED`/`QA_PENDING` and retries it from the recorded coding session and workspace. A successful or reconciled handoff remains in that verification-ready state until the next QA review starts, preventing the restart window from launching unrelated coding work. Provider failures are bounded to three continuation attempts, while resuming a `running` checkpoint after a runtime restart does not consume another failure allowance; exhaustion then follows the configured QA exhaustion policy instead of redispatching the task as unrelated coding or heartbeating forever. - records the original worker-branch baseline before invoking a QA coding follow-up and reuses it after restart, so provider commits made before host-branch publication are still exported and published instead of being mistaken for an empty follow-up +- treats coding-provider completion as an intermediate checkpoint until Git finalization records a pushed branch or a verified no-change result. Task QA waits for that evidence. If restart interrupts this window, startup uses recovered-session membership (including hard kills with no shutdown event), preserves the workspace, requeues the premature terminal projection, and continues at Git finalization without calling the coding provider again. - reconciles the recovered coding task-run and dispatch after a successful handoff, preventing an earlier transient failure marker from incorrectly failing the sprint during terminal evaluation - requeues interrupted worker-owned CI-fix and merge-conflict attention, clearing ownership left by the stopped virtual worker - closes the stopped repair attempt's provider-usage row before requeueing it, so a hard restart cannot leave a stale invocation occupying the provider concurrency limit. A durable `workspace_finalized`, `host_publishing`, or `host_published` checkpoint proves that the provider returned successfully, so recovery records that attempt as completed; an attempt interrupted before that boundary is recorded as cancelled. diff --git a/docs/sprint-loop/atomic-loop.md b/docs/sprint-loop/atomic-loop.md index c62934fd73..9217fd1f96 100644 --- a/docs/sprint-loop/atomic-loop.md +++ b/docs/sprint-loop/atomic-loop.md @@ -124,6 +124,7 @@ For `status` and `orchestrate`, each cycle follows the strict execution order de - Filters `PENDING` tasks, skips quota cooldowns, applies coding guardrails, and respects provider concurrency deferrals. - Evaluates the readiness gate: a task must be `PENDING`, dependencies completed and merged, provider concurrency available, and emergency stop inactive. - Provider concurrency admission uses global provider load from both running provider invocations and running task runs. This matters for CLI/Docker providers because a task run can reserve orchestration capacity before its provider invocation row starts. + - The scheduler asks the admission service for current purpose-aware capacity once per provider in the cycle, caches that budget, and decrements it after every successful start. Adaptive reservations therefore stop over-dispatch before lower-level claims become visible; configured capacity remains an upper bound rather than a promise that every slot is presently available. - Task dispatch creates DB task dispatch and task-run records, selects the provider based on settings (uses hosted provider for `jules` and CLI/Docker or host workflows for local providers). - Capacity deferral is never a task failure. If a lower provider stage reports the cap after dispatch rows were created, the dispatch returns to `queued`, the task run returns to `PENDING`, and the project task returns to `pending` for a later cycle. - Marks tasks `RUNNING`, records session id/name/provider, and resets consecutive failure count on success. Triggers emergency stop after repeated real dispatch failures. @@ -144,12 +145,15 @@ For `status` and `orchestrate`, each cycle follows the strict execution order de When `action=orchestrate`, `wait` is true, and `watchLoop` is enabled: - Orchestrator executes continuous cycles. - The default wait interval is 1 second between cycles and remains configurable per scope. Explicit existing project or sprint overrides are preserved. +- The watch loop holds one project Git-helper lease for its lifetime. The helper is created lazily on the first eligible Git command, shared by concurrent sprints in the same project, and removed after the final active sprint releases its lease. A single-cycle orchestration action uses the same scoped lease. - Checkpoint reports (based on `watchLoopOutputIntervalSeconds`) are emitted without ending the run. The checkpoint boundary is used to renew heartbeats and leases inside the same sprint run, keeping it alive while resetting the checkpoint window. - The loop continuously observes pause and cancel interventions at the top of each cycle. - Finalisation only runs on terminal conditions. - Startup recovery and dashboard **Resume** restart monitoring through the existing-run recovery path. A resumed paused run keeps its original sprint-run id, is moved back to `running`, and then starts the watch loop without creating a duplicate run. Resume is refused while another queued/running/cancel-pending run for the same sprint is active. -- If shutdown lands after a coding provider has completed but before Git finalization records the task as code-complete, recovery preserves the workspace and marks that exact crash window. The replacement task run resumes at Git finalization and reuses the completed provider result instead of invoking the coding agent a second time. A missing preserved workspace falls back to a normal fresh invocation rather than trusting unavailable changes. -- Under the restart invocation `continue` policy, startup treats QA review, CI fix, and merge-conflict repair as durable work streams just like task coding. It closes each interrupted audit invocation, preserves its logical/native session and workspace binding, and requeues only the work needed to create the correlated continuation. Claimed repair attention is released from the stopped virtual-worker endpoint before the next worker poll. +- Provider completion is only an intermediate CLI-workflow checkpoint. Task QA and branch merging wait for durable Git-finalization evidence (`cli_git_pushed` or `cli_git_no_changes`), so they never inspect or settle a branch that is still being materialized. +- If shutdown lands after a coding provider has completed but before Git finalization records the task as code-complete, startup identifies the exact run through the recovered CLI session even when a hard kill prevented the shutdown event from being written. Recovery preserves the workspace, requeues the prematurely projected terminal run, and marks the completed provider result. The replacement task run resumes at Git finalization instead of invoking the coding agent a second time. A missing preserved workspace falls back to a normal fresh invocation rather than trusting unavailable changes. +- Under the restart invocation `continue` policy, startup treats QA review, CI fix, and merge-conflict repair as durable work streams just like task coding. It closes each interrupted audit invocation, preserves its logical/native session and workspace binding, and requeues only the work needed to create the correlated continuation. Claimed repair attention is released from the stopped virtual-worker endpoint before the next worker poll. A task-coding session interrupted by the runtime restart receives an idempotent guardrail refund keyed to its task run, so repeated operational restarts cannot exhaust the coding-attempt budget; genuine provider failures remain charged. +- Sprint planning follows the same restart contract. Startup preserves the full planning request and options, then continues the exact native provider conversation in its stable planning workspace. It never replaces a recorded but unavailable planning conversation with a fresh one; only a request stopped before provider linkage is reissued from durable input because no conversation existed yet. - Existing-run recovery first checks the in-memory active-orchestrator registry and returns without starting another watch loop when the same project/sprint is already being monitored by the current process. - Sprint-run lifecycle updates are mirrored to the parent sprint row for dashboard/operator consistency. Active run states (`queued`, `running`, `cancel_requested`) keep the sprint `running`; pause, completion, failure, and cancellation transitions update the sprint row to the matching summary state. Heartbeats also repair drift after restarts, so a live run cannot remain hidden behind an `idle` sprint summary. - Human-escalated merge conflicts stop counting as worker activity. If a task conflict has already been handed to a human and no runnable work remains, the watch loop pauses the sprint instead of keeping the run alive with only heartbeat traffic. @@ -163,7 +167,9 @@ When `action=orchestrate`, `wait` is true, and `watchLoop` is enabled: - The watch loop reuses active project-attention rows already loaded by the cycle runner when evaluating terminal state. It only falls back to a direct attention read if an older or test cycle result does not provide the cycle-loaded rows. - LOCAL Git-finalization evidence is collected by the cycle runner after its final merge drain and passed to the watch loop for terminal-state evaluation. The watch loop does not rescan every task-run event history unless it is running with a legacy/test cycle result that lacks that snapshot. - Session synchronization compares provider state with persisted task-run, dispatch, and planning state before writing. Unchanged active sessions do not rewrite rows; dispatch liveness heartbeats are refreshed at most once per minute, while state, error, branch, PR, start, and finish changes are persisted immediately. -- Feature-PR gate evaluation uses one immutable local-Git event snapshot per task per cycle. Duplicate idempotent task-run events do not invalidate runtime wall-time caches or publish realtime refreshes. +- The local session snapshot used by the watch loop projects only routing and state metadata; it does not load stored CLI prompts. Full prompts remain available through direct session and invocation reads, but large QA/planning prompts are not decoded again on every one-second synchronization cycle. +- Feature-PR gate evaluation loads the latest task runs for the DAG in one chunked batch and uses one immutable, event-type-filtered local-Git snapshot per task per cycle. It reuses those run objects through branch recovery, merge settlement, and CI event writes instead of issuing per-task lookup queries. Duplicate idempotent task-run events do not invalidate runtime wall-time caches or publish realtime refreshes. +- LOCAL Git-finalization evidence uses the same chunked latest-run projection for the whole DAG. Repeated status-derivation and merge-protocol passes therefore avoid one latest-run query per task, including pending tasks that have no run yet. - A task or merge-state change triggers one bounded 250 ms follow-up cycle before the loop resumes its configured polling interval. This removes avoidable dependency-unlock and merge-drain delay without busy-polling unchanged provider work. - Dashboard live snapshots are optimized for high sprint concurrency: selected-sprint checks use targeted project/sprint lookups instead of hydrating every sprint, recent provider activities are cached until the project's latest provider activity event changes, and provider activity event reads use partial sqlite indexes for the activity-only paths. - Preview reconciliation also avoids full project execution snapshots in the common running-session path. It queries running sprint runs directly and memoizes that result while reconciling preview sessions and auto-starting previews. @@ -222,6 +228,8 @@ For `action=status`: - Recovered stale QA reviews in `failed`, `cancelled`, or `errored` state are retryable infrastructure signals even when the verdict budget is otherwise spent, but only below the hard total-attempt ceiling (`maxTaskReviewRuns + QA_INFRA_FAILURE_GRACE`). Every terminal review attempt counts toward that ceiling, so repeated container disappearance eventually applies the exhaustion policy instead of looping forever. - A same-session CLI QA fix must produce new mergeable Git progress from that invocation. A successful provider exit with no new patch records `follow_up_no_progress` and applies the exhaustion policy immediately rather than starting another review; existing branch commits cannot renew the cycle. Coding prompts also require a structured `CODE_UX_TASK_OUTCOME` marker. A no-change run reported as `blocked`, or one that omits the required outcome, becomes a blocked task dispatch instead of a completed no-output task. - QA review budgets count review cycles, not reviewer rows. Multiple reviewer rows with the same `run_index` spend one task or sprint QA attempt while still preserving reviewer-specific `agent_preset_id`, `agent_name`, payload details, and task-run events for dashboard history. Latest-cycle summaries prefer blocking rows (`running`, `changes_requested`, or `failed`) over passing rows, so a single passing reviewer cannot hide another reviewer that still blocks the cycle. +- Wide-DAG QA reconciliation and initial merge-gate evaluation load latest-cycle rows and decisive/terminal cycle counts for all tasks in one chunked query. Per-task review dispatch and post-review gates retain their exact fail-closed behavior without four SQLite reads per task on every watch cycle. +- Task QA uses bounded waves of at most four reviews per orchestration cycle, or a lower positive routed-provider capacity. After one wave is scheduled, the cycle settles its results, merges ready branches, and starts newly unblocked coding before admitting another QA wave. The provider admission service remains authoritative and can lower concurrency further under host pressure before an invocation claims a slot. - Starting or resuming orchestration resolves stale sprint-level `manual_attention` escalations from prior runs. The new run recomputes current blockers, while task-specific human attention remains open until explicitly handled. - Before task QA gates are evaluated, the sprint cycle reconciles running task QA invocations with provider runtime state. Missing provider linkage or a missing Docker session container makes the stale QA row retryable instead of blocking the task indefinitely at `QA_PENDING`. - Sprint-completion QA also uses the sprint trigger's `agentPresetIds` list, or one default fallback reviewer when the list is empty. Completion is allowed only after the latest sprint QA cycle has all reviewers passed; any running, failed, or changes-requested reviewer blocks completion. Before the final configured cycle, a changes-requested review may route follow-up task/session repair through the existing sprint QA logic. The final cycle is verification-only: a non-passing verdict records its findings and escalates without creating work that has no remaining review budget. diff --git a/electron-builder.config.cjs b/electron-builder.config.cjs index aeed8e4065..91cc08ef19 100644 --- a/electron-builder.config.cjs +++ b/electron-builder.config.cjs @@ -118,6 +118,7 @@ module.exports = { linux: { category: "Development", maintainer: "Pierre Voss ", + executableName: "codeux", target: [ "AppImage", "deb", diff --git a/package.json b/package.json index 6caad842d2..ea2607879a 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "electron:dist:win": "pnpm run build && pnpm run electron:prepare-deps && electron-builder --config electron-builder.config.cjs --win", "electron:benchmark:runtime": "node scripts/benchmark-electron-runtime.mjs", "electron:benchmark:win": "node scripts/benchmark-electron-windows.mjs", + "electron:smoke-installed": "node scripts/smoke-installed-electron.mjs", "electron:install-deps": "electron-builder install-app-deps", "test": "vitest run", "smoketest:codex": "node --import ./scripts/tsnode-register.mjs tests/smoketest/codex.smoketest.ts", @@ -88,6 +89,7 @@ "test:orchestration:full": "pnpm run build && pnpm run test:e2e:mockup-sprint-pentest", "test:orchestration:large-dag": "pnpm run build && node scripts/e2e/run-mockup-sprint-pentest.mjs --scenario large-dag-stress --timeout-ms 3600000", "test:orchestration:pentest": "pnpm run build && node scripts/e2e/run-mockup-sprint-pentest.mjs --scenario pentest --timeout-ms 3600000", + "test:orchestration:extreme-dag": "pnpm run build && node scripts/e2e/run-mockup-sprint-pentest.mjs --scenario extreme-dag-recovery --timeout-ms 7200000 --stall-timeout-ms 300000 --restart-every-ms 30000 --restart-count 8", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:backend:coverage": "vitest run tests/backend --coverage", @@ -120,7 +122,8 @@ "multer": "^2.1.1", "onnxruntime-node": "^1.24.3", "p-limit": "^7.3.0", - "pdf-parse": "^2.4.5" + "pdf-parse": "^2.4.5", + "zod": "^4.3.6" }, "devDependencies": { "@monaco-editor/react": "^4.7.0", diff --git a/playwright.config.ts b/playwright.config.ts index b0910e0b8e..af0671c0b8 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -114,7 +114,7 @@ export default defineConfig({ // Rebuild the dashboard with gated workspaces enabled so navigation specs // exercise the production route tree instead of depending on a caller's // previously built dashboard assets. - command: 'pnpm exec vite build && node dist/index.js', + command: 'node ./node_modules/vite/bin/vite.js build && node dist/index.js', // Poll the liveness probe (/health) rather than the readiness probe (/ready). // /ready only returns 200 once a project has a live-status timestamp, which // never happens in a clean CI checkout, so it would hang until timeout. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b216a29a7c..d86300e4c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,6 +90,9 @@ importers: pdf-parse: specifier: ^2.4.5 version: 2.4.5 + zod: + specifier: ^4.3.6 + version: 4.3.6 devDependencies: '@monaco-editor/react': specifier: ^4.7.0 diff --git a/scripts/e2e/mock-provider-cli.mjs b/scripts/e2e/mock-provider-cli.mjs index 3ca76770d5..0cb4a658a0 100644 --- a/scripts/e2e/mock-provider-cli.mjs +++ b/scripts/e2e/mock-provider-cli.mjs @@ -38,13 +38,20 @@ function parseArgs(argv) { } } - if (!parsed.prompt && argv.length > 0) { - parsed.prompt = argv[argv.length - 1] || ""; - } - return parsed; } +async function readStdin() { + if (process.stdin.isTTY) { + return ""; + } + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} + function readMarker(prompt, name) { const pattern = new RegExp(`\\[mock-provider:${name}=([^\\]]+)\\]`, "i"); const match = pattern.exec(prompt); @@ -161,6 +168,9 @@ function writeProviderStdout(run) { async function main() { const run = parseArgs(process.argv.slice(2)); + if (!run.prompt) { + run.prompt = await readStdin(); + } const cwd = process.cwd(); const sleepMs = clampSleepMs(readMarker(run.prompt, "sleep") || "0"); const noOp = hasMarker(run.prompt, "no-op"); diff --git a/scripts/e2e/mockup-sprint-pentest-scenarios.mjs b/scripts/e2e/mockup-sprint-pentest-scenarios.mjs index 334b1c80b5..fe33bfcdc9 100644 --- a/scripts/e2e/mockup-sprint-pentest-scenarios.mjs +++ b/scripts/e2e/mockup-sprint-pentest-scenarios.mjs @@ -196,6 +196,24 @@ export const PROJECT_FIXTURES = [ maxSprintReviewRuns: 0, }, }, + { + id: "extreme-wide-docker", + name: "mockup pentest project extreme wide docker", + directoryName: "mockup-pentest-extreme-wide-docker", + featureBranchPrefix: "mockup-extreme/", + watchLoopIntervalSeconds: 1, + watchLoopOutputIntervalSeconds: 15, + providerConcurrency: 16, + workerTimeoutSeconds: 1_200, + executionMode: "DOCKER", + qualityAssurance: { + enabled: true, + taskCompletion: true, + sprintCompletion: true, + maxTaskReviewRuns: 3, + maxSprintReviewRuns: 1, + }, + }, ]; for (const fixture of PROJECT_FIXTURES) { @@ -215,7 +233,10 @@ function validationScript(expectedExpression, importLine = "import { stableNumbe } function fileWrite(pathname, content) { - return `mockup-cli:write ${pathname} :: ${content}`; + // Directives are line-oriented. Encode constructed multiline content so a + // provider sees one complete write operation instead of silently truncating + // the file at its first physical newline. + return `mockup-cli:write ${pathname} :: ${String(content).replaceAll("\n", "\\n")}`; } function append(pathname, content) { @@ -367,6 +388,257 @@ function buildLargeDagStressTasks() { const LARGE_DAG_STRESS_TASKS = buildLargeDagStressTasks(); +function buildExtremeDagRecoveryTasks() { + const tasks = [ + { + key: "extreme-root", + title: "Initialize extreme DAG recovery workspace", + priority: "critical", + promptMarkdown: prompt([ + fileWrite("src/extreme/root.js", "export const extremeRoot = 'extreme-root-ready';"), + append("README.md", "Extreme DAG recovery scenario initialized."), + ]), + }, + ]; + + // 240 leaves create a deliberately wide ready queue. Priority skew verifies + // that scheduler ordering never violates the shared root dependency. + for (let index = 1; index <= 240; index += 1) { + const padded = padNumber(index); + const identifier = camelKey("extremeLeaf", index); + tasks.push({ + key: `extreme-leaf-${padded}`, + title: `Create extreme DAG leaf ${padded}`, + priority: index % 17 === 0 ? "critical" : index % 5 === 0 ? "high" : index % 3 === 0 ? "low" : "medium", + dependsOn: ["extreme-root"], + promptMarkdown: prompt([ + fileWrite( + `src/extreme/leaf-${padded}.js`, + `import { extremeRoot } from './root.js';\nexport const ${identifier} = extremeRoot + ':leaf-${padded}';`, + ), + ]), + }); + } + + // Eighty three-way joins exercise repeated fan-in after the wide wave. + for (let shard = 1; shard <= 80; shard += 1) { + const padded = padNumber(shard, 2); + const leafIndexes = [(shard - 1) * 3 + 1, (shard - 1) * 3 + 2, (shard - 1) * 3 + 3]; + const imports = leafIndexes.map((leafIndex) => ( + `import { ${camelKey("extremeLeaf", leafIndex)} } from './leaf-${padNumber(leafIndex)}.js';` + )).join("\n"); + const values = leafIndexes.map((leafIndex) => camelKey("extremeLeaf", leafIndex)).join(", "); + tasks.push({ + key: `extreme-shard-${padded}`, + title: `Aggregate extreme DAG shard ${padded}`, + priority: shard % 11 === 0 ? "high" : "medium", + dependsOn: leafIndexes.map((leafIndex) => `extreme-leaf-${padNumber(leafIndex)}`), + promptMarkdown: prompt([ + fileWrite( + `src/extreme/shard-${padded}.js`, + `${imports}\nexport const ${camelKey("extremeShard", shard, 2)} = [${values}].join('|');`, + ), + ]), + }); + } + + // Pair the first and second halves so each join has non-adjacent parents. + for (let cross = 1; cross <= 40; cross += 1) { + const padded = padNumber(cross, 2); + const shardIndexes = [cross, cross + 40]; + const imports = shardIndexes.map((shardIndex) => ( + `import { ${camelKey("extremeShard", shardIndex, 2)} } from './shard-${padNumber(shardIndex, 2)}.js';` + )).join("\n"); + const values = shardIndexes.map((shardIndex) => camelKey("extremeShard", shardIndex, 2)).join(", "); + tasks.push({ + key: `extreme-cross-${padded}`, + title: `Join distant extreme DAG shards ${padded}`, + priority: cross % 7 === 0 ? "critical" : "high", + dependsOn: shardIndexes.map((shardIndex) => `extreme-shard-${padNumber(shardIndex, 2)}`), + promptMarkdown: prompt([ + fileWrite( + `src/extreme/cross-${padded}.js`, + `${imports}\nexport const ${camelKey("extremeCross", cross, 2)} = [${values}].join('::');`, + ), + ]), + }); + } + + // Each layer joins distant cross nodes plus a diagonal shard dependency, + // producing diamonds and cross-layer edges without introducing cycles. + for (let layer = 1; layer <= 20; layer += 1) { + const padded = padNumber(layer, 2); + const crossIndexes = [layer, layer + 20]; + const diagonalShard = ((layer * 7) % 80) + 1; + const imports = [ + ...crossIndexes.map((crossIndex) => ( + `import { ${camelKey("extremeCross", crossIndex, 2)} } from './cross-${padNumber(crossIndex, 2)}.js';` + )), + `import { ${camelKey("extremeShard", diagonalShard, 2)} } from './shard-${padNumber(diagonalShard, 2)}.js';`, + ].join("\n"); + const values = [ + ...crossIndexes.map((crossIndex) => camelKey("extremeCross", crossIndex, 2)), + camelKey("extremeShard", diagonalShard, 2), + ].join(", "); + tasks.push({ + key: `extreme-layer-${padded}`, + title: `Resolve extreme DAG diamond layer ${padded}`, + priority: layer % 2 === 0 ? "medium" : "low", + dependsOn: [ + ...crossIndexes.map((crossIndex) => `extreme-cross-${padNumber(crossIndex, 2)}`), + `extreme-shard-${padNumber(diagonalShard, 2)}`, + ], + promptMarkdown: prompt([ + fileWrite( + `src/extreme/layer-${padded}.js`, + `${imports}\nexport const ${camelKey("extremeLayer", layer, 2)} = [${values}].join('@@');`, + ), + ...(layer === 10 ? [ + "mockup-qa:require-file src/extreme/qa-follow-up.js :: extremeQaFollowUp", + "mockup-qa:fix-write src/extreme/qa-follow-up.js :: export const extremeQaFollowUp = 'qa-follow-up-visible';", + ] : []), + ]), + }); + } + + for (let barrier = 1; barrier <= 10; barrier += 1) { + const padded = padNumber(barrier, 2); + const layerIndexes = [barrier, barrier + 10]; + const imports = layerIndexes.map((layerIndex) => ( + `import { ${camelKey("extremeLayer", layerIndex, 2)} } from './layer-${padNumber(layerIndex, 2)}.js';` + )).join("\n"); + const values = layerIndexes.map((layerIndex) => camelKey("extremeLayer", layerIndex, 2)).join(", "); + tasks.push({ + key: `extreme-barrier-${padded}`, + title: `Create extreme DAG priority barrier ${padded}`, + priority: barrier <= 2 ? "critical" : barrier <= 5 ? "high" : "medium", + dependsOn: layerIndexes.map((layerIndex) => `extreme-layer-${padNumber(layerIndex, 2)}`), + promptMarkdown: prompt([ + fileWrite( + `src/extreme/barrier-${padded}.js`, + `${imports}\nexport const ${camelKey("extremeBarrier", barrier, 2)} = [${values}].join('##');`, + ), + ]), + }); + } + + const regionalBarrierIndexes = [[1, 5, 9], [2, 6, 10], [3, 7], [4, 8]]; + for (let region = 1; region <= regionalBarrierIndexes.length; region += 1) { + const padded = padNumber(region, 2); + const barrierIndexes = regionalBarrierIndexes[region - 1]; + const imports = barrierIndexes.map((barrierIndex) => ( + `import { ${camelKey("extremeBarrier", barrierIndex, 2)} } from './barrier-${padNumber(barrierIndex, 2)}.js';` + )).join("\n"); + const values = barrierIndexes.map((barrierIndex) => camelKey("extremeBarrier", barrierIndex, 2)).join(", "); + tasks.push({ + key: `extreme-region-${padded}`, + title: `Aggregate extreme DAG region ${padded}`, + priority: "critical", + dependsOn: barrierIndexes.map((barrierIndex) => `extreme-barrier-${padNumber(barrierIndex, 2)}`), + promptMarkdown: prompt([ + fileWrite( + `src/extreme/region-${padded}.js`, + `${imports}\nexport const ${camelKey("extremeRegion", region, 2)} = [${values}].join('%%');`, + ), + ]), + }); + } + + const regionKeys = Array.from({ length: 4 }, (_, index) => `extreme-region-${padNumber(index + 1, 2)}`); + tasks.push({ + key: "extreme-no-change-gate", + title: "Exercise no-change dependency recovery", + priority: "low", + dependsOn: regionKeys, + promptMarkdown: prompt([ + run("node -e \"if (!process.versions.node) process.exit(1)\""), + ]), + }); + tasks.push({ + key: "extreme-tail-a", + title: "Start extreme DAG recovery tail", + priority: "critical", + dependsOn: ["extreme-no-change-gate"], + promptMarkdown: prompt([ + fileWrite("src/extreme/tail-a.js", "export const extremeTailA = 'tail-a-ready';"), + ]), + }); + tasks.push({ + key: "extreme-tail-b", + title: "Join extreme DAG recovery tail diamond", + priority: "critical", + dependsOn: ["extreme-tail-a", "extreme-leaf-240"], + promptMarkdown: prompt([ + fileWrite( + "src/extreme/tail-b.js", + "import { extremeTailA } from './tail-a.js';\nimport { extremeLeaf240 } from './leaf-240.js';\nexport const extremeTailB = extremeTailA + '::' + extremeLeaf240;", + ), + ]), + }); + + const regionImports = Array.from({ length: 4 }, (_, index) => { + const region = index + 1; + return `import { ${camelKey("extremeRegion", region, 2)} } from './region-${padNumber(region, 2)}.js';`; + }).join("\n"); + const regionValues = Array.from({ length: 4 }, (_, index) => camelKey("extremeRegion", index + 1, 2)).join(", "); + tasks.push({ + key: "extreme-final-manifest", + title: "Create extreme DAG final recovery manifest", + priority: "critical", + dependsOn: ["extreme-tail-b", ...regionKeys], + promptMarkdown: prompt([ + fileWrite( + "src/extreme/final.js", + `${regionImports}\nimport { extremeTailB } from './tail-b.js';\nimport { extremeQaFollowUp } from './qa-follow-up.js';\nexport const extremeDagSummary = [${regionValues}, extremeTailB, extremeQaFollowUp].join('&&');\nexport const extremeDagTaskCount = 400;`, + ), + append("README.md", "Extreme DAG recovery scenario produced 400 deterministic tasks."), + ]), + }); + tasks.push({ + key: "extreme-validation", + title: "Validate all extreme DAG recovery output", + priority: "critical", + dependsOn: ["extreme-final-manifest"], + promptMarkdown: prompt([ + fileWrite( + "test/run-validation.mjs", + validationScript( + "extremeDagTaskCount === 400 && extremeDagSummary.includes('leaf-001') && extremeDagSummary.includes('leaf-240') && extremeDagSummary.split('leaf-').length > 240", + "import { extremeDagSummary, extremeDagTaskCount } from '../src/extreme/final.js';", + ), + ), + "mockup-qa:require-file src/extreme/final.js :: extremeDagTaskCount = 400", + "mockup-sprint-qa:require-file src/extreme/final.js :: extremeDagTaskCount = 400", + "mockup-sprint-qa:require-file src/extreme/qa-follow-up.js :: qa-follow-up-visible", + run("node test/run-validation.mjs"), + ]), + }); + + if (tasks.length !== 400) { + throw new Error(`Extreme DAG fixture must contain exactly 400 tasks, received ${tasks.length}`); + } + // Double each instruction with inert, local-only context pressure. The full + // 400-task QA context then crosses the 100k estimated-token threshold, while + // the retained first half still contains every executable mockup directive. + return tasks.map((task) => ({ + ...task, + promptMarkdown: `${task.promptMarkdown}\n${"Local-only QA context pressure: ".padEnd(task.promptMarkdown.length + 12, "x")}`, + })); +} + +const EXTREME_DAG_RECOVERY_TASKS = buildExtremeDagRecoveryTasks(); +const EXTREME_QA_TASK_KEYS = EXTREME_DAG_RECOVERY_TASKS + .map((task) => task.key) + .filter((taskKey) => taskKey !== "extreme-no-change-gate"); +const EXTREME_QA_EXPECTATIONS = Object.fromEntries(EXTREME_QA_TASK_KEYS.map((taskKey) => [taskKey, { + outcomes: taskKey === "extreme-layer-10" ? ["changes_requested", "pass"] : ["pass"], + ...(taskKey === "extreme-layer-10" ? { requireFollowUp: true, requireSameWorkerBranch: true } : {}), +}])); +const EXTREME_TASK_CODING_COUNTS = Object.fromEntries(EXTREME_DAG_RECOVERY_TASKS.map((task) => [ + task.key, + task.key === "extreme-layer-10" ? 2 : 1, +])); + function buildCiSmallDagTasks() { const tasks = [ { @@ -1145,6 +1417,92 @@ export const SCENARIOS = [ }, ], }, + { + id: "extreme-dag-recovery", + name: "Mockup local-only 400-task DAG restart recovery sprint", + heavy: true, + localOnly: true, + timeoutMs: 2 * 60 * 60 * 1000, + projectRuns: [ + { + key: "extreme-dag-recovery-docker", + fixtureId: "extreme-wide-docker", + duringOrchestration: { + injectMainCiFix: { + markerPath: "src/extreme/ci-fix.js", + markerContent: "export const extremeCiFix = 'extreme-ci-fix-invoked';", + }, + }, + project: { + name: "mockup pentest project extreme dag recovery", + sourceType: "local", + initMode: "existing", + defaultBranch: "main", + }, + sprint: { + name: "Mockup 400-task DAG restart recovery", + goal: "Pentest a deterministic 400-task graph with wide fan-out, distant and diagonal edges, priority skew, task and sprint QA, QA-requested coding follow-up, routed CI repair, a no-change merge gate, layered fan-in, long-tail recovery, repeated full-runtime restarts, and final integration validation.", + }, + tasks: EXTREME_DAG_RECOVERY_TASKS, + expected: { + taskStatuses: expectedCompleted(EXTREME_DAG_RECOVERY_TASKS), + qa: { + tasks: EXTREME_QA_EXPECTATIONS, + sprintOutcomes: ["pass"], + }, + files: [ + { path: "src/extreme/ci-fix.js", contains: ["extreme-ci-fix-invoked"] }, + { path: "src/extreme/qa-follow-up.js", contains: ["qa-follow-up-visible"] }, + { path: "src/extreme/final.js", contains: ["extremeDagSummary", "extremeDagTaskCount = 400", "extremeQaFollowUp"] }, + { path: "README.md", contains: ["Extreme DAG recovery scenario produced 400 deterministic tasks"] }, + ], + invocations: { + minimumCompletedCiFixes: 1, + requireSprintLevelCiFix: true, + taskCodingCounts: EXTREME_TASK_CODING_COUNTS, + }, + statusReporting: { + taskKeys: EXTREME_DAG_RECOVERY_TASKS.map((task) => task.key), + qaTaskKeys: EXTREME_QA_TASK_KEYS, + followUpTaskKeys: ["extreme-layer-10"], + requireSprintQa: true, + }, + resources: { + maxRuntimeRssBytes: 1_073_741_824, + maxDatabaseBytes: 268_435_456, + maxWalBytes: 536_870_912, + maxSessionDatabaseBytes: 67_108_864, + maxSessionWalBytes: 67_108_864, + minTaskQaPromptChars: 5_000, + maxTaskQaPromptChars: 50_000, + minSprintQaPromptChars: 250_000, + maxSprintQaPromptChars: 400_000, + // Eight restarts can interrupt one complete 16-wide admission wave: + // 400 successful attempts + 128 interrupted attempts + margin. + maxTaskRuns: 560, + maxFailedTaskRuns: 160, + maxTaskAttemptAmplification: 1.45, + maxActiveTaskRuns: 16, + maxTaskRunEvents: 40_000, + maxProviderInvocations: 1_100, + maxProjectGitHelpers: 1, + maxWorkspaceHelpers: 16, + maxFinalNonRunningContainers: 0, + // A four-task Docker recovery profile on this host measured 4.3s preparation, + // 2.2s Git finalization, and 9.3s end-to-end workflow p50. Keep enough headroom for + // the 400-task contention profile while rejecting the former 55s/22s/94s regression. + maxPrepareP50Ms: 20_000, + maxGitFinalizeP50Ms: 10_000, + maxWorkflowP50Ms: 35_000, + // One project helper generation per compiled-runtime process, plus + // a small allowance for samples overlapping restart handoff. + maxProjectGitHelperGenerations: 10, + }, + commands: [{ command: "node test/run-validation.mjs", exitCode: 0 }], + }, + }, + ], + }, ]; export function listScenarioIds() { diff --git a/scripts/e2e/run-mockup-sprint-pentest.mjs b/scripts/e2e/run-mockup-sprint-pentest.mjs index dddbf05b79..9aa46d37a8 100644 --- a/scripts/e2e/run-mockup-sprint-pentest.mjs +++ b/scripts/e2e/run-mockup-sprint-pentest.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { constants as fsConstants } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { constants as fsConstants, createWriteStream } from "node:fs"; import fs from "node:fs/promises"; import http from "node:http"; import net from "node:net"; @@ -32,6 +32,8 @@ const DEFAULT_STALL_TIMEOUT_MS = 3 * 60 * 1000; const HTTP_REQUEST_TIMEOUT_MS = 60_000; const POLL_INTERVAL_MS = 2_000; const PROGRESS_LOG_INTERVAL_MS = 15_000; +const RESOURCE_SAMPLE_INTERVAL_MS = 5_000; +const MAX_RESOURCE_SAMPLES = 2_048; const MAX_STATUS_OBSERVATIONS = 1_024; const SERVER_READY_TIMEOUT_MS = 60_000; const SERVER_SHUTDOWN_TIMEOUT_MS = 15_000; @@ -202,10 +204,12 @@ function logEvent(type, payload = {}) { export function isMockupPollStateProgress({ progressChanged, expectedOutputReadinessChanged, + runtimeRestartCompleted = false, }) { // Admission heartbeats prove the waiter is alive and remain useful diagnostics, but they are not - // orchestration progress. Only externally visible state/output changes reset the stall watchdog. - return Boolean(progressChanged || expectedOutputReadinessChanged); + // orchestration progress. A finite, explicitly requested runtime restart is progress for watchdog + // accounting because task state is intentionally frozen while the process exits and recovers. + return Boolean(progressChanged || expectedOutputReadinessChanged || runtimeRestartCompleted); } const MAX_PROGRESS_CHANGED_TASKS = 32; @@ -537,6 +541,462 @@ async function writeFile(filePath, content) { await fs.writeFile(filePath, content, "utf8"); } +async function readFileSize(filePath) { + try { + return (await fs.stat(filePath)).size; + } catch { + return 0; + } +} + +async function readRuntimeRssBytes(pid) { + if (!Number.isInteger(pid) || pid <= 0) return null; + if (process.platform === "linux") { + try { + const status = await fs.readFile(`/proc/${pid}/status`, "utf8"); + const match = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status); + if (match) return Number(match[1]) * 1_024; + } catch { + return null; + } + } + if (process.platform !== "win32") { + try { + const result = await spawnLogged("ps", ["-o", "rss=", "-p", String(pid)]); + const rssKiB = Number(result.stdout.trim()); + return Number.isFinite(rssKiB) ? rssKiB * 1_024 : null; + } catch { + return null; + } + } + return null; +} + +function resolveRuntimeOwnerId(homeDir) { + const resolvedStateHome = path.resolve(homeDir, ".code-ux").replace(/\\/g, "/"); + const canonicalStateHome = process.platform === "win32" + ? resolvedStateHome.toLowerCase() + : resolvedStateHome; + return createHash("sha256").update(canonicalStateHome).digest("hex").slice(0, 24); +} + +export async function readRuntimeDockerHelperSnapshot(homeDir) { + try { + const ownerId = resolveRuntimeOwnerId(homeDir); + const result = await spawnLogged("docker", [ + "ps", + "-a", + "--filter", + `label=code-ux.runtime-owner=${ownerId}`, + "--format", + "{{.ID}}\t{{.State}}\t{{.Label \"code-ux.helper\"}}\t{{.Label \"code-ux.managed\"}}\t{{.Label \"code-ux.command\"}}", + ]); + const containers = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [id = "", state = "", type = "", managed = "", command = ""] = line.split("\t", 5); + return { id, state: state.toLowerCase(), type, managed, command }; + }); + const running = containers.filter((container) => container.state === "running"); + return { + projectGitHelperIds: running.filter((container) => container.type === "git").map((container) => container.id), + workspaceHelperIds: running.filter((container) => container.type === "volume").map((container) => container.id), + nonRunningContainerIds: containers.filter((container) => container.state !== "running").map((container) => container.id), + runningProviderContainerIds: running + .filter((container) => container.managed === "true" && container.command) + .map((container) => container.id), + containerStates: containers.map(({ id, state, type }) => ({ id, state, helperType: type || null })), + }; + } catch { + return { + projectGitHelperIds: [], + workspaceHelperIds: [], + nonRunningContainerIds: [], + runningProviderContainerIds: [], + containerStates: [], + }; + } +} + +export function readScopedResourceCounts(databasePath, projectId, sprintId) { + let database; + try { + database = new DatabaseSync(databasePath, { readOnly: true }); + const taskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs + WHERE project_id = ? AND sprint_id = ? + `).get(projectId, sprintId)?.count; + const runningTaskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs + WHERE project_id = ? AND sprint_id = ? AND state = 'RUNNING' + `).get(projectId, sprintId)?.count; + const completedTaskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs + WHERE project_id = ? AND sprint_id = ? AND state = 'COMPLETED' + `).get(projectId, sprintId)?.count; + const failedTaskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs + WHERE project_id = ? AND sprint_id = ? AND state = 'FAILED' + `).get(projectId, sprintId)?.count; + const cancelledTaskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs + WHERE project_id = ? AND sprint_id = ? AND state = 'CANCELLED' + `).get(projectId, sprintId)?.count; + const activeTaskRuns = database.prepare(` + SELECT COUNT(*) AS count + FROM task_runs tr + INNER JOIN task_dispatches td ON td.id = tr.dispatch_id + WHERE tr.project_id = ? + AND tr.sprint_id = ? + AND tr.state IN ('PENDING', 'RUNNING', 'PAUSED') + AND td.status IN ('queued', 'claimed', 'running', 'cancel_requested', 'paused') + AND ( + EXISTS ( + SELECT 1 + FROM provider_invocations active_pi + WHERE active_pi.task_run_id = tr.id + AND active_pi.status = 'running' + ) + OR NOT EXISTS ( + SELECT 1 + FROM provider_invocations pi + WHERE pi.task_run_id = tr.id + ) + ) + `).get(projectId, sprintId)?.count; + const taskRunEvents = database.prepare(` + SELECT COUNT(*) AS count + FROM task_run_events tre + INNER JOIN task_runs tr ON tr.id = tre.task_run_id + WHERE tr.project_id = ? AND tr.sprint_id = ? + `).get(projectId, sprintId)?.count; + const providerInvocations = database.prepare(` + SELECT COUNT(*) AS count + FROM provider_invocations + WHERE project_id = ? AND sprint_id = ? + `).get(projectId, sprintId)?.count; + const maxTaskQaPromptChars = database.prepare(` + SELECT COALESCE(MAX(prompt_chars), 0) AS count + FROM provider_invocations + WHERE project_id = ? + AND sprint_id = ? + AND task_id IS NOT NULL + AND purpose = 'qa_review' + `).get(projectId, sprintId)?.count; + const maxSprintQaPromptChars = database.prepare(` + SELECT COALESCE(MAX(prompt_chars), 0) AS count + FROM provider_invocations + WHERE project_id = ? + AND sprint_id = ? + AND task_id IS NULL + AND purpose = 'qa_review' + `).get(projectId, sprintId)?.count; + const recoveredProviderCompletions = database.prepare(` + SELECT COUNT(*) AS count + FROM task_run_events tre + INNER JOIN task_runs tr ON tr.id = tre.task_run_id + WHERE tr.project_id = ? + AND tr.sprint_id = ? + AND tre.event_type = 'cli_provider_completion_recovered' + `).get(projectId, sprintId)?.count; + return { + taskRuns: Number(taskRuns || 0), + runningTaskRuns: Number(runningTaskRuns || 0), + completedTaskRuns: Number(completedTaskRuns || 0), + failedTaskRuns: Number(failedTaskRuns || 0), + cancelledTaskRuns: Number(cancelledTaskRuns || 0), + activeTaskRuns: Number(activeTaskRuns || 0), + taskRunEvents: Number(taskRunEvents || 0), + providerInvocations: Number(providerInvocations || 0), + maxTaskQaPromptChars: Number(maxTaskQaPromptChars || 0), + maxSprintQaPromptChars: Number(maxSprintQaPromptChars || 0), + recoveredProviderCompletions: Number(recoveredProviderCompletions || 0), + }; + } catch { + return null; + } finally { + try { database?.close(); } catch { /* best effort diagnostic read */ } + } +} + +function summarizeDurationValues(values) { + const sorted = values.filter(Number.isFinite).sort((left, right) => left - right); + const at = (percentile) => sorted.length === 0 + ? 0 + : sorted[Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * percentile))]; + return { + count: sorted.length, + p50Ms: at(0.5), + p90Ms: at(0.9), + p99Ms: at(0.99), + maxMs: at(1), + }; +} + +export function readScopedWorkflowTimings(databasePath, projectId, sprintId) { + let database; + try { + database = new DatabaseSync(databasePath, { readOnly: true }); + const rows = database.prepare(` + SELECT tr.id, + tr.duration_ms, + MIN(CASE WHEN tre.event_type = 'cli_prepare_started' THEN tre.created_at END) AS prepare_started_at, + MIN(CASE WHEN tre.event_type = 'cli_prepare_completed' THEN tre.created_at END) AS prepare_completed_at, + MIN(CASE WHEN tre.event_type = 'cli_provider_started' THEN tre.created_at END) AS provider_started_at, + MIN(CASE WHEN tre.event_type = 'cli_provider_completed' THEN tre.created_at END) AS provider_completed_at, + MIN(CASE WHEN tre.event_type = 'cli_memory_capture_started' THEN tre.created_at END) AS memory_started_at, + MIN(CASE WHEN tre.event_type = 'cli_memory_capture_completed' THEN tre.created_at END) AS memory_completed_at, + MIN(CASE WHEN tre.event_type = 'cli_git_finalize_started' THEN tre.created_at END) AS git_finalize_started_at, + MIN(CASE WHEN tre.event_type IN ('cli_git_pushed', 'cli_git_no_changes') THEN tre.created_at END) AS git_finalized_at, + MIN(CASE WHEN tre.event_type = 'cli_workflow_completed' THEN tre.created_at END) AS workflow_completed_at + FROM task_runs tr + INNER JOIN task_run_events tre ON tre.task_run_id = tr.id + WHERE tr.project_id = ? + AND tr.sprint_id = ? + AND tr.state = 'COMPLETED' + GROUP BY tr.id, tr.duration_ms + `).all(projectId, sprintId); + const elapsed = (start, end) => { + const startMs = typeof start === "string" ? Date.parse(start) : Number.NaN; + const endMs = typeof end === "string" ? Date.parse(end) : Number.NaN; + return Number.isFinite(startMs) && Number.isFinite(endMs) && endMs >= startMs + ? endMs - startMs + : Number.NaN; + }; + return { + prepare: summarizeDurationValues(rows.map((row) => elapsed(row.prepare_started_at, row.prepare_completed_at))), + provider: summarizeDurationValues(rows.map((row) => elapsed(row.provider_started_at, row.provider_completed_at))), + memoryCapture: summarizeDurationValues(rows.map((row) => elapsed(row.memory_started_at, row.memory_completed_at))), + gitFinalize: summarizeDurationValues(rows.map((row) => elapsed( + row.git_finalize_started_at || row.provider_completed_at, + row.git_finalized_at, + ))), + workflow: summarizeDurationValues(rows.map((row) => elapsed(row.prepare_started_at, row.workflow_completed_at))), + taskRun: summarizeDurationValues(rows.map((row) => Number(row.duration_ms))), + }; + } catch { + return null; + } finally { + try { database?.close(); } catch { /* best effort diagnostic read */ } + } +} + +export async function captureMockupRuntimeResourceSample({ serverRef, homeDir, projectId, sprintId }) { + const pid = serverRef.current?.child?.pid || null; + const databasePath = path.join(homeDir, ".code-ux", "app.db"); + const sessionDatabasePath = path.join(homeDir, ".code-ux", "session-tracking.db"); + const [runtimeRssBytes, databaseBytes, walBytes, sessionDatabaseBytes, sessionWalBytes, dockerHelpers] = await Promise.all([ + readRuntimeRssBytes(pid), + readFileSize(databasePath), + readFileSize(`${databasePath}-wal`), + readFileSize(sessionDatabasePath), + readFileSize(`${sessionDatabasePath}-wal`), + readRuntimeDockerHelperSnapshot(homeDir), + ]); + return { + observedAt: new Date().toISOString(), + pid, + runtimeRssBytes, + databaseBytes, + walBytes, + sessionDatabaseBytes, + sessionWalBytes, + dockerHelpers, + counts: readScopedResourceCounts(databasePath, projectId, sprintId), + }; +} + +function maxSampleValue(samples, selector) { + return samples.reduce((maximum, sample) => { + const value = selector(sample); + return Number.isFinite(value) ? Math.max(maximum, value) : maximum; + }, 0); +} + +export function summarizeMockupRuntimeResources(samples) { + const pids = [...new Set(samples.map((sample) => sample.pid).filter((pid) => Number.isInteger(pid)))]; + const projectGitHelperIds = [...new Set(samples.flatMap( + (sample) => sample.dockerHelpers?.projectGitHelperIds || [], + ))]; + const workspaceHelperIds = [...new Set(samples.flatMap( + (sample) => sample.dockerHelpers?.workspaceHelperIds || [], + ))]; + return { + sampleCount: samples.length, + observedPids: pids, + peakRuntimeRssBytes: maxSampleValue(samples, (sample) => sample.runtimeRssBytes), + peakDatabaseBytes: maxSampleValue(samples, (sample) => sample.databaseBytes), + peakWalBytes: maxSampleValue(samples, (sample) => sample.walBytes), + peakSessionDatabaseBytes: maxSampleValue(samples, (sample) => sample.sessionDatabaseBytes), + peakSessionWalBytes: maxSampleValue(samples, (sample) => sample.sessionWalBytes), + peakTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.taskRuns), + peakRunningTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.runningTaskRuns), + peakCompletedTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.completedTaskRuns), + peakFailedTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.failedTaskRuns), + peakCancelledTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.cancelledTaskRuns), + peakActiveTaskRuns: maxSampleValue(samples, (sample) => sample.counts?.activeTaskRuns), + peakTaskRunEvents: maxSampleValue(samples, (sample) => sample.counts?.taskRunEvents), + peakProviderInvocations: maxSampleValue(samples, (sample) => sample.counts?.providerInvocations), + peakTaskQaPromptChars: maxSampleValue(samples, (sample) => sample.counts?.maxTaskQaPromptChars), + peakSprintQaPromptChars: maxSampleValue(samples, (sample) => sample.counts?.maxSprintQaPromptChars), + recoveredProviderCompletions: maxSampleValue( + samples, + (sample) => sample.counts?.recoveredProviderCompletions, + ), + peakProjectGitHelpers: maxSampleValue( + samples, + (sample) => sample.dockerHelpers?.projectGitHelperIds?.length, + ), + peakWorkspaceHelpers: maxSampleValue( + samples, + (sample) => sample.dockerHelpers?.workspaceHelperIds?.length, + ), + projectGitHelperGenerations: projectGitHelperIds.length, + workspaceHelperGenerations: workspaceHelperIds.length, + peakNonRunningContainers: maxSampleValue( + samples, + (sample) => sample.dockerHelpers?.nonRunningContainerIds?.length, + ), + finalNonRunningContainers: samples.at(-1)?.dockerHelpers?.nonRunningContainerIds?.length ?? null, + finalTaskAttemptAmplification: (() => { + const counts = samples.at(-1)?.counts; + return counts?.completedTaskRuns > 0 ? counts.taskRuns / counts.completedTaskRuns : null; + })(), + final: samples.at(-1) || null, + }; +} + +export function assertExpectedRuntimeResources(summary, expected = {}) { + const failures = []; + const checks = [ + ["peakRuntimeRssBytes", "runtime RSS", expected.maxRuntimeRssBytes], + ["peakDatabaseBytes", "SQLite database", expected.maxDatabaseBytes], + ["peakWalBytes", "SQLite WAL", expected.maxWalBytes], + ["peakSessionDatabaseBytes", "session database", expected.maxSessionDatabaseBytes], + ["peakSessionWalBytes", "session database WAL", expected.maxSessionWalBytes], + ["peakTaskRuns", "task runs", expected.maxTaskRuns], + ["peakActiveTaskRuns", "active task runs", expected.maxActiveTaskRuns], + ["peakTaskRunEvents", "task-run events", expected.maxTaskRunEvents], + ["peakProviderInvocations", "provider invocations", expected.maxProviderInvocations], + ["peakTaskQaPromptChars", "task QA prompt", expected.maxTaskQaPromptChars], + ["peakSprintQaPromptChars", "sprint QA prompt", expected.maxSprintQaPromptChars], + ["peakProjectGitHelpers", "concurrent project Git helpers", expected.maxProjectGitHelpers], + ["peakWorkspaceHelpers", "concurrent workspace helpers", expected.maxWorkspaceHelpers], + ["projectGitHelperGenerations", "project Git helper generations", expected.maxProjectGitHelperGenerations], + ]; + for (const [field, label, maximum] of checks) { + if (!Number.isFinite(maximum)) continue; + const actual = summary?.[field]; + if (!Number.isFinite(actual) || actual <= 0) { + failures.push(`expected ${label} resource telemetry, received ${actual ?? "missing"}`); + } else if (actual > maximum) { + failures.push(`expected ${label} to stay at or below ${maximum}, received ${actual}`); + } + } + if (Number.isFinite(expected.maxFailedTaskRuns)) { + const actual = summary?.peakFailedTaskRuns; + if (!Number.isFinite(actual) || actual < 0) { + failures.push(`expected failed task-run telemetry, received ${actual ?? "missing"}`); + } else if (actual > expected.maxFailedTaskRuns) { + failures.push(`expected failed task runs to stay at or below ${expected.maxFailedTaskRuns}, received ${actual}`); + } + } + if (Number.isFinite(expected.maxFinalNonRunningContainers)) { + const actual = summary?.finalNonRunningContainers; + if (!Number.isFinite(actual)) { + failures.push(`expected final non-running container telemetry, received ${actual ?? "missing"}`); + } else if (actual > expected.maxFinalNonRunningContainers) { + failures.push( + `expected final non-running containers to stay at or below ${expected.maxFinalNonRunningContainers}, received ${actual}`, + ); + } + } + if (Number.isFinite(expected.maxTaskAttemptAmplification)) { + const actual = summary?.finalTaskAttemptAmplification; + if (!Number.isFinite(actual)) { + failures.push(`expected task-attempt amplification telemetry, received ${actual ?? "missing"}`); + } else if (actual > expected.maxTaskAttemptAmplification) { + failures.push( + `expected task-attempt amplification to stay at or below ${expected.maxTaskAttemptAmplification}, received ${actual}`, + ); + } + } + const timingChecks = [ + ["prepare", "workspace preparation p50", expected.maxPrepareP50Ms], + ["gitFinalize", "Git finalization p50", expected.maxGitFinalizeP50Ms], + ["workflow", "CLI workflow p50", expected.maxWorkflowP50Ms], + ]; + for (const [phase, label, maximum] of timingChecks) { + if (!Number.isFinite(maximum)) continue; + const timing = summary?.workflowTimings?.[phase]; + if (!timing || !Number.isFinite(timing.p50Ms) || timing.count <= 0) { + failures.push(`expected ${label} telemetry, received missing`); + } else if (timing.p50Ms > maximum) { + failures.push(`expected ${label} to stay at or below ${maximum}ms, received ${timing.p50Ms}ms`); + } + } + if (Number.isFinite(expected.minTaskQaPromptChars)) { + const actual = summary?.peakTaskQaPromptChars; + if (!Number.isFinite(actual) || actual < expected.minTaskQaPromptChars) { + failures.push( + `expected the largest task QA prompt to contain at least ${expected.minTaskQaPromptChars} characters, received ${actual ?? "missing"}`, + ); + } + } + if (Number.isFinite(expected.minSprintQaPromptChars)) { + const actual = summary?.peakSprintQaPromptChars; + if (!Number.isFinite(actual) || actual < expected.minSprintQaPromptChars) { + failures.push( + `expected the largest sprint QA prompt to contain at least ${expected.minSprintQaPromptChars} characters, received ${actual ?? "missing"}`, + ); + } + } + if (failures.length > 0) throw new Error(failures.join("; ")); + return summary; +} + +function startRuntimeResourceSampler(context) { + if (!context.enabled) { + return { stop: async () => ({ samples: [], summary: null }) }; + } + const abortController = new AbortController(); + const samples = []; + const done = (async () => { + while (!abortController.signal.aborted) { + const sample = await captureMockupRuntimeResourceSample(context); + if (samples.length < MAX_RESOURCE_SAMPLES) { + samples.push(sample); + } else { + samples[MAX_RESOURCE_SAMPLES - 1] = sample; + } + if (await cancellableDelay(RESOURCE_SAMPLE_INTERVAL_MS, abortController.signal) === "aborted") break; + } + })(); + return { + stop: async () => { + abortController.abort(); + await done; + const finalSample = await captureMockupRuntimeResourceSample(context); + if (samples.length < MAX_RESOURCE_SAMPLES) samples.push(finalSample); + else samples[MAX_RESOURCE_SAMPLES - 1] = finalSample; + const summary = summarizeMockupRuntimeResources(samples); + summary.workflowTimings = readScopedWorkflowTimings( + path.join(context.homeDir, ".code-ux", "app.db"), + context.projectId, + context.sprintId, + ); + return { samples, summary }; + }, + }; +} + async function applyBeforeOrchestrationHook(repoDir, projectRun) { const hook = projectRun.beforeOrchestration; if (!hook) return; @@ -819,17 +1279,23 @@ async function startCodeUx(homeDir, port, artifactDir, executionMode) { }); activeChildren.add(child); const logChunks = []; - const appendLog = async (source, chunk) => { + let logChars = 0; + const logStream = createWriteStream(logPath, { flags: "a" }); + const appendLog = (source, chunk) => { const line = redact(`[${source}] ${chunk.toString("utf8")}`); logChunks.push(line); - if (logChunks.join("").length > 2_000_000) logChunks.splice(0, Math.max(1, Math.floor(logChunks.length / 3))); + logChars += line.length; + while (logChars > 2_000_000 && logChunks.length > 1) { + logChars -= logChunks.shift()?.length || 0; + } writeRuntimeLogToConsole(source, chunk); - await fs.appendFile(logPath, line, "utf8").catch(() => undefined); + logStream.write(line); }; - child.stdout?.on("data", (chunk) => { void appendLog("stdout", chunk); }); - child.stderr?.on("data", (chunk) => { void appendLog("stderr", chunk); }); + child.stdout?.on("data", (chunk) => { appendLog("stdout", chunk); }); + child.stderr?.on("data", (chunk) => { appendLog("stderr", chunk); }); child.on("close", () => { activeChildren.delete(child); + logStream.end(); }); const baseUrl = `http://127.0.0.1:${port}`; await waitForReady(baseUrl).catch((error) => { @@ -849,6 +1315,7 @@ async function startElectronCodeUx(homeDir, port, artifactDir, executionMode) { await fs.mkdir(path.join(homeDir, "AppData", "Roaming"), { recursive: true }); await fs.mkdir(path.join(homeDir, "AppData", "Local"), { recursive: true }); const logPath = path.join(artifactDir, "server.log"); + const logStream = createWriteStream(logPath, { flags: "a" }); const electronApp = await electron.launch({ executablePath: electronPath, args: [DIST_ELECTRON_ENTRYPOINT], @@ -869,13 +1336,18 @@ async function startElectronCodeUx(homeDir, port, artifactDir, executionMode) { activeChildren.add(child); child.stdout?.on("data", (chunk) => { writeRuntimeLogToConsole("electron stdout", chunk); - void fs.appendFile(logPath, redact(`[electron stdout] ${chunk.toString("utf8")}`), "utf8").catch(() => undefined); + logStream.write(redact(`[electron stdout] ${chunk.toString("utf8")}`)); }); child.stderr?.on("data", (chunk) => { writeRuntimeLogToConsole("electron stderr", chunk); - void fs.appendFile(logPath, redact(`[electron stderr] ${chunk.toString("utf8")}`), "utf8").catch(() => undefined); + logStream.write(redact(`[electron stderr] ${chunk.toString("utf8")}`)); + }); + child.on("close", () => { + activeChildren.delete(child); + logStream.end(); }); - child.on("close", () => activeChildren.delete(child)); + } else { + logStream.end(); } const baseUrl = `http://127.0.0.1:${port}`; await waitForReady(baseUrl).catch(async (error) => { @@ -920,7 +1392,7 @@ async function waitForRestartCheckpoint(baseUrl, projectId, sprintId, signal) { return null; } -function startRestartLoop({ serverRef, homeDir, port, artifactDir, everyMs, count, baseUrl, projectId, sprintId }) { +function startRestartLoop({ serverRef, homeDir, port, artifactDir, executionMode, everyMs, count, baseUrl, projectId, sprintId }) { if (!everyMs || !count) { return { events: [], @@ -952,7 +1424,7 @@ function startRestartLoop({ serverRef, homeDir, port, artifactDir, everyMs, coun console.log(redact({ type: "runtime_restart_started", ...event })); try { await stopProcess(serverRef.current?.child); - serverRef.current = await startCodeUx(homeDir, port, artifactDir); + serverRef.current = await startCodeUx(homeDir, port, artifactDir, executionMode); if (abortController.signal.aborted) { event.status = "aborted"; event.completedAt = new Date().toISOString(); @@ -1240,7 +1712,16 @@ async function expectedOutputFilesPresent(repoDir, projectRun) { } async function pollProjectRun(baseUrl, projectId, sprintId, timeoutMs, stallTimeoutMs, context = {}) { - const { repoDir, expectedProjectRun, homeDir, ...logContext } = context; + // Restart history is written once in the final artifact. Keeping it out of + // the per-poll context prevents an increasingly large array from being + // serialized into every progress/admission event during restart stress. + const { + repoDir, + expectedProjectRun, + homeDir, + restartEvents = [], + ...logContext + } = context; const startedAt = Date.now(); let latestTasks = []; let latestSprints = null; @@ -1254,6 +1735,7 @@ async function pollProjectRun(baseUrl, projectId, sprintId, timeoutMs, stallTime let lastProgressTaskStates = new Map(); let lastProgressAt = 0; let lastStateChangeAt = startedAt; + let observedCompletedRestartCount = 0; let stalled = false; let stallReason = null; while (Date.now() - startedAt < timeoutMs) { @@ -1295,9 +1777,13 @@ async function pollProjectRun(baseUrl, projectId, sprintId, timeoutMs, stallTime if (admissionWait.latestHeartbeatAt) { lastProviderAdmissionHeartbeatAt = admissionWait.latestHeartbeatAt; } + const completedRestartCount = restartEvents.filter((event) => event.status === "completed").length; + const runtimeRestartCompleted = completedRestartCount > observedCompletedRestartCount; + observedCompletedRestartCount = Math.max(observedCompletedRestartCount, completedRestartCount); if (isMockupPollStateProgress({ progressChanged, expectedOutputReadinessChanged, + runtimeRestartCompleted, })) { lastStateChangeAt = now; } @@ -1387,6 +1873,11 @@ async function pollProjectRun(baseUrl, projectId, sprintId, timeoutMs, stallTime } catch (error) { latestPollError = error; const now = Date.now(); + const completedRestartCount = restartEvents.filter((event) => event.status === "completed").length; + if (completedRestartCount > observedCompletedRestartCount) { + observedCompletedRestartCount = completedRestartCount; + lastStateChangeAt = now; + } if (now - lastProgressAt >= PROGRESS_LOG_INTERVAL_MS) { lastProgressAt = now; logEvent("mockup_pentest_progress", { @@ -1880,6 +2371,13 @@ async function runProjectRunInner(server, scenario, projectRun, options) { `/api/projects/${encodeURIComponent(records.project.id)}/sprints/${encodeURIComponent(records.sprint.id)}/orchestrate`, {}, ); + const resourceSampler = startRuntimeResourceSampler({ + enabled: Boolean(projectRun.expected?.resources), + serverRef: options.restart.serverRef, + homeDir: options.homeDir, + projectId: records.project.id, + sprintId: records.sprint.id, + }); const restartLoop = startRestartLoop({ ...options.restart, baseUrl: server.baseUrl, @@ -1905,10 +2403,13 @@ async function runProjectRunInner(server, scenario, projectRun, options) { repoDir: records.repoDir, expectedProjectRun: projectRun, homeDir: options.homeDir, + restartEvents: restartLoop.events, }, ); await duringHookPromise; await restartLoop.stop(); + const runtimeResources = await resourceSampler.stop(); + await writeFile(path.join(artifactDir, "resource-samples.json"), `${redact(runtimeResources.samples)}\n`); let assertionFailure = null; let commandResults = []; let qaHistory = null; @@ -1932,6 +2433,9 @@ async function runProjectRunInner(server, scenario, projectRun, options) { restartCount: options.restart?.count || 0, }); } + if (projectRun.expected?.resources) { + assertExpectedRuntimeResources(runtimeResources.summary, projectRun.expected.resources); + } } catch (error) { assertionFailure = error.message; } @@ -1957,6 +2461,7 @@ async function runProjectRunInner(server, scenario, projectRun, options) { qaHistory, invocationHistory, statusReporting, + runtimeResources: runtimeResources.summary, elapsedMs: Date.now() - startedAt, stalled: Boolean(polled.stalled), stallReason: polled.stallReason || null, @@ -1995,9 +2500,9 @@ async function runScenario(server, scenario, options) { return summary; } -function resolveScenarios(scenarioArg) { - if (scenarioArg === "all") return SCENARIOS.filter((scenario) => !scenario.heavy); - if (scenarioArg === "pentest") return SCENARIOS; +export function resolveScenarios(scenarioArg) { + if (scenarioArg === "all") return SCENARIOS.filter((scenario) => !scenario.heavy && !scenario.localOnly); + if (scenarioArg === "pentest") return SCENARIOS.filter((scenario) => !scenario.localOnly); const scenarioId = SCENARIO_ALIASES.get(scenarioArg) || scenarioArg; const scenario = getScenario(scenarioId); if (!scenario) throw new Error(`Unknown scenario '${scenarioArg}'. Expected one of: all, pentest, smoke, ${listScenarioIds().join(", ")}`); @@ -2075,6 +2580,7 @@ async function main() { homeDir, port, artifactDir, + executionMode: args.executionMode, everyMs: args.restartEveryMs, count: args.restartCount, }, diff --git a/scripts/measure-live-snapshot.ts b/scripts/measure-live-snapshot.ts index 05a7bc99ef..8f5ab4c083 100644 --- a/scripts/measure-live-snapshot.ts +++ b/scripts/measure-live-snapshot.ts @@ -1,5 +1,6 @@ import * as fs from "fs/promises"; import * as path from "path"; +import { performance } from "node:perf_hooks"; import { getProjectLiveSnapshot, ProjectLiveSnapshotDeps } from "../src/app/live/project-live-snapshot.js"; import { DashboardRealtimeService } from "../src/services/dashboard-realtime-service.js"; @@ -14,6 +15,9 @@ async function main() { const mockDeps: ProjectLiveSnapshotDeps = { projectManagementRepository: { getSelectedProjectId: () => fixtureData.projectId, + getSelectedSprintId: () => fixtureData.selectedSprintId, + sprintBelongsToProject: (_pid: string, sprintId: string) => + fixtureData.listSprintsResult.sprints.some((sprint: { id: string }) => sprint.id === sprintId), listSprints: (pid: string) => { return fixtureData.listSprintsResult; }, @@ -39,10 +43,6 @@ async function main() { metrics.executionMs.push(meta.executionMs); metrics.gitMs.push(meta.gitMs); - metrics.executionSizeBytes.push(meta.executionSizeBytes); - metrics.gitSizeBytes.push(meta.gitSizeBytes); - metrics.statusSizeBytes.push(meta.statusSizeBytes); - metrics.payloadSizeBytes.push(meta.payloadSizeBytes); } }, warn: () => {}, @@ -63,20 +63,28 @@ async function main() { gitSizeBytes: [] as number[], statusSizeBytes: [] as number[], payloadSizeBytes: [] as number[], + wallBuildTimes: [] as number[], publishCadenceMs: [] as number[], }; - const ITERATIONS = 100; + const ITERATIONS = 1_000; console.log(`\nRunning ${ITERATIONS} iterations of snapshot assembly...`); for (let i = 0; i < ITERATIONS; i++) { - await getProjectLiveSnapshot(mockDeps, projectId); + const startedAt = performance.now(); + const snapshot = await getProjectLiveSnapshot(mockDeps, projectId); + metrics.wallBuildTimes.push(performance.now() - startedAt); + metrics.executionSizeBytes.push(Buffer.byteLength(JSON.stringify(snapshot.execution), "utf8")); + metrics.gitSizeBytes.push(Buffer.byteLength(JSON.stringify(snapshot.gitStatus), "utf8")); + metrics.statusSizeBytes.push(Buffer.byteLength(JSON.stringify(snapshot.status), "utf8")); + metrics.payloadSizeBytes.push(Buffer.byteLength(JSON.stringify(snapshot), "utf8")); } const avg = (arr: number[]) => arr.reduce((a, b) => a + b, 0) / arr.length; console.log("\n--- Snapshot Assembly Latency (Average) ---"); - console.log(`Total Build Time: ${avg(metrics.buildTimes).toFixed(2)} ms`); + console.log(`Measured Wall Time: ${avg(metrics.wallBuildTimes).toFixed(3)} ms`); + console.log(`Instrumented Build Time: ${avg(metrics.buildTimes).toFixed(2)} ms`); console.log(` |- Project Mgmt: ${avg(metrics.projectMgmtMs).toFixed(2)} ms`); console.log(` |- Runtime Status: ${avg(metrics.runtimeMs).toFixed(2)} ms`); console.log(` |- Execution State: ${avg(metrics.executionMs).toFixed(2)} ms`); @@ -137,4 +145,4 @@ async function main() { main().catch(err => { console.error("Benchmark failed:", err); process.exit(1); -}); \ No newline at end of file +}); diff --git a/scripts/prepare-electron-runtime-deps.mjs b/scripts/prepare-electron-runtime-deps.mjs index 001252bde5..df5247d3ce 100644 --- a/scripts/prepare-electron-runtime-deps.mjs +++ b/scripts/prepare-electron-runtime-deps.mjs @@ -23,11 +23,17 @@ const fingerprintPath = path.join(runtimeDir, ".runtime-fingerprint"); const packageJson = JSON.parse(readFileSync(path.join(projectRoot, "package.json"), "utf8")); const lockfile = readFileSync(path.join(projectRoot, "pnpm-lock.yaml"), "utf8"); const workspace = readFileSync(path.join(projectRoot, "pnpm-workspace.yaml"), "utf8"); -const pruneVersion = 4; +const pruneVersion = 5; const targetPlatform = process.env.CODE_UX_ELECTRON_TARGET_PLATFORM || process.platform; const targetArch = process.env.CODE_UX_ELECTRON_TARGET_ARCH || process.arch; const keepAllNativeBinaries = process.env.CODE_UX_ELECTRON_KEEP_ALL_NATIVE_BINARIES === "1"; const onnxRuntimeInstallMode = "skip"; +const runtimeImportProbe = [ + "@modelcontextprotocol/sdk/server/index.js", + "@modelcontextprotocol/sdk/types.js", + "dotenv", + "zod", +]; const fingerprint = crypto .createHash("sha256") @@ -47,6 +53,7 @@ const fingerprint = crypto if (existsSync(nodeModulesDir) && existsSync(fingerprintPath)) { const currentFingerprint = readFileSync(fingerprintPath, "utf8").trim(); if (currentFingerprint === fingerprint) { + validateRuntimeTree(); console.log("Electron runtime dependencies are up to date."); process.exit(0); } @@ -74,13 +81,12 @@ writeFileSync( // flat hoisted top level, pairing packages with wrong dependency versions // (e.g. type-is@2 + media-typer@0.3, which silently disabled express.json() // parsing and broke MCP initialize for provider containers). -writeFileSync(path.join(runtimeDir, ".npmrc"), "node-linker=hoisted\n"); copyFileSync(path.join(projectRoot, "pnpm-lock.yaml"), path.join(runtimeDir, "pnpm-lock.yaml")); copyFileSync(path.join(projectRoot, "pnpm-workspace.yaml"), path.join(runtimeDir, "pnpm-workspace.yaml")); execFileSync( "pnpm", - ["install", "--prod", "--frozen-lockfile"], + ["install", "--prod", "--frozen-lockfile", "--config.node-linker=hoisted"], { cwd: runtimeDir, stdio: "inherit", @@ -133,6 +139,55 @@ function pruneTree(dir) { } } +function packageDirectory(packageName) { + return path.join(nodeModulesDir, ...packageName.split("/")); +} + +function collectSymlinks(dir, links = []) { + if (!existsSync(dir)) { + return links; + } + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const entryPath = path.join(dir, entry.name); + if (entry.isSymbolicLink()) { + links.push(path.relative(nodeModulesDir, entryPath)); + continue; + } + if (entry.isDirectory()) { + collectSymlinks(entryPath, links); + } + } + return links; +} + +function validateRuntimeTree() { + const missingDependencies = Object.keys(packageJson.dependencies || {}) + .filter((dependency) => !existsSync(path.join(packageDirectory(dependency), "package.json"))); + if (missingDependencies.length > 0) { + throw new Error( + `Electron runtime dependency tree is incomplete: ${missingDependencies.join(", ")}`, + ); + } + + const symlinks = collectSymlinks(nodeModulesDir); + if (symlinks.length > 0) { + throw new Error( + "Electron runtime dependency tree is not copy-safe; pnpm produced symbolic links: " + + symlinks.slice(0, 10).join(", "), + ); + } + + const importExpression = runtimeImportProbe + .map((specifier) => `import(${JSON.stringify(specifier)})`) + .join(","); + execFileSync( + process.execPath, + ["--input-type=module", "--eval", `await Promise.all([${importExpression}])`], + { cwd: runtimeDir, stdio: "inherit" }, + ); +} + function normalizeNativePlatform(platform) { if (platform === "darwin" || platform === "win32" || platform === "linux") { return platform; @@ -239,8 +294,12 @@ function directorySize(dir) { return size; } +// Runtime command shims are unused by the packaged app and are commonly +// symlinks even in an otherwise hoisted tree. +rmSync(path.join(nodeModulesDir, ".bin"), { recursive: true, force: true }); pruneTree(nodeModulesDir); pruneOnnxRuntimeNativeBinaries(nodeModulesDir); +validateRuntimeTree(); writeFileSync(fingerprintPath, `${fingerprint}\n`); const fileCount = countFiles(nodeModulesDir); diff --git a/scripts/smoke-installed-electron.mjs b/scripts/smoke-installed-electron.mjs new file mode 100644 index 0000000000..3939960eac --- /dev/null +++ b/scripts/smoke-installed-electron.mjs @@ -0,0 +1,231 @@ +import { spawn, spawnSync } from "node:child_process"; +import { + access, + mkdir, + mkdtemp, + readFile, + readdir, + rm, +} from "node:fs/promises"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const artifactDirectory = path.resolve( + process.env.CODE_UX_ELECTRON_ARTIFACT_DIR || path.join(projectRoot, "release", "electron"), +); +const timeoutMs = Number.parseInt(process.env.CODE_UX_ELECTRON_SMOKE_TIMEOUT_MS || "120000", 10); +const packageJson = JSON.parse(await readFile(path.join(projectRoot, "package.json"), "utf8")); + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: projectRoot, + encoding: "utf8", + stdio: "inherit", + ...options, + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`${command} exited with status ${result.status ?? "unknown"}.`); + } +} + +async function findArtifact(extension) { + const entries = await readdir(artifactDirectory, { withFileTypes: true }); + const versionMarker = `-${packageJson.version}-`; + const artifact = entries + .filter((entry) => ( + entry.isFile() + && entry.name.includes(versionMarker) + && entry.name.endsWith(extension) + )) + .map((entry) => path.join(artifactDirectory, entry.name)) + .sort()[0]; + if (!artifact) { + throw new Error( + `No Electron ${packageJson.version} ${extension} artifact found in ${artifactDirectory}.`, + ); + } + return artifact; +} + +async function installLinuxCandidate() { + const deb = await findArtifact(".deb"); + run("sudo", ["apt-get", "install", "--no-install-recommends", "-y", deb]); + const executable = path.join("/opt", "Code UX", "codeux"); + await access(executable); + return { command: "xvfb-run", args: ["-a", executable] }; +} + +async function installWindowsCandidate(temporaryRoot) { + const installer = await findArtifact(".exe"); + const installDirectory = path.join(temporaryRoot, "installed", "Code UX"); + await mkdir(installDirectory, { recursive: true }); + run(installer, ["/S", `/D=${installDirectory}`]); + const executable = path.join(installDirectory, "Code UX.exe"); + await access(executable); + return { command: executable, args: [] }; +} + +async function installMacCandidate(temporaryRoot) { + const dmg = await findArtifact(".dmg"); + const mountDirectory = path.join(temporaryRoot, "mounted"); + const installDirectory = path.join(temporaryRoot, "Applications"); + await mkdir(mountDirectory, { recursive: true }); + await mkdir(installDirectory, { recursive: true }); + run("hdiutil", ["attach", "-nobrowse", "-readonly", "-mountpoint", mountDirectory, dmg]); + try { + const mountedEntries = await readdir(mountDirectory, { withFileTypes: true }); + const appEntry = mountedEntries.find((entry) => entry.isDirectory() && entry.name.endsWith(".app")); + if (!appEntry) { + throw new Error(`No macOS app bundle found in ${dmg}.`); + } + const installedApp = path.join(installDirectory, appEntry.name); + run("ditto", [path.join(mountDirectory, appEntry.name), installedApp]); + const executable = path.join(installedApp, "Contents", "MacOS", "Code UX"); + await access(executable); + return { command: executable, args: [] }; + } finally { + run("hdiutil", ["detach", mountDirectory]); + } +} + +async function reserveDashboardPort() { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : null; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + if (!port) { + throw new Error("Unable to reserve a dashboard port for the Electron startup smoke test."); + } + return port; +} + +function appendBounded(current, chunk) { + const combined = current + chunk.toString(); + return combined.length <= 65_536 ? combined : combined.slice(-65_536); +} + +async function waitForInstalledApp(launch, temporaryRoot) { + const markerPath = path.join(temporaryRoot, "startup-ready.json"); + const isolatedHome = path.join(temporaryRoot, "home"); + const appData = path.join(isolatedHome, "AppData", "Roaming"); + const localAppData = path.join(isolatedHome, "AppData", "Local"); + const dashboardPort = await reserveDashboardPort(); + await Promise.all([ + mkdir(isolatedHome, { recursive: true }), + mkdir(appData, { recursive: true }), + mkdir(localAppData, { recursive: true }), + ]); + + const child = spawn(launch.command, launch.args, { + cwd: isolatedHome, + env: { + ...process.env, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + APPDATA: appData, + LOCALAPPDATA: localAppData, + DASHBOARD_PORT: String(dashboardPort), + MCP_HTTP_ENABLED: "false", + CODE_UX_DISABLE_MCP_STDIO: "1", + CODE_UX_ELECTRON_STARTUP_SMOKE_FILE: markerPath, + CODE_UX_ELECTRON_STARTUP_SMOKE_EXIT: "1", + }, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk) => { stdout = appendBounded(stdout, chunk); }); + child.stderr?.on("data", (chunk) => { stderr = appendBounded(stderr, chunk); }); + + let exitResult = null; + const exitPromise = new Promise((resolve) => { + child.once("error", (error) => { + exitResult = { code: null, signal: null, error }; + resolve(exitResult); + }); + child.once("exit", (code, signal) => { + exitResult = { code, signal, error: null }; + resolve(exitResult); + }); + }); + + const deadline = Date.now() + timeoutMs; + let marker = null; + while (Date.now() < deadline) { + try { + marker = JSON.parse(await readFile(markerPath, "utf8")); + break; + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + } + if (exitResult) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + + if (!marker) { + child.kill(); + throw new Error( + `Installed Electron app did not become renderer-ready within ${timeoutMs}ms. ` + + `Exit: ${JSON.stringify(exitResult)}\nstdout:\n${stdout}\nstderr:\n${stderr}`, + ); + } + if ( + marker.schemaVersion !== 1 + || marker.packaged !== true + || marker.version !== packageJson.version + || marker.platform !== process.platform + || new URL(marker.rendererUrl).origin !== marker.dashboardOrigin + ) { + child.kill(); + throw new Error(`Installed Electron app returned an invalid startup marker: ${JSON.stringify(marker)}`); + } + + const exit = await Promise.race([ + exitPromise, + new Promise((resolve) => setTimeout(() => resolve(null), 30_000)), + ]); + if (!exit || exit.error || exit.code !== 0) { + child.kill(); + throw new Error( + `Installed Electron app became ready but did not exit cleanly: ${JSON.stringify(exit)}\n` + + `stdout:\n${stdout}\nstderr:\n${stderr}`, + ); + } + + console.log( + `Installed Electron ${marker.version} became renderer-ready on ${marker.platform}/${marker.arch} ` + + `at ${marker.dashboardOrigin} and exited cleanly.`, + ); +} + +const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), "code-ux-electron-install-smoke-")); +try { + const launch = process.platform === "linux" + ? await installLinuxCandidate() + : process.platform === "win32" + ? await installWindowsCandidate(temporaryRoot) + : process.platform === "darwin" + ? await installMacCandidate(temporaryRoot) + : null; + if (!launch) { + throw new Error(`Unsupported Electron startup smoke platform: ${process.platform}`); + } + await waitForInstalledApp(launch, temporaryRoot); +} finally { + await rm(temporaryRoot, { recursive: true, force: true }).catch(() => undefined); +} diff --git a/src/app/dependency-factory/sprint-factory.ts b/src/app/dependency-factory/sprint-factory.ts index 38e1348653..26a4c9f946 100644 --- a/src/app/dependency-factory/sprint-factory.ts +++ b/src/app/dependency-factory/sprint-factory.ts @@ -245,6 +245,7 @@ export function createSprintDependencies( coreDeps.providerConcurrencyService, resolveDashboardSettings, logger.child({ component: "sprint-task-dispatch-service" }), + () => julesApi.getSessionsForCapacityCheck(), ); projectAttentionService.setWorkerAttentionOpenedCallback((projectId) => { diff --git a/src/app/lifecycle/dashboard-snapshot-cache-policy.ts b/src/app/lifecycle/dashboard-snapshot-cache-policy.ts index deb92cbbe1..79f79e630c 100644 --- a/src/app/lifecycle/dashboard-snapshot-cache-policy.ts +++ b/src/app/lifecycle/dashboard-snapshot-cache-policy.ts @@ -24,6 +24,9 @@ export class DashboardSnapshotCachePolicy { static readonly HEADER_TOKEN_THROUGHPUT_CACHE_TTL_MS = 1_000; static readonly OVERVIEW_CACHE_TTL_MS = 500; static readonly PROJECTS_CACHE_TTL_MS = 500; + static readonly PROJECT_EXECUTION_CACHE_MAX_ENTRIES = 64; + static readonly PROJECT_STATS_CACHE_MAX_ENTRIES = 128; + static readonly HEADER_TOKEN_THROUGHPUT_CACHE_MAX_ENTRIES = 128; static getProjectExecutionSnapshotCacheScope( projectId: string, diff --git a/src/app/lifecycle/dashboard-snapshot-cache.ts b/src/app/lifecycle/dashboard-snapshot-cache.ts index a1a9017e0d..48fc04de1b 100644 --- a/src/app/lifecycle/dashboard-snapshot-cache.ts +++ b/src/app/lifecycle/dashboard-snapshot-cache.ts @@ -104,20 +104,33 @@ export type DashboardSnapshotCacheDeps = Pick(); + private projectExecutionSnapshotCache = new Map(); private projectExecutionSnapshotKeysByProject = new Map>(); // Memoizes the feed-less view with the same explicit project/sprint scope as // the full snapshot cache. The source snapshot guard prevents stale lean views // from surviving a full snapshot rebuild after TTL expiry. private leanExecutionSnapshotCache = new Map(); private leanExecutionSnapshotKeysByProject = new Map>(); - private projectStatsSnapshotCache = new Map; expiresAt: number }>(); + private projectStatsSnapshotCache = new Map; + expiresAt: number; + }>(); private projectStatsSnapshotKeysByProject = new Map>(); - private headerTokenThroughputSnapshotCache = new Map; expiresAt: number }>(); + private headerTokenThroughputSnapshotCache = new Map; + expiresAt: number; + }>(); private headerTokenThroughputSnapshotKeysByProject = new Map>(); private overviewTelemetryCache: { snapshot: ReturnType; expiresAt: number } | null = null; private projectsSnapshotCache: { snapshot: ReturnType; expiresAt: number } | null = null; @@ -154,6 +167,38 @@ export class DashboardSnapshotCache { } } + private evictOldestProjectExecutionSnapshot(): void { + const oldestKey = this.projectExecutionSnapshotCache.keys().next().value; + if (oldestKey === undefined) return; + const entry = this.projectExecutionSnapshotCache.get(oldestKey); + this.projectExecutionSnapshotCache.delete(oldestKey); + this.leanExecutionSnapshotCache.delete(oldestKey); + if (entry) { + this.unregisterProjectCacheKey(this.projectExecutionSnapshotKeysByProject, entry.projectId, oldestKey); + this.unregisterProjectCacheKey(this.leanExecutionSnapshotKeysByProject, entry.projectId, oldestKey); + } + } + + private evictOldestProjectStatsSnapshot(): void { + const oldestKey = this.projectStatsSnapshotCache.keys().next().value; + if (oldestKey === undefined) return; + const entry = this.projectStatsSnapshotCache.get(oldestKey); + this.projectStatsSnapshotCache.delete(oldestKey); + if (entry) { + this.unregisterProjectCacheKey(this.projectStatsSnapshotKeysByProject, entry.projectId, oldestKey); + } + } + + private evictOldestHeaderTokenThroughputSnapshot(): void { + const oldestKey = this.headerTokenThroughputSnapshotCache.keys().next().value; + if (oldestKey === undefined) return; + const entry = this.headerTokenThroughputSnapshotCache.get(oldestKey); + this.headerTokenThroughputSnapshotCache.delete(oldestKey); + if (entry?.projectId) { + this.unregisterProjectCacheKey(this.headerTokenThroughputSnapshotKeysByProject, entry.projectId, oldestKey); + } + } + getProjectsSnapshot = () => { const now = Date.now(); if (this.projectsSnapshotCache && this.projectsSnapshotCache.expiresAt > now) { @@ -211,7 +256,14 @@ export class DashboardSnapshotCache { ), }; + if ( + !this.projectExecutionSnapshotCache.has(cacheKey) + && this.projectExecutionSnapshotCache.size >= DashboardSnapshotCachePolicy.PROJECT_EXECUTION_CACHE_MAX_ENTRIES + ) { + this.evictOldestProjectExecutionSnapshot(); + } this.projectExecutionSnapshotCache.set(cacheKey, { + projectId, snapshot, expiresAt: now + DashboardSnapshotCachePolicy.PROJECT_EXECUTION_CACHE_TTL_MS, }); @@ -246,6 +298,7 @@ export class DashboardSnapshotCache { } const lean: ExecutionDashboardSnapshot = { ...full, recentEvents: [], recentInvocations: [] }; this.leanExecutionSnapshotCache.set(cacheKey, { + projectId, sourceSnapshot: full, snapshot: lean, expiresAt: now + DashboardSnapshotCachePolicy.PROJECT_EXECUTION_CACHE_TTL_MS, @@ -262,7 +315,14 @@ export class DashboardSnapshotCache { return cached.snapshot; } const snapshot = this.deps.executionRepository.getProjectStatsSnapshot(projectId, query); + if ( + !this.projectStatsSnapshotCache.has(cacheKey) + && this.projectStatsSnapshotCache.size >= DashboardSnapshotCachePolicy.PROJECT_STATS_CACHE_MAX_ENTRIES + ) { + this.evictOldestProjectStatsSnapshot(); + } this.projectStatsSnapshotCache.set(cacheKey, { + projectId, snapshot, expiresAt: now + DashboardSnapshotCachePolicy.PROJECT_STATS_CACHE_TTL_MS, }); @@ -278,7 +338,14 @@ export class DashboardSnapshotCache { return cached.snapshot; } const snapshot = this.deps.executionRepository.getHeaderTokenThroughputSnapshot(query); + if ( + !this.headerTokenThroughputSnapshotCache.has(cacheKey) + && this.headerTokenThroughputSnapshotCache.size >= DashboardSnapshotCachePolicy.HEADER_TOKEN_THROUGHPUT_CACHE_MAX_ENTRIES + ) { + this.evictOldestHeaderTokenThroughputSnapshot(); + } this.headerTokenThroughputSnapshotCache.set(cacheKey, { + projectId: query.projectId ?? null, snapshot, expiresAt: now + DashboardSnapshotCachePolicy.HEADER_TOKEN_THROUGHPUT_CACHE_TTL_MS, }); diff --git a/src/domain/sprint/ci/feature-pr-gate.ts b/src/domain/sprint/ci/feature-pr-gate.ts index 7a97964168..b4a162c829 100644 --- a/src/domain/sprint/ci/feature-pr-gate.ts +++ b/src/domain/sprint/ci/feature-pr-gate.ts @@ -2,7 +2,7 @@ import { evaluateMergeReadiness } from "./feature-pr/merge-readiness-policy.js"; import { deriveChecksFromCiRuns } from "../../../sprint/ci-status-utils.js"; import { runCommandStrict } from "../../../services/cli-process-runner.js"; import type { GuardrailService } from "../../../services/guardrail-service.js"; -import { createTemporaryWorktreeBranchMerger, deleteBranchLocally, findRecoverableWorkerBranch, mergeBranchLocallyInTemporaryWorktree, workerBranchHasMergeWork, workerBranchIsMergedIntoFeature } from "../../../infrastructure/git/local-merge.js"; +import { createTemporaryWorktreeBranchMerger, deleteBranchLocally, findRecoverableWorkerBranch, mergeBranchLocallyInTemporaryWorktree, resolveWorkerBranchMergeState, workerBranchHasMergeWork } from "../../../infrastructure/git/local-merge.js"; import { buildWorkerBranchPrefix } from "../../../services/cli-workflow-utils.js"; import { matchMergedPrForTask, matchPrForTask } from "./feature-pr/pr-matcher.js"; import { attemptAutoMerge } from "./feature-pr/automerge-policy.js"; @@ -116,6 +116,15 @@ export class FeaturePrGateService { error?: unknown; } const taskCiInfoMap = new Map(); + const taskRecordIds = updatedSubtasks + .map((task) => task.record_id?.trim()) + .filter((taskId): taskId is string => Boolean(taskId)); + const listLatestTaskRuns = context.executionRepository + ? (context.executionRepository as Partial>).listLatestTaskRuns + : undefined; + const latestTaskRuns = context.executionRepository && context.sprintRunId && listLatestTaskRuns + ? listLatestTaskRuns.call(context.executionRepository, taskRecordIds, context.sprintRunId) + : null; for (const task of updatedSubtasks) { let pr: GitPullRequestStatus | undefined = undefined; @@ -134,7 +143,9 @@ export class FeaturePrGateService { } const taskRun = context.executionRepository && context.sprintRunId && task.record_id - ? context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId) + ? latestTaskRuns + ? latestTaskRuns.get(task.record_id) ?? null + : context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId) : null; // All CLI git decisions below use the same immutable event snapshot. A // gate evaluation never appends git-finalization events, so rereading the @@ -143,9 +154,15 @@ export class FeaturePrGateService { const listTaskRunEvents = taskRun?.id && context.executionRepository ? (taskRunId: string, limit?: number): TaskRunEventLike[] => { if (taskRunId !== taskRun.id) { - return context.executionRepository!.listTaskRunEvents(taskRunId, limit); + return context.executionRepository!.listTaskRunEvents(taskRunId, limit, { + eventTypes: ["cli_git_pushed", "cli_git_no_changes"], + skipValidation: true, + }); } - taskRunEvents ??= context.executionRepository!.listTaskRunEvents(taskRun.id, limit); + taskRunEvents ??= context.executionRepository!.listTaskRunEvents(taskRun.id, limit, { + eventTypes: ["cli_git_pushed", "cli_git_no_changes"], + skipValidation: true, + }); return taskRunEvents; } : undefined; @@ -266,9 +283,10 @@ export class FeaturePrGateService { } context.logger?.info(`LOCAL Mode: Recovered worker branch ${recovered} for task ${task.id} from local refs.`); if (context.executionRepository && context.sprintRunId && task.record_id) { - const taskRun = context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId); + const taskRun = info.taskRun; if (taskRun && !taskRun.workerBranch) { context.executionRepository.updateTaskRun(taskRun.id, { workerBranch: recovered }); + taskRun.workerBranch = recovered; } } } @@ -336,28 +354,25 @@ export class FeaturePrGateService { } const info = taskCiInfoMap.get(task.id)!; - const hasMergeWork = await workerBranchHasMergeWork({ + const mergeResolution = await resolveWorkerBranchMergeState({ repoPath: context.repoPath, featureBranch: context.featureBranch, workerBranch, }); - if (!hasMergeWork) { + if (mergeResolution.state !== "unmerged") { const recoveredCompletedMerge = context.githubMode === "LOCAL" && info.cliGitPushed && !info.cliGitNoChanges - && await workerBranchIsMergedIntoFeature({ - repoPath: context.repoPath, - featureBranch: context.featureBranch, - workerBranch, - }); + && mergeResolution.state === "merged"; task.status = "COMPLETED"; task.is_merged = recoveredCompletedMerge; task.merge_indicator = recoveredCompletedMerge ? "MERGED" : undefined; task.worker_branch = undefined; if (context.executionRepository && context.sprintRunId && task.record_id) { - const taskRun = context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId); + const taskRun = info.taskRun; if (taskRun?.id) { context.executionRepository.updateTaskRun(taskRun.id, { workerBranch: null }); + taskRun.workerBranch = null; context.executionRepository.appendTaskRunEvent(taskRun.id, "ci_gate_status", "system", { state: recoveredCompletedMerge ? "merged_branch" : "no_merge_work", taskId: task.id, @@ -429,9 +444,10 @@ export class FeaturePrGateService { task.intervention_owner = undefined; task.intervention_hint = undefined; if (context.executionRepository && context.sprintRunId && task.record_id) { - const taskRun = context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId); + const taskRun = info.taskRun; if (taskRun?.id) { context.executionRepository.updateTaskRun(taskRun.id, { workerBranch: null }); + taskRun.workerBranch = null; context.executionRepository.appendTaskRunEvent(taskRun.id, "ci_gate_status", "system", { state: "merged_branch", taskId: task.id, @@ -512,7 +528,7 @@ export class FeaturePrGateService { if (info.error) { return Promise.resolve({ reportText: "", events: [], attentionItem: undefined }); } - return this.processTask(task, context, info.pr, info.mergedPr).catch((err) => { + return this.processTask(task, context, info.pr, info.mergedPr, info.taskRun).catch((err) => { context.logger?.error(`Error processing task ${task.id}:`, { error: err }); return { reportText: "", events: [], attentionItem: undefined }; }); @@ -524,11 +540,12 @@ export class FeaturePrGateService { for (let i = 0; i < completedAwaitingMerge.length; i++) { const task = completedAwaitingMerge[i]; + const info = taskCiInfoMap.get(task.id)!; const result = processResults[i]; if (result) { reportText += result.reportText; for (const event of result.events) { - this.appendCiGateEvent(task, context, event.state, event.payload); + this.appendCiGateEvent(task, context, event.state, event.payload, info.taskRun); } if (result.attentionItem) { itemsToOpen.push({ task, payload: result.attentionItem }); @@ -548,7 +565,8 @@ export class FeaturePrGateService { task: Subtask, context: CiGateContext, cachedPr: GitPullRequestStatus | undefined, - cachedMergedPr: GitMergeStatus | undefined + cachedMergedPr: GitMergeStatus | undefined, + cachedTaskRun: TaskRunRecord | null, ): Promise<{ reportText: string; events: Array<{ state: string; payload: Record }>; @@ -568,9 +586,10 @@ export class FeaturePrGateService { if (!workerBranch && context.executionRepository && context.sprintRunId && task.record_id) { const headRef = pr?.headRefName || mergedPr?.headRefName; if (headRef) { - const taskRun = context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId); + const taskRun = cachedTaskRun; if (taskRun && !taskRun.workerBranch) { context.executionRepository.updateTaskRun(taskRun.id, { workerBranch: headRef }); + taskRun.workerBranch = headRef; } task.worker_branch = headRef; } @@ -591,10 +610,7 @@ export class FeaturePrGateService { } if (!pr) { - const taskRun = context.executionRepository && context.sprintRunId && task.record_id - ? context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId) - : null; - const isExecutionCompleted = isExecutionCompletedForCi(context, task, taskRun); + const isExecutionCompleted = isExecutionCompletedForCi(context, task, cachedTaskRun); if (isExecutionCompleted) { const qaGate = context.evaluateTaskQaGate?.(task); @@ -625,8 +641,9 @@ export class FeaturePrGateService { task.status = "COMPLETED"; task.merge_indicator = undefined; task.worker_branch = undefined; - if (taskRun?.id && context.executionRepository) { - context.executionRepository.updateTaskRun(taskRun.id, { workerBranch: null }); + if (cachedTaskRun?.id && context.executionRepository) { + context.executionRepository.updateTaskRun(cachedTaskRun.id, { workerBranch: null }); + cachedTaskRun.workerBranch = null; } await this.persistMergedTask(task, context); events.push({ state: "no_merge_work", payload: { @@ -900,12 +917,13 @@ export class FeaturePrGateService { context: CiGateContext, state: string, payload: Record, + cachedTaskRun: TaskRunRecord | null, ): void { if (!context.executionRepository || !context.sprintRunId || !task.record_id) { return; } - const taskRun = context.executionRepository.getLatestTaskRun(task.record_id, context.sprintRunId); + const taskRun = cachedTaskRun; if (!taskRun) { return; } diff --git a/src/domain/sprint/orchestrator/cycle-runner.ts b/src/domain/sprint/orchestrator/cycle-runner.ts index 419051dc2b..501f8fe0bb 100644 --- a/src/domain/sprint/orchestrator/cycle-runner.ts +++ b/src/domain/sprint/orchestrator/cycle-runner.ts @@ -1,7 +1,10 @@ import { applyActionRequiredAutomation } from "../../../sprint/action-required-automation.js"; import { runSessionSyncStep } from "../../../sprint/steps/session-sync-step.js"; import { runStatusDerivationStep } from "../../../sprint/steps/status-derivation-step.js"; -import { runStartReadyTasksStep } from "../../../sprint/steps/start-ready-tasks-step.js"; +import { + runStartReadyTasksStep, + type ProviderCapLogState, +} from "../../../sprint/steps/start-ready-tasks-step.js"; import { runStatusTableStep } from "../../../sprint/steps/status-table-step.js"; import { runProtocolStep } from "../../../sprint/steps/protocol-step.js"; import type { SprintCycleResult } from "../../../sprint/sprint-types.js"; @@ -26,7 +29,6 @@ import { FeaturePrGateService } from "../ci/feature-pr-gate.js"; import { CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, isCliTaskRun, - isCliTaskRunAwaitingGitFinalization, } from "../ci/cli-git-finalization.js"; import { MergeConflictDebouncer } from "../ci/merge-conflict-debouncer.js"; import { matchPrForTask } from "../ci/feature-pr/pr-matcher.js"; @@ -74,9 +76,40 @@ export interface LocalCliGitEvidence { settledTaskIds: Set; } +const DEFAULT_QA_REVIEW_PARALLELISM = 4; +const MAX_QA_REVIEW_PARALLELISM = 4; + +export function resolveTaskQaReviewParallelism(settings: ReturnType< + SprintOrchestratorDependencies["getDashboardSettings"] +>): number { + const route = settings.aiProvider.invocationRouting.qa_review; + const routedProviderConfigIds = [...new Set([ + ...(route.allowedProviders ?? []), + ...(route.provider ? [route.provider] : []), + ])]; + const providerConfigIds = routedProviderConfigIds.length > 0 + ? routedProviderConfigIds + : settings.aiProvider.provider + ? [settings.aiProvider.provider] + : []; + const configuredCapacity = providerConfigIds.reduce((total, providerConfigId) => { + const providerSettings = settings.aiProvider.providers[providerConfigId]; + if (!providerSettings?.enabled || route.providers[providerConfigId]?.enabled === false) { + return total; + } + const limit = providerSettings.maxConcurrentTasks; + return Number.isFinite(limit) && Number(limit) > 0 ? total + Math.floor(Number(limit)) : total; + }, 0); + return Math.min( + MAX_QA_REVIEW_PARALLELISM, + Math.max(1, configuredCapacity || DEFAULT_QA_REVIEW_PARALLELISM), + ); +} + export class CycleRunner { private readonly featurePrGate = new FeaturePrGateService(); private readonly lastAutomatedInterventionKeys = new Map(); + private readonly providerCapLogState: ProviderCapLogState = new Map(); private readonly stateCoordinator: CycleStateCoordinator; // Persists across cycles (CycleRunner is long-lived per orchestrator) so a // transient `DIRTY` PR state must persist before it escalates a conflict. @@ -218,7 +251,13 @@ export class CycleRunner { let qaFinishedTaskIds = new Set(); if (subtasks.length > 0) { if (args.loopSteps.statusDerivation && !isAutomaticRollback) { - qaFinishedTaskIds = await this.reviewCompletedTasks(subtasks, cycleEntryStates, args, dashboardSettings); + qaFinishedTaskIds = await this.reviewCompletedTasks( + subtasks, + cycleEntryStates, + args, + dashboardSettings, + localCliGitEvidence, + ); } const taskStateBeforeFastBranchGate = snapshotTaskState(subtasks); const fastBranchOnlyResult = await this.runFastBranchOnlyMergeGate( @@ -543,7 +582,7 @@ export class CycleRunner { args, resolvedWorkerMergeConflictSuppressionKeys, gitStatus, - ) || this.isCliTaskAwaitingGitFinalization(task, args) + ) || this.isCliTaskAwaitingGitFinalization(task, args, localCliGitEvidence) // Failed CI is not merge work. Once its repair guardrail is exhausted, // the CI gate owns a human handoff and the merge protocol must not open // a misleading worker merge_required item for the same task. @@ -616,6 +655,24 @@ export class CycleRunner { getRunningCounts: () => { return this.deps.providerConcurrencyService.getGlobalRunningCounts(); }, + getAvailableProviderCapacity: async (provider) => { + if (!(PROVIDER_IDS as readonly string[]).includes(provider) + || typeof this.deps.providerConcurrencyService.getAvailableCapacityCount !== "function") { + return null; + } + const providerSettings = dashboardSettings.aiProvider.providers[provider as ProviderId]; + return await this.deps.providerConcurrencyService.getAvailableCapacityCount( + provider as ProviderId, + providerSettings?.maxConcurrentTasks ?? 0, + "task_coding", + ); + }, + providerCapLogState: this.providerCapLogState, + providerCapLogScope: [ + args.executionContext.project.id, + args.executionContext.sprint.id, + args.sprintRunId ?? "no-run", + ].join(":"), startTask: (task) => { if (!args.sprintRunId) { throw new Error("Missing sprint run id for orchestrate action."); @@ -712,38 +769,89 @@ export class CycleRunner { }; } - private isCliTaskAwaitingGitFinalization(task: Subtask, args: CycleRunnerArgs): boolean { + private isCliTaskAwaitingGitFinalization( + task: Subtask, + args: CycleRunnerArgs, + evidence: LocalCliGitEvidence, + ): boolean { const taskId = task.record_id?.trim(); if (!taskId || !args.sprintRunId) { return false; } + const taskAliases = [taskId, task.id?.trim()].filter((value): value is string => Boolean(value)); + if (taskAliases.some((value) => evidence.pushedTaskIds.has(value) || evidence.settledTaskIds.has(value))) { + return false; + } const taskRun = this.deps.executionRepository.getLatestTaskRun(taskId, args.sprintRunId); - const listTaskRunEvents = this.deps.executionRepository.listTaskRunEvents?.bind(this.deps.executionRepository); - return isCliTaskRunAwaitingGitFinalization(taskRun, listTaskRunEvents); + return isCliTaskRun(taskRun); } private collectLocalCliGitEvidence(subtasks: Subtask[], args: CycleRunnerArgs): LocalCliGitEvidence { const pushedTaskIds = new Set(); const settledTaskIds = new Set(); - if (args.githubMode !== "LOCAL" || !args.sprintRunId) { + if (!args.sprintRunId) { return { pushedTaskIds, settledTaskIds }; } + const recordIds = subtasks + .map((task) => task.record_id?.trim()) + .filter((recordId): recordId is string => Boolean(recordId)); + const executionRepository = this.deps.executionRepository as Partial< + SprintOrchestratorDependencies["executionRepository"] + >; + const latestRuns = typeof executionRepository.listLatestTaskRuns === "function" + ? executionRepository.listLatestTaskRuns(recordIds, args.sprintRunId) + : null; + + const resolvedRuns: Array<{ + task: Subtask; + recordId: string; + taskRun: NonNullable>; + }> = []; for (const task of subtasks) { const recordId = task.record_id?.trim(); if (!recordId) { continue; } - const taskRun = this.deps.executionRepository.getLatestTaskRun(recordId, args.sprintRunId); + const taskRun = latestRuns + ? latestRuns.get(recordId) ?? null + : this.deps.executionRepository.getLatestTaskRun(recordId, args.sprintRunId); if (!isCliTaskRun(taskRun) || !taskRun?.id) { continue; } + resolvedRuns.push({ task, recordId, taskRun }); + } - let events: ReturnType; - try { - events = this.deps.executionRepository.listTaskRunEvents(taskRun.id, CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT); - } catch { - continue; + let eventsByTaskRunId: Map>; + try { + eventsByTaskRunId = this.deps.executionRepository.listTaskRunEventsForRuns( + resolvedRuns.map(({ taskRun }) => taskRun.id), + { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "ci_gate_status"], + limitPerRun: CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + }, + ); + } catch { + eventsByTaskRunId = new Map(); + } + + for (const { task, recordId, taskRun } of resolvedRuns) { + let events = eventsByTaskRunId.get(taskRun.id); + if (!events) { + // Keep partial test doubles and older adapters compatible while the + // production repository uses the single batched query above. + try { + events = this.deps.executionRepository.listTaskRunEvents( + taskRun.id, + CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "ci_gate_status"], + skipValidation: true, + }, + ); + } catch { + continue; + } } const taskIds = [recordId, task.id?.trim()].filter((taskId): taskId is string => Boolean(taskId)); @@ -1314,6 +1422,7 @@ export class CycleRunner { previousStates: Map, args: CycleRunnerArgs, settings: ReturnType, + cliGitEvidence?: LocalCliGitEvidence, ): Promise> { const qaFinishedTaskIds = new Set(); if (!this.deps.qualityAssuranceService || !settings.agents.qualityAssurance.enabled) { @@ -1325,18 +1434,38 @@ export class CycleRunner { sprintId: args.executionContext.sprint.id, tasks: subtasks, }); + const qaGateStatuses = typeof this.deps.qualityAssuranceService.getTaskMergeGateStatuses === "function" + ? this.deps.qualityAssuranceService.getTaskMergeGateStatuses({ + projectId: args.executionContext.project.id, + sprintId: args.executionContext.sprint.id, + tasks: subtasks, + }) + : new Map(); - const limit = pLimit(5); + const reviewParallelism = resolveTaskQaReviewParallelism(settings); + const limit = pLimit(reviewParallelism); const reviewPromises: Promise[] = []; for (const task of subtasks) { const prev = previousStates.get(task.id); - const qaGate = this.deps.qualityAssuranceService.getTaskMergeGateStatus({ - projectId: args.executionContext.project.id, - sprintId: args.executionContext.sprint.id, - task, - }); + const taskRecordId = task.record_id?.trim(); + const qaGate = (taskRecordId ? qaGateStatuses.get(taskRecordId) : null) + ?? this.deps.qualityAssuranceService.getTaskMergeGateStatus({ + projectId: args.executionContext.project.id, + sprintId: args.executionContext.sprint.id, + task, + }); const taskIsCodeComplete = isTaskCodeComplete(task); + // Provider completion is not task completion for a CLI workflow. QA must + // inspect the finalized worker branch, never a workspace that is still + // being committed/published or was interrupted in that crash window. + if ( + taskIsCodeComplete + && cliGitEvidence + && this.isCliTaskAwaitingGitFinalization(task, args, cliGitEvidence) + ) { + continue; + } const hasSameSessionFollowUpAfterLatestQaRequest = taskIsCodeComplete && this.hasCompletedTaskFollowUpAfterLatestQaRequest(task, qaGate, args.sprintRunId); const hasPendingQaFollowUp = isPendingQaContinuation(qaGate.latestRun); @@ -1373,6 +1502,13 @@ export class CycleRunner { if (!shouldRunQaReview) { continue; } + // A cycle should settle one resource-bounded QA wave, then merge those results and start + // newly unblocked coding work. Queueing the entire backlog behind p-limit would still block + // the cycle until every review finished and turn wide DAGs into coding/QA stop-the-world + // phases even though only `reviewParallelism` reviews can run at once. + if (reviewPromises.length >= reviewParallelism) { + break; + } const runReview = async () => { try { diff --git a/src/domain/sprint/orchestrator/sprint-action-runner.ts b/src/domain/sprint/orchestrator/sprint-action-runner.ts index 2b3520e4e7..ae894b99ac 100644 --- a/src/domain/sprint/orchestrator/sprint-action-runner.ts +++ b/src/domain/sprint/orchestrator/sprint-action-runner.ts @@ -4,6 +4,7 @@ import type { WatchLoopRunner } from "./watch-loop-runner.js"; import type { AutomationInterventionsSettings, AutomationLevel, CiIntelligenceSettings, SprintLoopStepSettings, Subtask, DashboardStatusSnapshot } from "../../../contracts/app-types.js"; import type { SprintAgentArgs } from "../../../sprint/sprint-types.js"; import type { SprintExecutionContext } from "../../../services/sprint-execution-state-service.js"; +import { acquireProjectGitHelperForSprint } from "../../../shared/subprocess/command-runner.js"; export class SprintActionRunner { constructor( @@ -58,6 +59,8 @@ export class SprintActionRunner { return { content: [{ type: "text", text: fullReport }] }; } + const releaseProjectGitHelper = acquireProjectGitHelperForSprint(options.repoPath); + try { const cycleResult = await this.cycleRunner.run({ action: "orchestrate", automationLevel: options.automationLevel, @@ -84,6 +87,9 @@ export class SprintActionRunner { watchLoopEnabled: options.watchLoopEnabled, cycleResult, }); + } finally { + await releaseProjectGitHelper(); + } } async runStatus(options: { diff --git a/src/domain/sprint/orchestrator/watch-loop-runner.ts b/src/domain/sprint/orchestrator/watch-loop-runner.ts index ab41ed3922..0e6883daa6 100644 --- a/src/domain/sprint/orchestrator/watch-loop-runner.ts +++ b/src/domain/sprint/orchestrator/watch-loop-runner.ts @@ -42,9 +42,10 @@ import type { SprintIssueService } from "../../../services/sprint-issue-service. import type { SprintRunLifecycleService } from "../../../services/sprint-run-lifecycle-service.js"; import { getFailedJobLabels, getFailedLogSnippets } from "../../../sprint/ci-status-utils.js"; import { resolveRollbackFinalizationCiIntelligence } from "./rollback-finalization-policy.js"; +import { acquireProjectGitHelperForSprint } from "../../../shared/subprocess/command-runner.js"; -export type WatchLoopExecutionDependencies = Pick; +export type WatchLoopExecutionDependencies = Pick; export type WatchLoopAttentionDependencies = Pick; export interface WatchLoopDependencies { @@ -212,6 +213,7 @@ export class WatchLoopRunner { sourceEventKey: `watch-loop-started:${sprintRunId}`, }); + const releaseProjectGitHelper = acquireProjectGitHelperForSprint(repoPath); this.deps.heartbeatService.startHeartbeat(sprintRunId, scopedExecutionContext.sprint.id, leaseToken); try { while (!allFinished) { @@ -372,6 +374,7 @@ export class WatchLoopRunner { } finally { this.deps.heartbeatService.stopHeartbeat(sprintRunId); this.lastStatusSnapshotFingerprints.delete(sprintRunId); + await releaseProjectGitHelper(); } return fullReport; } @@ -1273,6 +1276,11 @@ export class WatchLoopRunner { return { pushedTaskIds, settledTaskIds }; } + const resolvedRuns: Array<{ + task: Subtask; + recordId: string; + taskRun: NonNullable>; + }> = []; for (const task of args.subtasks) { const recordId = task.record_id?.trim(); if (!recordId) { @@ -1282,12 +1290,37 @@ export class WatchLoopRunner { if (!isCliTaskRun(taskRun) || !taskRun?.id) { continue; } + resolvedRuns.push({ task, recordId, taskRun }); + } - let events: ReturnType; - try { - events = this.deps.executionRepository.listTaskRunEvents(taskRun.id, CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT); - } catch { - continue; + let eventsByTaskRunId: Map>; + try { + eventsByTaskRunId = this.deps.executionRepository.listTaskRunEventsForRuns( + resolvedRuns.map(({ taskRun }) => taskRun.id), + { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "ci_gate_status"], + limitPerRun: CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + }, + ); + } catch { + eventsByTaskRunId = new Map(); + } + + for (const { task, recordId, taskRun } of resolvedRuns) { + let events = eventsByTaskRunId.get(taskRun.id); + if (!events) { + try { + events = this.deps.executionRepository.listTaskRunEvents( + taskRun.id, + CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "ci_gate_status"], + skipValidation: true, + }, + ); + } catch { + continue; + } } const taskIds = [recordId, task.id?.trim()].filter((taskId): taskId is string => Boolean(taskId)); @@ -1318,7 +1351,10 @@ export class WatchLoopRunner { } private resolveWorkspaceReferenceFromTaskRunEvents(taskRunId: string): string | undefined { - const events = this.deps.executionRepository.listTaskRunEvents(taskRunId, 200); + const events = this.deps.executionRepository.listTaskRunEvents(taskRunId, 200, { + eventTypes: ["cli_workspace_bound", "cli_prepare_completed", "cli_worktree_preserved"], + skipValidation: true, + }); for (const event of events) { if (event.eventType !== "cli_workspace_bound" && event.eventType !== "cli_prepare_completed" && event.eventType !== "cli_worktree_preserved") { continue; diff --git a/src/electron/main.ts b/src/electron/main.ts index e9517aee04..3d8ca5ebd0 100644 --- a/src/electron/main.ts +++ b/src/electron/main.ts @@ -18,6 +18,7 @@ import { createDebouncedSaver, loadWindowState, saveWindowState } from "./window import { ElectronCredentialKeyPersistence } from "./credential-key-persistence.js"; import { ElectronSafeStorageKeyProvider } from "../infrastructure/security/electron-safe-storage-key-provider.js"; import { setProcessCredentialKeyProvider } from "../services/credentials/key-provider-registry.js"; +import { writeElectronStartupSmoke } from "./startup-smoke.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -293,6 +294,28 @@ function createMainWindow(url: string): BrowserWindow { } }); + const startupSmokePath = process.env.CODE_UX_ELECTRON_STARTUP_SMOKE_FILE?.trim(); + if (startupSmokePath) { + window.webContents.once("did-finish-load", () => { + void writeElectronStartupSmoke(startupSmokePath, { + version: app.getVersion(), + platform: process.platform, + arch: process.arch, + packaged: app.isPackaged, + dashboardOrigin: url, + rendererUrl: window.webContents.getURL(), + }).then(() => { + if (process.env.CODE_UX_ELECTRON_STARTUP_SMOKE_EXIT === "1") { + app.quit(); + } + }).catch((error: unknown) => { + process.exitCode = 1; + console.error("Failed to record Electron startup smoke readiness", error); + app.quit(); + }); + }); + } + void window.loadURL(url); return window; } diff --git a/src/electron/startup-smoke.ts b/src/electron/startup-smoke.ts new file mode 100644 index 0000000000..2f193aa18e --- /dev/null +++ b/src/electron/startup-smoke.ts @@ -0,0 +1,62 @@ +import { mkdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export interface ElectronStartupSmokeRecord { + schemaVersion: 1; + version: string; + platform: NodeJS.Platform; + arch: string; + packaged: boolean; + dashboardOrigin: string; + rendererUrl: string; + pid: number; + readyAt: string; +} + +export interface ElectronStartupSmokeInput { + version: string; + platform: NodeJS.Platform; + arch: string; + packaged: boolean; + dashboardOrigin: string; + rendererUrl: string; + pid?: number; + now?: () => Date; +} + +export async function writeElectronStartupSmoke( + markerPath: string, + input: ElectronStartupSmokeInput, +): Promise { + if (!path.isAbsolute(markerPath)) { + throw new Error("Electron startup smoke marker path must be absolute."); + } + + const record: ElectronStartupSmokeRecord = { + schemaVersion: 1, + version: input.version, + platform: input.platform, + arch: input.arch, + packaged: input.packaged, + dashboardOrigin: input.dashboardOrigin, + rendererUrl: input.rendererUrl, + pid: input.pid ?? process.pid, + readyAt: (input.now ?? (() => new Date()))().toISOString(), + }; + const temporaryPath = `${markerPath}.${record.pid}.${Date.now()}.tmp`; + + await mkdir(path.dirname(markerPath), { recursive: true }); + try { + await writeFile(temporaryPath, `${JSON.stringify(record, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + await rename(temporaryPath, markerPath); + } catch (error) { + await rm(temporaryPath, { force: true }).catch(() => undefined); + throw error; + } + + return record; +} diff --git a/src/infrastructure/git/local-merge.ts b/src/infrastructure/git/local-merge.ts index ac6cf20a32..c28b0c00a5 100644 --- a/src/infrastructure/git/local-merge.ts +++ b/src/infrastructure/git/local-merge.ts @@ -25,6 +25,14 @@ export interface LocalMergeResult { error?: string; } +export type WorkerBranchMergeState = "unmerged" | "merged" | "missing"; + +export interface WorkerBranchMergeResolution { + state: WorkerBranchMergeState; + sourceCommit: string | null; + targetCommit: string | null; +} + /** A ref that was checked out before the orchestrator started mutating branches. */ export interface CheckedOutRef { ref: string; @@ -319,19 +327,6 @@ async function gitCommitExists( } } -async function gitRevListCount( - repoPath: string, - range: string, - runner: LocalMergeRunner, -): Promise { - try { - const res = await runner("git", ["rev-list", "--count", range], repoPath); - return Number.parseInt(res.stdout.trim(), 10) || 0; - } catch { - return 0; - } -} - async function gitResolveCommit( repoPath: string, ref: string, @@ -475,54 +470,51 @@ export async function workerBranchHasMergeWork(args: { workerBranch: string; runner?: LocalMergeRunner; }): Promise { + const resolution = await resolveWorkerBranchMergeState(args); + return resolution.state === "unmerged"; +} + +/** + * Resolves the worker/feature relationship once for merge gating. Prefer the + * local refs that the LOCAL merger actually publishes, then fall back to the + * corresponding origin refs. A comparison failure is treated as unmerged so + * the authoritative merge fails closed instead of discarding work. + */ +export async function resolveWorkerBranchMergeState(args: { + repoPath: string; + featureBranch: string; + workerBranch: string; + runner?: LocalMergeRunner; +}): Promise { const runner = args.runner ?? defaultRunner; const branch = args.workerBranch.trim(); - if (!branch) return false; - - const sourceRefs = [ - `refs/heads/${branch}`, - `refs/remotes/origin/${branch}`, - ]; - const existingSourceRefs: string[] = []; - for (const ref of sourceRefs) { - if (await gitRefExists(args.repoPath, ref, runner)) { - existingSourceRefs.push(ref); - } - } - if (existingSourceRefs.length === 0) { - return false; + if (!branch) { + return { state: "missing", sourceCommit: null, targetCommit: null }; } - const baseRefs = [ - `refs/remotes/origin/${args.featureBranch}`, - `refs/heads/${args.featureBranch}`, - ]; - const existingBaseRefs: string[] = []; - for (const ref of baseRefs) { - if (await gitRefExists(args.repoPath, ref, runner)) { - existingBaseRefs.push(ref); - } + const sourceCommit = await gitResolveCommit(args.repoPath, `refs/heads/${branch}`, runner) + ?? await gitResolveCommit(args.repoPath, `refs/remotes/origin/${branch}`, runner); + if (!sourceCommit) { + return { state: "missing", sourceCommit: null, targetCommit: null }; } - if (existingBaseRefs.length === 0) { - return true; + + const targetCommit = await gitResolveCommit(args.repoPath, `refs/heads/${args.featureBranch}`, runner) + ?? await gitResolveCommit(args.repoPath, `refs/remotes/origin/${args.featureBranch}`, runner); + if (!targetCommit) { + return { state: "unmerged", sourceCommit, targetCommit: null }; } - for (const sourceRef of existingSourceRefs) { - const sourceCommit = await gitResolveCommit(args.repoPath, sourceRef, runner); - if (!sourceCommit) { - continue; - } - for (const baseRef of existingBaseRefs) { - const baseCommit = await gitResolveCommit(args.repoPath, baseRef, runner); - if (!baseCommit) { - continue; - } - if ((await gitRevListCount(args.repoPath, `${baseCommit}..${sourceCommit}`, runner)) > 0) { - return true; - } - } + try { + const result = await runner("git", ["rev-list", "--count", `${targetCommit}..${sourceCommit}`], args.repoPath); + const aheadCount = Number.parseInt(result.stdout.trim(), 10); + return { + state: Number.isFinite(aheadCount) && aheadCount === 0 ? "merged" : "unmerged", + sourceCommit, + targetCommit, + }; + } catch { + return { state: "unmerged", sourceCommit, targetCommit }; } - return false; } /** @@ -537,38 +529,8 @@ export async function workerBranchIsMergedIntoFeature(args: { workerBranch: string; runner?: LocalMergeRunner; }): Promise { - const runner = args.runner ?? defaultRunner; - const branch = args.workerBranch.trim(); - if (!branch) return false; - - const sourceRefs = [ - `refs/heads/${branch}`, - `refs/remotes/origin/${branch}`, - ]; - const baseRefs = [ - `refs/remotes/origin/${args.featureBranch}`, - `refs/heads/${args.featureBranch}`, - ]; - - for (const sourceRef of sourceRefs) { - if (!(await gitRefExists(args.repoPath, sourceRef, runner))) continue; - const sourceCommit = await gitResolveCommit(args.repoPath, sourceRef, runner); - if (!sourceCommit) continue; - - for (const baseRef of baseRefs) { - if (!(await gitRefExists(args.repoPath, baseRef, runner))) continue; - const baseCommit = await gitResolveCommit(args.repoPath, baseRef, runner); - if (!baseCommit) continue; - try { - const result = await runner("git", ["rev-list", "--count", `${baseCommit}..${sourceCommit}`], args.repoPath); - if (Number.parseInt(result.stdout.trim(), 10) === 0) return true; - } catch { - // Try the next local/remote ref pair before treating the merge as unproven. - } - } - } - - return false; + const resolution = await resolveWorkerBranchMergeState(args); + return resolution.state === "merged"; } /** @@ -587,14 +549,8 @@ export async function deleteBranchLocally(args: { } const runner = args.runner ?? defaultRunner; try { - const current = await runner("git", ["rev-parse", "--abbrev-ref", "HEAD"], args.repoPath); - if (current.stdout.trim() === branch) { - return false; - } - } catch { - // If HEAD cannot be resolved, fall through and let the delete attempt decide. - } - try { + // Git itself refuses to delete a branch checked out by any worktree. Let + // that authoritative operation decide instead of paying for a racy probe. await runner("git", ["branch", "-D", branch], args.repoPath); return true; } catch { @@ -700,6 +656,7 @@ export function createTemporaryWorktreeBranchMerger(args: { let visibleCheckout: CheckedOutRef | null | undefined; let worktreePath: string | null = null; let worktreeHead: string | null = null; + let publishedTarget: string | null = null; let worktreeCreated = false; let mergedTarget = false; let closed = false; @@ -745,6 +702,7 @@ export function createTemporaryWorktreeBranchMerger(args: { // Real Git always resolves this to an object ID. The symbolic fallback keeps injected // runners usable for higher-level orchestration tests that intentionally stub empty output. worktreeHead = await gitResolveCommit(worktreePath, "HEAD", runner) ?? targetBranch; + publishedTarget = worktreeHead; return null; } catch (err) { return { ok: false, conflict: false, error: formatGitError(err) }; @@ -760,14 +718,6 @@ export function createTemporaryWorktreeBranchMerger(args: { if (!sourceBranch) { return { ok: false, conflict: false, error: "Source branch is required for local merge." }; } - if (!(await gitCommitExists(args.repoPath, sourceBranch, runner))) { - return { - ok: false, - conflict: false, - error: `Source branch or ref '${sourceBranch}' was not found or does not point to a commit.`, - }; - } - const opened = await openWorktree(sourceBranch); if (opened) { return opened; @@ -778,7 +728,7 @@ export function createTemporaryWorktreeBranchMerger(args: { const targetRef = `refs/heads/${targetBranch}`; for (let attempt = 1; attempt <= LOCAL_MERGE_REF_UPDATE_RETRY_ATTEMPTS; attempt++) { - const expectedTarget = await gitResolveCommit(args.repoPath, targetRef, runner) ?? targetBranch; + const expectedTarget = publishedTarget ?? targetBranch; if (worktreeHead !== expectedTarget) { try { @@ -813,13 +763,17 @@ export function createTemporaryWorktreeBranchMerger(args: { try { await runner("git", ["update-ref", targetRef, mergedHead, expectedTarget], worktreePath); mergedTarget = true; + publishedTarget = mergedHead; return mergeResult; } catch (error) { const actualTarget = await gitResolveCommit(args.repoPath, targetRef, runner); if (actualTarget !== expectedTarget && attempt < LOCAL_MERGE_REF_UPDATE_RETRY_ATTEMPTS) { // A concurrent CI repair or merge advanced the target. Re-run this merge from that // commit; the compare-and-swap prevents either writer from discarding the other. - continue; + if (actualTarget) { + publishedTarget = actualTarget; + continue; + } } const detail = actualTarget !== expectedTarget ? `Target branch '${targetBranch}' kept changing during local merge publication.` diff --git a/src/infrastructure/providers/cli/docker-helper-pool.ts b/src/infrastructure/providers/cli/docker-helper-pool.ts index 49706136e8..cc08ffaac0 100644 --- a/src/infrastructure/providers/cli/docker-helper-pool.ts +++ b/src/infrastructure/providers/cli/docker-helper-pool.ts @@ -1,39 +1,69 @@ -import { CommandResult, runStreamingCommand } from "../../../services/cli-process-runner.js"; +import { + type CommandResult, + runStreamingCommand, + type StreamingCommandOptions, +} from "../../../services/cli-process-runner.js"; +import { getRuntimeOwnerId } from "../../../shared/config/runtime-owner.js"; +import pLimit from "p-limit"; -export type HelperCommandRunner = (command: string, args: string[]) => Promise; +/** Options accepted by the host-side process that invokes Docker. */ +export type HelperRunnerOptions = StreamingCommandOptions; -export const defaultHelperRunner: HelperCommandRunner = (command, args) => - runStreamingCommand(command, args, process.cwd(), process.env); +export type HelperCommandRunner = ( + command: string, + args: string[], + options?: HelperRunnerOptions, +) => Promise; + +export const defaultHelperRunner: HelperCommandRunner = (command, args, options = {}) => + runStreamingCommand(command, args, process.cwd(), process.env, options); /** Shared docker label so every persistent helper container can be reaped together. */ export const HELPER_LABEL = "code-ux.helper"; +/** Suffix helper names so runtimes backed by different state homes cannot replace each other. */ +export const HELPER_OWNER_NAME_SUFFIX = getRuntimeOwnerId().slice(0, 12); +const HELPER_DOCKER_LIFECYCLE_CONCURRENCY = 4; +const helperDockerLifecycleLimit = pLimit(HELPER_DOCKER_LIFECYCLE_CONCURRENCY); + export interface HelperPoolSpec { /** Deterministic container name for a pool key (used so a previous process's helper is reclaimed). */ nameFor: (key: string) => string; - /** `docker run -d …` args (including --name, label, mounts, image and keep-alive entrypoint). */ + /** `docker run -d ...` args (including --name, labels, mounts, image and keep-alive command). */ buildCreateArgs: (key: string, name: string) => string[]; idleTtlMs?: number; reapIntervalMs?: number; + /** Hard bound for tracked helper generations. Idle helpers are evicted before admission waits. */ + maxContainers?: number; } interface HelperEntry { id: string; lastUsed: number; + activeUses: number; creating?: Promise; + releasing: boolean; + idleWaiters: Set<() => void>; } /** - * Generic manager for long-lived `docker` helper containers keyed by an arbitrary string. - * Containers are created lazily, reused across operations, reaped after an idle window, and - * recreated transparently if they disappear. Callers run their actual work via `docker exec` - * into {@link ensure}'s container id; this class only owns lifecycle. + * Generic manager for short-lived Docker sidecars keyed by an arbitrary string. + * + * Containers are created lazily, shared while commands are active, and removed after a bounded + * idle window. {@link withContainer} pins a container generation for the duration of a command so + * neither the idle reaper nor an explicit {@link release} can remove it in flight. Callers that + * only need lifecycle compatibility may still use {@link ensure}, but command execution should use + * {@link withContainer}. */ export class DockerHelperContainerPool { private readonly helpers = new Map(); + private readonly releases = new Map>(); + private readonly reservations = new Map(); private reaper: NodeJS.Timeout | null = null; private readonly idleTtlMs: number; private readonly reapIntervalMs: number; + private readonly maxContainers: number; + private readonly capacityWaiters = new Set<() => void>(); private shuttingDown = false; constructor( @@ -42,86 +72,201 @@ export class DockerHelperContainerPool { ) { this.idleTtlMs = spec.idleTtlMs ?? 120_000; this.reapIntervalMs = spec.reapIntervalMs ?? 60_000; + this.maxContainers = spec.maxContainers ?? Number.POSITIVE_INFINITY; + if (!Number.isFinite(this.idleTtlMs) || this.idleTtlMs < 0) { + throw new Error("Helper idle TTL must be a finite non-negative number."); + } + if (!Number.isFinite(this.reapIntervalMs) || this.reapIntervalMs <= 0) { + throw new Error("Helper reap interval must be a finite positive number."); + } + if (this.maxContainers !== Number.POSITIVE_INFINITY + && (!Number.isInteger(this.maxContainers) || this.maxContainers <= 0)) { + throw new Error("Helper container capacity must be a positive integer."); + } } /** Returns the id of the live helper container for `key`, creating it if necessary. */ async ensure(key: string): Promise { - if (this.shuttingDown) { - throw new Error("Helper container pool is shutting down."); - } - const existing = this.helpers.get(key); - if (existing?.creating) { - return existing.creating; - } - if (existing) { - existing.lastUsed = Date.now(); - return existing.id; + for (;;) { + if (this.shuttingDown) { + throw new Error("Helper container pool is shutting down."); + } + + const releasing = this.releases.get(key); + if (releasing) { + await releasing; + continue; + } + + const existing = this.helpers.get(key); + if (existing?.creating) { + return existing.creating; + } + if (existing && !existing.releasing) { + existing.lastUsed = Date.now(); + return existing.id; + } + + if (this.helpers.size >= this.maxContainers) { + const idleKey = this.findLeastRecentlyUsedIdleKey(); + if (idleKey !== undefined) { + await this.release(idleKey); + } else { + await this.waitForCapacity(); + } + continue; + } + + const entry: HelperEntry = { + id: "", + lastUsed: Date.now(), + activeUses: 0, + releasing: false, + idleWaiters: new Set(), + }; + const creating = this.create(key); + entry.creating = creating; + this.helpers.set(key, entry); + try { + const id = await creating; + const current = this.helpers.get(key); + if (this.shuttingDown || current !== entry || entry.releasing) { + await this.removeContainerWithVolumes(id || this.spec.nameFor(key)); + throw new Error("Helper container was released before startup completed."); + } + entry.id = id; + entry.creating = undefined; + entry.lastUsed = Date.now(); + this.startReaper(); + return id; + } catch (error) { + if (this.helpers.get(key) === entry) { + this.helpers.delete(key); + this.notifyCapacityWaiters(); + } + throw error; + } } + } - const creating = this.create(key); - this.helpers.set(key, { id: "", lastUsed: Date.now(), creating }); - try { - const id = await creating; - const current = this.helpers.get(key); - if (this.shuttingDown || current?.creating !== creating) { - await this.removeContainerWithVolumes(id || this.spec.nameFor(key)); - throw new Error("Helper container was released before startup completed."); + /** + * Pins one container generation while `operation` runs. Idle and explicit cleanup wait until the + * operation settles. A generation removed between `ensure` and pinning is retried transparently. + */ + async withContainer(key: string, operation: (containerId: string) => Promise): Promise { + for (;;) { + const id = await this.ensure(key); + const entry = this.helpers.get(key); + if (!entry || entry.creating || entry.releasing || entry.id !== id) { + const releasing = this.releases.get(key); + if (releasing) { + await releasing; + } + continue; + } + + entry.activeUses += 1; + entry.lastUsed = Date.now(); + try { + return await operation(id); + } finally { + entry.activeUses = Math.max(0, entry.activeUses - 1); + entry.lastUsed = Date.now(); + if (entry.activeUses === 0) { + for (const resolve of entry.idleWaiters) { + resolve(); + } + entry.idleWaiters.clear(); + this.notifyCapacityWaiters(); + } } - this.helpers.set(key, { id, lastUsed: Date.now() }); - this.startReaper(); - return id; - } catch (error) { - this.helpers.delete(key); - throw error; } } + /** + * Protects the helper generation for `key` from capacity eviction and idle reaping across a + * logical workflow. The reservation does not create a container or consume capacity until the + * first command needs one. Callers must release the returned lease when that workflow settles. + */ + reserve(key: string): () => void { + if (this.shuttingDown) { + throw new Error("Helper container pool is shutting down."); + } + this.reservations.set(key, (this.reservations.get(key) || 0) + 1); + let released = false; + return () => { + if (released) { + return; + } + released = true; + const remaining = (this.reservations.get(key) || 1) - 1; + if (remaining > 0) { + this.reservations.set(key, remaining); + } else { + this.reservations.delete(key); + this.notifyCapacityWaiters(); + } + }; + } + /** Marks the helper for `key` as recently used so the idle reaper leaves it alone. */ touch(key: string): void { const entry = this.helpers.get(key); - if (entry && !entry.creating) { + if (entry && !entry.creating && !entry.releasing) { entry.lastUsed = Date.now(); } } - /** Drops the tracked helper for `key` without removing the container (used before a retry). */ - invalidate(key: string): void { + /** + * Drops the tracked helper generation for `key` without removing the container (used before a + * retry). With `expectedId`, a concurrent replacement is left intact. + */ + invalidate(key: string, expectedId?: string): boolean { + const current = this.helpers.get(key); + if (!current || current.releasing) { + return false; + } + if (expectedId !== undefined && (current.creating !== undefined || current.id !== expectedId)) { + return false; + } this.helpers.delete(key); + this.notifyCapacityWaiters(); + return true; } - /** Removes the helper container for `key` (by tracked id and by deterministic name). */ + /** + * Removes the helper for `key` after its active commands settle. Concurrent releases share one + * drain operation, and new acquisitions wait for that operation before creating a replacement. + */ async release(key: string): Promise { - const entry = this.helpers.get(key); - this.helpers.delete(key); - await this.removeContainerWithVolumes(this.spec.nameFor(key)); - if (entry?.creating) { - await entry.creating.catch(() => undefined); - await this.removeContainerWithVolumes(this.spec.nameFor(key)); + const existingRelease = this.releases.get(key); + if (existingRelease) { + await existingRelease; + return; + } + + const releasePromise = this.releaseGeneration(key); + this.releases.set(key, releasePromise); + try { + await releasePromise; + } finally { + if (this.releases.get(key) === releasePromise) { + this.releases.delete(key); + } } } - /** Removes every helper container this pool is tracking (call on graceful shutdown). */ + /** Removes every helper after active commands finish (call on graceful shutdown). */ async shutdown(): Promise { this.shuttingDown = true; - const entries = [...this.helpers.entries()]; - this.helpers.clear(); + this.reservations.clear(); + this.notifyCapacityWaiters(); if (this.reaper) { clearInterval(this.reaper); this.reaper = null; } - await Promise.all(entries.map(async ([key, entry]) => { - const refs = new Set([this.spec.nameFor(key)]); - if (entry.id) { - refs.add(entry.id); - } - if (entry.creating) { - const createdId = await entry.creating.catch(() => ""); - if (createdId) { - refs.add(createdId); - } - } - await Promise.all([...refs].map((ref) => this.removeContainerWithVolumes(ref))); - })); + const keys = new Set([...this.helpers.keys(), ...this.releases.keys()]); + await Promise.all([...keys].map((key) => this.release(key))); } isContainerGone(result: CommandResult): boolean { @@ -133,15 +278,89 @@ export class DockerHelperContainerPool { private async create(key: string): Promise { const name = this.spec.nameFor(key); - // Reclaim a same-named helper left behind by a previous process before starting a fresh one. - await this.removeContainerWithVolumes(name); - const result = await this.runner("docker", this.spec.buildCreateArgs(key, name)); + const createArgs = this.spec.buildCreateArgs(key, name); + let result = await helperDockerLifecycleLimit(() => this.runner("docker", createArgs)); + if (!result.ok && this.isContainerNameConflict(result)) { + // A deterministic name can survive a crashed process. Reclaim only when Docker proves that + // this exact create collided; speculative removal doubles control-plane mutations in wide DAGs. + await this.removeContainerWithVolumes(name); + result = await helperDockerLifecycleLimit(() => this.runner("docker", createArgs)); + } if (!result.ok) { throw new Error(result.stderr || result.stdout || "Failed to start helper container."); } return (result.stdout || "").trim() || name; } + private isContainerNameConflict(result: CommandResult): boolean { + const text = `${result.stderr || ""} ${result.stdout || ""}`.toLowerCase(); + return text.includes("container name") && text.includes("already in use"); + } + + private async releaseGeneration(key: string): Promise { + const entry = this.helpers.get(key); + if (entry) { + entry.releasing = true; + if (entry.creating) { + await entry.creating.catch(() => undefined); + } + await this.waitForIdle(entry); + } + + try { + // Acquisitions for this key remain blocked by `releases`, so one exact generation reference + // is sufficient. A create that was interrupted before publishing its id falls back to the + // deterministic name while the ensure path separately reaps any late id it observed. + await this.removeContainerWithVolumes(entry?.id || this.spec.nameFor(key)); + } finally { + // Keep the releasing generation in the capacity count until Docker removal has settled. This + // prevents a wide waiter set from creating replacements while the old containers still exist. + if (entry && this.helpers.get(key) === entry) { + this.helpers.delete(key); + this.notifyCapacityWaiters(); + } + } + } + + private waitForIdle(entry: HelperEntry): Promise { + if (entry.activeUses === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + entry.idleWaiters.add(resolve); + }); + } + + private findLeastRecentlyUsedIdleKey(): string | undefined { + let candidate: { key: string; lastUsed: number } | undefined; + for (const [key, entry] of this.helpers) { + if (entry.creating || entry.releasing || entry.activeUses > 0 || this.reservations.has(key)) { + continue; + } + if (!candidate || entry.lastUsed < candidate.lastUsed) { + candidate = { key, lastUsed: entry.lastUsed }; + } + } + return candidate?.key; + } + + private waitForCapacity(): Promise { + if (this.shuttingDown || this.helpers.size < this.maxContainers + || this.findLeastRecentlyUsedIdleKey() !== undefined) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.capacityWaiters.add(resolve); + }); + } + + private notifyCapacityWaiters(): void { + for (const resolve of this.capacityWaiters) { + resolve(); + } + this.capacityWaiters.clear(); + } + private startReaper(): void { if (this.reaper) { return; @@ -156,24 +375,24 @@ export class DockerHelperContainerPool { private async reapIdle(): Promise { const now = Date.now(); - const removals: Array> = []; + const removals: Array> = []; for (const [key, entry] of [...this.helpers.entries()]) { - if (entry.creating || now - entry.lastUsed < this.idleTtlMs) { + if (entry.creating || entry.releasing || entry.activeUses > 0 || this.reservations.has(key) + || now - entry.lastUsed < this.idleTtlMs) { continue; } - this.helpers.delete(key); - if (entry.id) { - removals.push(this.removeContainerWithVolumes(entry.id)); - } + removals.push(this.release(key)); } await Promise.all(removals); - if (this.helpers.size === 0 && this.reaper) { + if (this.helpers.size === 0 && this.releases.size === 0 && this.reaper) { clearInterval(this.reaper); this.reaper = null; } } private async removeContainerWithVolumes(containerRef: string): Promise { - await this.runner("docker", ["rm", "-f", "-v", containerRef]).catch(() => undefined); + await helperDockerLifecycleLimit( + () => this.runner("docker", ["rm", "-f", "-v", containerRef]), + ).catch(() => undefined); } } diff --git a/src/infrastructure/providers/cli/docker-runner.ts b/src/infrastructure/providers/cli/docker-runner.ts index abb6165e94..d077277b5c 100644 --- a/src/infrastructure/providers/cli/docker-runner.ts +++ b/src/infrastructure/providers/cli/docker-runner.ts @@ -31,6 +31,7 @@ import { workspaceVolumeHelperPool, type WorkspaceVolumeHelperPool } from "./wor import { CONTAINER_RUNTIME_HOME, CONTAINER_WORKSPACE_ROOT } from "./provider-runtime-artifacts.js"; import type { CliProviderId } from "./provider-command-specs.js"; import { getHomeCodeUxPath, getRepoCodeUxPath } from "../../../shared/config/code-ux-paths.js"; +import { getRuntimeOwnerDockerArgs } from "../../../shared/config/runtime-owner.js"; import { ensureDefaultCodeUxAssetsInstalled } from "../../../services/code-ux-default-assets-service.js"; import { DEFAULT_PLAYWRIGHT_MCP_SERVER_ID } from "../../../repositories/settings-defaults.js"; import { sanitizeInvocationOutputText } from "../../../services/invocation-output-sanitizer.js"; @@ -62,6 +63,8 @@ const BUNDLED_CONTAINER_SETUP_SCRIPT = path.resolve( ); const CONTAINER_PROVIDER_ARGV_FILE = "/opt/code-ux/provider-argv.sh"; +const PROVIDER_PROMPT_ARG_MAX_BYTES = 48 * 1024; +const PROVIDER_PROMPT_SINGLE_ARG_HARD_LIMIT_BYTES = 120 * 1024; const PROVIDER_CPU_SHARES = "768"; const LAUNCH_ARTIFACT_INVALID_PREFIX = "CODE_UX_LAUNCH_ARTIFACT_INVALID:"; @@ -80,6 +83,7 @@ export interface IDockerRunner { runProviderInDocker(args: { command: string; args: string[]; + prompt?: string; cwd: string; providerEnv: NodeJS.ProcessEnv; sessionId: string; @@ -158,6 +162,7 @@ export class DockerRunner implements IDockerRunner { async runProviderInDocker(input: { command: string; args: string[]; + prompt?: string; cwd: string; providerEnv: NodeJS.ProcessEnv; sessionId: string; @@ -222,9 +227,15 @@ export class DockerRunner implements IDockerRunner { this.mapDockerSourcePathForDaemon(sourcePath, repoPath, sessionId, label, emitActivity), }); + const launch = this.prepareProviderLaunch(providerLabel, args, input.prompt); const argvFilePath = path.join(tempRoot, "provider-argv.sh"); - await this.writeRestrictiveFile(argvFilePath, this.buildProviderArgvFile(args)); + await this.writeRestrictiveFile(argvFilePath, this.buildProviderArgvFile(launch.args)); const argvFileSource = this.mapDockerSourcePathForDaemon(argvFilePath, repoPath, sessionId, "provider argv", emitActivity); + let promptFilePath: string | undefined; + if (launch.stdinPrompt !== null) { + promptFilePath = path.join(tempRoot, "provider-prompt.txt"); + await this.writeRestrictiveFile(promptFilePath, launch.stdinPrompt); + } const envFilePath = path.join(tempRoot, "provider.env"); await writeDockerEnvFile(envFilePath, pickContainerEnv(providerEnv)); const envFileSource = this.mapDockerSourcePathForDaemon(envFilePath, repoPath, sessionId, "provider env", emitActivity); @@ -246,12 +257,13 @@ export class DockerRunner implements IDockerRunner { CONTAINER_WORKSPACE_ROOT, "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", `code-ux.session-id=${sessionId}`, "--label", `code-ux.command=${command}`, "--label", - `code-ux.args-count=${args.length}`, + `code-ux.args-count=${launch.args.length}`, "--mount", toDockerMountArg({ source: workspace.volumeName, @@ -444,6 +456,7 @@ export class DockerRunner implements IDockerRunner { } } + let finalDockerResult: CommandResult | null = null; try { const runDocker = async (): Promise => { await this.workspaceManager.ensureRuntimeVolume(cwd, { @@ -452,11 +465,13 @@ export class DockerRunner implements IDockerRunner { }); return runStreamingCommand("docker", dockerArgs, process.cwd(), process.env, { signal, + stdinFile: promptFilePath, onStdoutLine: (line) => emitActivity(line, "agent"), onStderrLine: (line) => emitActivity(`[${providerLabel}] ${line}`, "provider"), }); }; let result = await runDocker(); + finalDockerResult = result; const repairedArtifacts = new Set(); let reclaimedContainerName = false; for (;;) { @@ -504,6 +519,7 @@ export class DockerRunner implements IDockerRunner { launchImage = repairedImage.image; } result = await runDocker(); + finalDockerResult = result; continue; } if (!reclaimedContainerName && this.isDockerNameConflict(result, containerName)) { @@ -512,6 +528,7 @@ export class DockerRunner implements IDockerRunner { await this.removeProviderContainer(containerName); await this.sleep(500); result = await runDocker(); + finalDockerResult = result; continue; } break; @@ -521,6 +538,12 @@ export class DockerRunner implements IDockerRunner { if (signal) { signal.removeEventListener("abort", killContainerOnAbort); } + // A stopped `docker run --rm` normally removes itself. The explicit + // cleanup also covers the narrow restart window where Docker created + // the container but the client died before the container could start. + if (abortKillIssued || (finalDockerResult && !finalDockerResult.ok)) { + await this.removeProviderContainer(containerName); + } } } finally { await fs.rm(tempRoot, { recursive: true, force: true }).catch(() => undefined); @@ -536,6 +559,73 @@ export class DockerRunner implements IDockerRunner { ].join("\n"); } + private prepareProviderLaunch( + provider: CliProviderId, + args: string[], + prompt: string | undefined, + ): { args: string[]; stdinPrompt: string | null } { + if (!prompt) { + return { args, stdinPrompt: null }; + } + + const promptIndex = args.findIndex((arg) => arg === prompt); + if (promptIndex < 0) { + return { args, stdinPrompt: null }; + } + + const isE2eShim = promptIndex > 0 && args[promptIndex - 1] === "--prompt"; + const shouldExternalize = provider === "mockup-cli" + || isE2eShim + || Buffer.byteLength(prompt, "utf8") > PROVIDER_PROMPT_ARG_MAX_BYTES; + if (!shouldExternalize) { + return { args, stdinPrompt: null }; + } + + const launchArgs = [...args]; + if (isE2eShim) { + launchArgs.splice(promptIndex - 1, 2); + return { args: launchArgs, stdinPrompt: prompt }; + } + + if (provider === "mockup-cli") { + launchArgs.splice(promptIndex, 1); + return { args: launchArgs, stdinPrompt: prompt }; + } + + if (provider === "gemini" || provider === "qwen-code") { + const promptFlagIndex = promptIndex - 1; + if (promptFlagIndex >= 0 && ["-p", "--p", "--prompt"].includes(launchArgs[promptFlagIndex])) { + launchArgs.splice(promptFlagIndex, 2); + } else { + launchArgs.splice(promptIndex, 1); + } + return { args: launchArgs, stdinPrompt: prompt }; + } + + if (provider === "claude-code" || provider === "opencode") { + launchArgs.splice(promptIndex, 1); + return { args: launchArgs, stdinPrompt: prompt }; + } + + if (provider === "codex") { + launchArgs[promptIndex] = "-"; + return { args: launchArgs, stdinPrompt: prompt }; + } + + // Antigravity print mode requires one value for -p/--print and does not + // publish a file/stdin prompt contract. Splitting the value would turn the + // remaining chunks into unrelated positional arguments, so retain the exact + // invocation while it is below Linux MAX_ARG_STRLEN and fail this invocation + // explicitly before execve can abort with an opaque E2BIG error. + if (Buffer.byteLength(prompt, "utf8") <= PROVIDER_PROMPT_SINGLE_ARG_HARD_LIMIT_BYTES) { + return { args, stdinPrompt: null }; + } + throw new Error( + `Antigravity cannot safely accept a ${Buffer.byteLength(prompt, "utf8")}-byte prompt in Docker: ` + + `its -p/--print contract requires one argument and the safe limit is ${PROVIDER_PROMPT_SINGLE_ARG_HARD_LIMIT_BYTES} bytes.`, + ); + } + private buildLaunchArtifactValidation(args: { runtimeOwner: string | null; providerBinary: string | null; diff --git a/src/infrastructure/providers/cli/provider-execution-loop.ts b/src/infrastructure/providers/cli/provider-execution-loop.ts index 46d680d1f7..fcc34e4eb0 100644 --- a/src/infrastructure/providers/cli/provider-execution-loop.ts +++ b/src/infrastructure/providers/cli/provider-execution-loop.ts @@ -7,6 +7,7 @@ export interface ProviderExecutionLoopOptions { command: string; args: string[]; continueSession: boolean; + allowFreshSessionFallback?: boolean; antigravityLogPath?: string | null; runCmd: (command: string, args: string[]) => Promise; trackingOnActivity: (desc: string, originator?: string) => void; @@ -22,6 +23,7 @@ export async function runProviderExecutionLoop(options: ProviderExecutionLoopOpt const { provider, continueSession, + allowFreshSessionFallback = true, antigravityLogPath, runCmd, trackingOnActivity, @@ -46,7 +48,13 @@ export async function runProviderExecutionLoop(options: ProviderExecutionLoopOpt // `claude --resume ` fails with "No conversation found" when the prior // conversation is gone. Retry once with a fresh session instead. - if (!result.ok && provider === "claude-code" && continueSession && isClaudeConversationNotFoundError(result)) { + if ( + !result.ok + && provider === "claude-code" + && continueSession + && allowFreshSessionFallback + && isClaudeConversationNotFoundError(result) + ) { trackingOnActivity("Claude Code could not resume the previous conversation (no conversation found). Retrying once with a fresh session...", "provider"); const freshSpec = buildFreshClaudeSpec(); command = freshSpec.command; @@ -57,7 +65,13 @@ export async function runProviderExecutionLoop(options: ProviderExecutionLoopOpt // `opencode run --session ` fails with "Session not found" if the native // session was removed from OpenCode's local store. Preserve the workspace and // retry once as a new OpenCode session instead of failing the task immediately. - if (!result.ok && provider === "opencode" && continueSession && isOpenCodeSessionNotFoundError(result)) { + if ( + !result.ok + && provider === "opencode" + && continueSession + && allowFreshSessionFallback + && isOpenCodeSessionNotFoundError(result) + ) { trackingOnActivity("OpenCode could not resume the previous session (session not found). Retrying once with a fresh session...", "provider"); const freshSpec = buildFreshOpenCodeSpec(); command = freshSpec.command; diff --git a/src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.ts b/src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.ts index dcfd33625f..fb2d89854e 100644 --- a/src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.ts +++ b/src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.ts @@ -22,6 +22,10 @@ export interface ClaudeUsageTotals { export interface ClaudeCodeLogResult extends ParsedProviderLogResult { /** Raw object of the last usage seen, for telemetry storage. */ rawUsageJson: Record | null; + /** Monotonic revision for append-efficient downstream message mapping. */ + conversationRevision?: number; + /** First normalized turn changed by the latest append. */ + conversationChangedFromIndex?: number; } function asRecord(value: unknown): Record | null { @@ -246,138 +250,196 @@ function turnSignature(turns: ParsedConversationTurn[]): string { * (matches codex / qwen conventions). Entries older than `sinceMs - 2000ms` * are skipped so only the current invocation's turns are included. */ -export function parseClaudeCodeSessionJsonl( - jsonl: string, - sinceMs?: number, -): ClaudeCodeLogResult { - const lines = jsonl.split("\n"); - const usageByMessageId = new Map(); - const assistantMessageRanges = new Map(); - const conversation: ParsedConversationTurn[] = []; - - let totalInputTokens = 0; - let totalOutputTokens = 0; - let totalCacheCreation = 0; - let totalCacheRead = 0; - let latestRawUsage: Record | null = null; - let nativeSessionId: string | null = null; - let hasUsage = false; - - const applyUsage = (messageId: string | null, usage: Record | null): void => { - if (!usage || !claudeUsageHasTokens(usage)) return; - const next = { - inputTokens: toNumber(usage.input_tokens), - outputTokens: toNumber(usage.output_tokens), - cacheCreationTokens: toNumber(usage.cache_creation_input_tokens), - cacheReadTokens: toNumber(usage.cache_read_input_tokens), - }; - const previous = messageId ? usageByMessageId.get(messageId) : undefined; - totalInputTokens += next.inputTokens - (previous?.inputTokens ?? 0); - totalOutputTokens += next.outputTokens - (previous?.outputTokens ?? 0); - totalCacheCreation += next.cacheCreationTokens - (previous?.cacheCreationTokens ?? 0); - totalCacheRead += next.cacheReadTokens - (previous?.cacheReadTokens ?? 0); - if (messageId) usageByMessageId.set(messageId, next); - latestRawUsage = usage; - hasUsage = true; - }; +interface ClaudeCodeParserState { + usageByMessageId: Map; + assistantMessageRanges: Map; + conversation: ParsedConversationTurn[]; + totalInputTokens: number; + totalOutputTokens: number; + totalCacheCreation: number; + totalCacheRead: number; + latestRawUsage: Record | null; + nativeSessionId: string | null; + hasUsage: boolean; + minMs: number | null; + conversationRevision: number; + conversationChangedFromIndex: number | null; +} - const upsertAssistantTurns = (messageId: string | null, turns: ParsedConversationTurn[]): void => { - if (!messageId || turns.length === 0) { - conversation.push(...turns); - return; - } - const signature = turnSignature(turns); - const existing = assistantMessageRanges.get(messageId); - if (!existing) { - assistantMessageRanges.set(messageId, { start: conversation.length, count: turns.length, signature }); - conversation.push(...turns); - return; - } - if (existing.signature === signature) return; - conversation.splice(existing.start, existing.count, ...turns); - const delta = turns.length - existing.count; - for (const [id, range] of assistantMessageRanges.entries()) { - if (id !== messageId && range.start > existing.start) { - assistantMessageRanges.set(id, { ...range, start: range.start + delta }); - } - } - assistantMessageRanges.set(messageId, { start: existing.start, count: turns.length, signature }); +function createClaudeCodeParserState(sinceMs?: number): ClaudeCodeParserState { + return { + usageByMessageId: new Map(), + assistantMessageRanges: new Map(), + conversation: [], + totalInputTokens: 0, + totalOutputTokens: 0, + totalCacheCreation: 0, + totalCacheRead: 0, + latestRawUsage: null, + nativeSessionId: null, + hasUsage: false, + minMs: typeof sinceMs === "number" ? sinceMs - 2000 : null, + conversationRevision: 0, + conversationChangedFromIndex: null, }; +} - // 2-second grace window, same as codex / qwen parsers. - const minMs = typeof sinceMs === "number" ? sinceMs - 2000 : null; - - for (const rawLine of lines) { - const trimmed = rawLine.trim(); - if (!trimmed.startsWith("{")) continue; +function markClaudeConversationChanged(state: ClaudeCodeParserState, changedFrom: number): void { + state.conversationRevision += 1; + state.conversationChangedFromIndex = state.conversationChangedFromIndex === null + ? changedFrom + : Math.min(state.conversationChangedFromIndex, changedFrom); +} - const entry = parseJsonObject(trimmed); - if (!entry) continue; +function applyClaudeUsage( + state: ClaudeCodeParserState, + messageId: string | null, + usage: Record | null, +): void { + if (!usage || !claudeUsageHasTokens(usage)) return; + const next = { + inputTokens: toNumber(usage.input_tokens), + outputTokens: toNumber(usage.output_tokens), + cacheCreationTokens: toNumber(usage.cache_creation_input_tokens), + cacheReadTokens: toNumber(usage.cache_read_input_tokens), + }; + const previous = messageId ? state.usageByMessageId.get(messageId) : undefined; + state.totalInputTokens += next.inputTokens - (previous?.inputTokens ?? 0); + state.totalOutputTokens += next.outputTokens - (previous?.outputTokens ?? 0); + state.totalCacheCreation += next.cacheCreationTokens - (previous?.cacheCreationTokens ?? 0); + state.totalCacheRead += next.cacheReadTokens - (previous?.cacheReadTokens ?? 0); + if (messageId) state.usageByMessageId.set(messageId, next); + state.latestRawUsage = usage; + state.hasUsage = true; +} - const entryType = typeof entry.type === "string" ? entry.type : null; +function appendClaudeTurns(state: ClaudeCodeParserState, turns: ParsedConversationTurn[]): void { + if (turns.length === 0) return; + const changedFrom = state.conversation.length; + state.conversation.push(...turns); + markClaudeConversationChanged(state, changedFrom); +} - // Timestamp extraction and filtering (applied to all entry types). - const timestampMs = parseTimestampMs(entry.timestamp); - if (minMs !== null && timestampMs !== null && timestampMs < minMs) { - continue; +function upsertClaudeAssistantTurns( + state: ClaudeCodeParserState, + messageId: string | null, + turns: ParsedConversationTurn[], +): void { + if (!messageId || turns.length === 0) { + appendClaudeTurns(state, turns); + return; + } + const signature = turnSignature(turns); + const existing = state.assistantMessageRanges.get(messageId); + if (!existing) { + const start = state.conversation.length; + state.assistantMessageRanges.set(messageId, { start, count: turns.length, signature }); + state.conversation.push(...turns); + markClaudeConversationChanged(state, start); + return; + } + if (existing.signature === signature) return; + state.conversation.splice(existing.start, existing.count, ...turns); + const delta = turns.length - existing.count; + for (const [id, range] of state.assistantMessageRanges.entries()) { + if (id !== messageId && range.start > existing.start) { + state.assistantMessageRanges.set(id, { ...range, start: range.start + delta }); } + } + state.assistantMessageRanges.set(messageId, { start: existing.start, count: turns.length, signature }); + markClaudeConversationChanged(state, existing.start); +} - // Capture the session id only from records eligible for this invocation. - if (!nativeSessionId) { - nativeSessionId = claudeSessionId(entry); +function processClaudeCodeLine(state: ClaudeCodeParserState, rawLine: string): boolean { + const trimmed = rawLine.trim(); + if (!trimmed.startsWith("{")) return false; + const entry = parseJsonObject(trimmed); + if (!entry) return false; + + const entryType = typeof entry.type === "string" ? entry.type : null; + const timestampMs = parseTimestampMs(entry.timestamp); + if (state.minMs !== null && timestampMs !== null && timestampMs < state.minMs) return true; + if (!state.nativeSessionId) state.nativeSessionId = claudeSessionId(entry); + + if (!entryType && asRecord(entry.message)) { + const message = asRecord(entry.message)!; + const messageId = typeof message.id === "string" ? message.id : null; + applyClaudeUsage(state, messageId, claudeMessageUsage(message)); + if (message.role === "user") { + appendClaudeTurns(state, claudeUserTurns(message, timestampMs)); + } else { + upsertClaudeAssistantTurns(state, messageId, claudeAssistantTurns(message, timestampMs)); } + return true; + } - // ── Legacy bare-message format ─────────────────────────────────────────── - // Older Claude Code sessions and container artifact dumps write - // `{ message: { usage, content } }` with no `type` wrapper. Treat these - // as assistant turns so we stay backwards-compatible. - if (!entryType && asRecord(entry.message)) { - const legacyMessage = asRecord(entry.message)!; - const messageId = typeof legacyMessage.id === "string" ? legacyMessage.id : null; - applyUsage(messageId, claudeMessageUsage(legacyMessage)); - const role = typeof legacyMessage.role === "string" ? legacyMessage.role : "assistant"; - if (role === "user") { - conversation.push(...claudeUserTurns(legacyMessage, timestampMs)); - } else { - upsertAssistantTurns(messageId, claudeAssistantTurns(legacyMessage, timestampMs)); - } - continue; - } + if (entryType === "assistant") { + const message = asRecord(entry.message); + if (!message) return true; + const messageId = typeof message.id === "string" ? message.id : null; + applyClaudeUsage(state, messageId, claudeMessageUsage(message)); + upsertClaudeAssistantTurns(state, messageId, claudeAssistantTurns(message, timestampMs)); + return true; + } - // ── Assistant turns ────────────────────────────────────────────────────── - if (entryType === "assistant") { - const message = asRecord(entry.message); - if (!message) continue; + if (entryType === "user") { + const message = asRecord(entry.message); + if (message) appendClaudeTurns(state, claudeUserTurns(message, timestampMs)); + } + return true; +} - const messageId = typeof message.id === "string" ? message.id : null; +function processClaudeCodeChunk(state: ClaudeCodeParserState, chunk: string, pendingLine: string): string { + const lines = (pendingLine + chunk).split("\n"); + const finalLine = lines.pop() ?? ""; + for (const line of lines) processClaudeCodeLine(state, line); + if (!finalLine) return ""; + return processClaudeCodeLine(state, finalLine) ? "" : finalLine; +} - // ── Token usage ───────────────────────────────────────────────────── - applyUsage(messageId, claudeMessageUsage(message)); +function buildClaudeCodeLogResult(state: ClaudeCodeParserState): ClaudeCodeLogResult { + const usage: ClaudeUsageTotals | null = state.hasUsage + ? { + inputTokens: state.totalInputTokens, + outputTokens: state.totalOutputTokens, + cacheCreationTokens: state.totalCacheCreation, + cacheReadTokens: state.totalCacheRead, + } + : null; + return { + usage, + rawUsageJson: state.latestRawUsage, + conversation: state.conversation, + nativeSessionId: state.nativeSessionId, + conversationRevision: state.conversationRevision, + ...(state.conversationChangedFromIndex !== null + ? { conversationChangedFromIndex: state.conversationChangedFromIndex } + : {}), + }; +} - // ── Conversation turns ─────────────────────────────────────────────── - const turns = claudeAssistantTurns(message, timestampMs); - upsertAssistantTurns(messageId, turns); - continue; - } +/** Append-only parser used by live Claude telemetry to avoid full-history joins and reparses. */ +export class ClaudeCodeLogAccumulator { + private state: ClaudeCodeParserState; + private pendingLine = ""; + private sourceId: string | null = null; - // ── User turns (may carry tool results) ────────────────────────────────── - if (entryType === "user") { - const message = asRecord(entry.message); - if (!message) continue; + constructor(private readonly sinceMs?: number) { + this.state = createClaudeCodeParserState(sinceMs); + } - conversation.push(...claudeUserTurns(message, timestampMs)); - continue; + appendChunk(text: string, sourceId: string, reset = false): ClaudeCodeLogResult { + if (reset || (this.sourceId !== null && this.sourceId !== sourceId)) { + this.state = createClaudeCodeParserState(this.sinceMs); + this.pendingLine = ""; } + this.state.conversationChangedFromIndex = null; + this.pendingLine = processClaudeCodeChunk(this.state, text, this.pendingLine); + this.sourceId = sourceId; + return buildClaudeCodeLogResult(this.state); } +} - const usage: ClaudeUsageTotals | null = hasUsage - ? { - inputTokens: totalInputTokens, - outputTokens: totalOutputTokens, - cacheCreationTokens: totalCacheCreation, - cacheReadTokens: totalCacheRead, - } - : null; - - return { usage, rawUsageJson: latestRawUsage, conversation, nativeSessionId }; +export function parseClaudeCodeSessionJsonl(jsonl: string, sinceMs?: number): ClaudeCodeLogResult { + return new ClaudeCodeLogAccumulator(sinceMs).appendChunk(jsonl, "full"); } diff --git a/src/infrastructure/providers/cli/provider-logs/codex-log-parser.ts b/src/infrastructure/providers/cli/provider-logs/codex-log-parser.ts index 409e337dcf..7c01144613 100644 --- a/src/infrastructure/providers/cli/provider-logs/codex-log-parser.ts +++ b/src/infrastructure/providers/cli/provider-logs/codex-log-parser.ts @@ -570,11 +570,16 @@ function processCodexRolloutLine(state: CodexRolloutParserState, rawLine: string } if (type === "event_msg" && payload) { + // event_msg user/assistant rows duplicate the canonical response_item + // stream. Keep them only until the first canonical turn arrives so + // fallback-only Codex versions still work without retaining a second copy + // of every message for the lifetime of a long invocation. + if (state.conversationGroups.length > 0) { + return true; + } const turns = eventMsgToTurns(payload, timestampMs); if (turns.length > 0 && isInWindow(timestampMs)) { - const changedFrom = state.conversationGroups.length > 0 - ? state.conversationGroups.reduce((count, group) => count + group.turns.length, 0) - : state.fallbackEventConversation.length; + const changedFrom = state.fallbackEventConversation.length; state.fallbackEventConversation.push(...turns); state.conversationRevision += 1; state.conversationChangedFromIndex = state.conversationChangedFromIndex === null @@ -594,6 +599,9 @@ function processCodexRolloutLine(state: CodexRolloutParserState, rawLine: string turnsFromCodexItem(payload, timestampMs), ); if (changedGroupIndex !== null) { + if (state.fallbackEventConversation.length > 0) { + state.fallbackEventConversation = []; + } let changedFrom = 0; for (let index = 0; index < changedGroupIndex; index += 1) { changedFrom += state.conversationGroups[index]!.turns.length; diff --git a/src/infrastructure/providers/cli/provider-runner.ts b/src/infrastructure/providers/cli/provider-runner.ts index 5e4b439d85..f83eb4ca97 100644 --- a/src/infrastructure/providers/cli/provider-runner.ts +++ b/src/infrastructure/providers/cli/provider-runner.ts @@ -120,6 +120,10 @@ export interface ProviderRunInput { * Claude Code: uses --resume. Gemini: adds --resume. Codex: uses exec resume --last. * Qwen Code uses project-scoped --continue because Code UX logical ids are not Qwen saved-session ids. */ continueSessionId?: string | null; + /** Whether a missing resumable provider conversation may be replaced by a + * fresh conversation. Disable this when the caller promises strict + * same-session continuity, such as restart recovery for sprint planning. */ + allowFreshSessionFallback?: boolean; /** The previous invocation's raw opencode export snapshot (`{ tokens, cost }`) * for this same session, when `continueSessionId` resumes it. `opencode * export` reports cumulative session totals, so this is subtracted out to @@ -262,6 +266,7 @@ export class ProviderRunner implements IProviderRunner { nativeSessionOperation?: NativeSessionOperation; codexOutputPath?: string | null; continueSessionId?: string | null; + allowFreshSessionFallback?: boolean; openCodeBaselineUsage?: Record | null; mcpConnection?: McpConnectionInfo | null; customMcpServers?: CustomMcpServer[]; @@ -385,7 +390,7 @@ export class ProviderRunner implements IProviderRunner { const runCmd = async () => { if (workflowSettings.executionMode === "DOCKER") { const result = await this.dockerRunner.runProviderInDocker({ - command, args, cwd, providerEnv, sessionId, + command, args, prompt, cwd, providerEnv, sessionId, providerLabel: provider, workflowSettings, repoPath, signal, onActivity: trackingOnActivity, providerMountAuth, providerAuthPath, @@ -538,6 +543,7 @@ export class ProviderRunner implements IProviderRunner { command, args, continueSession: !!continueSession, + allowFreshSessionFallback: input.allowFreshSessionFallback, antigravityLogPath, runCmd: async (cmd, a) => { command = cmd; diff --git a/src/infrastructure/providers/cli/provider-telemetry-watcher.ts b/src/infrastructure/providers/cli/provider-telemetry-watcher.ts index 6f1ec414d3..3dab3b2514 100644 --- a/src/infrastructure/providers/cli/provider-telemetry-watcher.ts +++ b/src/infrastructure/providers/cli/provider-telemetry-watcher.ts @@ -13,6 +13,10 @@ import { ParsedConversationTurn, } from "./provider-usage.js"; import { CodexRolloutAccumulator, type CodexLogResult } from "./provider-logs/codex-log-parser.js"; +import { + ClaudeCodeLogAccumulator, + type ClaudeCodeLogResult, +} from "./provider-logs/claude-code-log-parser.js"; import { ProviderTranscriptChunkDecoder, type ProviderTranscriptChunk, @@ -62,6 +66,8 @@ type ProviderMetadataSignature = { available: true; signature: string } | { avai interface FullReadInputs { resolvedNativeSessionId: string | null; claudeSessionJsonl: string | null; + claudeLog: ClaudeCodeLogResult | null; + claudeIncrementalSignature: string | null; codexSessionJson: string | null; codexRollout: CodexLogResult | null; codexIncrementalSignature: string | null; @@ -108,6 +114,7 @@ async function buildTelemetrySourceSignature(args: { stdout: string; stderr: string; claudeSessionJsonl: string | null; + claudeIncrementalSignature: string | null; codexSessionJson: string | null; codexIncrementalSignature: string | null; qwenLog: { usage: QwenUsageTotals | null; conversation: ParsedConversationTurn[] } | null; @@ -121,6 +128,7 @@ async function buildTelemetrySourceSignature(args: { signatureForString(args.stdout), signatureForString(args.stderr), signatureForString(args.claudeSessionJsonl || ""), + args.claudeIncrementalSignature || "", signatureForString(args.codexSessionJson || ""), args.codexIncrementalSignature || "", signatureForString(args.antigravityTranscriptJsonl || ""), @@ -180,15 +188,18 @@ export class ProviderTelemetryWatcher { private failureBackoff: FailureBackoffState | null = null; private resolvedNativeSessionId: string | null = null; private readonly codexRolloutAccumulator: CodexRolloutAccumulator | null; + private readonly claudeLogAccumulator: ClaudeCodeLogAccumulator | null; private readonly codexChunkDecoder = new ProviderTranscriptChunkDecoder(); private readonly claudeChunkDecoder = new ProviderTranscriptChunkDecoder(); - private claudeJsonlChunks: string[] = []; private wakeWait: (() => void) | null = null; constructor(private readonly opts: TelemetryWatcherOptions) { this.codexRolloutAccumulator = opts.provider === "codex" ? new CodexRolloutAccumulator(opts.startedMs) : null; + this.claudeLogAccumulator = opts.provider === "claude-code" + ? new ClaudeCodeLogAccumulator(opts.startedMs) + : null; } start() { @@ -216,6 +227,8 @@ export class ProviderTelemetryWatcher { const result = await this.collectIncrementalCodexInputs(null, { resolvedNativeSessionId: this.resolvedNativeSessionId || this.opts.nativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, @@ -295,6 +308,8 @@ export class ProviderTelemetryWatcher { const { resolvedNativeSessionId, claudeSessionJsonl, + claudeLog, + claudeIncrementalSignature, codexSessionJson, codexRollout, codexIncrementalSignature, @@ -309,6 +324,7 @@ export class ProviderTelemetryWatcher { stdout, stderr, claudeSessionJsonl, + claudeIncrementalSignature, codexSessionJson, codexIncrementalSignature, qwenLog, @@ -339,6 +355,7 @@ export class ProviderTelemetryWatcher { capturedText: "", nativeSessionId: resolvedNativeSessionId || this.opts.nativeSessionId, claudeSessionJsonl, + claudeSessionLog: claudeLog, codexSessionJson, codexRollout: parsedCodexRollout, qwenReportedUsage: qwenLog?.usage ?? null, @@ -384,6 +401,8 @@ export class ProviderTelemetryWatcher { const emptyInputs: FullReadInputs = { resolvedNativeSessionId: this.resolvedNativeSessionId || this.opts.nativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, @@ -487,6 +506,11 @@ export class ProviderTelemetryWatcher { preReadSourceSignature: string | null, emptyInputs: FullReadInputs, ): Promise { + let latestLog: ClaudeCodeLogResult | null = null; + let signature: string | null = null; + let sourceId: string | null = null; + let reset = false; + const decodedParts: string[] = []; for (let index = 0; index < MAX_INCREMENTAL_CHUNKS_PER_POLL; index += 1) { const chunk = await this.opts.readClaudeSessionJsonlChunk!( this.opts.nativeSessionId!, @@ -494,17 +518,23 @@ export class ProviderTelemetryWatcher { ); if (!chunk) break; const decoded = this.claudeChunkDecoder.consume(chunk); - if (decoded.reset) this.claudeJsonlChunks = []; - if (decoded.text) this.claudeJsonlChunks.push(decoded.text); + signature = `${chunk.sourceId}:${chunk.nextOffset}:${chunk.totalBytes}`; + sourceId = decoded.sourceId; + reset ||= decoded.reset; + if (decoded.text) decodedParts.push(decoded.text); if (decoded.complete || chunk.nextOffset === chunk.startOffset) break; } + if (sourceId && this.claudeLogAccumulator) { + latestLog = this.claudeLogAccumulator.appendChunk(decodedParts.join(""), sourceId, reset); + } return { skipped: false, preReadSourceSignature, inputs: { ...emptyInputs, resolvedNativeSessionId: this.opts.nativeSessionId, - claudeSessionJsonl: this.claudeJsonlChunks.length > 0 ? this.claudeJsonlChunks.join("") : null, + claudeLog: latestLog, + claudeIncrementalSignature: signature, }, }; } @@ -523,6 +553,8 @@ export class ProviderTelemetryWatcher { inputs: { resolvedNativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, @@ -549,6 +581,8 @@ export class ProviderTelemetryWatcher { inputs: { resolvedNativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, @@ -566,6 +600,8 @@ export class ProviderTelemetryWatcher { inputs: { resolvedNativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, @@ -583,6 +619,8 @@ export class ProviderTelemetryWatcher { inputs: { resolvedNativeSessionId, claudeSessionJsonl: null, + claudeLog: null, + claudeIncrementalSignature: null, codexSessionJson: null, codexRollout: null, codexIncrementalSignature: null, diff --git a/src/infrastructure/providers/cli/provider-usage.ts b/src/infrastructure/providers/cli/provider-usage.ts index 5260354e35..d7fcd7d255 100644 --- a/src/infrastructure/providers/cli/provider-usage.ts +++ b/src/infrastructure/providers/cli/provider-usage.ts @@ -288,8 +288,18 @@ function claudeJsonlToTelemetry( sinceMs?: number, ): ProviderUsageTelemetry | null { if (!raw.trim()) return null; + return claudeLogToTelemetry( + parseClaudeCodeSessionJsonl(raw, sinceMs), + nativeSessionId, + rawUsageJson, + ); +} - const parsed = parseClaudeCodeSessionJsonl(raw, sinceMs); +function claudeLogToTelemetry( + parsed: ClaudeCodeLogResult, + nativeSessionId: string, + rawUsageJson: Record | null, +): ProviderUsageTelemetry { // Prefer the session id embedded in the JSONL entries over the caller-supplied one. const resolvedSessionId = parsed.nativeSessionId ?? nativeSessionId; @@ -310,6 +320,8 @@ function claudeJsonlToTelemetry( transcriptText, nativeSessionId: resolvedSessionId, conversation, + conversationRevision: parsed.conversationRevision, + conversationChangedFromIndex: parsed.conversationChangedFromIndex, }; } @@ -328,6 +340,8 @@ function claudeJsonlToTelemetry( transcriptText, nativeSessionId: resolvedSessionId, conversation, + conversationRevision: parsed.conversationRevision, + conversationChangedFromIndex: parsed.conversationChangedFromIndex, }; } @@ -341,6 +355,8 @@ export async function collectProviderUsageTelemetry(args: { capturedText?: string; nativeSessionId?: string | null; claudeSessionJsonl?: string | null; + /** Pre-parsed append-only Claude session state supplied by the live watcher. */ + claudeSessionLog?: ClaudeCodeLogResult | null; codexSessionJson?: string | null; /** Pre-parsed live rollout state supplied by the incremental watcher. */ codexRollout?: CodexLogResult | null; @@ -589,19 +605,30 @@ export async function collectProviderUsageTelemetry(args: { } if (args.nativeSessionId) { - if (args.claudeSessionJsonl) { - const usage = claudeJsonlToTelemetry( - args.claudeSessionJsonl, - args.nativeSessionId, - { source: "container-session-jsonl" }, - args.startTimeMs, - ); + if (args.claudeSessionLog || args.claudeSessionJsonl) { + const usage = args.claudeSessionLog + ? claudeLogToTelemetry( + args.claudeSessionLog, + args.nativeSessionId, + { source: "container-session-jsonl" }, + ) + : claudeJsonlToTelemetry( + args.claudeSessionJsonl!, + args.nativeSessionId, + { source: "container-session-jsonl" }, + args.startTimeMs, + ); if (usage) { const conversation = withLeadingUserTurn(usage.conversation, args.prompt); if (usage.totalTokens > 0) { return { ...usage, conversation }; } - return estimateTelemetry("claude-code", args.model, args.prompt, usage.transcriptText || fallbackOutput); + const estimated = estimateTelemetry("claude-code", args.model, args.prompt, usage.transcriptText || fallbackOutput); + estimated.nativeSessionId = usage.nativeSessionId; + estimated.conversation = conversation; + estimated.conversationRevision = usage.conversationRevision; + estimated.conversationChangedFromIndex = usage.conversationChangedFromIndex; + return estimated; } } const usage = await parseClaudeSessionTelemetry(args.cwd, args.nativeSessionId, args.startTimeMs); @@ -610,7 +637,12 @@ export async function collectProviderUsageTelemetry(args: { if (usage.totalTokens > 0) { return { ...usage, conversation }; } - return estimateTelemetry("claude-code", args.model, args.prompt, usage.transcriptText || fallbackOutput); + const estimated = estimateTelemetry("claude-code", args.model, args.prompt, usage.transcriptText || fallbackOutput); + estimated.nativeSessionId = usage.nativeSessionId; + estimated.conversation = conversation; + estimated.conversationRevision = usage.conversationRevision; + estimated.conversationChangedFromIndex = usage.conversationChangedFromIndex; + return estimated; } } diff --git a/src/infrastructure/providers/cli/workspace-manager.ts b/src/infrastructure/providers/cli/workspace-manager.ts index 356e8f1df9..d9b50a7c15 100644 --- a/src/infrastructure/providers/cli/workspace-manager.ts +++ b/src/infrastructure/providers/cli/workspace-manager.ts @@ -10,6 +10,7 @@ import { extractPathHints, normalizePathHint } from "../../../services/cli-workf import { workspaceVolumeHelperPool } from "./workspace-volume-helper.js"; import { CONTAINER_RUNTIME_HOME } from "./provider-runtime-artifacts.js"; import { getHomeCodeUxPath } from "../../../shared/config/code-ux-paths.js"; +import { getRuntimeOwnerDockerArgs } from "../../../shared/config/runtime-owner.js"; import { buildGitHttpAuthEnvForRepoWithFallbacks, buildNonInteractiveGitEnv, @@ -88,6 +89,8 @@ export interface IWorkspaceManager { prepareWorktree(repoPath: string, worktreePath: string, workerBranch: string, featureBranch: string, resumeSessionId?: string, gitAuth?: GitHttpAuthOptions, options?: PrepareWorktreeOptions): Promise<{ worktreePath: string; resumed: boolean }>; fastForwardResumedWorkspace(worktreePath: string, workerBranch: string, repoPath: string, gitAuth?: GitHttpAuthOptions): Promise; removeWorktree(repoPath: string, worktreePath: string): Promise; + reserveWorkspaceHelper(worktreePath: string): () => void; + releaseWorkspaceHelper(worktreePath: string): Promise; buildWorkspaceGuidance(taskPrompt: string, worktreePath: string): Promise; runWorkspaceCommand(worktreePath: string, command: string, args: string[], options?: WorkspaceCommandOptions): Promise; readWorkspaceFile(worktreePath: string, relativePath: string): Promise; @@ -169,6 +172,10 @@ const DOCKER_WORKSPACE_ENV_KEYS = new Set([ "GIT_AUTHOR_NAME", "GIT_COMMITTER_EMAIL", "GIT_COMMITTER_NAME", + "GIT_ASKPASS", + "GIT_CONFIG_COUNT", + "GIT_INDEX_FILE", + "GIT_TERMINAL_PROMPT", "GCM_INTERACTIVE", "SSH_ASKPASS", ]); @@ -181,13 +188,14 @@ const DEFAULT_WORKSPACE_GIT_IDENTITY: Record = { }; const shouldForwardWorkspaceEnv = (key: string): boolean => ( - key.startsWith("GIT_") - || key.startsWith("GIT_CONFIG_") - || DOCKER_WORKSPACE_ENV_KEYS.has(key) + DOCKER_WORKSPACE_ENV_KEYS.has(key) + || /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/.test(key) ); -const buildWorkspaceDockerEnvArgs = (env: NodeJS.ProcessEnv): string[] => { - const args: string[] = []; +const buildWorkspaceEnvironment = (env: NodeJS.ProcessEnv): NodeJS.ProcessEnv => { + const result: NodeJS.ProcessEnv = { + HOME: CONTAINER_WORKSPACE_HELPER_HOME, + }; const dockerEnv = { ...DEFAULT_WORKSPACE_GIT_IDENTITY, ...env, @@ -196,9 +204,39 @@ const buildWorkspaceDockerEnvArgs = (env: NodeJS.ProcessEnv): string[] => { if (typeof value !== "string" || !shouldForwardWorkspaceEnv(key)) { continue; } - args.push("-e", `${key}=${value}`); + result[key] = value; + } + return result; +}; + +const NETWORK_GIT_COMMANDS = new Set(["clone", "fetch", "ls-remote", "pull", "push", "submodule"]); + +const resolveGitSubcommand = (args: readonly string[]): string | null => { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (!arg) continue; + if (arg === "-C" || arg === "-c" || arg === "--config-env" || arg === "--git-dir" || arg === "--work-tree") { + index += 1; + continue; + } + if (arg.startsWith("-")) continue; + return arg; } - return args; + return null; +}; + +const requiresNetworkedWorkspaceContainer = (command: string, args: readonly string[]): boolean => ( + command === "git" && NETWORK_GIT_COMMANDS.has(resolveGitSubcommand(args) || "") +); + +const assertWorkspaceHelperResult = ( + result: CommandResult, + command: string, + args: readonly string[], +): CommandResult => { + if (result.ok) return result; + const detail = result.stderr || result.stdout || `Unknown error (exit code ${result.code ?? "unknown"}, no output captured)`; + throw new Error(`${command} ${args.join(" ")} failed: ${detail}`); }; export class WorkspaceManager implements IWorkspaceManager { @@ -631,6 +669,20 @@ export class WorkspaceManager implements IWorkspaceManager { await runCommandStrict("git", ["worktree", "prune"], repoPath).catch(() => undefined); } + async releaseWorkspaceHelper(worktreePath: string): Promise { + if (!isWorkspaceHandle(worktreePath)) return; + const { volumeName } = parseWorkspaceHandle(worktreePath); + await workspaceVolumeHelperPool.releaseVolume(volumeName); + } + + reserveWorkspaceHelper(worktreePath: string): () => void { + if (!isWorkspaceHandle(worktreePath)) return () => undefined; + const { volumeName } = parseWorkspaceHandle(worktreePath); + // Normal workspace commands mount both the task workspace and its provider-runtime volume. + // Reservations must use that exact composite key or the live helper remains LRU-evictable. + return workspaceVolumeHelperPool.reserve(volumeName, buildRuntimeVolumeName(volumeName)); + } + async buildWorkspaceGuidance(taskPrompt: string, worktreePath: string): Promise { const hints = extractPathHints(taskPrompt).slice(0, 10); const isDockerWorkspace = isWorkspaceHandle(worktreePath); @@ -700,6 +752,33 @@ export class WorkspaceManager implements IWorkspaceManager { const { volumeName } = parseWorkspaceHandle(worktreePath); const ownerSpec = getWorkspaceOwnerSpec(); await this.ensurePublicHelperImage(WORKSPACE_HELPER_IMAGE, process.cwd(), options.env ?? process.env); + if (requiresNetworkedWorkspaceContainer(command, args)) { + return await this.runNetworkedWorkspaceCommand(volumeName, command, args, options, ownerSpec); + } + const result = await workspaceVolumeHelperPool.exec( + volumeName, + [command, ...args], + buildRuntimeVolumeName(volumeName), + { + environment: buildWorkspaceEnvironment(options.env ?? process.env), + signal: options.signal, + stdinFile: options.stdinFile, + trimOutput: options.trimOutput, + user: ownerSpec, + workdir: CONTAINER_WORKSPACE_ROOT, + }, + ); + return assertWorkspaceHelperResult(result, command, args); + } + + private async runNetworkedWorkspaceCommand( + volumeName: string, + command: string, + args: string[], + options: WorkspaceCommandOptions, + ownerSpec: string, + ): Promise { + const environment = buildWorkspaceEnvironment(options.env ?? process.env); const dockerArgs = [ "run", "--rm", @@ -710,9 +789,7 @@ export class WorkspaceManager implements IWorkspaceManager { `type=volume,source=${volumeName},target=${CONTAINER_WORKSPACE_ROOT}`, "--entrypoint", command, - "-e", - `HOME=${CONTAINER_WORKSPACE_HELPER_HOME}`, - ...buildWorkspaceDockerEnvArgs(options.env ?? process.env), + ...Object.entries(environment).flatMap(([key, value]) => typeof value === "string" ? ["-e", `${key}=${value}`] : []), WORKSPACE_HELPER_IMAGE, ...args, ]; @@ -808,6 +885,7 @@ export class WorkspaceManager implements IWorkspaceManager { RUNTIME_VOLUME_LABEL, "--label", `${WORKSPACE_SESSION_LABEL_PREFIX}${sessionKey}`, + ...getRuntimeOwnerDockerArgs(), runtimeVolumeName, ], process.cwd(), @@ -881,13 +959,17 @@ export class WorkspaceManager implements IWorkspaceManager { WORKSPACE_VOLUME_LABEL, "--label", `${WORKSPACE_SESSION_LABEL_PREFIX}${sessionKey}`, + ...getRuntimeOwnerDockerArgs(), volumeName, ], process.cwd(), ); } - private async initializeRuntimeVolumeOwnership(runtimeVolumeName: string, ownerSpec: string): Promise { + private async initializeRuntimeVolumeOwnership( + runtimeVolumeName: string, + ownerSpec: string, + ): Promise { await this.ensurePublicHelperImage(WORKSPACE_HELPER_IMAGE, process.cwd(), process.env); await runCommandStrict( "docker", @@ -1104,7 +1186,7 @@ export class WorkspaceManager implements IWorkspaceManager { "set -e", "tmp=$(mktemp)", "cat > \"$tmp\"", - "rm -rf /workspace/* /workspace/.[!.]* /workspace/..?* 2>/dev/null || true", + "(rm -rf /workspace/* /workspace/.[!.]* /workspace/..?* 2>/dev/null || true)", // See the single-branch seed path above: the helper is root while the // persistent volume root belongs to the provider UID/GID. "git config --global --add safe.directory /workspace", @@ -1115,7 +1197,7 @@ export class WorkspaceManager implements IWorkspaceManager { "rm -f \"$tmp\"", originUrl ? `git -C /workspace remote set-url origin ${shellQuote(originUrl)}` - : "git -C /workspace remote remove origin >/dev/null 2>&1 || true", + : "(git -C /workspace remote remove origin >/dev/null 2>&1 || true)", ...this.buildLocalBranchAliasCommands(localBranchAliases), "git -C /workspace config user.name \"${CODE_UX_GIT_USER_NAME:-Code UX}\"", "git -C /workspace config user.email \"${CODE_UX_GIT_USER_EMAIL:-agents@codeux.ai}\"", @@ -1126,26 +1208,18 @@ export class WorkspaceManager implements IWorkspaceManager { ownerSpec ? this.buildRuntimeOwnershipMarkerCommand(ownerSpec) : null, ].filter((step): step is string => Boolean(step)).join(" && "); - await runCommandStrict( - "docker", - [ - "run", - "--rm", - "-i", - "--mount", - `type=volume,source=${volumeName},target=${CONTAINER_WORKSPACE_ROOT}`, - "--mount", - `type=volume,source=${runtimeVolumeName},target=${CONTAINER_RUNTIME_HOME}`, - "--entrypoint", - "sh", - WORKSPACE_HELPER_IMAGE, - "-lc", - initScript, - ], - repoPath, - process.env, - { stdinFile: bundlePath }, + const commandArgs = ["sh", "-lc", initScript]; + const result = await workspaceVolumeHelperPool.exec( + volumeName, + commandArgs, + runtimeVolumeName, + { + environment: buildWorkspaceEnvironment(process.env), + stdinFile: bundlePath, + workdir: CONTAINER_WORKSPACE_ROOT, + }, ); + assertWorkspaceHelperResult(result, commandArgs[0], commandArgs.slice(1)); if (ownerSpec) this.runtimeVolumeOwners.set(runtimeVolumeName, ownerSpec); this.runtimeVolumesKnownPresent.add(runtimeVolumeName); }); diff --git a/src/infrastructure/providers/cli/workspace-volume-helper.ts b/src/infrastructure/providers/cli/workspace-volume-helper.ts index 5a9813bbc4..48d792b153 100644 --- a/src/infrastructure/providers/cli/workspace-volume-helper.ts +++ b/src/infrastructure/providers/cli/workspace-volume-helper.ts @@ -1,21 +1,59 @@ -import { CommandResult } from "../../../services/cli-process-runner.js"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import type { CommandResult } from "../../../services/cli-process-runner.js"; import { DOCKER_NETWORK_NONE_ARGS, DOCKER_NO_NEW_PRIVILEGES_ARGS, toDockerMountArg, } from "../../../services/cli-docker-utils.js"; +import { getRuntimeOwnerDockerArgs } from "../../../shared/config/runtime-owner.js"; import { DockerHelperContainerPool, HELPER_LABEL, + HELPER_OWNER_NAME_SUFFIX, defaultHelperRunner, type HelperCommandRunner, + type HelperRunnerOptions, } from "./docker-helper-pool.js"; const CONTAINER_WORKSPACE_ROOT = "/workspace"; const CONTAINER_RUNTIME_HOME = "/code-ux-runtime-home"; -const HELPER_IMAGE = "alpine:3.20"; +const CONTAINER_HELPER_HOME = "/tmp/code-ux-home"; +const HELPER_IMAGE = "alpine/git"; const KEEPALIVE_COMMAND = "tail -f /dev/null"; +const HELPER_HOME_MOUNT = `type=tmpfs,target=${CONTAINER_HELPER_HOME},tmpfs-mode=1777,tmpfs-size=1048576`; const HELPER_KEY_DELIMITER = "\n"; +const DEFAULT_IDLE_TTL_MS = 30_000; +const DEFAULT_REAP_INTERVAL_MS = 5_000; +const DEFAULT_MAX_CONTAINERS = 16; +const DOCKER_ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const DOCKER_USER_PATTERN = /^[A-Za-z0-9_][A-Za-z0-9_.-]*(?::[A-Za-z0-9_][A-Za-z0-9_.-]*)?$/; +const DOCKER_VOLUME_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/; + +export interface WorkspaceSidecarExecOptions extends HelperRunnerOptions { + /** Explicit command-only container environment. Ambient host variables are never inherited. */ + environment?: Readonly; + /** Docker user/user:group applied only to this command. */ + user?: string; + /** Absolute command workdir below /workspace (or the mounted runtime home). */ + workdir?: string; +} + +export interface WorkspaceSidecarLifecycleOptions { + idleTtlMs?: number; + reapIntervalMs?: number; + maxContainers?: number; +} + +interface VolumeGate { + activeCommands: number; + idleWaiters: Set<() => void>; + releasePromise?: Promise; +} + +type WorkspaceExecAttempt = + | { kind: "result"; containerId: string; result: CommandResult } + | { kind: "runner-error"; error: unknown }; const buildHelperKey = (workspaceVolumeName: string, runtimeVolumeName?: string): string => [workspaceVolumeName, runtimeVolumeName || ""].join(HELPER_KEY_DELIMITER); @@ -26,25 +64,30 @@ const parseHelperKey = (key: string): { workspaceVolumeName: string; runtimeVolu }; /** - * Maintains one long-lived `alpine:3.20` helper container per workspace volume and runs - * read/maintenance commands inside it via `docker exec`, instead of spawning a fresh - * `docker run --rm` container for every operation. + * Maintains one short-lived, Git-capable sidecar per workspace/runtime-volume pair. * - * The provider telemetry watcher polls workspace logs every ~1.5s per active session; with the - * previous per-operation containers that produced a steady stream of container create/start/stop - * churn. Reusing a single warm container per volume removes that churn entirely while keeping the - * exact same commands and output. + * Commands use `docker exec` against a warm sidecar, while all identity, environment, working + * directory, stdin, and cancellation settings remain command-scoped. Sidecars have no network and + * no-new-privileges, are automatically reaped after a bounded idle window, and fall back to an + * equivalent one-shot container if a helper generation cannot be created or disappears twice. */ export class WorkspaceVolumeHelperPool { private readonly pool: DockerHelperContainerPool; private readonly keysByWorkspaceVolume = new Map>(); + private readonly volumeGates = new Map(); + private shuttingDown = false; + private shutdownPromise: Promise | null = null; constructor( private readonly runner: HelperCommandRunner = defaultHelperRunner, private readonly image: string = HELPER_IMAGE, + lifecycle: WorkspaceSidecarLifecycleOptions = {}, ) { + if (!image || image.includes("\0")) { + throw new Error("Workspace sidecar image cannot be empty or contain null bytes."); + } this.pool = new DockerHelperContainerPool({ - nameFor: (volumeName) => `code-ux-vol-helper-${volumeName.replace(/[^A-Za-z0-9_.-]/g, "-").slice(0, 200)}`, + nameFor: (key) => `code-ux-vol-helper-${HELPER_OWNER_NAME_SUFFIX}-${createHash("sha1").update(key).digest("hex").slice(0, 24)}`, buildCreateArgs: (key, name) => { const { workspaceVolumeName, runtimeVolumeName } = parseHelperKey(key); const args = [ @@ -58,10 +101,20 @@ export class WorkspaceVolumeHelperPool { "code-ux.managed=true", "--label", `${HELPER_LABEL}=volume`, + ...getRuntimeOwnerDockerArgs(), "--workdir", CONTAINER_WORKSPACE_ROOT, "--mount", toDockerMountArg({ source: workspaceVolumeName, destination: CONTAINER_WORKSPACE_ROOT, readonly: false, type: "volume" }), + // alpine/git declares /git as a volume. Mask it so the warm sidecar does not allocate an + // anonymous Docker volume; `docker rm -v` remains a second line of cleanup defense. + "--mount", + "type=tmpfs,target=/git", + // WorkspaceManager deliberately gives Git an isolated HOME instead of inheriting a + // provider home. Mount it explicitly so both root bootstrap commands and later + // uid-scoped commands can create/read the transient global Git config. + "--mount", + HELPER_HOME_MOUNT, ]; if (runtimeVolumeName) { args.push( @@ -72,56 +125,219 @@ export class WorkspaceVolumeHelperPool { args.push("--entrypoint", "sh", this.image, "-c", KEEPALIVE_COMMAND); return args; }, + idleTtlMs: lifecycle.idleTtlMs ?? DEFAULT_IDLE_TTL_MS, + reapIntervalMs: lifecycle.reapIntervalMs ?? DEFAULT_REAP_INTERVAL_MS, + maxContainers: lifecycle.maxContainers ?? DEFAULT_MAX_CONTAINERS, }, runner); } - /** Runs a command inside the persistent helper container for the given workspace volume. */ - async exec(volumeName: string, commandArgs: string[], runtimeVolumeName?: string): Promise { - const key = buildHelperKey(volumeName, runtimeVolumeName); - const keys = this.keysByWorkspaceVolume.get(volumeName) || new Set(); - keys.add(key); - this.keysByWorkspaceVolume.set(volumeName, keys); + /** Keeps a logical workspace helper generation reusable until the returned lease is released. */ + reserve(volumeName: string, runtimeVolumeName?: string): () => void { + this.validateVolumeName(volumeName, "workspace"); + if (runtimeVolumeName !== undefined) { + this.validateVolumeName(runtimeVolumeName, "runtime"); + } + return this.pool.reserve(buildHelperKey(volumeName, runtimeVolumeName)); + } - let id: string; - try { - id = await this.pool.ensure(key); - } catch { - return this.fallbackRun(volumeName, commandArgs, runtimeVolumeName); + /** + * Runs an executable and arguments inside the workspace sidecar. + * + * A non-zero command resolves to a non-ok CommandResult. Invalid sidecar options reject before + * Docker is invoked. When `stdinFile` is present, Docker receives `-i` and the host runner streams + * that file to the command's stdin. + */ + async exec( + volumeName: string, + commandArgs: readonly string[], + runtimeVolumeName?: string, + options: WorkspaceSidecarExecOptions = {}, + ): Promise { + this.validateVolumeName(volumeName, "workspace"); + if (runtimeVolumeName !== undefined) { + this.validateVolumeName(runtimeVolumeName, "runtime"); } - this.pool.touch(key); + this.validateCommandArgs(commandArgs); + const commandDockerArgs = this.buildCommandDockerArgs(options, runtimeVolumeName); + const runnerOptions = this.buildRunnerOptions(options); + const releaseCommand = await this.acquireVolumeCommand(volumeName); - let result = await this.runner("docker", ["exec", id, ...commandArgs]); - if (!result.ok && this.pool.isContainerGone(result)) { - // The helper vanished (e.g. external prune or daemon restart) — drop it and retry once. - this.pool.invalidate(key); + try { + const key = buildHelperKey(volumeName, runtimeVolumeName); + const keys = this.keysByWorkspaceVolume.get(volumeName) || new Set(); + keys.add(key); + this.keysByWorkspaceVolume.set(volumeName, keys); + + const runViaExec = (): Promise => ( + this.pool.withContainer(key, async (containerId) => { + try { + return { + kind: "result" as const, + containerId, + result: await this.runner( + "docker", + ["exec", ...commandDockerArgs, containerId, ...commandArgs], + runnerOptions, + ), + }; + } catch (error) { + // Only sidecar lifecycle failures use the one-shot fallback. A host-runner exception + // may have happened after the command started, so repeating it could duplicate writes. + return { kind: "runner-error" as const, error }; + } + }) + ); + + let attempt: WorkspaceExecAttempt; try { - id = await this.pool.ensure(key); - result = await this.runner("docker", ["exec", id, ...commandArgs]); + attempt = await runViaExec(); } catch { - return this.fallbackRun(volumeName, commandArgs, runtimeVolumeName); + return this.fallbackRun(volumeName, commandArgs, runtimeVolumeName, commandDockerArgs, runnerOptions); + } + if (attempt.kind === "runner-error") { + throw attempt.error; } + + if (!attempt.result.ok && this.pool.isContainerGone(attempt.result)) { + // Invalidate only the generation that failed. A concurrent caller may already have + // installed a replacement, in which case the next acquisition joins that generation. + this.pool.invalidate(key, attempt.containerId); + try { + attempt = await runViaExec(); + } catch { + return this.fallbackRun(volumeName, commandArgs, runtimeVolumeName, commandDockerArgs, runnerOptions); + } + if (attempt.kind === "runner-error") { + throw attempt.error; + } + if (!attempt.result.ok && this.pool.isContainerGone(attempt.result)) { + this.pool.invalidate(key, attempt.containerId); + return this.fallbackRun(volumeName, commandArgs, runtimeVolumeName, commandDockerArgs, runnerOptions); + } + } + return attempt.result; + } finally { + releaseCommand(); } - return result; } /** - * Removes the helper container holding the given volume. Must be called before deleting the - * underlying workspace volume, otherwise `docker volume rm` fails because the helper still has - * it mounted. Safe to call when no helper exists. + * Drains commands using `volumeName`, then removes all sidecars that mount it. The named workspace + * and runtime volumes themselves are preserved. A concurrent release joins the same drain; a new + * command waits until release completes before creating a fresh sidecar generation. */ async releaseVolume(volumeName: string): Promise { - const keys = this.keysByWorkspaceVolume.get(volumeName) || new Set([buildHelperKey(volumeName)]); - await Promise.all([...keys].map((key) => this.pool.release(key).catch(() => undefined))); - this.keysByWorkspaceVolume.delete(volumeName); + this.validateVolumeName(volumeName, "workspace"); + const gate = this.getOrCreateVolumeGate(volumeName); + if (gate.releasePromise) { + await gate.releasePromise; + return; + } + + const releasePromise = this.releaseVolumeGeneration(volumeName, gate); + gate.releasePromise = releasePromise; + try { + await releasePromise; + } finally { + if (gate.releasePromise === releasePromise) { + gate.releasePromise = undefined; + } + if (gate.activeCommands === 0 && this.volumeGates.get(volumeName) === gate) { + this.volumeGates.delete(volumeName); + } + } } - /** Removes every helper container this pool is tracking (call on graceful shutdown). */ - async shutdown(): Promise { + /** Drains active commands and removes every sidecar. Idempotent. */ + shutdown(): Promise { + if (this.shutdownPromise) { + return this.shutdownPromise; + } + this.shuttingDown = true; + this.shutdownPromise = this.shutdownAll(); + return this.shutdownPromise; + } + + private async shutdownAll(): Promise { + const volumes = new Set([ + ...this.keysByWorkspaceVolume.keys(), + ...this.volumeGates.keys(), + ]); + await Promise.all([...volumes].map((volumeName) => this.releaseVolume(volumeName))); await this.pool.shutdown(); } - /** One-shot `docker run --rm` so a pool failure never blocks the underlying operation. */ - private fallbackRun(volumeName: string, commandArgs: string[], runtimeVolumeName?: string): Promise { + private async releaseVolumeGeneration(volumeName: string, gate: VolumeGate): Promise { + await this.waitForVolumeIdle(gate); + const keys = [...(this.keysByWorkspaceVolume.get(volumeName) || new Set([buildHelperKey(volumeName)]))]; + try { + await Promise.all(keys.map((key) => this.pool.release(key))); + } finally { + this.keysByWorkspaceVolume.delete(volumeName); + } + } + + private async acquireVolumeCommand(volumeName: string): Promise<() => void> { + for (;;) { + if (this.shuttingDown) { + throw new Error("Workspace sidecar pool is shutting down."); + } + const gate = this.getOrCreateVolumeGate(volumeName); + if (gate.releasePromise) { + await gate.releasePromise; + continue; + } + gate.activeCommands += 1; + let released = false; + return () => { + if (released) { + return; + } + released = true; + gate.activeCommands = Math.max(0, gate.activeCommands - 1); + if (gate.activeCommands === 0) { + for (const resolve of gate.idleWaiters) { + resolve(); + } + gate.idleWaiters.clear(); + if (!gate.releasePromise && this.volumeGates.get(volumeName) === gate) { + this.volumeGates.delete(volumeName); + } + } + }; + } + } + + private getOrCreateVolumeGate(volumeName: string): VolumeGate { + const existing = this.volumeGates.get(volumeName); + if (existing) { + return existing; + } + const created: VolumeGate = { + activeCommands: 0, + idleWaiters: new Set(), + }; + this.volumeGates.set(volumeName, created); + return created; + } + + private waitForVolumeIdle(gate: VolumeGate): Promise { + if (gate.activeCommands === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + gate.idleWaiters.add(resolve); + }); + } + + /** One-shot equivalent so a pool failure never blocks the underlying operation. */ + private fallbackRun( + volumeName: string, + commandArgs: readonly string[], + runtimeVolumeName: string | undefined, + commandDockerArgs: string[], + runnerOptions: HelperRunnerOptions, + ): Promise { const args = [ "run", "--rm", @@ -131,10 +347,13 @@ export class WorkspaceVolumeHelperPool { "code-ux.managed=true", "--label", `${HELPER_LABEL}=volume`, - "--workdir", - CONTAINER_WORKSPACE_ROOT, + ...getRuntimeOwnerDockerArgs(), "--mount", toDockerMountArg({ source: volumeName, destination: CONTAINER_WORKSPACE_ROOT, readonly: false, type: "volume" }), + "--mount", + "type=tmpfs,target=/git", + "--mount", + HELPER_HOME_MOUNT, ]; if (runtimeVolumeName) { args.push( @@ -142,13 +361,91 @@ export class WorkspaceVolumeHelperPool { toDockerMountArg({ source: runtimeVolumeName, destination: CONTAINER_RUNTIME_HOME, readonly: false, type: "volume" }), ); } - args.push(this.image, ...commandArgs); - return this.runner("docker", args); + args.push( + ...commandDockerArgs, + "--entrypoint", + commandArgs[0], + this.image, + ...commandArgs.slice(1), + ); + return this.runner("docker", args, runnerOptions); + } + + private buildCommandDockerArgs( + options: WorkspaceSidecarExecOptions, + runtimeVolumeName?: string, + ): string[] { + const args: string[] = []; + if (options.stdinFile) { + args.push("-i"); + } + if (options.workdir !== undefined) { + args.push("--workdir", this.validateWorkdir(options.workdir, Boolean(runtimeVolumeName))); + } + if (options.user !== undefined) { + if (!DOCKER_USER_PATTERN.test(options.user) || options.user.includes("\0")) { + throw new Error(`Invalid Docker sidecar user: ${options.user}`); + } + args.push("--user", options.user); + } + for (const [name, value] of Object.entries(options.environment || {})) { + if (value === undefined || value.length === 0) { + continue; + } + if (!DOCKER_ENV_NAME_PATTERN.test(name)) { + throw new Error(`Invalid Docker sidecar environment name: ${name}`); + } + if (value.includes("\0")) { + throw new Error(`Docker sidecar environment value contains a null byte: ${name}`); + } + args.push("--env", `${name}=${value}`); + } + return args; + } + + private buildRunnerOptions(options: WorkspaceSidecarExecOptions): HelperRunnerOptions { + return { + ...(options.signal !== undefined ? { signal: options.signal } : {}), + ...(options.stdinFile !== undefined ? { stdinFile: options.stdinFile } : {}), + ...(options.trimOutput !== undefined ? { trimOutput: options.trimOutput } : {}), + ...(options.maxStdoutChars !== undefined ? { maxStdoutChars: options.maxStdoutChars } : {}), + ...(options.onStdoutLine !== undefined ? { onStdoutLine: options.onStdoutLine } : {}), + ...(options.onStderrLine !== undefined ? { onStderrLine: options.onStderrLine } : {}), + }; + } + + private validateVolumeName(volumeName: string, kind: "workspace" | "runtime"): void { + if (!DOCKER_VOLUME_NAME_PATTERN.test(volumeName) || volumeName.includes("\0")) { + throw new Error(`Invalid ${kind} Docker volume name.`); + } + } + + private validateCommandArgs(commandArgs: readonly string[]): void { + if (commandArgs.length === 0 || !commandArgs[0]) { + throw new Error("Workspace sidecar command must include an executable."); + } + if (commandArgs.some((argument) => argument.includes("\0"))) { + throw new Error("Workspace sidecar command arguments cannot contain null bytes."); + } + } + + private validateWorkdir(workdir: string, runtimeMounted: boolean): string { + if (!workdir || workdir.includes("\0") || !path.posix.isAbsolute(workdir)) { + throw new Error("Workspace sidecar workdir must be an absolute container path."); + } + const normalized = path.posix.normalize(workdir); + const roots = runtimeMounted + ? [CONTAINER_WORKSPACE_ROOT, CONTAINER_RUNTIME_HOME] + : [CONTAINER_WORKSPACE_ROOT]; + if (!roots.some((root) => normalized === root || normalized.startsWith(`${root}/`))) { + throw new Error("Workspace sidecar workdir must stay inside a mounted workspace or runtime volume."); + } + return normalized; } } -/** - * Process-wide singleton so every DockerRunner instance shares one helper container per volume - * (otherwise two runners would fight over the same deterministic container name). - */ +/** Preferred name for new call sites; the old export remains source-compatible. */ +export { WorkspaceVolumeHelperPool as WorkspaceSidecarPool }; + +/** Process-wide pool so all Docker runners share one sidecar generation per workspace key. */ export const workspaceVolumeHelperPool = new WorkspaceVolumeHelperPool(); diff --git a/src/integrations/jules-api-client.ts b/src/integrations/jules-api-client.ts index ba3e952376..244c4ec6cc 100644 --- a/src/integrations/jules-api-client.ts +++ b/src/integrations/jules-api-client.ts @@ -14,6 +14,100 @@ export class JulesNotFoundError extends Error { } } +const MAX_JULES_API_ERROR_MESSAGE_CHARS = 2_048; +const JULES_SESSION_CAPACITY_PATTERN = /(?:concurren\w*|too many|max(?:imum)?|limit|quota|capacity|resource\s+exhausted).{0,80}(?:active\s+)?sessions?|(?:active\s+)?sessions?.{0,80}(?:concurren\w*|too many|max(?:imum)?|limit|quota|capacity|resource\s+exhausted)/i; +const JULES_CONCURRENT_TASK_STATES = new Set(["QUEUED", "PLANNING", "IN_PROGRESS"]); + +/** + * Jules exposes session state but no dedicated subscription-slot endpoint. + * Waiting/paused sessions do not represent executing work and can accumulate + * well beyond a plan's concurrent-task limit, so counting every non-terminal + * session permanently starves admission on established accounts. + */ +export function isJulesSessionConsumingConcurrentTask(session: Pick): boolean { + const state = String(session.state || "STATE_UNSPECIFIED").trim().toUpperCase(); + if (state === "STATE_UNSPECIFIED") { + // Unknown states are counted conservatively until Jules reports a known one. + return true; + } + return JULES_CONCURRENT_TASK_STATES.has(state); +} + +/** + * An actionable, bounded Jules API failure. Axios' default message only includes + * the HTTP status, which previously discarded the provider's explanation and + * made capacity responses indistinguishable from malformed requests. + */ +export class JulesApiRequestError extends Error { + readonly cause?: unknown; + + constructor( + message: string, + public readonly status: number | null, + public readonly apiStatus: string | null, + cause?: unknown, + ) { + super(message); + this.name = "JulesApiRequestError"; + this.cause = cause; + } +} + +export function isJulesSessionCapacityError(error: unknown): boolean { + if (!(error instanceof JulesApiRequestError)) { + return false; + } + return (error.status === 400 || error.status === 409 || error.status === 429) + && JULES_SESSION_CAPACITY_PATTERN.test(error.message); +} + +function boundJulesApiErrorText(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const sanitized = value + .replace(/[\r\n\t]+/g, " ") + .replace(/(api[_-]?key|x-goog-api-key)\s*[:=]\s*[^\s,;]+/gi, "$1=[REDACTED]") + .replace(/\s+/g, " ") + .trim(); + if (!sanitized) { + return null; + } + return sanitized.slice(0, MAX_JULES_API_ERROR_MESSAGE_CHARS); +} + +function toJulesApiRequestError(error: unknown, operation: string): JulesApiRequestError { + const candidate = error && typeof error === "object" + ? error as { + message?: unknown; + response?: { + status?: unknown; + data?: unknown; + }; + } + : null; + const status = typeof candidate?.response?.status === "number" ? candidate.response.status : null; + const data = candidate?.response?.data; + const payload = data && typeof data === "object" ? data as Record : null; + const nestedError = payload?.error && typeof payload.error === "object" + ? payload.error as Record + : null; + const apiStatus = boundJulesApiErrorText(nestedError?.status ?? payload?.status); + const providerMessage = boundJulesApiErrorText( + nestedError?.message + ?? payload?.message + ?? (typeof data === "string" ? data : null), + ); + const fallbackMessage = boundJulesApiErrorText(candidate?.message) || "Unknown Jules API error"; + const statusLabel = status === null ? "" : ` (HTTP ${status}${apiStatus ? ` ${apiStatus}` : ""})`; + return new JulesApiRequestError( + `Jules API ${operation} failed${statusLabel}: ${providerMessage || fallbackMessage}`, + status, + apiStatus, + error, + ); +} + export function isNotFoundError(error: unknown): boolean { if (!error || typeof error !== "object") { return false; @@ -64,10 +158,17 @@ export interface JulesApiClientOptions { * concurrent loops share one fetch while state stays near-real-time). */ sessionsCacheTtlMs?: number; + /** + * Maximum age of a session snapshot used to admit a new Jules session. + * Capacity checks are stricter than watch-loop synchronization and never + * serve stale data after a failed refresh. Defaults to 10 seconds; local + * atomic claims account for sessions created inside that window. + */ + sessionsCapacityCacheTtlMs?: number; /** * Upper bound on how many sessions the shared snapshot paginates through per - * refresh. Active sessions are always the most recent, so this caps work on - * accounts with thousands of historical sessions. Defaults to 300. + * refresh. This bounds watch-loop work on accounts with thousands of + * historical sessions. Defaults to 300. */ maxSnapshotSessions?: number; /** Injectable clock for deterministic tests. Defaults to `Date.now`. */ @@ -183,17 +284,21 @@ export class JulesApiClient implements JulesClient { private readonly minRequestIntervalMs: number; private readonly maxTransientRetries: number; private readonly sessionsCacheTtlMs: number; + private readonly sessionsCapacityCacheTtlMs: number; private readonly maxSnapshotSessions: number; private readonly now: () => number; private nextRequestSlot = 0; private sessionSnapshot: { at: number; sessions: JulesSession[] } | null = null; private sessionSnapshotInFlight: Promise | null = null; + private sessionCapacitySnapshot: { at: number; sessions: JulesSession[] } | null = null; + private sessionCapacitySnapshotInFlight: Promise | null = null; constructor(options: JulesApiClientOptions) { this.apiKey = this.normalizeApiKey(options.apiKey); this.minRequestIntervalMs = Math.max(0, options.minRequestIntervalMs ?? 250); this.maxTransientRetries = Math.max(0, options.maxTransientRetries ?? 4); this.sessionsCacheTtlMs = Math.max(0, options.sessionsCacheTtlMs ?? 12_000); + this.sessionsCapacityCacheTtlMs = Math.max(0, options.sessionsCapacityCacheTtlMs ?? 10_000); this.maxSnapshotSessions = Math.max(1, options.maxSnapshotSessions ?? 300); this.now = options.now ?? Date.now; this.axiosInstance = axios.create({ @@ -391,9 +496,17 @@ export class JulesApiClient implements JulesClient { async createSession(data: JulesCreateSessionRequest): Promise { this.ensureApiKey(); - const response = await this.axiosInstance.post("/sessions", data); - this.invalidateSessionsCache(); - return response.data; + try { + const response = await this.axiosInstance.post("/sessions", data); + this.invalidateSessionsCache(); + return response.data; + } catch (error) { + // A rejected create can be a subscription-cap race. Force any admission + // diagnostic that follows to read the provider again instead of reusing + // the optimistic pre-create snapshot. + this.invalidateSessionsCache(); + throw toJulesApiRequestError(error, "create session"); + } } async getSession(sessionId: string): Promise { @@ -443,12 +556,45 @@ export class JulesApiClient implements JulesClient { return this.sessionSnapshotInFlight; } + /** + * Returns a bounded, API-backed preflight snapshot for admission control. + * The Jules list API has pagination but no state filter or subscription-slot + * counter, and old waiting sessions can occur deep in account history. A + * complete history scan before every dispatch would make admission slower as + * the account ages. Instead concurrent dispatches share one fresh first-page + * preflight, local claims provide the atomic hard cap, and a provider-side + * FAILED_PRECONDITION remains an authoritative retryable capacity deferral. + * Unlike watch-loop synchronization this path never serves stale data after + * an API error. + */ + async getSessionsForCapacityCheck(): Promise { + const fresh = this.sessionCapacitySnapshot + && (this.now() - this.sessionCapacitySnapshot.at) < this.sessionsCapacityCacheTtlMs; + if (fresh) { + return this.sessionCapacitySnapshot!.sessions; + } + if (this.sessionCapacitySnapshotInFlight) { + return this.sessionCapacitySnapshotInFlight; + } + this.sessionCapacitySnapshotInFlight = this.refreshSessionCapacitySnapshot() + .finally(() => { this.sessionCapacitySnapshotInFlight = null; }); + return this.sessionCapacitySnapshotInFlight; + } + /** Drops the cached session snapshot so the next read re-fetches fresh state. */ invalidateSessionsCache(): void { this.sessionSnapshot = null; + this.sessionCapacitySnapshot = null; + } + + private async refreshSessionCapacitySnapshot(): Promise { + const response = await this.listSessions({ page_size: 100 }); + const sessions = response.sessions || []; + this.sessionCapacitySnapshot = { at: this.now(), sessions }; + return sessions; } - private async refreshSessionSnapshot(): Promise { + private async refreshSessionSnapshot(allowStaleOnError = true): Promise { try { const all: JulesSession[] = []; let pageToken: string | undefined = undefined; @@ -461,7 +607,7 @@ export class JulesApiClient implements JulesClient { this.sessionSnapshot = { at: this.now(), sessions: all }; return all; } catch (error) { - if (this.sessionSnapshot) { + if (allowStaleOnError && this.sessionSnapshot) { // Serve stale rather than failing every sprint's sync on a blip; the // timestamp is left untouched so the next call retries promptly. return this.sessionSnapshot.sessions; diff --git a/src/repositories/app-db-storage.ts b/src/repositories/app-db-storage.ts index 1c1819a58b..26eb5e2620 100644 --- a/src/repositories/app-db-storage.ts +++ b/src/repositories/app-db-storage.ts @@ -30,6 +30,7 @@ const MAINTENANCE_CRITICAL_INDEXES = new Set([ "idx_provider_invocations_task_run", "idx_qa_review_runs_task_run", "idx_task_run_events_task_run_created_id", + "idx_task_run_events_task_run_type_created_id", ]); export function resolveAppDbPath(dbPath?: string): string { diff --git a/src/repositories/db/app-db-migrations.ts b/src/repositories/db/app-db-migrations.ts index 25f46a9b05..3d1423fdb5 100644 --- a/src/repositories/db/app-db-migrations.ts +++ b/src/repositories/db/app-db-migrations.ts @@ -1311,6 +1311,7 @@ function runMigrationsInternal(db: DatabaseAdapter): void { ensureIndex(db, "idx_execution_leases_scope", "execution_leases", "scope_type, scope_id"); ensureIndex(db, "idx_task_run_events_task_run_created", "task_run_events", "task_run_id, created_at DESC"); ensureIndex(db, "idx_task_run_events_task_run_created_id", "task_run_events", "task_run_id, created_at DESC, id DESC"); + ensureIndex(db, "idx_task_run_events_task_run_type_created_id", "task_run_events", "task_run_id, event_type, created_at DESC, id DESC"); ensureReadIndexSql( db, "idx_task_run_events_provider_activity_run_created", @@ -1564,6 +1565,25 @@ function runMigrationsInternal(db: DatabaseAdapter): void { ensureUniqueIndex(db, "idx_guardrail_ledger_task_purpose", "guardrail_ledger", "task_id, purpose"); ensureIndex(db, "idx_guardrail_ledger_project", "guardrail_ledger", "project_id, task_id"); + // Runtime restarts are operational interruptions, not failed agent attempts. Keep each + // guardrail refund durable and idempotent so a second startup recovery cannot refund the + // same interrupted task run twice. + db.exec(` + CREATE TABLE IF NOT EXISTS guardrail_ledger_adjustments ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + task_id TEXT NOT NULL, + purpose TEXT NOT NULL, + adjustment INTEGER NOT NULL, + source_key TEXT NOT NULL UNIQUE, + reason TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE, + FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE + ) + `); + ensureIndex(db, "idx_guardrail_ledger_adjustments_task", "guardrail_ledger_adjustments", "task_id, purpose"); + db.exec(` CREATE TABLE IF NOT EXISTS knowledge_documents ( id TEXT PRIMARY KEY, diff --git a/src/repositories/db/app-db-schema.ts b/src/repositories/db/app-db-schema.ts index 7c6f3bc001..b262d295b2 100644 --- a/src/repositories/db/app-db-schema.ts +++ b/src/repositories/db/app-db-schema.ts @@ -1101,6 +1101,7 @@ CREATE INDEX IF NOT EXISTS idx_task_self_reflection_ratings_task_latest ON task_ CREATE INDEX IF NOT EXISTS idx_task_self_reflection_ratings_project_task_latest ON task_self_reflection_ratings (project_id, task_id, captured_at DESC); CREATE INDEX IF NOT EXISTS idx_task_run_events_project_created ON task_run_events (project_id, created_at DESC, id DESC); CREATE INDEX IF NOT EXISTS idx_task_run_events_task_run_created_id ON task_run_events (task_run_id, created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_task_run_events_task_run_type_created_id ON task_run_events (task_run_id, event_type, created_at DESC, id DESC); CREATE INDEX IF NOT EXISTS idx_task_run_events_provider_activity_run_created ON task_run_events (task_run_id, created_at DESC, id DESC) WHERE event_type = 'provider_activity'; CREATE INDEX IF NOT EXISTS idx_task_run_events_provider_activity_project_created ON task_run_events (project_id, created_at DESC, id DESC) WHERE event_type = 'provider_activity'; CREATE INDEX IF NOT EXISTS idx_project_attention_items_project_owner_status ON project_attention_items (project_id, owner_type, status); diff --git a/src/repositories/execution-repository.ts b/src/repositories/execution-repository.ts index 523d4bdd7e..b45d2ab894 100644 --- a/src/repositories/execution-repository.ts +++ b/src/repositories/execution-repository.ts @@ -33,7 +33,13 @@ import { import { randomUUID } from "crypto"; import { createLogger, type Logger } from "../shared/logging/logger.js"; -import { ConcurrencyConflictError, EntityNotFoundError, RepositoryError, ValidationError } from "./repository-utils.js"; +import { + ConcurrencyConflictError, + EntityNotFoundError, + RepositoryError, + ValidationError, + executeChunkedInQuery, +} from "./repository-utils.js"; import { DatabaseAdapter } from "./db/database-adapter.js"; import { AppDbStorage } from "./app-db-storage.js"; import { toNumber, parsePayloadJson } from "./repository-utils.js"; @@ -1486,18 +1492,76 @@ export class ExecutionRepository { return inserted; } - listTaskRunEvents(taskRunId: string, limit: number = 50): TaskRunEventRecord[] { - requireTaskRun((id) => this.getTaskRun(id), taskRunId); + listTaskRunEvents( + taskRunId: string, + limit: number = 50, + options?: { eventTypes?: string[]; skipValidation?: boolean }, + ): TaskRunEventRecord[] { + if (!options?.skipValidation) { + requireTaskRun((id) => this.getTaskRun(id), taskRunId); + } + const eventTypes = [...new Set(options?.eventTypes?.map((value) => value.trim()).filter(Boolean) ?? [])]; + const eventTypeClause = eventTypes.length > 0 + ? `AND event_type IN (${eventTypes.map(() => "?").join(", ")})` + : ""; const rows = this.db.prepare(` SELECT * FROM task_run_events WHERE task_run_id = ? + ${eventTypeClause} ORDER BY created_at DESC, rowid DESC LIMIT ? - `).all(taskRunId, Math.max(1, limit)) as unknown as TaskRunEventRow[]; + `).all(taskRunId, ...eventTypes, Math.max(1, limit)) as unknown as TaskRunEventRow[]; return rows.map((row) => this.mapTaskRunEventRow(row)); } + /** + * Loads a bounded event slice for many already-resolved task runs in one SQL + * pass. Orchestration uses this after loading the task runs themselves, so + * missing IDs intentionally map to empty arrays instead of triggering one + * existence query per task. + */ + listTaskRunEventsForRuns( + taskRunIds: string[], + options: { eventTypes: string[]; limitPerRun?: number }, + ): Map { + const normalizedTaskRunIds = [...new Set(taskRunIds.map((value) => value.trim()).filter(Boolean))]; + const eventTypes = [...new Set(options.eventTypes.map((value) => value.trim()).filter(Boolean))]; + const eventsByTaskRunId = new Map( + normalizedTaskRunIds.map((taskRunId) => [taskRunId, []]), + ); + if (normalizedTaskRunIds.length === 0 || eventTypes.length === 0) { + return eventsByTaskRunId; + } + + const eventTypePlaceholders = eventTypes.map(() => "?").join(", "); + const rankedRows = executeChunkedInQuery( + (sql) => this.db.prepare(sql), + { + sqlPrefix: ` + SELECT * FROM ( + SELECT task_run_events.*, + ROW_NUMBER() OVER ( + PARTITION BY task_run_id + ORDER BY created_at DESC, rowid DESC + ) AS event_rank + FROM task_run_events + WHERE task_run_id`, + sqlSuffix: ` + AND event_type IN (${eventTypePlaceholders}) + ) ranked_events + WHERE event_rank <= ? + ORDER BY task_run_id ASC, created_at DESC, id DESC`, + items: normalizedTaskRunIds, + bindParamsAfter: [...eventTypes, Math.max(1, options.limitPerRun ?? 500)], + }, + ); + for (const row of rankedRows) { + eventsByTaskRunId.get(row.task_run_id)?.push(this.mapTaskRunEventRow(row)); + } + return eventsByTaskRunId; + } + listSprintRunEvents(sprintRunId: string, limit: number = 50): SprintRunEventRecord[] { requireSprintRun((id) => this.getSprintRun(id), sprintRunId); const rows = this.db.prepare(` diff --git a/src/repositories/guardrail-repository.ts b/src/repositories/guardrail-repository.ts index 6d5039e11c..9fa98844f5 100644 --- a/src/repositories/guardrail-repository.ts +++ b/src/repositories/guardrail-repository.ts @@ -13,6 +13,11 @@ import { GUARDRAIL_JOB_TYPES } from "./settings-defaults.js"; */ export type GuardrailLedgerPurpose = GuardrailJobType | "qa_review"; +export interface GuardrailRefundResult { + applied: boolean; + count: number; +} + export const GUARDRAIL_LEDGER_PURPOSES: GuardrailLedgerPurpose[] = [...GUARDRAIL_JOB_TYPES, "qa_review"]; interface GuardrailLedgerRow { @@ -49,6 +54,44 @@ export class GuardrailRepository { return this.getCount(input.taskId, input.purpose); } + /** + * Atomically refunds one recorded invocation exactly once. Operational recovery uses the + * interrupted task-run id as `sourceKey`, preventing repeated startup passes from reducing + * a task's real attempt count more than once. + */ + refund(input: { + projectId: string; + taskId: string; + purpose: GuardrailLedgerPurpose; + sourceKey: string; + reason?: string; + }): GuardrailRefundResult { + return this.db.transaction(() => { + const now = new Date().toISOString(); + const inserted = this.db.prepare(` + INSERT OR IGNORE INTO guardrail_ledger_adjustments + (id, project_id, task_id, purpose, adjustment, source_key, reason, created_at) + VALUES (?, ?, ?, ?, -1, ?, ?, ?) + `).run( + `gra_${randomUUID().replace(/-/g, "")}`, + input.projectId, + input.taskId, + input.purpose, + input.sourceKey, + input.reason ?? null, + now, + ).changes > 0; + if (inserted) { + this.db.prepare(` + UPDATE guardrail_ledger + SET count = MAX(0, count - 1), updated_at = ? + WHERE task_id = ? AND purpose = ? + `).run(now, input.taskId, input.purpose); + } + return { applied: inserted, count: this.getCount(input.taskId, input.purpose) }; + }); + } + getCount(taskId: string, purpose: GuardrailLedgerPurpose): number { const row = this.db.prepare(` SELECT count FROM guardrail_ledger WHERE task_id = ? AND purpose = ? @@ -80,10 +123,16 @@ export class GuardrailRepository { } reset(taskId: string): void { - this.db.prepare(`DELETE FROM guardrail_ledger WHERE task_id = ?`).run(taskId); + this.db.transaction(() => { + this.db.prepare(`DELETE FROM guardrail_ledger_adjustments WHERE task_id = ?`).run(taskId); + this.db.prepare(`DELETE FROM guardrail_ledger WHERE task_id = ?`).run(taskId); + }); } resetPurpose(taskId: string, purpose: GuardrailLedgerPurpose): void { - this.db.prepare(`DELETE FROM guardrail_ledger WHERE task_id = ? AND purpose = ?`).run(taskId, purpose); + this.db.transaction(() => { + this.db.prepare(`DELETE FROM guardrail_ledger_adjustments WHERE task_id = ? AND purpose = ?`).run(taskId, purpose); + this.db.prepare(`DELETE FROM guardrail_ledger WHERE task_id = ? AND purpose = ?`).run(taskId, purpose); + }); } } diff --git a/src/repositories/project-runtime/runtime-status-projection.ts b/src/repositories/project-runtime/runtime-status-projection.ts index 7dc56f2a62..55a53409cb 100644 --- a/src/repositories/project-runtime/runtime-status-projection.ts +++ b/src/repositories/project-runtime/runtime-status-projection.ts @@ -16,6 +16,50 @@ export type ProjectStatus = "running" | "failed" | "intervention" | "idle"; export type TaskRunState = Exclude; export type JulesPlan = { steps?: Array<{ title?: string }> }; +const MAX_PROJECTED_ACTIVITY_TEXT_CHARS = 8 * 1024; +const MAX_PROJECTED_ACTIVITY_ID_CHARS = 2 * 1024; +const MAX_PROJECTED_PLAN_STEPS = 32; +const MAX_PROJECTED_PLAN_TITLE_CHARS = 512; +const MAX_PROJECTED_COMPLETION_JSON_CHARS = 8 * 1024; +const MAX_RECENT_ACTIVITY_CACHE_ENTRIES = 32; +const MAX_RECENT_ACTIVITY_CACHE_CHARS = 32 * 1024 * 1024; + +function asBoundedString(value: unknown, maxChars = MAX_PROJECTED_ACTIVITY_TEXT_CHARS): string | undefined { + const normalized = asString(value); + if (!normalized || normalized.length <= maxChars) { + return normalized; + } + const marker = "\n… [activity preview truncated] …\n"; + const retainedChars = Math.max(maxChars - marker.length, 0); + const headChars = Math.ceil(retainedChars / 2); + const tailChars = retainedChars - headChars; + return `${normalized.slice(0, headChars)}${marker}${normalized.slice(-tailChars)}`.slice(0, maxChars); +} + +function boundProjectedPlan(value: unknown): JulesPlan | undefined { + const plan = asRecord(value); + if (!plan) return undefined; + const steps = Array.isArray(plan.steps) + ? plan.steps.slice(0, MAX_PROJECTED_PLAN_STEPS).map((step) => ({ + title: asBoundedString(asRecord(step)?.title, MAX_PROJECTED_PLAN_TITLE_CHARS), + })) + : undefined; + return steps ? { steps } : {}; +} + +function boundProjectedCompletion(value: unknown): unknown { + if (value === undefined) return undefined; + try { + const serialized = JSON.stringify(value); + if (serialized.length <= MAX_PROJECTED_COMPLETION_JSON_CHARS) { + return value; + } + return { truncated: true, originalChars: serialized.length }; + } catch { + return { truncated: true, serializationFailed: true }; + } +} + export interface ProjectRow { id: string; base_dir: string; @@ -91,6 +135,7 @@ export interface MappedTask { interface RecentActivitiesCacheEntry { version: string; activitiesByTaskId: Map; + estimatedChars: number; } export function asString(value: unknown): string | undefined { @@ -129,6 +174,7 @@ function resolveProjectedTaskStatus(row: TaskRow, run?: TaskRunRow): Subtask["st export class RuntimeStatusProjection { private readonly recentActivitiesCache = new Map(); + private recentActivitiesCacheChars = 0; constructor( private readonly storage: AppDbStorage, @@ -334,6 +380,7 @@ export class RuntimeStatusProjection { this.setRecentActivitiesCache(cacheKey, { version, activitiesByTaskId: this.cloneActivitiesByTaskId(activitiesByTaskId), + estimatedChars: this.estimateActivitiesChars(activitiesByTaskId), }); return activitiesByTaskId; @@ -361,15 +408,40 @@ export class RuntimeStatusProjection { } private setRecentActivitiesCache(cacheKey: string, entry: RecentActivitiesCacheEntry): void { - this.recentActivitiesCache.delete(cacheKey); + const replaced = this.recentActivitiesCache.get(cacheKey); + if (replaced) { + this.recentActivitiesCacheChars -= replaced.estimatedChars; + this.recentActivitiesCache.delete(cacheKey); + } this.recentActivitiesCache.set(cacheKey, entry); - while (this.recentActivitiesCache.size > 100) { + this.recentActivitiesCacheChars += entry.estimatedChars; + while ( + this.recentActivitiesCache.size > MAX_RECENT_ACTIVITY_CACHE_ENTRIES + || (this.recentActivitiesCacheChars > MAX_RECENT_ACTIVITY_CACHE_CHARS && this.recentActivitiesCache.size > 1) + ) { const oldestKey = this.recentActivitiesCache.keys().next().value; if (typeof oldestKey !== "string") { return; } + const oldest = this.recentActivitiesCache.get(oldestKey); this.recentActivitiesCache.delete(oldestKey); + this.recentActivitiesCacheChars -= oldest?.estimatedChars ?? 0; + } + } + + private estimateActivitiesChars(activitiesByTaskId: Map): number { + let total = 0; + for (const [taskId, activities] of activitiesByTaskId) { + total += taskId.length; + for (const activity of activities) { + try { + total += JSON.stringify(activity).length; + } catch { + total += MAX_PROJECTED_ACTIVITY_TEXT_CHARS; + } + } } + return total; } mapTaskActivityRow(row: TaskActivityRow): JulesActivity | null { @@ -380,8 +452,10 @@ export class RuntimeStatusProjection { const planGenerated = asRecord(payload?.planGenerated); const planApproved = asRecord(payload?.planApproved); const sessionFailed = asRecord(payload?.sessionFailed); - const activityId = asString(row.activity_id) || asString(payload?.activityId); - const sessionName = asString(row.session_name) || asString(payload?.sessionName); + const activityId = asBoundedString(row.activity_id, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || asBoundedString(payload?.activityId, MAX_PROJECTED_ACTIVITY_ID_CHARS); + const sessionName = asBoundedString(row.session_name, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || asBoundedString(payload?.sessionName, MAX_PROJECTED_ACTIVITY_ID_CHARS); if (!activityId) { return null; @@ -389,20 +463,24 @@ export class RuntimeStatusProjection { return { id: activityId, - name: asString(row.activity_name) || asString(payload?.activityName) || (sessionName ? `${sessionName}/activities/${activityId}` : `activities/${activityId}`), + name: asBoundedString(row.activity_name, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || asBoundedString(payload?.activityName, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || (sessionName ? `${sessionName}/activities/${activityId}` : `activities/${activityId}`), createTime: row.created_at, - originator: asString(row.originator) || asString(payload?.originator) || "provider", - description: asString(payload?.description), - agentMessaged: agentMessaged ? { agentMessage: asString(agentMessaged.agentMessage) } : undefined, - userMessaged: userMessaged ? { userMessage: asString(userMessaged.userMessage) } : undefined, + originator: asBoundedString(row.originator, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || asBoundedString(payload?.originator, MAX_PROJECTED_ACTIVITY_ID_CHARS) + || "provider", + description: asBoundedString(payload?.description), + agentMessaged: agentMessaged ? { agentMessage: asBoundedString(agentMessaged.agentMessage) } : undefined, + userMessaged: userMessaged ? { userMessage: asBoundedString(userMessaged.userMessage) } : undefined, progressUpdated: progressUpdated ? { - title: asString(progressUpdated.title), - description: asString(progressUpdated.description), + title: asBoundedString(progressUpdated.title), + description: asBoundedString(progressUpdated.description), } : undefined, - planGenerated: planGenerated ? { plan: asRecord(planGenerated.plan) as JulesPlan | undefined } : undefined, - planApproved: planApproved ? { planId: asString(planApproved.planId) } : undefined, - sessionFailed: sessionFailed ? { reason: asString(sessionFailed.reason) } : undefined, - sessionCompleted: payload?.sessionCompleted ?? undefined, + planGenerated: planGenerated ? { plan: boundProjectedPlan(planGenerated.plan) } : undefined, + planApproved: planApproved ? { planId: asBoundedString(planApproved.planId, MAX_PROJECTED_ACTIVITY_ID_CHARS) } : undefined, + sessionFailed: sessionFailed ? { reason: asBoundedString(sessionFailed.reason) } : undefined, + sessionCompleted: boundProjectedCompletion(payload?.sessionCompleted), }; } diff --git a/src/repositories/qa-review-repository.ts b/src/repositories/qa-review-repository.ts index dd45806c97..88c1bce838 100644 --- a/src/repositories/qa-review-repository.ts +++ b/src/repositories/qa-review-repository.ts @@ -31,6 +31,13 @@ export interface QaReviewRunRecord { updatedAt: string; } +export interface QaTaskReviewSnapshot { + latestRun: QaReviewRunRecord | null; + latestCycleRuns: QaReviewRunRecord[]; + runsUsed: number; + decisiveRuns: number; +} + interface QaReviewRunRow { id: string; project_id: string; @@ -69,9 +76,11 @@ function parsePayload(value: string | null): Record | null { } export class QaReviewRepository { + private readonly storage: AppDbStorage; private readonly db: DatabaseAdapter; constructor(storage: AppDbStorage = new AppDbStorage()) { + this.storage = storage; this.db = storage.getDatabase(); } @@ -328,6 +337,78 @@ export class QaReviewRepository { return rows.map((row) => this.mapRow(row)); } + /** + * Load the merge-gate state for a whole DAG in one chunked query. This keeps + * the watch loop from issuing latest/count/decisive SQL separately for every + * task on every cycle while preserving the single-task ordering semantics. + */ + listTaskReviewSnapshots(taskIds: string[]): Map { + const uniqueTaskIds = [...new Set(taskIds.map((taskId) => taskId.trim()).filter(Boolean))]; + const snapshots = new Map(uniqueTaskIds.map((taskId) => [taskId, { + latestRun: null, + latestCycleRuns: [], + runsUsed: 0, + decisiveRuns: 0, + }])); + if (uniqueTaskIds.length === 0) { + return snapshots; + } + + const rows = this.storage.executeChunkedInQuery({ + sqlPrefix: `SELECT * + FROM qa_review_runs + WHERE task_id`, + sqlSuffix: `AND trigger_type IN ('task_completion', 'completed_task_without_pr') + ORDER BY task_id ASC, + run_index DESC, + CASE + WHEN status = 'running' THEN 0 + WHEN outcome = 'changes_requested' THEN 1 + WHEN status = 'failed' THEN 2 + WHEN outcome = 'pass' THEN 3 + ELSE 4 + END, + started_at DESC`, + items: uniqueTaskIds, + }); + const rowsByTaskId = new Map(); + for (const row of rows) { + if (!row.task_id) { + continue; + } + const mapped = this.mapRow(row); + const taskRows = rowsByTaskId.get(row.task_id) ?? []; + taskRows.push(mapped); + rowsByTaskId.set(row.task_id, taskRows); + } + + for (const [taskId, taskRows] of rowsByTaskId) { + const latestRun = taskRows[0] ?? null; + const latestRunIndex = latestRun?.runIndex ?? 0; + const latestCycleRuns = taskRows + .filter((run) => run.runIndex === latestRunIndex) + .sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id)); + const terminalRunIndexes = new Set(); + const decisiveRunIndexes = new Set(); + for (const run of taskRows) { + if (["completed", "failed", "cancelled", "errored"].includes(run.status)) { + terminalRunIndexes.add(run.runIndex); + } + if (run.status === "completed") { + decisiveRunIndexes.add(run.runIndex); + } + } + snapshots.set(taskId, { + latestRun, + latestCycleRuns, + runsUsed: terminalRunIndexes.size, + decisiveRuns: decisiveRunIndexes.size, + }); + } + + return snapshots; + } + getLatestSprintRun(sprintId: string): QaReviewRunRecord | null { const row = this.db.prepare(` SELECT * diff --git a/src/repositories/session-tracking-repository.ts b/src/repositories/session-tracking-repository.ts index 91a5d7941c..88364d3a52 100644 --- a/src/repositories/session-tracking-repository.ts +++ b/src/repositories/session-tracking-repository.ts @@ -85,6 +85,7 @@ export interface UpdateTrackedSessionInput { } const SESSION_DB_PATH = getHomeCodeUxPath("session-tracking.db"); +const SESSION_TRACKING_SCHEMA_VERSION = 1; const resolveDbPath = (dbPath?: string): string => { if (dbPath && dbPath.trim().length > 0) { @@ -131,14 +132,37 @@ export class SessionTrackingRepository { CREATE INDEX IF NOT EXISTS idx_provider_activities_session_time ON provider_activities (session_id, create_time DESC); `); + const schemaVersionRow = this.db.prepare("PRAGMA user_version").get() as { + user_version?: number; + } | undefined; + if (Number(schemaVersionRow?.user_version || 0) < SESSION_TRACKING_SCHEMA_VERSION) { + this.db.transaction(() => { + // Local prompts are already stored with the durable execution invocation. Remove legacy + // duplicates once so upgraded installations stop carrying wide-DAG prompt blobs forward. + this.db.prepare(` + UPDATE provider_sessions + SET prompt = NULL + WHERE provider != 'jules' AND prompt IS NOT NULL + `).run(); + this.db.exec(`PRAGMA user_version = ${SESSION_TRACKING_SCHEMA_VERSION}`); + }); + } } getDatabase(): DatabaseAdapter { return this.db; } + close(): void { + this.db.close(); + } + createSession(input: CreateTrackedSessionInput): JulesSession { const now = new Date().toISOString(); + // CLI prompts are already durable in the execution invocation/message store and can contain + // an entire wide-DAG context. The legacy session database only needs prompts for Jules usage + // estimation, so avoid writing a second multi-megabyte copy for every local provider session. + const storedPrompt = input.provider === "jules" ? input.prompt ?? null : null; this.db.prepare(` INSERT INTO provider_sessions ( id, provider, task_id, title, prompt, state, create_time, update_time, feature_branch, worker_branch, pr_url, repo_path @@ -159,7 +183,7 @@ export class SessionTrackingRepository { input.provider, input.taskId ?? null, input.title ?? null, - input.prompt ?? null, + storedPrompt, input.state ?? "RUNNING", now, now, @@ -263,9 +287,13 @@ export class SessionTrackingRepository { return row ? this.rowToSession(row) : null; } - listSessions(limit: number = 200): { sessions: JulesSession[] } { + listSessions( + limit: number = 200, + options: { includePrompt?: boolean } = {}, + ): { sessions: JulesSession[] } { + const promptSelection = options.includePrompt === false ? "NULL AS prompt" : "prompt"; const rows = this.db.prepare(` - SELECT id, provider, task_id, title, prompt, state, create_time, update_time, feature_branch, worker_branch, pr_url, repo_path + SELECT id, provider, task_id, title, ${promptSelection}, state, create_time, update_time, feature_branch, worker_branch, pr_url, repo_path FROM provider_sessions ORDER BY create_time DESC LIMIT ? diff --git a/src/server/activity-cache-service.ts b/src/server/activity-cache-service.ts index 9c54ed0933..3e61ced2b2 100644 --- a/src/server/activity-cache-service.ts +++ b/src/server/activity-cache-service.ts @@ -8,6 +8,22 @@ import type { Logger } from "../shared/logging/logger.js"; const DEFAULT_LIVE_ACTIVITY_FETCH_TIMEOUT_MS = 30_000; const LIVE_ACTIVITY_FETCH_TIMEOUT_ERROR_NAME = "ActivityFetchTimeoutError"; +const MAX_LIVE_ACTIVITY_DESCRIPTION_CHARS = 64 * 1024; + +const boundLiveActivity = (activity: JulesActivity): JulesActivity => { + const description = activity.description; + if (typeof description !== "string" || description.length <= MAX_LIVE_ACTIVITY_DESCRIPTION_CHARS) { + return activity; + } + const marker = "\n… [activity preview truncated] …\n"; + const retainedChars = MAX_LIVE_ACTIVITY_DESCRIPTION_CHARS - marker.length; + const headChars = Math.ceil(retainedChars / 2); + const tailChars = retainedChars - headChars; + return { + ...activity, + description: `${description.slice(0, headChars)}${marker}${description.slice(-tailChars)}`, + }; +}; const getFetchFailureMetadata = ( sessionName: string, @@ -87,6 +103,13 @@ export class ActivityCacheService { ) ); + const activeSessionSet = new Set(activeSessionNames); + for (const cachedSessionName of this.liveActivitiesCache.keys()) { + if (!activeSessionSet.has(cachedSessionName)) { + this.liveActivitiesCache.delete(cachedSessionName); + } + } + if (activeSessionNames.length === 0) { return {}; } @@ -124,7 +147,13 @@ export class ActivityCacheService { }, }, ); - return { sessionName, activities, isNegative: activities.length === 0, failed: false }; + const boundedActivities = activities.map(boundLiveActivity); + return { + sessionName, + activities: boundedActivities, + isNegative: boundedActivities.length === 0, + failed: false, + }; } catch (error) { const cached = this.liveActivitiesCache.get(sessionName); if (cached && !cached.isNegative) { diff --git a/src/server/code-ux-server.ts b/src/server/code-ux-server.ts index 70b1fd1898..c1421a0489 100644 --- a/src/server/code-ux-server.ts +++ b/src/server/code-ux-server.ts @@ -360,6 +360,10 @@ export class CodeUxServer { scope.sprintId, ).settings; }, + listDurableRemoteSessions: () => this.julesApi.getSessionsForCapacityCheck(), + resumeInterruptedPlanningInvocation: (invocationId, mode) => ( + this.planningAgentService.recoverInterruptedInvocation(invocationId, mode) + ), logger: this.logger.child({ component: "runtime-startup-recovery-service" }), }); this.dashboardRealtimeService = deps.dashboardRealtimeService; @@ -426,6 +430,12 @@ export class CodeUxServer { this.startupTaskTimers.clear(); this.virtualWorkerService.stop(); this.schedulerService.stop(); + const requestedDispatchStops = await this.shutdownContainerService.requestActiveDispatchStops().catch((error) => { + this.logger.warn("Failed to request active dispatch stops during shutdown", { + error: error instanceof Error ? error.message : String(error), + }); + return 0; + }); disposeCommandSpawner(); await shutdownGitHelperPool().catch((error) => { this.logger.warn("Failed to stop Docker git helper containers during shutdown", { @@ -437,7 +447,7 @@ export class CodeUxServer { error: error instanceof Error ? error.message : String(error), }); }); - await this.shutdownContainerService.stopRunningContainers().catch((error) => { + await this.shutdownContainerService.stopRemainingContainers(requestedDispatchStops).catch((error) => { this.logger.warn("Failed to stop running containers during shutdown", { error: error instanceof Error ? error.message : String(error), }); @@ -466,6 +476,40 @@ export class CodeUxServer { } } await this.releaseProjectManagerRuntimeLock(); + + // Flush the final write burst and close every SQLite connection explicitly. The CLI exits via + // process.exit() after this method, so relying on process teardown leaves large WAL files behind + // after busy sprints and skips SQLite's normal last-connection checkpoint. + try { + const failures = new DatabaseMaintenanceService({ + appDbStorage: this.appDbStorage, + sessionTracking: this.sessionTracking, + settingsRepository: this.settingsRepository, + logger: this.logger.child({ component: "database-maintenance-service" }), + }).checkpointWalDatabases(); + if (failures.length > 0) { + this.logger.warn("Final WAL checkpoint completed with failures", { databases: failures }); + } + } catch (error) { + this.logger.warn("Final WAL checkpoint failed during shutdown", { + error: error instanceof Error ? error.message : String(error), + }); + } + + for (const [label, closeDatabase] of [ + ["app", () => this.appDbStorage.close()], + ["session tracking", () => this.sessionTracking.close()], + ["settings", () => this.settingsRepository.close()], + ] as const) { + try { + closeDatabase(); + } catch (error) { + this.logger.warn("Failed to close SQLite database during shutdown", { + database: label, + error: error instanceof Error ? error.message : String(error), + }); + } + } } private async closeHttpServer(server: HttpServer): Promise { @@ -627,29 +671,31 @@ export class CodeUxServer { } private startRuntimeCleanupLoop(): void { - if (this.appConfig.runtimeRole !== "project_manager" || this.runtimeCleanupInterval) { + if (this.appConfig.runtimeRole !== "project_manager" || this.runtimeCleanupInterval || this.isClosing) { return; } const runCleanup = (): void => { + if (this.isClosing) { + return; + } void this.runtimeCleanupService.cleanup().catch((error) => { this.logger.error("Runtime cleanup sweep failed", { error }); }); }; - const initialTimer = setTimeout(runCleanup, CodeUxServer.LOOP_INITIAL_DELAY_MS); - initialTimer.unref?.(); + this.scheduleTrackedTimer(CodeUxServer.LOOP_INITIAL_DELAY_MS, runCleanup); this.runtimeCleanupInterval = setInterval(runCleanup, CodeUxServer.RUNTIME_CLEANUP_INTERVAL_MS); this.runtimeCleanupInterval.unref?.(); } private startSprintPreviewLoop(): void { - if (this.appConfig.runtimeRole !== "project_manager" || this.sprintPreviewInterval) { + if (this.appConfig.runtimeRole !== "project_manager" || this.sprintPreviewInterval || this.isClosing) { return; } const reconcile = (): void => { - if (this.sprintPreviewReconcileInFlight) { + if (this.isClosing || this.sprintPreviewReconcileInFlight) { return; } this.sprintPreviewReconcileInFlight = true; @@ -669,18 +715,20 @@ export class CodeUxServer { }); }; - const initialTimer = setTimeout(reconcile, CodeUxServer.LOOP_INITIAL_DELAY_MS); - initialTimer.unref?.(); + this.scheduleTrackedTimer(CodeUxServer.LOOP_INITIAL_DELAY_MS, reconcile); this.sprintPreviewInterval = setInterval(reconcile, CodeUxServer.RUNTIME_CLEANUP_INTERVAL_MS); this.sprintPreviewInterval.unref?.(); } private startLiveSnapshotLoop(): void { - if (this.appConfig.runtimeRole !== "project_manager" || this.liveSnapshotInterval) { + if (this.appConfig.runtimeRole !== "project_manager" || this.liveSnapshotInterval || this.isClosing) { return; } const refreshLiveSnapshot = (): void => { + if (this.isClosing) { + return; + } const projectId = this.projectManagementRepository.getSelectedProjectId(); if (!projectId) { return; @@ -691,14 +739,13 @@ export class CodeUxServer { this.dashboardRealtimeService.scheduleProjectGitRefresh(projectId); }; - const initialTimer = setTimeout(refreshLiveSnapshot, 250); - initialTimer.unref?.(); + this.scheduleTrackedTimer(250, refreshLiveSnapshot); this.liveSnapshotInterval = setInterval(refreshLiveSnapshot, CodeUxServer.LIVE_SNAPSHOT_REFRESH_INTERVAL_MS); this.liveSnapshotInterval.unref?.(); } private startWalCheckpointLoop(): void { - if (this.appConfig.runtimeRole !== "project_manager" || this.walCheckpointInterval) { + if (this.appConfig.runtimeRole !== "project_manager" || this.walCheckpointInterval || this.isClosing) { return; } @@ -710,6 +757,9 @@ export class CodeUxServer { }); const checkpoint = (): void => { + if (this.isClosing) { + return; + } try { this.advanceDeferredDatabaseMigrations(); if (!this.appDbStorage.hasPendingMaintenanceCriticalIndexes()) { @@ -1072,7 +1122,10 @@ export class CodeUxServer { } private async listSessionsForSync(): Promise<{ sessions?: JulesSession[] }> { - const tracked = this.sessionTracking.listSessions(300).sessions; + // Session sync matches durable task/run metadata and never reads local CLI prompts. + // QA prompts can contain a full 400-task context, so projecting them into every + // one-second watch cycle creates a large transient heap proportional to session count. + const tracked = this.sessionTracking.listSessions(300, { includePrompt: false }).sessions; let julesSessions: JulesSession[] = []; if (this.isJulesApiConfigured()) { try { @@ -1317,20 +1370,26 @@ export class CodeUxServer { return await this.activityCacheService.getLiveActivitiesForActiveTasks(); } - private scheduleStartupTask(label: string, delayMs: number, task: () => Promise): void { + private scheduleTrackedTimer(delayMs: number, task: () => void): void { const timer = setTimeout(() => { this.startupTaskTimers.delete(timer); if (this.isClosing) { return; } - void task().catch((error) => { - this.logger.error?.(`${label} failed`, { error }); - }); + task(); }, delayMs); timer.unref?.(); this.startupTaskTimers.add(timer); } + private scheduleStartupTask(label: string, delayMs: number, task: () => Promise): void { + this.scheduleTrackedTimer(delayMs, () => { + void task().catch((error) => { + this.logger.error?.(`${label} failed`, { error }); + }); + }); + } + private scheduleBackgroundStartupTasks(): void { this.scheduleStartupTask( "Startup recovery", diff --git a/src/server/dashboard-realtime-websocket-server.ts b/src/server/dashboard-realtime-websocket-server.ts index 27d34c1c62..249c1fe644 100644 --- a/src/server/dashboard-realtime-websocket-server.ts +++ b/src/server/dashboard-realtime-websocket-server.ts @@ -50,8 +50,8 @@ function encodeFrame(payload: string): Buffer { return Buffer.concat([header, message]); } -function sendJson(socket: Socket, payload: DashboardRealtimeServerMessage): void { - socket.write(encodeFrame(JSON.stringify(payload))); +function encodeJsonFrame(payload: DashboardRealtimeServerMessage): Buffer { + return encodeFrame(JSON.stringify(payload)); } function encodeEventFrame(event: DashboardRealtimeEvent): Buffer { @@ -195,6 +195,46 @@ export function bootDashboardRealtimeWebSocketServer(args: { return false; }); + const writeFrame = ( + client: RealtimeClientState, + frame: Buffer, + context: { eventType: string; sequence?: number; scope?: string; projectId?: string | null; correlationId?: string | null }, + ): boolean => { + const pendingLimit = Math.max(MAX_WS_PENDING_WRITE_BYTES, frame.length * 2); + const writableLength = client.socket.writableLength ?? 0; + const projectedWritableLength = writableLength + frame.length; + if ( + client.socket.destroyed + || client.socket.writable === false + || projectedWritableLength > pendingLimit + ) { + clients.delete(client.socket); + args.logger.warn("dashboard_realtime_websocket_backpressure_disconnect", { + logPurpose: "realtime", + eventType: context.eventType, + sequence: context.sequence, + scope: context.scope, + projectId: context.projectId, + correlationId: context.correlationId ?? client.correlationId, + clientId: client.socket.remoteAddress || "unknown", + writableLength, + frameLength: frame.length, + projectedWritableLength, + pendingLimit, + }); + client.socket.destroy(); + return false; + } + client.socket.write(frame); + return true; + }; + + const sendClientJson = ( + client: RealtimeClientState, + payload: DashboardRealtimeServerMessage, + eventType: string, + ): boolean => writeFrame(client, encodeJsonFrame(payload), { eventType }); + const broadcastEvent = (event: DashboardRealtimeEvent): void => { const subscribers = selectSubscribedClients(clients.values(), event.scope); if (subscribers.length === 0) { @@ -205,28 +245,16 @@ export function bootDashboardRealtimeWebSocketServer(args: { // after subscriber selection and reuse the same Buffer for all connected dashboards. const frame = encodeEventFrame(event); for (const client of subscribers) { - client.lastPushedSequence = event.sequence; try { - const pendingLimit = Math.max(MAX_WS_PENDING_WRITE_BYTES, frame.length * 2); - const writableLength = client.socket.writableLength ?? 0; - if (client.socket.destroyed || client.socket.writable === false || writableLength > pendingLimit) { - clients.delete(client.socket); - args.logger.warn("dashboard_realtime_websocket_backpressure_disconnect", { - logPurpose: "realtime", - eventType: event.eventType, - sequence: event.sequence, - scope: event.scope, - projectId: event.projectId, - correlationId: event.correlationId, - clientId: client.socket.remoteAddress || "unknown", - writableLength, - pendingLimit, - }); - client.socket.destroy(); - continue; + if (writeFrame(client, frame, { + eventType: event.eventType, + sequence: event.sequence, + scope: event.scope, + projectId: event.projectId, + correlationId: event.correlationId, + })) { + client.lastPushedSequence = event.sequence; } - - client.socket.write(frame); } catch (error) { clients.delete(client.socket); args.logger.warn("dashboard_realtime_websocket_broadcast_failed", { @@ -304,7 +332,8 @@ export function bootDashboardRealtimeWebSocketServer(args: { correlationId, }; clients.set(socket, client); - sendJson(socket, { type: "ready" }); + socket.setKeepAlive?.(true, 30_000); + sendClientJson(client, { type: "ready" }, "websocket.ready"); let buffered: Buffer = Buffer.alloc(0); socket.on("data", (chunk: Buffer) => { @@ -378,26 +407,28 @@ export function bootDashboardRealtimeWebSocketServer(args: { client.recoveryAttempts = []; } - sendJson(socket, { + sendClientJson(client, { type: "snapshot_required", reason, - }); + }, "websocket.snapshot_required"); } else { for (const replayEvent of replayEvents) { - sendJson(socket, { + if (!sendClientJson(client, { type: "event", event: replayEvent, - }); + }, replayEvent.eventType)) { + break; + } } } } } - sendJson(socket, { + sendClientJson(client, { type: "subscribed", scopes: validScopes, lastSequence: args.realtimeService.getLatestSequence(), - }); + }, "websocket.subscribed"); } catch (error) { args.logger.warn("Invalid dashboard realtime websocket message", { logPurpose: "realtime", @@ -424,10 +455,10 @@ export function bootDashboardRealtimeWebSocketServer(args: { client.recoveryAttempts = []; } - sendJson(socket, { + sendClientJson(client, { type: "snapshot_required", reason, - }); + }, "websocket.snapshot_required"); } } }); diff --git a/src/server/terminal-routes.ts b/src/server/terminal-routes.ts index fabbed0b1e..906f976543 100644 --- a/src/server/terminal-routes.ts +++ b/src/server/terminal-routes.ts @@ -15,6 +15,7 @@ import { getDockerUserSpec } from "../services/cli-docker-utils.js"; import { assertSafePathSegment } from "../utils/path-validator.js"; import { managedRuntimeService } from "../services/managed-runtime-service.js"; import { PROVIDER_TOOL_MOUNT, providerToolManager } from "../services/provider-tool-manager.js"; +import { getRuntimeOwnerDockerArgs, getRuntimeOwnerLabel } from "../shared/config/runtime-owner.js"; interface TerminalSession { sessionId: string; @@ -331,11 +332,20 @@ async function cleanupAllRunningLoginSessions(logger?: Logger): Promise { // 2. Clean up any leftover docker containers labeled code-ux.login=true try { const cp = await import("child_process"); - if (!cp || typeof cp.exec !== "function") { + if (!cp || typeof cp.execFile !== "function") { return; } + const execFile = cp.execFile; await new Promise((resolve) => { - cp.exec("docker ps -a -q --filter 'label=code-ux.login=true'", (err, stdout) => { + execFile("docker", [ + "ps", + "-a", + "-q", + "--filter", + "label=code-ux.login=true", + "--filter", + `label=${getRuntimeOwnerLabel()}`, + ], (err, stdout) => { if (err) { logger?.error(`[DEBUG] Failed to query leftover login containers: ${String(err)}`); resolve(); @@ -344,7 +354,7 @@ async function cleanupAllRunningLoginSessions(logger?: Logger): Promise { const containerIds = stdout.trim().split(/\s+/).filter(Boolean); if (containerIds.length > 0) { logger?.info(`[DEBUG] Preemptively removing active/stray login containers: ${containerIds.join(", ")}`); - cp.exec(`docker rm -f -v ${containerIds.join(" ")}`, (rmErr) => { + execFile("docker", ["rm", "-f", "-v", ...containerIds], (rmErr) => { if (rmErr) { logger?.error(`[DEBUG] Failed to force-remove leftover login containers: ${String(rmErr)}`); } @@ -356,7 +366,7 @@ async function cleanupAllRunningLoginSessions(logger?: Logger): Promise { }); }); } catch (_) { - // Ignore in environments where child_process dynamic import or exec is unavailable + // Ignore in environments where child_process dynamic import or execFile is unavailable } } @@ -590,6 +600,7 @@ export function registerTerminalRoutes(app: Express, options: DashboardDependenc `code-ux-login-${providerId}-${sessionId}`, "--label", "code-ux.login=true", + ...getRuntimeOwnerDockerArgs(), "--label", `code-ux.session-id=${sessionId}`, "--label", diff --git a/src/services/activity-write-coalescer.ts b/src/services/activity-write-coalescer.ts index 32149ebf6b..c013964ad2 100644 --- a/src/services/activity-write-coalescer.ts +++ b/src/services/activity-write-coalescer.ts @@ -51,7 +51,11 @@ export class ActivityWriteCoalescer { } push(description: string, originator?: string): void { - this.buffer.push({ description, originator, createTime: new Date().toISOString() }); + this.buffer.push({ + description: this.boundDescription(description), + originator, + createTime: new Date().toISOString(), + }); if (this.buffer.length >= this.maxBuffer) { this.flush(); return; @@ -111,6 +115,18 @@ export class ActivityWriteCoalescer { return compacted; } + private boundDescription(description: string): string { + if (description.length <= this.maxChunkChars) { + return description; + } + const marker = "\n… [activity truncated] …\n"; + const retainedChars = Math.max(this.maxChunkChars - marker.length, 0); + const headChars = Math.ceil(retainedChars / 2); + const tailChars = retainedChars - headChars; + return `${description.slice(0, headChars)}${marker}${tailChars > 0 ? description.slice(-tailChars) : ""}` + .slice(0, this.maxChunkChars); + } + /** Flush any remaining buffered activities and cancel the pending timer. */ stop(): void { this.flush(); diff --git a/src/services/cli-process-runner.ts b/src/services/cli-process-runner.ts index c347371968..5854da2912 100644 --- a/src/services/cli-process-runner.ts +++ b/src/services/cli-process-runner.ts @@ -4,6 +4,7 @@ export { commandRunner, type CommandResult, type CommandOptions }; export interface StreamingCommandOptions { signal?: AbortSignal; + stdinFile?: string; trimOutput?: boolean; maxStdoutChars?: number; onStdoutLine?: (line: string) => void; @@ -21,6 +22,7 @@ export const runStreamingCommand = async ( cwd, env, signal: options.signal, + stdinFile: options.stdinFile, trimOutput: options.trimOutput, maxStdoutChars: options.maxStdoutChars, onStdoutLine: options.onStdoutLine, diff --git a/src/services/cli-workflow-service.ts b/src/services/cli-workflow-service.ts index a9f98b9090..f0529c97ac 100644 --- a/src/services/cli-workflow-service.ts +++ b/src/services/cli-workflow-service.ts @@ -374,9 +374,13 @@ export class CliWorkflowService { }, }) : undefined; + let releaseWorkspaceHelperReservation = (): void => undefined; let preserveWorkspaceForShutdown = false; try { + releaseWorkspaceHelperReservation = this.workspaceManager.reserveWorkspaceHelper( + ctx.worktreePath, + ); if (taskRun && invocationModel && this.deps.executionRepository) { const invocation = this.deps.executionRepository.createExecutionInvocation({ projectId: taskRun.projectId, @@ -455,7 +459,14 @@ export class CliWorkflowService { worktreePath: ctx.worktreePath, }, `cli:provider:completed:${ctx.worktreePath}`); + this.appendExecutionEvent(args, "cli_memory_capture_started", { + provider: args.provider, + }, `cli:memory:capture:started:${args.sessionId}`); const { memoriesCaptured } = await executeMemoryCaptureStage(ctx); + this.appendExecutionEvent(args, "cli_memory_capture_completed", { + provider: args.provider, + memoriesCaptured, + }, `cli:memory:capture:completed:${args.sessionId}`); if (memoriesCaptured > 0) { this.appendExecutionEvent(args, "cli_memory_captured", { provider: args.provider, @@ -464,6 +475,10 @@ export class CliWorkflowService { } } + this.appendExecutionEvent(args, "cli_git_finalize_started", { + provider: args.provider, + worktreePath: ctx.worktreePath, + }, `cli:git:finalize:started:${ctx.worktreePath}`); const { hasChanges, committedChanges, pushedBranch, stats } = await executeGitFinalizeStage(ctx); if (!hasChanges) { @@ -739,6 +754,7 @@ export class CliWorkflowService { worktreePath: ctx.worktreePath, }, `cli:cleanup:${cleanupResult.cleanedUp ? "cleaned" : "preserved"}:${ctx.worktreePath}`); } finally { + releaseWorkspaceHelperReservation(); unregisterDispatch?.(); const taskRun = this.resolveTaskRun(args); if (taskRun?.sprintRunId) { @@ -793,10 +809,16 @@ export class CliWorkflowService { return null; } - const events = repository.listTaskRunEvents(providerInvocation.taskRunId, 200); + const events = repository.listTaskRunEvents(providerInvocation.taskRunId, 200, { + eventTypes: ["task_dispatch_reconciled", "cli_workspace_bound", "cli_workflow_completed"], + skipValidation: true, + }); const recoveredCompletedProvider = events.some((event) => ( event.eventType === "task_dispatch_reconciled" - && event.payload?.reason === "terminal_provider_active_dispatch_mismatch" + && ( + event.payload?.reason === "terminal_provider_active_dispatch_mismatch" + || event.payload?.reason === "shutdown_interrupted_after_provider_completion" + ) && event.payload?.providerStatus === "completed" )); const sameWorkspace = events.some((event) => ( diff --git a/src/services/cli-workflow/pipeline/cleanup-stage.ts b/src/services/cli-workflow/pipeline/cleanup-stage.ts index 743fec1596..2899855d29 100644 --- a/src/services/cli-workflow/pipeline/cleanup-stage.ts +++ b/src/services/cli-workflow/pipeline/cleanup-stage.ts @@ -12,6 +12,11 @@ export async function executeCleanupStage(ctx: PipelineContext): Promise<{ clean return { cleanedUp: true }; } + // A successful task workspace may remain available for QA/restart recovery, + // but its helper container is only a command sidecar. Release that sidecar + // immediately so preserved volumes do not retain one container per task. + await ctx.workspaceManager.releaseWorkspaceHelper(ctx.worktreePath); + ctx.deps.sessionTracking.appendActivity(ctx.sessionId, { originator: "system", description: `Preserving worktree: ${ctx.worktreePath}`, diff --git a/src/services/custom-dashboard-docker-plan.ts b/src/services/custom-dashboard-docker-plan.ts index 0a637140c4..b2aac7685f 100644 --- a/src/services/custom-dashboard-docker-plan.ts +++ b/src/services/custom-dashboard-docker-plan.ts @@ -4,6 +4,7 @@ import { DOCKER_NO_NEW_PRIVILEGES_ARGS, toDockerMountArg, } from "./cli-docker-utils.js"; +import { getRuntimeOwnerDockerArgs } from "../shared/config/runtime-owner.js"; export const CUSTOM_DASHBOARD_VALIDATION_CONTAINER_PORT = 4173; export const CUSTOM_DASHBOARD_VALIDATION_LOG_DRIVER = "local"; @@ -49,6 +50,7 @@ export function buildCustomDashboardValidationDockerRunArgs( ...DOCKER_NO_NEW_PRIVILEGES_ARGS, "--workdir", CUSTOM_DASHBOARD_VALIDATION_CONTAINER_WORKSPACE, "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "code-ux.custom-dashboard-validation-build=true", "--label", `code-ux.project-id=${args.projectId}`, "--label", `code-ux.dashboard-id=${args.dashboardId}`, @@ -95,6 +97,7 @@ export function buildCustomDashboardValidationDockerCreateArgs( "-p", `127.0.0.1:${args.hostPort}:${CUSTOM_DASHBOARD_VALIDATION_CONTAINER_PORT}`, "--workdir", CUSTOM_DASHBOARD_VALIDATION_CONTAINER_WORKSPACE, "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "code-ux.custom-dashboard-validation=true", "--label", `code-ux.project-id=${args.projectId}`, "--label", `code-ux.dashboard-id=${args.dashboardId}`, diff --git a/src/services/custom-nodes/custom-node-runtime-service.ts b/src/services/custom-nodes/custom-node-runtime-service.ts index 6170e8dc6f..046a9070a6 100644 --- a/src/services/custom-nodes/custom-node-runtime-service.ts +++ b/src/services/custom-nodes/custom-node-runtime-service.ts @@ -15,6 +15,7 @@ import { runCommandStrict, type CommandResult } from "../cli-process-runner.js"; import type { CredentialBroker } from "../credentials/credential-broker.js"; import type { EgressPolicyService } from "../node-flows/egress-policy-service.js"; import { CUSTOM_NODE_EGRESS_SOCKET_DIRECTORY, CustomNodeEgressBroker } from "./custom-node-egress-broker.js"; +import { getRuntimeOwnerDockerArgs } from "../../shared/config/runtime-owner.js"; export const CUSTOM_NODE_CONTAINER_SCRATCH = "/tmp/codeux"; @@ -42,6 +43,7 @@ export function buildCustomNodeDockerRunArgs(input: CustomNodeDockerPlanInput): "--tmpfs", `${CUSTOM_NODE_CONTAINER_SCRATCH}:rw,nosuid,nodev,noexec,size=${limits.scratchMb}m,mode=700,uid=65532,gid=65532`, "--log-driver", "none", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "code-ux.custom-node=true", "--label", `code-ux.custom-node-digest=${input.artifact.digest}`, ]; diff --git a/src/services/dashboard-realtime-service.ts b/src/services/dashboard-realtime-service.ts index 1b6481997d..2ecec60307 100644 --- a/src/services/dashboard-realtime-service.ts +++ b/src/services/dashboard-realtime-service.ts @@ -424,6 +424,19 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi } if (options.shouldPublish && !options.shouldPublish()) { + // Keep an in-memory non-replayable watermark even when no current client needs the payload. + // A previously connected client may reconnect with an older cursor and must be told to + // refresh its REST snapshot. The repository does not persist these events, so this preserves + // recovery correctness without running the loader, serializing the payload, or growing WAL. + this.publishRawEvent({ + scopeType: options.scopeType, + scopeId: options.scopeId, + eventType: options.eventType, + entityType: options.entityType, + entityId: options.entityId, + ...(options.projectId ? { projectId: options.projectId } : {}), + replayable: false, + }); this.incrementMetric(options.eventType, "skipped"); return { task: null, waitMs: 0 }; } @@ -554,6 +567,7 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi entityType: "project_collection", entityId: (id: string) => "projects", loader: () => loaders.getProjectsSnapshot(), + shouldPublish: () => this.hasScopeInterest("projects"), cacheKey: (id: string) => `${id}:projects.updated`, skipDuplicate: true, lastPublishedAt: (id: string) => this.projectsPublishedAt, @@ -627,6 +641,7 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi entityId: (projectId: string) => projectId, projectId: (projectId: string) => projectId, loader: (projectId: string) => loaders.getProjectExecutionSnapshot(projectId), + shouldPublish: (projectId: string) => this.hasScopeInterest(`project:${projectId}`), cacheKey: (projectId: string) => `project:${projectId}:project.execution.updated`, skipDuplicate: true, lastPublishedAt: (projectId: string) => this.projectExecutionPublishedAt.get(projectId) ?? 0, @@ -647,6 +662,7 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi entityId: (projectId: string) => projectId, projectId: (projectId: string) => projectId, loader: (projectId: string) => loaders.getProjectStatusSnapshot(projectId), + shouldPublish: (projectId: string) => this.hasScopeInterest(`project:${projectId}`), cacheKey: (projectId: string) => `project:${projectId}:project.runtime_status.updated`, skipDuplicate: true, lastPublishedAt: (projectId: string) => this.projectRuntimeStatusPublishedAt.get(projectId) ?? 0, @@ -670,6 +686,7 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi projectId, updatedAt: new Date().toISOString(), }), + shouldPublish: (projectId: string) => this.hasScopeInterest(`project:${projectId}`), lastPublishedAt: (projectId: string) => this.projectStructurePublishedAt.get(projectId) ?? 0, onPublished: (projectId: string, publishedAt: number) => { this.projectStructurePublishedAt.set(projectId, publishedAt); @@ -722,6 +739,7 @@ export class DashboardRealtimeService implements DashboardRealtimeMutationNotifi entityType: "overview", entityId: "overview", loader: () => loaders.getOverviewTelemetrySnapshot(), + shouldPublish: () => this.hasScopeInterest("overview"), cacheKey: `overview:overview:overview.telemetry.updated`, skipDuplicate: true, logType: "realtime_background_refresh", diff --git a/src/services/database-maintenance-service.ts b/src/services/database-maintenance-service.ts index eafc6077ca..64c79fef31 100644 --- a/src/services/database-maintenance-service.ts +++ b/src/services/database-maintenance-service.ts @@ -141,7 +141,11 @@ export class DatabaseMaintenanceService { */ runPeriodicMaintenance(): void { if (this.hasActiveProviderInvocations()) { - this.deps.logger.debug("Skipping periodic database maintenance while provider invocations are active."); + // PASSIVE checkpoints never wait for readers or writers. Keep the write-heavy retention + // work deferred, but bound WAL growth for continuously busy runtimes where there may not be + // an idle maintenance window for hours. + this.deps.logger.debug("Skipping periodic database pruning while provider invocations are active."); + this.checkpointWalDatabases(); return; } diff --git a/src/services/docker-asset-prune-service.ts b/src/services/docker-asset-prune-service.ts index ed234f6dc2..400ed00299 100644 --- a/src/services/docker-asset-prune-service.ts +++ b/src/services/docker-asset-prune-service.ts @@ -5,11 +5,13 @@ import { runCommandStrict, type CommandResult } from "./cli-process-runner.js"; import { SessionTrackingRepository } from "../repositories/session-tracking-repository.js"; import { AsyncSemaphore } from "../shared/async-semaphore.js"; import type { Logger } from "../shared/logging/logger.js"; +import { getRuntimeOwnerLabel } from "../shared/config/runtime-owner.js"; export interface DockerAssetPruneResult { prunedWorkspaceVolumes: string[]; prunedSetupImages: string[]; prunedLoginContainers: string[]; + prunedProviderContainers?: string[]; prunedHelperContainers?: string[]; prunedTempCredentialsDirs?: string[]; prunedProviderToolVolumes?: string[]; @@ -81,12 +83,15 @@ export class DockerAssetPruneService { .map((session) => session.id), ); + // Old helper generations may still be running after a hard process exit; + // remove them first so the generic non-running scan cannot race the same ID. + const prunedHelperContainers = await this.pruneOrphanedHelperContainers(); const [ - prunedHelperContainers, + prunedProviderContainers, prunedLoginContainers, prunedTempCredentialsDirs, ] = await Promise.all([ - this.pruneOrphanedHelperContainers(), + this.pruneOrphanedProviderContainers(), this.pruneOrphanedLoginContainers(), this.pruneTemporaryCredentialsDirectories(), ]); @@ -108,6 +113,7 @@ export class DockerAssetPruneService { prunedWorkspaceVolumes.length > 0 || prunedSetupImages.length > 0 || prunedLoginContainers.length > 0 || + prunedProviderContainers.length > 0 || prunedHelperContainers.length > 0 || prunedTempCredentialsDirs.length > 0 || prunedProviderToolVolumes.length > 0 || @@ -117,6 +123,7 @@ export class DockerAssetPruneService { prunedWorkspaceVolumes: prunedWorkspaceVolumes.length, prunedSetupImages: prunedSetupImages.length, prunedLoginContainers: prunedLoginContainers.length, + prunedProviderContainers: prunedProviderContainers.length, prunedHelperContainers: prunedHelperContainers.length, prunedTempCredentialsDirs: prunedTempCredentialsDirs.length, prunedProviderToolVolumes: prunedProviderToolVolumes.length, @@ -128,6 +135,7 @@ export class DockerAssetPruneService { prunedWorkspaceVolumes, prunedSetupImages, prunedLoginContainers, + prunedProviderContainers, prunedHelperContainers, prunedTempCredentialsDirs, prunedProviderToolVolumes, @@ -137,8 +145,8 @@ export class DockerAssetPruneService { private async pruneWorkspaceVolumes(startupSessionIds: ReadonlySet): Promise { const [workspaceResult, runtimeResult] = await Promise.all([ - this.runDocker(["volume", "ls", "-q", "--filter", `label=${WORKSPACE_VOLUME_LABEL}`]), - this.runDocker(["volume", "ls", "-q", "--filter", `label=${RUNTIME_VOLUME_LABEL}`]), + this.runDocker(["volume", "ls", "-q", "--filter", `label=${WORKSPACE_VOLUME_LABEL}`, "--filter", `label=${getRuntimeOwnerLabel()}`]), + this.runDocker(["volume", "ls", "-q", "--filter", `label=${RUNTIME_VOLUME_LABEL}`, "--filter", `label=${getRuntimeOwnerLabel()}`]), ]); const volumeNames = [ @@ -190,7 +198,26 @@ export class DockerAssetPruneService { } private async pruneOrphanedLoginContainers(): Promise { - const result = await this.runDocker(["ps", "-aq", "--filter", "label=code-ux.login=true"]); + const result = await this.runDocker(["ps", "-aq", "--filter", "label=code-ux.login=true", "--filter", `label=${getRuntimeOwnerLabel()}`]); + if (!result) { + return []; + } + + return await this.removeDockerItems(["rm", "-f", "-v"], this.parseLines(result.stdout)); + } + + private async pruneOrphanedProviderContainers(): Promise { + // Provider clients cannot be reattached after the Code UX process exits. + // Remove their owner-scoped container generation in every state. This also + // covers `docker run --rm` clients interrupted after daemon create but + // before start, which otherwise remain in `created` forever. + const result = await this.runDocker([ + "ps", + "-aq", + "--filter", "label=code-ux.managed=true", + "--filter", "label=code-ux.command", + "--filter", `label=${getRuntimeOwnerLabel()}`, + ]); if (!result) { return []; } @@ -199,7 +226,7 @@ export class DockerAssetPruneService { } private async pruneProviderToolVolumes(): Promise { - const listed = await this.runDocker(["volume", "ls", "-q", "--filter", "label=ai.codeux.asset=provider-tool"]); + const listed = await this.runDocker(["volume", "ls", "-q", "--filter", "label=ai.codeux.asset=provider-tool", "--filter", `label=${getRuntimeOwnerLabel()}`]); const names = this.parseLines(listed?.stdout); if (names.length === 0) return []; const [active, inspections] = await Promise.all([ @@ -241,7 +268,7 @@ export class DockerAssetPruneService { } private async prunePlaywrightBrowserVolumes(): Promise { - const listed = await this.runDocker(["volume", "ls", "-q", "--filter", "label=ai.codeux.asset=playwright-browser"]); + const listed = await this.runDocker(["volume", "ls", "-q", "--filter", "label=ai.codeux.asset=playwright-browser", "--filter", `label=${getRuntimeOwnerLabel()}`]); const names = this.parseLines(listed?.stdout); if (names.length === 0) return []; const [active, inspections] = await Promise.all([ @@ -279,9 +306,9 @@ export class DockerAssetPruneService { } private async pruneOrphanedHelperContainers(): Promise { - // Persistent git/file helper containers (`code-ux.helper`) from a previous process are - // recreated on demand, so any survivors at startup are safe to remove. - const result = await this.runDocker(["ps", "-aq", "--filter", "label=code-ux.helper"]); + // Scope cleanup to this state home. An isolated test runtime may share the daemon with a live + // app, and unscoped removal would terminate Git operations in that other runtime. + const result = await this.runDocker(["ps", "-aq", "--filter", "label=code-ux.helper", "--filter", `label=${getRuntimeOwnerLabel()}`]); if (!result) { return []; } diff --git a/src/services/docker-orphan-cleanup-utils.ts b/src/services/docker-orphan-cleanup-utils.ts index cbfca190a0..2d12e4ed16 100644 --- a/src/services/docker-orphan-cleanup-utils.ts +++ b/src/services/docker-orphan-cleanup-utils.ts @@ -1,5 +1,6 @@ import { runCommandStrict } from "./cli-process-runner.js"; import type { Logger } from "../shared/logging/logger.js"; +import { getRuntimeOwnerLabel } from "../shared/config/runtime-owner.js"; export async function pruneOrphanedDockerVolumes(args: { prefix: string; @@ -9,7 +10,7 @@ export async function pruneOrphanedDockerVolumes(args: { }): Promise { const result = await runCommandStrict( "docker", - ["volume", "ls", "-q"], + ["volume", "ls", "-q", "--filter", `label=${getRuntimeOwnerLabel()}`], process.cwd(), ).catch(() => null); diff --git a/src/services/git-status-service.ts b/src/services/git-status-service.ts index a7957a7911..370728ee52 100644 --- a/src/services/git-status-service.ts +++ b/src/services/git-status-service.ts @@ -33,6 +33,7 @@ import { } from "../infrastructure/git/git-status-policy.js"; import { buildGitHttpAuthEnvForRepoWithFallbacks } from "./git-http-auth.js"; import { commandRunner, type CommandResult } from "../shared/subprocess/command-runner.js"; +import { createHash } from "node:crypto"; export type { GitTrackingRequest }; @@ -104,6 +105,8 @@ export class GitStatusService { this.queryClient.setProvider(provider, hostDomain, repoTarget, this.preferApi && !!token); return { provider, token }; } + private static readonly STATUS_CACHE_LIMIT = 128; + private static readonly STATUS_CACHE_MAX_RETENTION_MS = 60_000; private static statusCache = new Map }>(); // Local git plumbing (rev-parse / branch / remote / status) is repository-level and does not @@ -111,12 +114,35 @@ export class GitStatusService { // sprint/dashboard caller. This stops the same project from spinning up a fresh batch of // `alpine/git` containers on every watch-loop cycle for each feature branch. private static readonly REPO_PLUMBING_CACHE_MS = 10_000; + private static readonly REPO_PLUMBING_CACHE_LIMIT = 128; private static repoPlumbingCache = new Map }>(); + private static pruneCache( + cache: Map }>, + maxAgeMs: number, + limit: number, + ): void { + const oldestAllowed = Date.now() - maxAgeMs; + for (const [key, entry] of cache) { + if (entry.timestamp < oldestAllowed) cache.delete(key); + } + while (cache.size >= limit) { + const oldest = cache.keys().next(); + if (oldest.done) break; + cache.delete(oldest.value); + } + } + public static invalidateCache(repoPath?: string): void { if (repoPath) { for (const key of GitStatusService.statusCache.keys()) { - if (key.includes(`"repoPath":"${repoPath}"`)) { + let cachedRepoPath: unknown; + try { + cachedRepoPath = (JSON.parse(key) as { repoPath?: unknown }).repoPath; + } catch { + cachedRepoPath = undefined; + } + if (cachedRepoPath === repoPath) { GitStatusService.statusCache.delete(key); } } @@ -139,8 +165,16 @@ export class GitStatusService { const key = this.repoPath; const cached = GitStatusService.repoPlumbingCache.get(key); if (cached && Date.now() - cached.timestamp < GitStatusService.REPO_PLUMBING_CACHE_MS) { + GitStatusService.repoPlumbingCache.delete(key); + GitStatusService.repoPlumbingCache.set(key, cached); return cached.promise; } + if (cached) GitStatusService.repoPlumbingCache.delete(key); + GitStatusService.pruneCache( + GitStatusService.repoPlumbingCache, + GitStatusService.REPO_PLUMBING_CACHE_MS, + GitStatusService.REPO_PLUMBING_CACHE_LIMIT, + ); const promise = this.fetchRepoPlumbing(); GitStatusService.repoPlumbingCache.set(key, { timestamp: Date.now(), promise }); try { @@ -339,19 +373,25 @@ export class GitStatusService { async getStatus(mode: "REMOTE" | "LOCAL", tokens: GitHostTokens | string = {}, trackingRequest?: GitTrackingRequest, cacheTtlMs?: number): Promise { const normalizedTokens = this.normalizeTokens(tokens); + const tokenFingerprint = (value: string | null | undefined): string | undefined => value?.trim() + ? createHash("sha256").update(value.trim()).digest("base64url") + : undefined; const cacheKey = JSON.stringify({ repoPath: this.repoPath, mode, - githubToken: normalizedTokens.githubToken?.trim() || undefined, - gitlabToken: normalizedTokens.gitlabToken?.trim() || undefined, + githubTokenHash: tokenFingerprint(normalizedTokens.githubToken), + gitlabTokenHash: tokenFingerprint(normalizedTokens.gitlabToken), trackingRequest, }); if (cacheTtlMs && cacheTtlMs > 0) { const cached = GitStatusService.statusCache.get(cacheKey); if (cached && Date.now() - cached.timestamp < cacheTtlMs) { + GitStatusService.statusCache.delete(cacheKey); + GitStatusService.statusCache.set(cacheKey, cached); return cached.promise; } + if (cached) GitStatusService.statusCache.delete(cacheKey); } const fetchPromise = (async () => { @@ -486,6 +526,11 @@ export class GitStatusService { })(); if (cacheTtlMs && cacheTtlMs > 0) { + GitStatusService.pruneCache( + GitStatusService.statusCache, + GitStatusService.STATUS_CACHE_MAX_RETENTION_MS, + GitStatusService.STATUS_CACHE_LIMIT, + ); GitStatusService.statusCache.set(cacheKey, { timestamp: Date.now(), promise: fetchPromise }); } diff --git a/src/services/guardrail-service.ts b/src/services/guardrail-service.ts index 7b7401d62d..a05b781648 100644 --- a/src/services/guardrail-service.ts +++ b/src/services/guardrail-service.ts @@ -117,6 +117,31 @@ export class GuardrailService { return count; } + /** Refunds an operationally interrupted invocation without erasing genuine failures. */ + refund( + scope: GuardrailScope, + taskId: string, + purpose: GuardrailLedgerPurpose, + sourceKey: string, + reason?: string, + ): number { + const result = this.repo.refund({ + projectId: scope.projectId, + taskId, + purpose, + sourceKey, + reason, + }); + this.logger?.debug?.("Guardrail invocation refunded", { + taskId, + purpose, + sourceKey, + applied: result.applied, + count: result.count, + }); + return result.count; + } + getCounts(taskId: string): Record { return this.repo.getCounts(taskId); } diff --git a/src/services/planning-agent-service.ts b/src/services/planning-agent-service.ts index 797a7a4f38..af0c39fb9f 100644 --- a/src/services/planning-agent-service.ts +++ b/src/services/planning-agent-service.ts @@ -138,9 +138,21 @@ function finalizePlanningInvocationError( interface PlanningContinuationContext { promptOverride?: string; + provider: Exclude; continueSessionId: string; logicalSessionId: string; openCodeBaselineRawUsageJson?: Record | null; + requireExistingSession?: boolean; +} + +interface PersistedPlanSprintRequest { + kind: "plan_sprint"; + autoStart: boolean; + replan: boolean; + sprintRunId?: string; + planningAgentPresetId?: string; + quicksprintTemplateId?: string; + overrides?: PlanningOverrides; } export class PlanningAgentService { @@ -322,16 +334,111 @@ export class PlanningAgentService { preservedAt: invocation.preservedAt || new Date().toISOString(), }); + const continuationProvider = this.requirePlanningContinuationProvider(providerUsage.provider); return await this.runPlanSprint(invocation.projectId, invocation.sprintId, { autoStart: false, replan: true, planningAgentPresetId: invocation.agentPresetId || undefined, + overrides: { + virtualProvider: continuationProvider, + ...(providerUsage.model || invocation.model + ? { virtualModel: providerUsage.model || invocation.model || undefined } + : {}), + }, }, signal, { + provider: continuationProvider, continueSessionId, logicalSessionId: providerUsage.sessionId, openCodeBaselineRawUsageJson: providerUsage.provider === "opencode" ? providerUsage.rawUsageJson : null, promptOverride: mode === "continue_session" ? "continue_session" : undefined, + requireExistingSession: true, + }); + } + + /** + * Reissues a sprint-planning request interrupted by a runtime restart. When + * provider session metadata survived, the new invocation continues that + * native conversation with the complete original planning instructions. A + * request interrupted before provider linkage can be reissued from durable + * input because no provider conversation existed yet. + */ + async recoverInterruptedInvocation( + invocationId: string, + mode: PlanningInvocationRestartMode = "continue_session", + ): Promise { + const invocation = this.deps.executionRepository?.getExecutionInvocation(invocationId); + if (!invocation) { + throw new Error(`Execution invocation not found: ${invocationId}`); + } + if (invocation.status !== "failed" && invocation.status !== "cancelled") { + throw new Error("Only failed or cancelled planning invocations can be recovered."); + } + if (invocation.type !== "planning" || !invocation.sprintId) { + throw new Error("Only sprint-linked planning invocations support automatic restart recovery."); + } + + const options = this.readPersistedPlanSprintRequest(invocation.id, invocation.projectId, invocation.sprintId); + const providerUsage = invocation.providerInvocationId + ? this.deps.executionRepository?.getProviderInvocationUsage(invocation.providerInvocationId) + : null; + const continueSessionId = providerUsage + ? providerUsage.nativeSessionId || (providerUsage.provider === "claude-code" ? null : providerUsage.sessionId) + : null; + if (mode === "continue_session" && providerUsage && !continueSessionId) { + throw new Error( + `Interrupted ${providerUsage.provider} planning invocation does not have a resumable provider session id. Refusing to start a fresh session.`, + ); + } + const continuationProvider = providerUsage + ? this.requirePlanningContinuationProvider(providerUsage.provider) + : null; + const continuation: PlanningContinuationContext | undefined = mode === "continue_session" && providerUsage && continueSessionId && continuationProvider + ? { + provider: continuationProvider, + continueSessionId, + logicalSessionId: providerUsage.sessionId, + openCodeBaselineRawUsageJson: providerUsage.provider === "opencode" ? providerUsage.rawUsageJson : null, + promptOverride: "continue_session", + requireExistingSession: true, + } + : undefined; + const recoveredAt = new Date().toISOString(); + this.deps.executionRepository?.updateExecutionInvocation(invocation.id, { + preservedAt: invocation.preservedAt || recoveredAt, }); + this.deps.executionRepository?.appendExecutionInvocationMessage(invocation.id, { + role: "system", + contentMarkdown: continuation + ? "Runtime restart interrupted this planning request. Code UX is continuing it in the preserved provider session." + : "Runtime restart interrupted this planning request before a resumable provider session was persisted. Code UX is reissuing the complete request.", + metadata: { + recovery: "startup_planning_request_resumed", + continuationMode: continuation ? "continue_session" : "retry_full_prompt", + }, + createdAt: recoveredAt, + }); + + const recoveredOptions: PlanSprintOptions = continuation + ? { + ...options, + planningAgentPresetId: options.planningAgentPresetId || invocation.agentPresetId || undefined, + overrides: { + ...options.overrides, + virtualProvider: continuation.provider, + ...(providerUsage?.model || invocation.model + ? { virtualModel: providerUsage?.model || invocation.model || undefined } + : {}), + }, + } + : options; + + return await this.runPlanSprint( + invocation.projectId, + invocation.sprintId, + recoveredOptions, + undefined, + continuation, + ); } async planSprint(projectId: string, sprintId: string, options: PlanSprintOptions, signal?: AbortSignal): Promise { @@ -404,6 +511,9 @@ export class PlanningAgentService { this.deps.executionRepository?.appendExecutionInvocationMessage(invocation.id, { role: "user", contentMarkdown: prompt, + metadata: { + planningRequest: this.buildPersistedPlanSprintRequest(options), + }, }); } @@ -551,7 +661,7 @@ export class PlanningAgentService { private buildPlanningContinuationPrompt(fullPlanningPrompt: string): string { return [ "Continue the previous planning attempt in this same provider session.", - "If the previous provider conversation cannot be resumed, use the original planning instructions below as the complete source of truth.", + "Use the original planning instructions below as the complete source of truth while continuing this conversation.", "", "Output the complete valid JSON sprint definition now. Requirements:", "- Output raw JSON only — no markdown fences, no commentary, no prose before or after.", @@ -563,6 +673,61 @@ export class PlanningAgentService { ].join("\n"); } + private buildPersistedPlanSprintRequest(options: PlanSprintOptions): PersistedPlanSprintRequest { + return { + kind: "plan_sprint", + autoStart: options.autoStart === true, + replan: options.replan === true, + ...(options.sprintRunId ? { sprintRunId: options.sprintRunId } : {}), + ...(options.planningAgentPresetId ? { planningAgentPresetId: options.planningAgentPresetId } : {}), + ...(options.quicksprintTemplateId ? { quicksprintTemplateId: options.quicksprintTemplateId } : {}), + ...(options.overrides ? { overrides: options.overrides } : {}), + }; + } + + private readPersistedPlanSprintRequest( + invocationId: string, + projectId: string, + sprintId: string, + ): PlanSprintOptions { + const messages = this.deps.executionRepository?.listExecutionInvocationMessages(invocationId) || []; + const raw = messages + .map((message) => message.metadata?.planningRequest) + .find((value) => value && typeof value === "object") as Record | undefined; + const existingTasks = this.deps.projectManagementRepository.listTasks(projectId, sprintId); + if (!raw || raw.kind !== "plan_sprint") { + return { + autoStart: false, + replan: existingTasks.length > 0, + }; + } + return { + autoStart: raw.autoStart === true, + replan: raw.replan === true || existingTasks.length > 0, + sprintRunId: typeof raw.sprintRunId === "string" ? raw.sprintRunId : undefined, + planningAgentPresetId: typeof raw.planningAgentPresetId === "string" ? raw.planningAgentPresetId : undefined, + quicksprintTemplateId: typeof raw.quicksprintTemplateId === "string" ? raw.quicksprintTemplateId : undefined, + overrides: raw.overrides && typeof raw.overrides === "object" + ? raw.overrides as PlanningOverrides + : undefined, + }; + } + + private requirePlanningContinuationProvider(provider: string): Exclude { + switch (provider) { + case "gemini": + case "codex": + case "claude-code": + case "qwen-code": + case "opencode": + case "antigravity": + case "mockup-cli": + return provider; + default: + throw new Error(`Provider ${provider} does not support provider-native sprint-planning continuation.`); + } + } + private resolvePlanningRuntime(projectId: string, overrides?: PlanningOverrides): { mode: "VIRTUAL"; settings: DashboardSettings; @@ -698,6 +863,12 @@ export class PlanningAgentService { const providerSettings = { ...baseProviderSettings }; const provider = providerSettings.provider as Exclude; + if (args.continuation && args.continuation.provider !== provider) { + throw new Error( + `Planning continuation provider changed from ${args.continuation.provider} to ${provider}. Refusing to start a fresh provider session.`, + ); + } + if (args.overrides?.virtualModel) { providerSettings.model = args.overrides.virtualModel; } @@ -817,6 +988,7 @@ export class PlanningAgentService { sessionIdPrefix: "planning", logicalSessionId: args.continuation?.logicalSessionId, continueSessionId: args.continuation?.continueSessionId, + allowFreshSessionFallback: args.continuation?.requireExistingSession !== true, openCodeBaselineRawUsageJson: args.continuation?.openCodeBaselineRawUsageJson, invocationId: args.invocationId, systemRoutingMessage, @@ -830,7 +1002,7 @@ export class PlanningAgentService { invocationId: args.invocationId, provider, originator: originator || "system", - description, + descriptionChars: description.length, }); }, }); diff --git a/src/services/playwright-browser-manager.ts b/src/services/playwright-browser-manager.ts index 45d4b3f2c6..1712037bd3 100644 --- a/src/services/playwright-browser-manager.ts +++ b/src/services/playwright-browser-manager.ts @@ -9,6 +9,7 @@ import type { import { commandRunner, runStreamingCommand, type CommandResult } from "./cli-process-runner.js"; import { managedRuntimeService, type ManagedRuntimeService } from "./managed-runtime-service.js"; import type { Logger } from "../shared/logging/logger.js"; +import { getRuntimeOwnerDockerArgs } from "../shared/config/runtime-owner.js"; export const PLAYWRIGHT_BROWSERS_MOUNT = "/ms-playwright"; @@ -147,6 +148,7 @@ export class PlaywrightBrowserManager { const created = await this.commands.run("docker", [ "volume", "create", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "ai.codeux.asset=playwright-browser", "--label", `ai.codeux.version=${this.labelValue(version)}`, "--label", `ai.codeux.compatibility=${compatibilityKey}`, @@ -164,6 +166,7 @@ export class PlaywrightBrowserManager { const install = await this.commands.stream("docker", [ "run", "--rm", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "ai.codeux.browser-installer=playwright", "--mount", `type=volume,source=${volumeName},target=${PLAYWRIGHT_BROWSERS_MOUNT}`, image, diff --git a/src/services/provider-execution-service.ts b/src/services/provider-execution-service.ts index 6297ea84fe..e2b0c26848 100644 --- a/src/services/provider-execution-service.ts +++ b/src/services/provider-execution-service.ts @@ -311,6 +311,9 @@ export interface ExecutionProviderRunArgs { onActivity?: (description: string, originator?: string) => void; signal?: AbortSignal; continueSessionId?: string | null; + /** Defaults to true. Set false when recovery must fail rather than silently + * replacing a missing provider conversation with a fresh session. */ + allowFreshSessionFallback?: boolean; /** Native in-session operation forwarded through the shared provider boundary. */ nativeSessionOperation?: NativeSessionOperation; /** The previous invocation's raw opencode export snapshot for this session, @@ -578,6 +581,7 @@ export class ProviderExecutionService { gitlabToken: args.gitlabToken, signal: args.signal, continueSessionId, + allowFreshSessionFallback: args.allowFreshSessionFallback, nativeSessionOperation: args.nativeSessionOperation, openCodeBaselineUsage: openCodeBaselineRawUsageJson, invocationId: execInvocationId, diff --git a/src/services/provider-tool-manager.ts b/src/services/provider-tool-manager.ts index f571d59b57..d7271115fb 100644 --- a/src/services/provider-tool-manager.ts +++ b/src/services/provider-tool-manager.ts @@ -10,6 +10,7 @@ import type { import { commandRunner, runStreamingCommand, type CommandResult } from "./cli-process-runner.js"; import { managedRuntimeService, type ManagedRuntimeService } from "./managed-runtime-service.js"; import type { Logger } from "../shared/logging/logger.js"; +import { getRuntimeOwnerDockerArgs } from "../shared/config/runtime-owner.js"; export const PROVIDER_TOOL_MOUNT = "/opt/code-ux/provider-tool"; @@ -271,6 +272,7 @@ export class ProviderToolManager { const create = await this.commands.run("docker", [ "volume", "create", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "ai.codeux.asset=provider-tool", "--label", `ai.codeux.provider=${provider}`, "--label", `ai.codeux.version=${this.labelValue(release.version)}`, @@ -410,6 +412,7 @@ export class ProviderToolManager { return await this.commands.stream("docker", [ "run", "--rm", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", `ai.codeux.provider-installer=${provider}`, "-e", "DISABLE_AUTOUPDATER=1", "-e", "OPENCODE_DISABLE_AUTOUPDATE=true", diff --git a/src/services/quality-assurance-service.ts b/src/services/quality-assurance-service.ts index e73ddcc750..eb461ff33b 100644 --- a/src/services/quality-assurance-service.ts +++ b/src/services/quality-assurance-service.ts @@ -78,6 +78,8 @@ import { workerClarificationAgentMcpAccess } from "./agent-mcp-access.js"; type CliQaProvider = Exclude; const SPRINT_RUN_KEEPALIVE_MS = 30_000; +const QA_TASK_LIST_TOKEN_THRESHOLD = 100_000; +const QA_TOKEN_ESTIMATE_CHARACTERS_PER_TOKEN = 4; interface QaFixContinuationResult { applied: boolean; @@ -602,10 +604,12 @@ export class QualityAssuranceService { sprintId: string; tasks: Subtask[]; }): Promise { - const runningRuns = args.tasks + const taskIds = args.tasks .map((task) => task.record_id?.trim()) - .filter((taskId): taskId is string => Boolean(taskId)) - .flatMap((taskId) => this.deps.qaReviewRepository.listLatestTaskCycleRuns(taskId)) + .filter((taskId): taskId is string => Boolean(taskId)); + const snapshots = this.deps.qaReviewRepository.listTaskReviewSnapshots(taskIds); + const runningRuns = [...snapshots.values()] + .flatMap((snapshot) => snapshot.latestCycleRuns) .filter((run): run is QaReviewRunRecord => Boolean(run && run.status === "running")); if (runningRuns.length === 0) { @@ -1149,6 +1153,43 @@ export class QualityAssuranceService { }); } + getTaskMergeGateStatuses(args: { + projectId: string; + sprintId: string; + tasks: Subtask[]; + }): Map { + const settings = this.deps.getDashboardSettings({ projectId: args.projectId, sprintId: args.sprintId }); + const qaSettings = settings.agents.qualityAssurance; + const taskIds = args.tasks + .map((task) => task.record_id?.trim()) + .filter((taskId): taskId is string => Boolean(taskId)); + const snapshots = this.deps.qaReviewRepository.listTaskReviewSnapshots(taskIds); + const statuses = new Map(); + + for (const task of args.tasks) { + const taskId = task.record_id?.trim(); + if (!taskId) { + continue; + } + const triggerType = resolveTaskTriggerType(task, qaSettings); + const isReviewRequired = Boolean(qaSettings.enabled && triggerType); + const snapshot = snapshots.get(taskId); + const latestRun = isReviewRequired && snapshot?.latestRun + ? this.reconcileRunningQaRun(snapshot.latestRun) + : null; + statuses.set(taskId, computeTaskMergeGateStatus({ + taskId, + triggerType, + qaSettings, + latestRun, + runsUsed: isReviewRequired ? snapshot?.runsUsed ?? 0 : 0, + decisiveRuns: isReviewRequired ? snapshot?.decisiveRuns ?? 0 : 0, + })); + } + + return statuses; + } + private findResumableQaReviewerRun( runs: QaReviewRunRecord[], agentPresetId: string | null, @@ -1327,6 +1368,8 @@ export class QualityAssuranceService { }); } } + let releaseSnapshotHelperReservation: (() => void) | null = null; + try { let snapshotWorkspace = args.repoPath; let shouldCleanupSnapshot = false; if (workflowSettings.executionMode === "DOCKER") { @@ -1337,6 +1380,14 @@ export class QualityAssuranceService { fallbackBranch: args.baseBranch, useDefaultBranch: false, }); + const plannedSnapshotWorkspace = this.workspaceManager.buildWorktreePath( + args.repoPath, + `${snapshotSessionId}-snapshot`, + "DOCKER", + ); + releaseSnapshotHelperReservation = this.workspaceManager.reserveWorkspaceHelper( + plannedSnapshotWorkspace, + ); snapshotWorkspace = await this.invocationWorkspacePreparer.createSnapshotWorkspace({ repoPath: args.repoPath, sessionId: snapshotSessionId, @@ -1456,6 +1507,9 @@ export class QualityAssuranceService { } return result.parsed; + } finally { + releaseSnapshotHelperReservation?.(); + } }); } @@ -1649,6 +1703,7 @@ export class QualityAssuranceService { `Provider: ${args.currentTask.provider || "unknown"}`, `Worker branch: ${args.currentTask.worker_branch || "none"}`, `PR URL: ${args.currentTask.pr_url || "none"}`, + `Depends on: ${args.currentTask.depends_on.length > 0 ? args.currentTask.depends_on.join(", ") : "none"}`, "", "Prompt:", args.currentTask.prompt, @@ -1660,23 +1715,9 @@ export class QualityAssuranceService { "## CURRENT TASK", "No single task is preselected. If fixes are required, choose the best target task from the sprint task list and return its task key in `targetTaskKey`.", ]; - const fullTaskInstructionsHeading = isTaskLevelReview - ? "## FULL TASK INSTRUCTIONS (SPRINT CONTEXT; ONLY CURRENT TASK IS UNDER REVIEW)" - : "## FULL TASK INSTRUCTIONS"; - const fullTaskContextSections = args.subtasks.map((task) => [ - `### ${task.id}: ${task.title}`, - `Status: ${task.status || "unknown"}`, - `Provider: ${task.provider || "unknown"}`, - `Worker branch: ${task.worker_branch || "none"}`, - `PR URL: ${task.pr_url || "none"}`, - `Depends on: ${task.depends_on.length > 0 ? task.depends_on.join(", ") : "none"}`, - "", - "Instruction:", - task.prompt || "No task instruction provided.", - "", - "Recent activity excerpts:", - this.renderActivityExcerpt(task), - ].join("\n")); + const sprintTaskContextSection = isTaskLevelReview + ? this.buildTaskReviewSiblingContext(args.subtasks, args.currentTask) + : this.buildSprintReviewTaskContext(args.subtasks); return [ "## QUALITY ASSURANCE AGENT INSTRUCTIONS", @@ -1694,13 +1735,7 @@ export class QualityAssuranceService { `Project: ${args.projectName}`, `Sprint goal: ${args.sprintGoal || "No sprint goal provided."}`, "", - "## SPRINT TASKS", - args.subtasks.map((task) => ( - `- [${task.status || "unknown"}] ${task.id}: ${task.title} | provider=${task.provider || "unknown"} | branch=${task.worker_branch || "none"} | pr=${task.pr_url || "none"}` - )).join("\n"), - "", - fullTaskInstructionsHeading, - fullTaskContextSections.join("\n\n"), + ...sprintTaskContextSection, "", ...currentTaskSection, "", @@ -1739,6 +1774,80 @@ export class QualityAssuranceService { ].join("\n"); } + private buildTaskReviewSiblingContext(subtasks: Subtask[], currentTask: Subtask | null): string[] { + const completedSiblingTitles = subtasks + .filter((task) => task.id !== currentTask?.id && task.status?.trim().toLowerCase() === "completed") + .map((task) => `- ${task.title}`); + return [ + "## PREVIOUSLY COMPLETED SPRINT TASKS (TITLES ONLY)", + completedSiblingTitles.length > 0 + ? completedSiblingTitles.join("\n") + : "- No other sprint tasks have completed.", + ]; + } + + private buildSprintReviewTaskContext(subtasks: Subtask[]): string[] { + const sprintTaskList = subtasks.map((task) => ( + `- [${task.status || "unknown"}] ${task.id}: ${task.title} | provider=${task.provider || "unknown"} | branch=${task.worker_branch || "none"} | pr=${task.pr_url || "none"}` + )).join("\n"); + const taskContextInputs = subtasks.map((task) => { + const instruction = task.prompt || "No task instruction provided."; + const sectionWithoutInstruction = this.renderFullTaskContextSection(task, ""); + return { task, instruction, sectionWithoutInstruction }; + }); + const fullTaskContextCharacters = sprintTaskList.length + + Math.max(0, taskContextInputs.length - 1) * 2 + + taskContextInputs.reduce( + (total, input) => total + input.sectionWithoutInstruction.length + input.instruction.length, + 0, + ); + const estimatedFullTaskContextTokens = Math.ceil( + fullTaskContextCharacters / QA_TOKEN_ESTIMATE_CHARACTERS_PER_TOKEN, + ); + const shortenTaskInstructions = estimatedFullTaskContextTokens > QA_TASK_LIST_TOKEN_THRESHOLD; + const taskContextPolicy = shortenTaskInstructions + ? [ + `Context policy: the full sprint task context is approximately ${estimatedFullTaskContextTokens.toLocaleString("en-US")} tokens, exceeding the 100,000-token threshold.`, + "Every task remains listed in order, but each task instruction below contains only its first half.", + "Use the visible first half of each task instruction together with repository and validation evidence.", + "Recent activity excerpts are not shortened.", + ].join("\n") + : ""; + const fullTaskContextSections = taskContextInputs.map(({ task, instruction }) => { + if (!shortenTaskInstructions) { + return this.renderFullTaskContextSection(task, instruction); + } + const firstHalf = instruction.slice(0, Math.ceil(instruction.length / 2)); + return this.renderFullTaskContextSection(task, firstHalf); + }); + return [ + "## SPRINT TASKS", + sprintTaskList, + "", + "## FULL TASK INSTRUCTIONS", + taskContextPolicy, + "", + fullTaskContextSections.join("\n\n"), + ]; + } + + private renderFullTaskContextSection(task: Subtask, instruction: string): string { + return [ + `### ${task.id}: ${task.title}`, + `Status: ${task.status || "unknown"}`, + `Provider: ${task.provider || "unknown"}`, + `Worker branch: ${task.worker_branch || "none"}`, + `PR URL: ${task.pr_url || "none"}`, + `Depends on: ${task.depends_on.length > 0 ? task.depends_on.join(", ") : "none"}`, + "", + "Instruction:", + instruction, + "", + "Recent activity excerpts:", + this.renderActivityExcerpt(task), + ].join("\n"); + } + private renderActivityExcerpt(task: Subtask): string { const activities = Array.isArray(task.activities) ? task.activities.slice(-8) : []; if (activities.length === 0) { @@ -3074,7 +3183,8 @@ function buildReviewScopeInstructions(triggerType: QaReviewTriggerType, currentT return [ `- This is a single-task QA review. The only task under review is ${currentTaskKey}.`, - "- Treat `SPRINT TASKS` and non-current entries in `FULL TASK INSTRUCTIONS` as context only, not as deliverables for this review.", + "- `PREVIOUSLY COMPLETED SPRINT TASKS` contains title-only historical context. Those sibling tasks are not deliverables for this review.", + "- The complete current-task details, prompt, dependencies, and recent activity are provided in `CURRENT TASK UNDER REVIEW`.", "- Assume the current workspace/branch contains only the current task's changes on top of its base branch. Independent sibling tasks may be completed in separate branches or PRs and may be absent here.", "- A task-level review must pass when the current task satisfies its own prompt, even if other completed sprint tasks are not present in this branch.", "- Do not request changes because files, commits, PRs, or behavior from other completed sibling tasks are missing from this branch.", diff --git a/src/services/runtime-recovery/durable-remote-recovery.ts b/src/services/runtime-recovery/durable-remote-recovery.ts new file mode 100644 index 0000000000..04df6ab36d --- /dev/null +++ b/src/services/runtime-recovery/durable-remote-recovery.ts @@ -0,0 +1,193 @@ +import type { JulesSession } from "../../contracts/app-types.js"; +import type { ExecutionRepository } from "../../repositories/execution-repository.js"; +import type { ProjectManagementRepository } from "../../repositories/project-management-repository.js"; +import type { SprintRunLifecycleService } from "../sprint-run-lifecycle-service.js"; + +const ACTIVE_JULES_SESSION_STATES = new Set([ + "QUEUED", + "PLANNING", + "AWAITING_PLAN_APPROVAL", + "AWAITING_USER_FEEDBACK", + "IN_PROGRESS", + "PAUSED", +]); + +interface DurableRemoteRecoveryDeps { + executionRepository: ExecutionRepository; + projectManagementRepository: ProjectManagementRepository; + sprintRunLifecycleService: SprintRunLifecycleService; +} + +export interface DurableRemoteRecoveryResult { + reactivatedTaskRunIds: string[]; + reactivatedSprintRunIds: string[]; +} + +export interface DurableRemoteRecoveryOptions { + evidence?: "remote_snapshot" | "local_fail_safe"; +} + +/** + * Restores local projections that an earlier process may have terminalized + * while their hosted Jules session kept running. The remote active state is + * authoritative; completed/cancelled local sprints and merged tasks remain + * untouched so an intentional stop cannot be undone on startup. + */ +export class DurableRemoteRecoveryService { + constructor(private readonly deps: DurableRemoteRecoveryDeps) {} + + reconcileActiveJulesSessions( + sessions: readonly JulesSession[], + options: DurableRemoteRecoveryOptions = {}, + ): DurableRemoteRecoveryResult { + const now = new Date().toISOString(); + const evidence = options.evidence ?? "remote_snapshot"; + const reactivatedTaskRunIds = new Set(); + const reactivatedSprintRunIds = new Set(); + + for (const session of sessions) { + if (!ACTIVE_JULES_SESSION_STATES.has(String(session.state || "").toUpperCase())) { + continue; + } + const sessionId = this.resolveSessionId(session); + if (!sessionId) { + continue; + } + const taskRun = this.deps.executionRepository.getLatestTaskRunBySessionId(sessionId); + if (!taskRun || taskRun.provider !== "jules" || taskRun.mode !== "jules") { + continue; + } + const task = this.deps.projectManagementRepository.getTask(taskRun.taskId); + if (!task || task.isMerged || task.status === "QA_REVIEW_FAILED") { + continue; + } + const rawSprintStatus = this.deps.projectManagementRepository.getRawSprintStatus(taskRun.sprintId); + if (rawSprintStatus === null || rawSprintStatus === "completed" || rawSprintStatus === "cancelled") { + continue; + } + const sprintRun = taskRun.sprintRunId + ? this.deps.executionRepository.getSprintRun(taskRun.sprintRunId) + : null; + if (!sprintRun || sprintRun.status === "completed" || sprintRun.status === "cancelled") { + continue; + } + + const dispatch = taskRun.dispatchId + ? this.deps.executionRepository.getTaskDispatch(taskRun.dispatchId) + : null; + let changed = false; + const taskRunNeedsReactivation = taskRun.state !== "RUNNING" || taskRun.finishedAt !== null; + if (taskRunNeedsReactivation) { + this.deps.executionRepository.updateTaskRun(taskRun.id, { + state: "RUNNING", + finishedAt: null, + durationMs: null, + }); + changed = true; + } + if (dispatch && ( + dispatch.status !== "running" + || dispatch.finishedAt !== null + || dispatch.errorMessage !== null + )) { + this.deps.executionRepository.updateTaskDispatch(dispatch.id, { + status: "running", + startedAt: dispatch.startedAt || taskRun.startedAt || now, + finishedAt: null, + lastHeartbeatAt: now, + errorMessage: null, + }); + changed = true; + } + if (task.status !== "in_progress") { + this.deps.projectManagementRepository.updateTask(task.id, { status: "in_progress" }); + changed = true; + } + if (sprintRun.status !== "running" || sprintRun.finishedAt !== null) { + this.deps.sprintRunLifecycleService.updateRun(sprintRun.id, { + status: "running", + startedAt: sprintRun.startedAt || now, + finishedAt: null, + lastHeartbeatAt: now, + }); + reactivatedSprintRunIds.add(sprintRun.id); + changed = true; + } + if (rawSprintStatus === "failed") { + this.deps.projectManagementRepository.updateSprint(taskRun.sprintId, { status: "running" }); + changed = true; + } + + const usage = this.deps.executionRepository.getLatestProviderInvocationUsageBySession( + sessionId, + "task_coding", + ); + if (usage?.provider === "jules") { + if (usage.status !== "running" || usage.finishedAt !== null) { + this.deps.executionRepository.updateProviderInvocationUsage(usage.id, { + status: "running", + finishedAt: null, + durationMs: null, + }); + changed = true; + } + for (const invocation of this.deps.executionRepository.listExecutionInvocationsByProviderInvocationId(usage.id)) { + if (invocation.status === "running" && invocation.finishedAt === null) { + continue; + } + this.deps.executionRepository.updateExecutionInvocation(invocation.id, { + status: "running", + finishedAt: null, + errorMessage: null, + lastErrorCategory: null, + lastErrorMessage: null, + lastRetryAfterIso: null, + }); + this.deps.executionRepository.appendExecutionInvocationMessage(invocation.id, { + role: "system", + contentMarkdown: evidence === "remote_snapshot" + ? "Restored the invocation because its Jules session is still active after restart." + : "Preserved the invocation while the Jules startup snapshot is unavailable. Session sync will verify the remote state.", + metadata: { + recovery: evidence === "remote_snapshot" + ? "startup_durable_remote_session_reactivated" + : "startup_durable_remote_session_preserved_unverified", + provider: "jules", + sessionId, + remoteState: evidence === "remote_snapshot" ? session.state || null : null, + }, + createdAt: now, + }); + changed = true; + } + } + + if (changed) { + this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "task_run_rehydrated", "system", { + reason: evidence === "remote_snapshot" + ? "durable_remote_session_still_active" + : "durable_remote_session_preserved_pending_verification", + provider: "jules", + sessionId, + remoteState: evidence === "remote_snapshot" ? session.state || null : null, + previousTaskRunState: taskRun.state, + previousDispatchStatus: dispatch?.status || null, + previousSprintRunStatus: sprintRun.status, + }, { + sourceEventKey: `startup-recovery:durable-remote-${evidence}:${taskRun.id}:${session.state || "active"}`, + }); + reactivatedTaskRunIds.add(taskRun.id); + } + } + + return { + reactivatedTaskRunIds: [...reactivatedTaskRunIds], + reactivatedSprintRunIds: [...reactivatedSprintRunIds], + }; + } + + private resolveSessionId(session: JulesSession): string | null { + const raw = (session.id || session.name || "").trim().replace(/^sessions\//, ""); + return raw || null; + } +} diff --git a/src/services/runtime-recovery/invocation-recovery.ts b/src/services/runtime-recovery/invocation-recovery.ts index 91a31541a0..5ebd7e72af 100644 --- a/src/services/runtime-recovery/invocation-recovery.ts +++ b/src/services/runtime-recovery/invocation-recovery.ts @@ -222,6 +222,17 @@ export class InvocationRecoveryService { invocation: ExecutionInvocationRecord, activeContainerSessionIds: ReadonlySet, ): { status: "completed" | "failed" | "cancelled"; message: string } | null { + const durableProviderInvocation = invocation.providerInvocationId + ? this.deps.executionRepository.getProviderInvocationUsage(invocation.providerInvocationId) + : null; + if (this.isRunningDurableRemoteInvocation(durableProviderInvocation)) { + // Hosted sessions survive the Code UX process. Local sprint/task + // projections can be terminal because of an interrupted scheduler, but + // they are not evidence that the remote work stopped. Session sync owns + // the authoritative terminal transition for this invocation. + return null; + } + const taskRun = invocation.taskRunId ? this.deps.executionRepository.getTaskRun(invocation.taskRunId) : null; if (taskRun && isTerminalTaskRunState(taskRun)) { return { @@ -373,6 +384,9 @@ export class InvocationRecoveryService { if (providerInvocation.purpose !== "task_coding" || providerInvocation.status !== "running") { return null; } + if (this.isRunningDurableRemoteInvocation(providerInvocation)) { + return null; + } if (providerInvocation.taskRunId) { const taskRun = this.deps.executionRepository.getTaskRun(providerInvocation.taskRunId); if (taskRun && !isTerminalTaskRunState(taskRun)) { @@ -441,6 +455,21 @@ export class InvocationRecoveryService { return null; } + private isRunningDurableRemoteInvocation( + invocation: ProviderInvocationUsageRecord | null, + ): boolean { + if ( + !invocation + || invocation.provider !== "jules" + || invocation.status !== "running" + || invocation.invocationSource !== "EXTERNAL_API" + ) { + return false; + } + const sessionId = (invocation.nativeSessionId || invocation.sessionId || "").trim(); + return sessionId.length > 0 && !sessionId.startsWith("jules-pending:"); + } + private resolveInterruptedStructuredInvocation( invocation: ExecutionInvocationRecord, activeContainerSessionIds: ReadonlySet, @@ -450,7 +479,7 @@ export class InvocationRecoveryService { const purpose = invocation.type === "qa_review" ? "QA review" : "planning"; if (!invocation.providerInvocationId) { - if (ageMs < QA_RUN_START_TIMEOUT_MS) { + if (invocation.type !== "planning" && ageMs < QA_RUN_START_TIMEOUT_MS) { return null; } return { @@ -461,7 +490,7 @@ export class InvocationRecoveryService { const providerInvocation = this.deps.executionRepository.getProviderInvocationUsage(invocation.providerInvocationId); if (!providerInvocation) { - if (ageMs < QA_RUN_START_TIMEOUT_MS) { + if (invocation.type !== "planning" && ageMs < QA_RUN_START_TIMEOUT_MS) { return null; } return { diff --git a/src/services/runtime-startup-recovery-service.ts b/src/services/runtime-startup-recovery-service.ts index 20ffb1b331..102cb6a4e7 100644 --- a/src/services/runtime-startup-recovery-service.ts +++ b/src/services/runtime-startup-recovery-service.ts @@ -4,6 +4,7 @@ import type { DashboardSettings, DashboardSettingsScope, DockerContainer, + JulesSession, ProviderId, RestartInvocationPolicy, RestartSprintPolicy, @@ -19,6 +20,7 @@ import type { ProjectAttentionService } from "../domain/workers/project-attentio import { sanitizeToken } from "./cli-workflow-utils.js"; import { QaReviewRecoveryService } from "./runtime-recovery/qa-review-recovery.js"; import { InvocationRecoveryService } from "./runtime-recovery/invocation-recovery.js"; +import { DurableRemoteRecoveryService } from "./runtime-recovery/durable-remote-recovery.js"; import { calculateInvocationDurationMs, isTerminalTaskRunState } from "./runtime-recovery/recovery-utils.js"; import { cancelStaleProviderInvocation, @@ -28,6 +30,10 @@ import { import type { GuardrailService } from "./guardrail-service.js"; import type { SprintRunLifecycleService } from "./sprint-run-lifecycle-service.js"; import { runCommandStrict } from "./cli-process-runner.js"; +import { + CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + isCliTaskRun, +} from "../domain/sprint/ci/cli-git-finalization.js"; const ACTIVE_SPRINT_RUN_STATUSES = ["queued", "running"] as const; const ACTIVE_DISPATCH_STATUSES = ["queued", "claimed", "running", "cancel_requested"] as const; @@ -39,6 +45,7 @@ const TERMINAL_PROVIDER_INVOCATION_STATUSES = new Set(["completed", "failed", "c const CLI_PROVIDERS = new Set(["gemini", "codex", "claude-code", "qwen-code", "opencode", "antigravity"]); const DURABLE_REMOTE_PROVIDERS = new Set(["jules"]); const QA_RUN_START_TIMEOUT_MS = 60_000; +const DURABLE_REMOTE_STARTUP_RECONCILIATION_TIMEOUT_MS = 5_000; interface RestartPolicies { sprintPolicy: RestartSprintPolicy; @@ -65,6 +72,7 @@ export interface RuntimeStartupRecoveryResult { reconciledStructuredInvocationIds: string[]; reconciledTaskCodingInvocationIds: string[]; reconciledTaskCodingProviderIds: string[]; + reconciledPostProviderTaskRunIds: string[]; reconciledTerminalProviderDispatchIds: string[]; reconciledTerminalDispatchIds: string[]; rehydratedSprintRunIds: string[]; @@ -80,6 +88,9 @@ export interface RuntimeStartupRecoveryResult { reconciledDuplicateDispatchIds: string[]; reconciledInterruptedRepairProviderInvocationIds: string[]; requeuedInterruptedRepairAttentionItemIds: string[]; + reactivatedDurableRemoteTaskRunIds: string[]; + reactivatedDurableRemoteSprintRunIds: string[]; + resumedPlanningInvocationIds: string[]; } interface RuntimeStartupRecoveryServiceDeps { @@ -89,12 +100,17 @@ interface RuntimeStartupRecoveryServiceDeps { qaReviewRepository?: QaReviewRepository; projectManagementRepository: ProjectManagementRepository; projectAttentionService?: Pick; - guardrailService?: Pick; + guardrailService?: Pick; sprintOrchestrator: SprintOrchestrator; dockerService?: Pick<{ listContainers: () => Promise; removeContainers: (containerIds: string[], options?: { removeVolumes?: boolean }) => Promise }, "listContainers"> & { removeContainers?: (containerIds: string[], options?: { removeVolumes?: boolean }) => Promise; }; getDashboardSettings?: (scope?: DashboardSettingsScope) => DashboardSettings; + listDurableRemoteSessions?: () => Promise; + resumeInterruptedPlanningInvocation?: ( + invocationId: string, + mode: "continue_session" | "retry_full_prompt", + ) => Promise; isProcessAlive?: (pid: number) => boolean; logger?: Logger; } @@ -123,6 +139,13 @@ export class RuntimeStartupRecoveryService { const restartPolicySyncedPausedSprintIds = this.syncPausedSprintProjections(); const restartPolicyResult = this.applyRestartSprintPolicy(restartPolicies.sprintPolicy); const shouldRecoverInterruptedInvocations = restartPolicies.sprintPolicy === "continue"; + const interruptedPlanningInvocationIds = shouldRecoverInterruptedInvocations + ? this.deps.executionRepository.listActiveExecutionInvocationsByTypes(["planning"]).map((invocation) => invocation.id) + : []; + const durableRemoteRecoveryResult = shouldRecoverInterruptedInvocations + && restartPolicies.invocationPolicy === "continue" + ? await this.reconcileDurableRemoteSessions() + : { reactivatedTaskRunIds: [], reactivatedSprintRunIds: [] }; const reconciledLocalDispatchIds = shouldRecoverInterruptedInvocations ? await this.reconcileInterruptedLocalDispatches(new Set(recoveredCliSessionIds), activeContainerSessionIds, restartPolicies.invocationPolicy) : []; @@ -135,10 +158,26 @@ export class RuntimeStartupRecoveryService { const reconciledContainerInvocationIds = shouldRecoverInterruptedInvocations ? await this.reconcileInterruptedCliInvocations(new Set(recoveredCliSessionIds), activeContainerSessionIds, restartPolicies.invocationPolicy) : []; + // A provider can finish while the outer CLI workflow is still committing and + // publishing its branch. Reclassify that exact restart window before generic + // terminal-invocation recovery treats the prematurely projected task run as + // authoritative and strands it outside the Git merge gate. + const reconciledPostProviderTaskRunIds = shouldRecoverInterruptedInvocations + ? this.reconcileRecoveredCliProviderCompletionsAwaitingGitFinalization( + new Set(recoveredCliSessionIds), + restartPolicies.invocationPolicy, + ) + : []; const reconciledQaReviewRunIds = await qaReviewRecovery.reconcileInterruptedQaReviewRuns(activeContainerSessionIds); const reconciledTerminalProviderLinkedInvocationIds = invocationRecovery.reconcileTerminalProviderLinkedInvocations(); const demotedPrematureMergeConflictEscalationIds = await this.demotePrematureMergeConflictEscalations(); const reconciledStructuredInvocationIds = await invocationRecovery.reconcileInterruptedStructuredInvocations(activeContainerSessionIds); + const resumedPlanningInvocationIds = shouldRecoverInterruptedInvocations + && restartPolicies.invocationPolicy !== "cancel" + ? this.resumeInterruptedPlanningInvocations([ + ...new Set([...interruptedPlanningInvocationIds, ...reconciledStructuredInvocationIds]), + ], restartPolicies.invocationPolicy) + : []; const rehydratedSprintRunIds = this.rehydrateDurableProviderSprintRuns(); const restartPolicySyncedOrphanedSprintIds = this.syncOrphanedRunningSprintProjections(); const reconciledTerminalProviderDispatchIds = this.reconcileTerminalProviderBackedDispatches(); @@ -173,11 +212,15 @@ export class RuntimeStartupRecoveryService { || reconciledStructuredInvocationIds.length > 0 || reconciledTaskCodingInvocationIds.length > 0 || reconciledTaskCodingProviderIds.length > 0 + || reconciledPostProviderTaskRunIds.length > 0 || reconciledTerminalProviderDispatchIds.length > 0 || reconciledTerminalDispatchIds.length > 0 || reconciledDuplicateDispatchIds.length > 0 || reconciledInterruptedRepairProviderInvocationIds.length > 0 || requeuedInterruptedRepairAttentionItemIds.length > 0 + || durableRemoteRecoveryResult.reactivatedTaskRunIds.length > 0 + || durableRemoteRecoveryResult.reactivatedSprintRunIds.length > 0 + || resumedPlanningInvocationIds.length > 0 || rehydratedSprintRunIds.length > 0 || reconciledTaskRunIds.length > 0 || reconciledPausedSprintRunIds.length > 0 @@ -200,11 +243,15 @@ export class RuntimeStartupRecoveryService { reconciledStructuredInvocations: reconciledStructuredInvocationIds.length, reconciledTaskCodingInvocations: reconciledTaskCodingInvocationIds.length, reconciledTaskCodingProviders: reconciledTaskCodingProviderIds.length, + reconciledPostProviderTaskRuns: reconciledPostProviderTaskRunIds.length, reconciledTerminalProviderDispatches: reconciledTerminalProviderDispatchIds.length, reconciledTerminalDispatches: reconciledTerminalDispatchIds.length, reconciledDuplicateDispatches: reconciledDuplicateDispatchIds.length, reconciledInterruptedRepairProviderInvocations: reconciledInterruptedRepairProviderInvocationIds.length, requeuedInterruptedRepairAttentionItems: requeuedInterruptedRepairAttentionItemIds.length, + reactivatedDurableRemoteTaskRuns: durableRemoteRecoveryResult.reactivatedTaskRunIds.length, + reactivatedDurableRemoteSprintRuns: durableRemoteRecoveryResult.reactivatedSprintRunIds.length, + resumedPlanningInvocations: resumedPlanningInvocationIds.length, rehydratedSprintRuns: rehydratedSprintRunIds.length, reconciledTaskRuns: reconciledTaskRunIds.length, reconciledPausedSprintRuns: reconciledPausedSprintRunIds.length, @@ -229,6 +276,7 @@ export class RuntimeStartupRecoveryService { reconciledStructuredInvocationIds, reconciledTaskCodingInvocationIds, reconciledTaskCodingProviderIds, + reconciledPostProviderTaskRunIds, reconciledTerminalProviderDispatchIds, reconciledTerminalDispatchIds, rehydratedSprintRunIds, @@ -239,6 +287,9 @@ export class RuntimeStartupRecoveryService { reconciledDuplicateDispatchIds, reconciledInterruptedRepairProviderInvocationIds, requeuedInterruptedRepairAttentionItemIds, + reactivatedDurableRemoteTaskRunIds: durableRemoteRecoveryResult.reactivatedTaskRunIds, + reactivatedDurableRemoteSprintRunIds: durableRemoteRecoveryResult.reactivatedSprintRunIds, + resumedPlanningInvocationIds, restartPolicyPausedSprintRunIds: restartPolicyResult.pausedSprintRunIds, restartPolicyCancelledSprintRunIds: restartPolicyResult.cancelledSprintRunIds, resumedSprintRunIds, @@ -246,6 +297,165 @@ export class RuntimeStartupRecoveryService { }; } + private resumeInterruptedPlanningInvocations( + reconciledInvocationIds: readonly string[], + invocationPolicy: RestartInvocationPolicy, + ): string[] { + if (!this.deps.resumeInterruptedPlanningInvocation || reconciledInvocationIds.length === 0) { + return []; + } + const latestBySprint = new Map(); + for (const invocationId of reconciledInvocationIds) { + const invocation = this.deps.executionRepository.getExecutionInvocation(invocationId); + if ( + !invocation + || invocation.type !== "planning" + || !invocation.sprintId + || (invocation.status !== "failed" && invocation.status !== "cancelled") + ) { + continue; + } + const existing = latestBySprint.get(invocation.sprintId); + if (!existing || Date.parse(invocation.startedAt) > Date.parse(existing.startedAt)) { + latestBySprint.set(invocation.sprintId, invocation); + } + } + + const resumedIds: string[] = []; + for (const invocation of latestBySprint.values()) { + resumedIds.push(invocation.id); + const mode = invocationPolicy === "restart" ? "retry_full_prompt" : "continue_session"; + this.deps.resumeInterruptedPlanningInvocation(invocation.id, mode).catch((error) => { + this.deps.logger?.error("Failed to resume interrupted planning invocation after startup", { + projectId: invocation.projectId, + sprintId: invocation.sprintId, + invocationId: invocation.id, + error: error instanceof Error ? error.message : String(error), + }); + }); + } + return resumedIds; + } + + private async reconcileDurableRemoteSessions(): Promise<{ + reactivatedTaskRunIds: string[]; + reactivatedSprintRunIds: string[]; + }> { + if (!this.deps.listDurableRemoteSessions) { + return { reactivatedTaskRunIds: [], reactivatedSprintRunIds: [] }; + } + const sessionsPromise = this.deps.listDurableRemoteSessions(); + try { + const sessions = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error( + `Durable remote session reconciliation timed out after ${DURABLE_REMOTE_STARTUP_RECONCILIATION_TIMEOUT_MS}ms`, + )); + }, DURABLE_REMOTE_STARTUP_RECONCILIATION_TIMEOUT_MS); + timeout.unref?.(); + sessionsPromise.then( + (result) => { + clearTimeout(timeout); + resolve(result); + }, + (error) => { + clearTimeout(timeout); + reject(error); + }, + ); + }); + return new DurableRemoteRecoveryService({ + executionRepository: this.deps.executionRepository, + projectManagementRepository: this.deps.projectManagementRepository, + sprintRunLifecycleService: this.deps.sprintRunLifecycleService, + }).reconcileActiveJulesSessions(sessions); + } catch (error) { + this.deps.logger?.warn("Could not reconcile durable remote sessions during startup", { + provider: "jules", + error: error instanceof Error ? error.message : String(error), + }); + const failSafeSessions = this.listLocallyDurableJulesSessions(); + const failSafeResult = new DurableRemoteRecoveryService({ + executionRepository: this.deps.executionRepository, + projectManagementRepository: this.deps.projectManagementRepository, + sprintRunLifecycleService: this.deps.sprintRunLifecycleService, + }).reconcileActiveJulesSessions(failSafeSessions, { evidence: "local_fail_safe" }); + + // A timeout bounds readiness, not the provider request. If that request + // eventually succeeds, repair any additional remotely-active sessions + // and attach their runs to the already-started orchestrator registry. + if (error instanceof Error && error.message.includes("timed out after")) { + void sessionsPromise.then((sessions) => { + const repaired = new DurableRemoteRecoveryService({ + executionRepository: this.deps.executionRepository, + projectManagementRepository: this.deps.projectManagementRepository, + sprintRunLifecycleService: this.deps.sprintRunLifecycleService, + }).reconcileActiveJulesSessions(sessions); + for (const sprintRunId of repaired.reactivatedSprintRunIds) { + void this.deps.sprintOrchestrator.recoverSprintRun(sprintRunId).catch((recoveryError) => { + this.deps.logger?.error("Failed to resume a late-reconciled durable remote sprint run", { + sprintRunId, + error: recoveryError instanceof Error ? recoveryError.message : String(recoveryError), + }); + }); + } + if (repaired.reactivatedTaskRunIds.length > 0 || repaired.reactivatedSprintRunIds.length > 0) { + this.deps.logger?.info("Reconciled durable remote sessions after startup readiness", { + provider: "jules", + reactivatedTaskRuns: repaired.reactivatedTaskRunIds.length, + reactivatedSprintRuns: repaired.reactivatedSprintRunIds.length, + }); + } + }).catch((lateError) => { + this.deps.logger?.warn("Late durable remote session reconciliation failed", { + provider: "jules", + error: lateError instanceof Error ? lateError.message : String(lateError), + }); + }); + } + + return failSafeResult; + } + } + + private listLocallyDurableJulesSessions(): JulesSession[] { + const executionRepository = this.deps.executionRepository as ExecutionRepository & { + listTaskRunsByStates?: (states: TaskRunRecord["state"][]) => TaskRunRecord[]; + }; + const candidates = new Map(); + if (typeof executionRepository.listTaskRunsByStates === "function") { + for (const taskRun of executionRepository.listTaskRunsByStates([...ACTIVE_TASK_RUN_STATES])) { + if (taskRun.provider === "jules" && taskRun.mode === "jules" && this.resolveTaskRunSessionKey(taskRun)) { + candidates.set(taskRun.id, taskRun); + } + } + } + for (const usage of this.deps.executionRepository.listRunningProviderInvocationUsages(["jules"])) { + if (usage.invocationSource !== "EXTERNAL_API" || !usage.taskRunId) { + continue; + } + const taskRun = this.deps.executionRepository.getTaskRun(usage.taskRunId); + if (taskRun?.provider === "jules" && taskRun.mode === "jules" && this.resolveTaskRunSessionKey(taskRun)) { + candidates.set(taskRun.id, taskRun); + } + } + + const sessions = new Map(); + for (const taskRun of candidates.values()) { + const sessionId = this.resolveTaskRunSessionKey(taskRun); + if (!sessionId || sessions.has(sessionId)) { + continue; + } + sessions.set(sessionId, { + id: sessionId, + name: taskRun.sessionName || `sessions/${sessionId}`, + prompt: "", + state: "IN_PROGRESS", + }); + } + return [...sessions.values()]; + } + private async requeueInterruptedVirtualRepairAttention(): Promise<{ attentionItemIds: string[]; providerInvocationIds: string[]; @@ -689,6 +899,118 @@ export class RuntimeStartupRecoveryService { return sessionName.replace(/^sessions\//, ""); } + /** + * The tracked CLI session covers the whole provider -> Git -> PR workflow, but + * task projections can observe provider completion before Git finalization has + * durably recorded its result. If shutdown lands in that window, a COMPLETED + * task run is not actually mergeable. Turn only recovered sessions with exact + * completed-provider evidence back into retryable runs; the replacement then + * resumes the preserved workspace at Git finalization. + */ + private reconcileRecoveredCliProviderCompletionsAwaitingGitFinalization( + recoveredCliSessionIds: ReadonlySet, + invocationPolicy: RestartInvocationPolicy, + ): string[] { + if (recoveredCliSessionIds.size === 0) { + return []; + } + + const candidates = this.deps.executionRepository.listTaskRunsByStates(["COMPLETED"]) + .filter((taskRun) => { + const sessionId = this.resolveTaskRunSessionKey(taskRun); + return Boolean(sessionId && recoveredCliSessionIds.has(sessionId) && isCliTaskRun(taskRun)); + }); + if (candidates.length === 0) { + return []; + } + + const eventsByTaskRunId = this.deps.executionRepository.listTaskRunEventsForRuns( + candidates.map((taskRun) => taskRun.id), + { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "cli_workflow_completed"], + limitPerRun: CLI_GIT_FINALIZATION_EVENT_SCAN_LIMIT, + }, + ); + const reconciledAt = new Date().toISOString(); + const reconciledTaskRunIds: string[] = []; + const retryTask = invocationPolicy !== "cancel"; + + for (const taskRun of candidates) { + const finalizationEvents = eventsByTaskRunId.get(taskRun.id) || []; + if (finalizationEvents.some((event) => ( + event.eventType === "cli_git_pushed" + || event.eventType === "cli_git_no_changes" + || event.eventType === "cli_workflow_completed" + ))) { + continue; + } + + const completedProviderInvocation = this.deps.executionRepository + .listProviderInvocationsForTask(taskRun.projectId, taskRun.taskId) + .filter((invocation) => ( + invocation.taskRunId === taskRun.id + && invocation.purpose === "task_coding" + && invocation.status === "completed" + )) + .sort((left, right) => this.providerInvocationActivityMs(right) - this.providerInvocationActivityMs(left))[0]; + if (!completedProviderInvocation) { + continue; + } + + const task = this.deps.projectManagementRepository.getTask(taskRun.taskId); + if (!task || task.isMerged) { + continue; + } + + const dispatch = taskRun.dispatchId + ? this.deps.executionRepository.getTaskDispatch(taskRun.dispatchId) + : null; + if (dispatch) { + this.deps.executionRepository.releaseLease("task_dispatch", dispatch.id); + this.deps.executionRepository.updateTaskDispatch(dispatch.id, { + connectionId: null, + status: "cancelled", + startedAt: dispatch.startedAt || taskRun.startedAt || reconciledAt, + finishedAt: reconciledAt, + lastHeartbeatAt: reconciledAt, + errorMessage: retryTask + ? null + : "Restart policy cancelled a CLI workflow before Git finalization completed.", + }); + } + + this.deps.executionRepository.updateTaskRun(taskRun.id, { + connectionId: null, + state: retryTask ? "FAILED" : "BLOCKED", + finishedAt: reconciledAt, + durationMs: calculateDurationMs(taskRun, reconciledAt), + }); + this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "task_dispatch_reconciled", "system", { + reason: "shutdown_interrupted_after_provider_completion", + providerInvocationId: completedProviderInvocation.id, + providerStatus: "completed", + previousTaskRunState: taskRun.state, + nextTaskRunState: retryTask ? "FAILED" : "BLOCKED", + previousDispatchStatus: dispatch?.status || null, + nextDispatchStatus: dispatch ? "cancelled" : null, + }, { + sourceEventKey: `startup-recovery:post-provider-git-finalization:${taskRun.id}`, + }); + + if (retryTask) { + this.refundRestartInterruptedCodingAttempt(taskRun, taskRun.projectId, taskRun.taskId); + this.resetTaskToPending(taskRun.taskId); + } else { + this.deps.projectManagementRepository.updateTask(taskRun.taskId, { + status: "QA_REVIEW_FAILED", + }); + } + reconciledTaskRunIds.push(taskRun.id); + } + + return reconciledTaskRunIds; + } + private reconcileInterruptedTaskRuns(): string[] { const executionRepository = this.deps.executionRepository as ExecutionRepository & { listTaskRunsByStates?: (states: TaskRunRecord["state"][]) => TaskRunRecord[]; @@ -1071,7 +1393,7 @@ export class RuntimeStartupRecoveryService { if (!interruptionReason) { continue; } - if (invocationPolicy !== "continue") { + if (invocationPolicy !== "continue" || invocation.purpose === "planning") { await this.removeContainersForSessions(new Set([invocation.sessionId])); } @@ -1247,6 +1569,7 @@ export class RuntimeStartupRecoveryService { } if (retryTask) { + this.refundRestartInterruptedCodingAttempt(taskRun, dispatch.projectId, dispatch.taskId); this.resetTaskToPending(dispatch.taskId); } else { this.deps.projectManagementRepository.updateTask(dispatch.taskId, { @@ -1780,6 +2103,10 @@ export class RuntimeStartupRecoveryService { return `Restart policy restarted ${invocation.purpose} invocation after Code UX restart. Session ${invocation.sessionId} was stopped so orchestration can dispatch a fresh attempt.`; } + if (invocation.purpose === "planning") { + return `Recovered interrupted planning invocation after Code UX restart. The local provider process cannot be reattached, so the durable planning request will continue in a replacement invocation.`; + } + if (recoveredCliSessionIds.has(invocation.sessionId)) { return `Recovered stale ${invocation.purpose} invocation after Code UX restart. The backing CLI session (${invocation.sessionId}) was interrupted before completion.`; } @@ -1837,8 +2164,10 @@ export class RuntimeStartupRecoveryService { } } - if (invocation.taskRunId) { - const taskRun = this.deps.executionRepository.getTaskRun(invocation.taskRunId); + const taskRun = invocation.taskRunId + ? this.deps.executionRepository.getTaskRun(invocation.taskRunId) + : null; + if (taskRun) { if (taskRun && !isTerminalTaskRunState(taskRun)) { this.deps.executionRepository.updateTaskRun(taskRun.id, { connectionId: null, @@ -1847,20 +2176,19 @@ export class RuntimeStartupRecoveryService { durationMs: calculateDurationMs(taskRun, reconciledAt), }); } - if (taskRun) { - this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "cli_workflow_cancelled", "system", { - dispatchId: invocation.dispatchId || null, - providerInvocationId: invocation.id, - reason: "runtime_restart_interrupted", - recoveredSessionId: invocation.sessionId, - message: failureReason, - }, { - sourceEventKey: `startup-recovery:cli-invocation:${invocation.id}:${taskRun.id}`, - }); - } + this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "cli_workflow_cancelled", "system", { + dispatchId: invocation.dispatchId || null, + providerInvocationId: invocation.id, + reason: "runtime_restart_interrupted", + recoveredSessionId: invocation.sessionId, + message: failureReason, + }, { + sourceEventKey: `startup-recovery:cli-invocation:${invocation.id}:${taskRun.id}`, + }); } if (retryTask) { + this.refundRestartInterruptedCodingAttempt(taskRun, invocation.projectId, invocation.taskId); this.resetTaskToPending(invocation.taskId); } else { this.deps.projectManagementRepository.updateTask(invocation.taskId, { @@ -1903,8 +2231,10 @@ export class RuntimeStartupRecoveryService { } } - if (invocation.taskRunId) { - const taskRun = this.deps.executionRepository.getTaskRun(invocation.taskRunId); + const taskRun = invocation.taskRunId + ? this.deps.executionRepository.getTaskRun(invocation.taskRunId) + : null; + if (taskRun) { if (taskRun && !isTerminalTaskRunState(taskRun)) { this.deps.executionRepository.updateTaskRun(taskRun.id, { connectionId: null, @@ -1913,20 +2243,19 @@ export class RuntimeStartupRecoveryService { durationMs: calculateDurationMs(taskRun, reconciledAt), }); } - if (taskRun) { - this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "cli_workflow_cancelled", "system", { - dispatchId: invocation.dispatchId || null, - executionInvocationId: invocation.id, - providerInvocationId: invocation.providerInvocationId || null, - reason: "runtime_restart_interrupted_retry_wait", - message: failureReason, - }, { - sourceEventKey: `startup-recovery:retry-wait:${invocation.id}:${taskRun.id}`, - }); - } + this.deps.executionRepository.appendTaskRunEvent(taskRun.id, "cli_workflow_cancelled", "system", { + dispatchId: invocation.dispatchId || null, + executionInvocationId: invocation.id, + providerInvocationId: invocation.providerInvocationId || null, + reason: "runtime_restart_interrupted_retry_wait", + message: failureReason, + }, { + sourceEventKey: `startup-recovery:retry-wait:${invocation.id}:${taskRun.id}`, + }); } if (retryTask) { + this.refundRestartInterruptedCodingAttempt(taskRun, invocation.projectId, invocation.taskId); this.resetTaskToPending(invocation.taskId); } else { this.deps.projectManagementRepository.updateTask(invocation.taskId, { @@ -1952,6 +2281,23 @@ export class RuntimeStartupRecoveryService { }).cliWorkflow.executionMode; } + private refundRestartInterruptedCodingAttempt( + taskRun: TaskRunRecord | null, + projectId: string, + taskId: string, + ): void { + if (!taskRun || (!taskRun.sessionId && !taskRun.sessionName)) { + return; + } + this.deps.guardrailService?.refund( + { projectId, sprintId: taskRun.sprintId }, + taskId, + "task_coding", + `runtime-restart:${taskRun.id}`, + "runtime_restart_interrupted", + ); + } + private resetTaskToPending(taskId: string): void { this.deps.projectManagementRepository.updateTask(taskId, { status: "pending", diff --git a/src/services/shutdown-container-service.ts b/src/services/shutdown-container-service.ts index 94808727f3..873e021988 100644 --- a/src/services/shutdown-container-service.ts +++ b/src/services/shutdown-container-service.ts @@ -2,6 +2,8 @@ import { execFile } from "node:child_process"; import type { ActiveDispatchRegistry } from "./active-dispatch-registry.js"; import { SERVER_SHUTDOWN_STOP_REASON } from "./active-dispatch-registry.js"; import type { Logger } from "../shared/logging/logger.js"; +import { getRuntimeOwnerId, RUNTIME_OWNER_LABEL } from "../shared/config/runtime-owner.js"; +import { AsyncSemaphore } from "../shared/async-semaphore.js"; export type ShutdownCommandRunner = (command: string, args: string[], cwd: string) => Promise<{ stdout: string }>; @@ -23,43 +25,25 @@ export interface ShutdownContainerStopResult { } const DOCKER_SHUTDOWN_COMMAND_TIMEOUT_MS = 5_000; +const DOCKER_SHUTDOWN_REMOVE_BATCH_SIZE = 8; +const DOCKER_SHUTDOWN_REMOVE_CONCURRENCY = 4; export class ShutdownContainerService { + private readonly removalSemaphore = new AsyncSemaphore(DOCKER_SHUTDOWN_REMOVE_CONCURRENCY); + constructor(private readonly deps: ShutdownContainerServiceDeps) {} async stopRunningContainers(reason = SERVER_SHUTDOWN_STOP_REASON): Promise { const requestedDispatchStops = await this.requestActiveDispatchStops(reason); - const containers = await this.listRunningCodeUxContainers(); - const killedContainerIds: string[] = []; - - for (const container of containers) { - await this.runCommand("docker", ["kill", container.id]) - .then(() => { - killedContainerIds.push(container.id); - }) - .catch((error: unknown) => { - this.deps.logger?.warn("Failed to kill Code UX container during shutdown", { - containerId: container.id, - containerName: container.names, - error: error instanceof Error ? error.message : String(error), - }); - }); - } - - if (requestedDispatchStops > 0 || killedContainerIds.length > 0) { - this.deps.logger?.info("Stopped Code UX containers during shutdown", { - requestedDispatchStops, - killedContainerIds, - }); - } - - return { - requestedDispatchStops, - killedContainerIds, - }; + return await this.stopRemainingContainers(requestedDispatchStops); } - private async requestActiveDispatchStops(reason: string): Promise { + /** + * Signals active workflows before shutdown starts draining helper pools. Keeping this operation + * separate lets the server overlap workflow cancellation with helper quiescence instead of + * waiting for commands that have not yet been told to stop. + */ + async requestActiveDispatchStops(reason = SERVER_SHUTDOWN_STOP_REASON): Promise { const handles = this.deps.activeDispatchRegistry.listHandles(); await Promise.all(handles.map(async (handle) => { const result = await Promise.resolve(handle.requestStop(reason)).catch((error: unknown) => { @@ -81,8 +65,38 @@ export class ShutdownContainerService { return handles.length; } - private async listRunningCodeUxContainers(): Promise { - const result = await this.runCommand("docker", ["ps", "--format", "{{json .}}"]) + /** Removes owner-scoped containers after active dispatch cancellation has already been sent. */ + async stopRemainingContainers(requestedDispatchStops: number): Promise { + const containers = await this.listCodeUxContainers(); + const killedContainerIds: string[] = []; + + if (containers.length > 0) { + // `docker run --rm` cannot remove a container interrupted between daemon + // creation and process start. Force-remove every owner-scoped container so + // running, exited, dead, and never-started generations share one cleanup path. + // Small bounded-parallel batches keep restart latency low without letting a full 16-task + // wave turn one oversized Docker request into a five-second timeout. A failed batch falls + // back to isolated removals so one daemon race cannot hide the other container outcomes. + const batches = this.toContainerBatches(containers); + const removed = await Promise.all(batches.map((batch) => this.removeContainerBatch(batch))); + killedContainerIds.push(...removed.flat()); + } + + if (requestedDispatchStops > 0 || killedContainerIds.length > 0) { + this.deps.logger?.info("Stopped Code UX containers during shutdown", { + requestedDispatchStops, + killedContainerIds, + }); + } + + return { + requestedDispatchStops, + killedContainerIds, + }; + } + + private async listCodeUxContainers(): Promise { + const result = await this.runCommand("docker", ["ps", "-a", "--format", "{{json .}}"]) .catch((error: unknown) => { this.deps.logger?.warn("Failed to inspect Docker containers during shutdown", { error: error instanceof Error ? error.message : String(error), @@ -105,6 +119,11 @@ export class ShutdownContainerService { } private isCodeUxContainer(container: ShutdownContainerSummary): boolean { + // Multiple isolated Code UX runtimes can share one Docker daemon (for example a live app and + // a local stress test). Never let one runtime's shutdown kill another runtime's containers. + if (container.labels[RUNTIME_OWNER_LABEL] !== getRuntimeOwnerId()) { + return false; + } if (Object.keys(container.labels).some((key) => key.startsWith("code-ux."))) { return true; } @@ -150,6 +169,55 @@ export class ShutdownContainerService { return labels; } + private isIdempotentContainerRemovalError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /no such container|is not running|removal of container .* is already in progress/i.test(message); + } + + private toContainerBatches(containers: ShutdownContainerSummary[]): ShutdownContainerSummary[][] { + const batches: ShutdownContainerSummary[][] = []; + for (let index = 0; index < containers.length; index += DOCKER_SHUTDOWN_REMOVE_BATCH_SIZE) { + batches.push(containers.slice(index, index + DOCKER_SHUTDOWN_REMOVE_BATCH_SIZE)); + } + return batches; + } + + private async removeContainerBatch(batch: ShutdownContainerSummary[]): Promise { + const ids = batch.map((container) => container.id); + try { + await this.runRemovalCommand(ids); + return ids; + } catch (error) { + if (this.isIdempotentContainerRemovalError(error)) { + return ids; + } + } + + const results = await Promise.all(batch.map(async (container) => { + try { + await this.runRemovalCommand([container.id]); + return container.id; + } catch (error) { + if (this.isIdempotentContainerRemovalError(error)) { + return container.id; + } + this.deps.logger?.warn("Failed to kill Code UX container during shutdown", { + containerIds: [container.id], + containerNames: [container.names], + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + })); + return results.filter((id): id is string => id !== null); + } + + private async runRemovalCommand(containerIds: string[]): Promise { + await this.removalSemaphore.run(async () => { + await this.runCommand("docker", ["rm", "-f", "-v", ...containerIds]); + }); + } + private async runCommand(command: string, args: string[]): Promise<{ stdout: string }> { if (this.deps.commandRunner) { return this.deps.commandRunner(command, args, process.cwd()); diff --git a/src/services/sprint-file-browser-service.ts b/src/services/sprint-file-browser-service.ts index c97330a121..5477a50b52 100644 --- a/src/services/sprint-file-browser-service.ts +++ b/src/services/sprint-file-browser-service.ts @@ -34,6 +34,7 @@ import { fetchOriginIfAvailable } from "./git-branch-sync-service.js"; import { buildGitHttpAuthEnvForRepoWithFallbacks, type GitHttpAuthOptions } from "./git-http-auth.js"; import { resolveLanguageForPath } from "./file-browser-language.js"; import { MAX_TREE_ENTRIES, MAX_FILE_BYTES, PRUNED_DIRECTORIES, normalizeAndValidatePath, isPrunedPath } from "./file-browser-scan-policy.js"; +import { getRuntimeOwnerDockerArgs, getRuntimeOwnerLabel } from "../shared/config/runtime-owner.js"; const FILE_BROWSER_LABEL = "code-ux.file-browser=true"; const FILE_BROWSER_IMAGE = "alpine:3.20"; @@ -157,6 +158,7 @@ export class SprintFileBrowserService { ...DOCKER_DROP_ALL_CAPS_ARGS, "--label", FILE_BROWSER_LABEL, "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", `code-ux.project-id=${projectId}`, "--label", `code-ux.sprint-id=${sprintId}`, "--label", `code-ux.session-id=${session.id}`, @@ -399,6 +401,7 @@ export class SprintFileBrowserService { "create", "--label", "code-ux.file-browser-volume=true", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", `code-ux.project-id=${projectId}`, "--label", `code-ux.sprint-id=${sprintId}`, "--label", `code-ux.session-id=${sessionId}`, @@ -903,6 +906,7 @@ export class SprintFileBrowserService { "ps", "-a", "--filter", `label=${FILE_BROWSER_LABEL}`, + "--filter", `label=${getRuntimeOwnerLabel()}`, "--format", "{{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Label \"code-ux.project-id\"}}\t{{.Label \"code-ux.sprint-id\"}}\t{{.Label \"code-ux.session-id\"}}", ], diff --git a/src/services/sprint-preview-docker-plan.ts b/src/services/sprint-preview-docker-plan.ts index 8f12e2f18e..a4f8b6ba3f 100644 --- a/src/services/sprint-preview-docker-plan.ts +++ b/src/services/sprint-preview-docker-plan.ts @@ -5,6 +5,7 @@ import { toDockerMountArg, } from "./cli-docker-utils.js"; import type { SprintPreviewPortMapping } from "../contracts/app-types.js"; +import { getRuntimeOwnerDockerArgs } from "../shared/config/runtime-owner.js"; export const CONTAINER_PREVIEW_PROXY_PORT = 39000; export const CONTAINER_PREVIEW_RUNTIME_ROOT = "/code-ux-preview-runtime"; @@ -64,6 +65,7 @@ export function buildSprintPreviewDockerCreateArgs(args: SprintPreviewDockerPlan ]), "--workdir", args.containerWorkspacePath, "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", "code-ux.preview=true", "--label", `code-ux.project-id=${args.projectId}`, "--label", `code-ux.sprint-id=${args.sprintId}`, diff --git a/src/services/sprint-preview-service.ts b/src/services/sprint-preview-service.ts index 6aea2d240e..4bf87357b0 100644 --- a/src/services/sprint-preview-service.ts +++ b/src/services/sprint-preview-service.ts @@ -51,6 +51,7 @@ import { ensureDefaultCodeUxAssetsInstalled } from "./code-ux-default-assets-ser import { fetchOriginIfAvailable } from "./git-branch-sync-service.js"; import { buildGitHttpAuthEnvForRepoWithFallbacks, type GitHttpAuthOptions } from "./git-http-auth.js"; import { mergePreviewEnvironmentVariables, sanitizePreviewEnvironmentVariables } from "../shared/preview-environment.js"; +import { getRuntimeOwnerDockerArgs, getRuntimeOwnerLabel } from "../shared/config/runtime-owner.js"; const BUNDLED_CONTAINER_SETUP_SCRIPT = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -440,6 +441,7 @@ export class SprintPreviewService { "create", "--label", "code-ux.preview-volume=true", "--label", "code-ux.managed=true", + ...getRuntimeOwnerDockerArgs(), "--label", `code-ux.project-id=${projectId}`, "--label", `code-ux.sprint-id=${sprintId}`, "--label", `code-ux.session-id=${sessionId}`, @@ -1203,7 +1205,7 @@ export class SprintPreviewService { try { const result = await runCommandStrict( "docker", - ["ps", "-aq", "--filter", "label=code-ux.preview=true"], + ["ps", "-aq", "--filter", "label=code-ux.preview=true", "--filter", `label=${getRuntimeOwnerLabel()}`], process.cwd(), ); return result.stdout @@ -1223,6 +1225,7 @@ export class SprintPreviewService { "ps", "-a", "--filter", "label=code-ux.preview=true", + "--filter", `label=${getRuntimeOwnerLabel()}`, "--format", "{{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Label \"code-ux.project-id\"}}\t{{.Label \"code-ux.sprint-id\"}}\t{{.Label \"code-ux.session-id\"}}\t{{.Label \"code-ux.host-port\"}}", ], diff --git a/src/services/sprint-task-dispatch-service.ts b/src/services/sprint-task-dispatch-service.ts index 42e9d94719..883969c58e 100644 --- a/src/services/sprint-task-dispatch-service.ts +++ b/src/services/sprint-task-dispatch-service.ts @@ -7,6 +7,10 @@ import { TaskService } from "./task-service.js"; import type { GuardrailService } from "./guardrail-service.js"; import type { ProviderConcurrencyService } from "./provider-concurrency-service.js"; import type { Logger } from "../shared/logging/logger.js"; +import { + isJulesSessionCapacityError, + isJulesSessionConsumingConcurrentTask, +} from "../integrations/jules-api-client.js"; /** * Thrown when a task cannot be dispatched because the provider's global concurrency cap is @@ -23,6 +27,23 @@ export class ProviderCapReachedError extends Error { } } +export class ProviderCapacityCheckUnavailableError extends Error { + readonly retryableDispatchDeferral = true; + readonly deferralReason = "provider_concurrency_cap" as const; + + constructor( + public readonly provider: string, + public readonly limit: number, + public readonly currentCount: number, + cause: unknown, + ) { + super(`Provider capacity could not be verified for ${provider}; task deferred to avoid exceeding limit ${limit}.`, { + cause, + }); + this.name = "ProviderCapacityCheckUnavailableError"; + } +} + export interface TaskDispatchDeferral { reason: "provider_concurrency_cap"; provider?: string; @@ -92,8 +113,16 @@ const DUPLICATE_BLOCKING_TASK_DISPATCH_STATUSES = new Set(); + constructor( private readonly executionRepository: ExecutionRepository, private readonly projectManagementRepository: ProjectManagementRepository, @@ -102,6 +131,7 @@ export class SprintTaskDispatchService { private readonly providerConcurrencyService: ProviderConcurrencyService, private readonly getDashboardSettings: (scope?: DashboardSettingsScope) => DashboardSettings, private readonly logger?: Logger, + private readonly listJulesSessionsForCapacity?: () => Promise, ) {} async startTask(args: StartSprintDispatchArgs): Promise { @@ -148,12 +178,27 @@ export class SprintTaskDispatchService { : undefined; const limit = providerSettings?.maxConcurrentTasks ?? 0; - if (provider && limit > 0) { + if (provider) { const counts = this.providerConcurrencyService.getGlobalRunningCounts([provider]); const currentCount = counts[provider] || 0; - if (currentCount >= limit) { + if (limit > 0 && currentCount >= limit) { throw this.deferForProviderCapacity(args, taskRecordId, provider, executorType, limit, currentCount); } + const providerBackoff = this.providerReportedCapacityBackoff.get(provider); + if ( + providerBackoff + && Date.now() < providerBackoff.retryAfterMs + && currentCount >= providerBackoff.maxRunningBeforeProbe + ) { + throw this.deferForProviderCapacity( + args, + taskRecordId, + provider, + executorType, + providerBackoff.maxRunningBeforeProbe, + currentCount, + ); + } } // Jules sessions run remotely and are not gated by the CLI execution path's atomic slot @@ -166,11 +211,19 @@ export class SprintTaskDispatchService { ? await this.claimJulesSlot(args, taskRecordId, settingsScope) : null; } catch (error) { - if (error instanceof ProviderCapReachedError) { + const deferral = getTaskDispatchDeferral(error); + if (deferral) { const pStr = provider || "jules"; const counts = this.providerConcurrencyService.getGlobalRunningCounts([pStr]); const currentCount = counts[pStr] || 0; - throw this.deferForProviderCapacity(args, taskRecordId, pStr, executorType, error.limit, currentCount); + throw this.deferForProviderCapacity( + args, + taskRecordId, + pStr, + executorType, + deferral.limit ?? 0, + Math.max(currentCount, deferral.currentCount ?? 0), + ); } throw error; } @@ -282,6 +335,9 @@ export class SprintTaskDispatchService { const sessionName = session.name || null; const sessionId = session.id || null; const nextProvider = session.provider || provider; + if (nextProvider) { + this.providerReportedCapacityBackoff.delete(nextProvider); + } // Re-key the claimed concurrency slot onto the real Jules session id so the session-sync // terminal handler can release it when the session completes or fails. @@ -331,9 +387,67 @@ export class SprintTaskDispatchService { provider: nextProvider || undefined, }; } catch (error) { - const deferral = getTaskDispatchDeferral(error); + let deferral = getTaskDispatchDeferral(error); + if (!deferral && executorType === "jules" && isJulesSessionCapacityError(error)) { + const counts = this.providerConcurrencyService.getGlobalRunningCounts(["jules"]); + const claimedCount = counts.jules || 0; + const currentCount = Math.max(0, claimedCount - (julesClaim?.status === "running" ? 1 : 0)); + this.providerReportedCapacityBackoff.set("jules", { + maxRunningBeforeProbe: currentCount, + retryAfterMs: Date.now() + PROVIDER_REPORTED_CAPACITY_RETRY_MS, + }); + deferral = { + reason: "provider_concurrency_cap", + provider: "jules", + limit: currentCount, + currentCount, + }; + this.logger?.info("Jules task dispatch deferred after the provider reported session capacity", { + projectId: args.projectId, + sprintId: args.sprintId, + sprintRunId: args.sprintRunId, + taskId: taskRecordId, + currentCount, + retryAfterMs: PROVIDER_REPORTED_CAPACITY_RETRY_MS, + providerError: error instanceof Error ? error.message : String(error), + }); + } + if (!deferral && executorType === "jules" && this.isGenericJulesPreconditionError(error)) { + // Jules currently returns a generic FAILED_PRECONDITION with no quota + // detail when createSession loses a subscription-capacity race. The + // list API has no state filter/count endpoint and may place old running + // sessions beyond its first page, so a bounded post-error snapshot + // cannot disprove the provider's authoritative rejection. Always make + // this retryable: INVALID_ARGUMENT remains the terminal validation path. + const confirmedCapacity = await this.confirmJulesCapacityAfterRejectedCreate( + limit, + julesClaim?.id || null, + ).catch(() => null); + const locallyRunning = this.providerConcurrencyService.getGlobalRunningCounts(["jules"]).jules || 0; + const currentCount = Math.max(limit, locallyRunning, confirmedCapacity?.currentCount ?? 0); + this.providerReportedCapacityBackoff.set("jules", { + maxRunningBeforeProbe: Math.max(0, locallyRunning - (julesClaim?.status === "running" ? 1 : 0)), + retryAfterMs: Date.now() + PROVIDER_REPORTED_CAPACITY_RETRY_MS, + }); + deferral = { + reason: "provider_concurrency_cap", + provider: "jules", + limit, + currentCount, + }; + this.logger?.info("Jules generic precondition response treated as retryable provider capacity", { + projectId: args.projectId, + sprintId: args.sprintId, + sprintRunId: args.sprintRunId, + taskId: taskRecordId, + configuredLimit: limit, + currentCount, + capacitySnapshotConfirmed: confirmedCapacity !== null, + }); + } if (deferral) { const deferredProvider = deferral.provider || provider || executorType; + this.releaseJulesClaimForDeferral(julesClaim, julesExecutionInvocation?.id || null, error); throw this.deferForProviderCapacity( args, taskRecordId, @@ -394,6 +508,41 @@ export class SprintTaskDispatchService { } } + private releaseJulesClaimForDeferral( + claim: ProviderInvocationUsageRecord | null, + executionInvocationId: string | null, + cause: unknown, + ): void { + if (!claim) { + return; + } + const deferredAt = new Date().toISOString(); + this.executionRepository.updateProviderInvocationUsage(claim.id, { + status: "cancelled", + finishedAt: deferredAt, + }); + if (!executionInvocationId) { + return; + } + this.executionRepository.updateExecutionInvocation(executionInvocationId, { + status: "cancelled", + finishedAt: deferredAt, + errorMessage: null, + lastErrorMessage: null, + }); + this.executionRepository.appendExecutionInvocationMessage(executionInvocationId, { + role: "system", + contentMarkdown: "Jules dispatch deferred because the provider reported that session capacity is currently full.", + metadata: { + provider: "jules", + model: "jules-agent", + kind: "dispatch_deferred", + providerMessage: cause instanceof Error ? cause.message : String(cause), + }, + createdAt: deferredAt, + }); + } + private canRecordStartedSession(sprintRunId: string, dispatchId: string): boolean { const sprintRun = this.executionRepository.getSprintRun(sprintRunId); if (sprintRun?.status === "cancelled" || sprintRun?.status === "failed" || sprintRun?.status === "completed" || sprintRun?.status === "cancel_requested") { @@ -419,7 +568,8 @@ export class SprintTaskDispatchService { ?? Object.values(settings.aiProvider.providers).find((entry) => entry.provider === "jules"); const limit = julesSettings?.maxConcurrentTasks ?? 0; - const claim = await this.providerConcurrencyService.tryClaimSlot("jules" as ProviderId, limit, { + const admission = await this.resolveJulesAdmission(limit); + const claim = await this.providerConcurrencyService.tryClaimSlot("jules" as ProviderId, admission.localLimit, { projectId: args.projectId, sprintId: args.sprintId, taskId: taskRecordId, @@ -434,13 +584,129 @@ export class SprintTaskDispatchService { if (!claim) { const counts = this.providerConcurrencyService.getGlobalRunningCounts(["jules"]); - const currentCount = counts["jules"] || 0; + const currentCount = (counts["jules"] || 0) + admission.remoteOnlyCount; throw new ProviderCapReachedError("jules", limit, currentCount); } return claim; } + private async resolveJulesAdmission(limit: number, excludedLocalInvocationId?: string | null): Promise<{ + localLimit: number; + remoteOnlyCount: number; + }> { + if (limit <= 0 || !this.listJulesSessionsForCapacity) { + return { localLimit: limit, remoteOnlyCount: 0 }; + } + + let sessions: JulesSession[]; + try { + sessions = await this.listJulesSessionsForCapacity(); + } catch (error) { + const localCount = this.providerConcurrencyService.getGlobalRunningCounts(["jules"]).jules || 0; + this.logger?.warn("Jules dispatch deferred because remote session capacity could not be verified", { + provider: "jules", + configuredLimit: limit, + localRunningCount: localCount, + error: error instanceof Error ? error.message : String(error), + }); + throw new ProviderCapacityCheckUnavailableError("jules", limit, localCount, error); + } + + const remoteActive = this.uniqueActiveJulesSessions(sessions); + const localRunning = this.executionRepository.listRunningProviderInvocationUsages(["jules"]) + .filter((invocation) => invocation.id !== excludedLocalInvocationId); + const localSessionIds = new Set(); + for (const invocation of localRunning) { + this.addJulesSessionIdentity(localSessionIds, invocation.sessionId); + this.addJulesSessionIdentity(localSessionIds, invocation.nativeSessionId); + } + const remoteOnlyCount = remoteActive.filter((session) => { + const identities = new Set(); + this.addJulesSessionIdentity(identities, session.id); + this.addJulesSessionIdentity(identities, session.name); + return !Array.from(identities).some((identity) => localSessionIds.has(identity)); + }).length; + const totalRunningCount = localRunning.length + remoteOnlyCount; + + if (totalRunningCount >= limit) { + throw new ProviderCapReachedError("jules", limit, totalRunningCount); + } + + if (remoteOnlyCount > 0) { + this.logger?.info("Jules admission included remotely active sessions outside local runtime accounting", { + provider: "jules", + configuredLimit: limit, + remoteActiveCount: remoteActive.length, + remoteOnlyCount, + localRunningCount: localRunning.length, + availableSlots: Math.max(0, limit - totalRunningCount), + }); + } + + // Atomic local claims now enforce the subscription limit after reserving + // room for sessions that are active in Jules but absent from this DB. + return { + localLimit: Math.max(0, limit - remoteOnlyCount), + remoteOnlyCount, + }; + } + + private isGenericJulesPreconditionError(error: unknown): boolean { + if (!error || typeof error !== "object") { + return false; + } + const candidate = error as { status?: unknown; apiStatus?: unknown; message?: unknown }; + return candidate.status === 400 + && String(candidate.apiStatus || "").toUpperCase() === "FAILED_PRECONDITION" + && String(candidate.message || "").toLowerCase().includes("precondition"); + } + + private async confirmJulesCapacityAfterRejectedCreate( + limit: number, + excludedLocalInvocationId: string | null, + ): Promise<{ currentCount: number } | null> { + if (limit <= 0 || !this.listJulesSessionsForCapacity) { + return null; + } + try { + await this.resolveJulesAdmission(limit, excludedLocalInvocationId); + return null; + } catch (error) { + const deferral = getTaskDispatchDeferral(error); + if (!deferral) { + throw error; + } + return { currentCount: deferral.currentCount ?? limit }; + } + } + + private uniqueActiveJulesSessions(sessions: readonly JulesSession[]): JulesSession[] { + const unique = new Map(); + for (const session of sessions) { + if (!isJulesSessionConsumingConcurrentTask(session)) { + continue; + } + const key = session.id || session.name; + if (key && !unique.has(key)) { + unique.set(key, session); + } + } + return Array.from(unique.values()); + } + + private addJulesSessionIdentity(target: Set, value: string | null | undefined): void { + const normalized = value?.trim(); + if (!normalized) { + return; + } + target.add(normalized); + const slashIndex = normalized.lastIndexOf("/"); + if (slashIndex >= 0 && slashIndex < normalized.length - 1) { + target.add(normalized.slice(slashIndex + 1)); + } + } + private requireTaskRecordId(task: Subtask): string { if (typeof task.record_id === "string" && task.record_id.trim().length > 0) { return task.record_id; diff --git a/src/services/structured-agent-request-service.ts b/src/services/structured-agent-request-service.ts index 9f57a33bba..a92d13fa2c 100644 --- a/src/services/structured-agent-request-service.ts +++ b/src/services/structured-agent-request-service.ts @@ -61,6 +61,7 @@ export interface StructuredRequestArgs { sessionIdPrefix: string; logicalSessionId?: string; continueSessionId?: string | null; + allowFreshSessionFallback?: boolean; openCodeBaselineRawUsageJson?: Record | null; invocationId?: string; systemRoutingMessage?: string; @@ -195,6 +196,7 @@ export class StructuredAgentRequestService { signal: args.signal, invocationId, continueSessionId: args.continueSessionId, + allowFreshSessionFallback: args.allowFreshSessionFallback, openCodeBaselineRawUsageJson: args.openCodeBaselineRawUsageJson, onActivity: args.onActivity, agentMcpAccess: args.agentMcpAccess, diff --git a/src/shared/config/runtime-owner.ts b/src/shared/config/runtime-owner.ts new file mode 100644 index 0000000000..2021b5e1a8 --- /dev/null +++ b/src/shared/config/runtime-owner.ts @@ -0,0 +1,30 @@ +import { createHash } from "node:crypto"; +import * as os from "node:os"; +import * as path from "node:path"; + +/** Docker label used to keep assets from independent Code UX runtimes isolated. */ +export const RUNTIME_OWNER_LABEL = "code-ux.runtime-owner"; + +let cachedOwnerId: string | null = null; + +/** + * Stable identity for the current Code UX state directory. Test/pentest runtimes use an isolated + * HOME, so their Docker cleanup must never remove assets owned by the user's live runtime. + */ +export function getRuntimeOwnerId(): string { + if (cachedOwnerId) { + return cachedOwnerId; + } + const resolvedHome = path.resolve(os.homedir(), ".code-ux").replace(/\\/g, "/"); + const canonicalHome = process.platform === "win32" ? resolvedHome.toLowerCase() : resolvedHome; + cachedOwnerId = createHash("sha256").update(canonicalHome).digest("hex").slice(0, 24); + return cachedOwnerId; +} + +export function getRuntimeOwnerLabel(): string { + return `${RUNTIME_OWNER_LABEL}=${getRuntimeOwnerId()}`; +} + +export function getRuntimeOwnerDockerArgs(): string[] { + return ["--label", getRuntimeOwnerLabel()]; +} diff --git a/src/shared/logging/logger.ts b/src/shared/logging/logger.ts index b1520fb577..bb6c48dda7 100644 --- a/src/shared/logging/logger.ts +++ b/src/shared/logging/logger.ts @@ -58,6 +58,14 @@ interface StructuredLogRecord { } const logFileStreams = new Map(); +const MAX_PENDING_LOG_STREAM_BYTES = 8 * 1024 * 1024; + +const canQueueLogWrite = (stream: NodeJS.WritableStream, text: string): boolean => { + const writableLength = typeof (stream as { writableLength?: unknown }).writableLength === "number" + ? (stream as unknown as { writableLength: number }).writableLength + : 0; + return writableLength + Buffer.byteLength(text, "utf8") + 1 <= MAX_PENDING_LOG_STREAM_BYTES; +}; const LOG_LEVEL_PRIORITY: Record = { debug: 10, @@ -279,13 +287,16 @@ export const createLogger = (options: StructuredLoggerOptions = {}): Logger => { // In Node.js, console.info/log goes to stdout. // MCP uses stdout for its protocol. // We must redirect ALL logs to stderr. - if (shouldLogToConsole(level, purpose)) { + if (shouldLogToConsole(level, purpose) && canQueueLogWrite(process.stderr, consoleText)) { process.stderr.write(consoleText + "\n"); } if (shouldLogToFile(level)) { try { - getLogFileStream()?.write(fileText + "\n"); + const stream = getLogFileStream(); + if (stream && canQueueLogWrite(stream, fileText)) { + stream.write(fileText + "\n"); + } } catch { // Silently ignore log file write errors to avoid crashing } diff --git a/src/shared/subprocess/command-runner.ts b/src/shared/subprocess/command-runner.ts index 98cbb58779..7b0b3f6a0a 100644 --- a/src/shared/subprocess/command-runner.ts +++ b/src/shared/subprocess/command-runner.ts @@ -7,7 +7,9 @@ import { createHash } from "crypto"; import { DockerHelperContainerPool, HELPER_LABEL, + HELPER_OWNER_NAME_SUFFIX, } from "../../infrastructure/providers/cli/docker-helper-pool.js"; +import { getRuntimeOwnerDockerArgs } from "../config/runtime-owner.js"; import { CommandSpawnerClient, HostUnavailableError, @@ -16,6 +18,7 @@ import type { SpawnerCommandOptions, SpawnerRawResult } from "./command-spawner- import { isRuntimeShutdownInProgress } from "../../services/shutdown-state.js"; import { BoundedTextBuffer } from "./bounded-text-buffer.js"; import { expandHomePath } from "../config/home-path.js"; +import pLimit from "p-limit"; declare const spawnCommandBrand: unique symbol; declare const spawnArgumentBrand: unique symbol; @@ -101,10 +104,21 @@ const GIT_PATH_ENV_KEYS = new Set([ * cli-process-runner resolves before first use. */ let gitHelperPool: DockerHelperContainerPool | null = null; +const PROJECT_GIT_EXEC_CONCURRENCY = 4; +type GitExecLimit = ReturnType; +interface ProjectGitHelperLease { + holders: number; + releaseReservation: (() => void) | null; +} +const projectGitHelperLeases = new Map(); +const projectGitExecLimits = new Map(); +const projectGitInFlight = new Map>>(); +const projectGitHelpersReleasing = new Set(); + function getGitHelperPool(): DockerHelperContainerPool { if (!gitHelperPool) { gitHelperPool = new DockerHelperContainerPool({ - nameFor: (key) => `code-ux-git-helper-${createHash("sha1").update(key).digest("hex").slice(0, 24)}`, + nameFor: (key) => `code-ux-git-helper-${HELPER_OWNER_NAME_SUFFIX}-${createHash("sha1").update(key).digest("hex").slice(0, 24)}`, buildCreateArgs: (key, name) => { const parsed = JSON.parse(key) as { mountRoot: string; uid?: number; gid?: number }; const userArgs = parsed.uid !== undefined && parsed.gid !== undefined && parsed.uid !== 0 @@ -117,6 +131,7 @@ function getGitHelperPool(): DockerHelperContainerPool { name, "--label", `${HELPER_LABEL}=git`, + ...getRuntimeOwnerDockerArgs(), "--workdir", CONTAINER_REPO_ROOT, "--mount", @@ -138,6 +153,77 @@ function getGitHelperPool(): DockerHelperContainerPool { return gitHelperPool; } +function getProjectGitExecLimit(poolKey: string): GitExecLimit { + const existing = projectGitExecLimits.get(poolKey); + if (existing) { + return existing; + } + const created = pLimit(PROJECT_GIT_EXEC_CONCURRENCY); + projectGitExecLimits.set(poolKey, created); + return created; +} + +async function drainProjectGitExecutions(poolKey: string): Promise { + for (;;) { + const current = [...(projectGitInFlight.get(poolKey) || [])]; + if (current.length === 0) { + return; + } + await Promise.allSettled(current); + } +} + +/** + * Keeps one lazy Git helper warm while a project has an active sprint. Multiple sprints for the + * same project share a reference-counted lease; the final release drains commands and removes the + * helper. Outside an active lease, Git uses the isolated one-shot path and leaves no warm helper. + */ +export function acquireProjectGitHelperForSprint(cwd: string): () => Promise { + const context = CommandRunner.resolveGitPoolContextForPath(cwd); + if (!context) { + return async () => undefined; + } + const existing = projectGitHelperLeases.get(context.poolKey); + if (existing) { + existing.holders += 1; + } else { + projectGitHelperLeases.set(context.poolKey, { + holders: 1, + releaseReservation: null, + }); + } + + let released = false; + return async () => { + if (released) { + return; + } + released = true; + const lease = projectGitHelperLeases.get(context.poolKey); + if (!lease) { + return; + } + lease.holders = Math.max(0, lease.holders - 1); + if (lease.holders > 0) { + return; + } + + projectGitHelperLeases.delete(context.poolKey); + projectGitHelpersReleasing.add(context.poolKey); + try { + await drainProjectGitExecutions(context.poolKey); + lease.releaseReservation?.(); + await gitHelperPool?.release(context.poolKey); + } finally { + projectGitHelpersReleasing.delete(context.poolKey); + if (!projectGitHelperLeases.has(context.poolKey)) { + projectGitExecLimits.delete(context.poolKey); + projectGitInFlight.delete(context.poolKey); + } + } + }; +} + /** Removes the persistent git helper container bound to a project root. */ export async function releaseGitHelperForCwd(cwd: string): Promise { if (!gitHelperPool) { @@ -147,14 +233,40 @@ export async function releaseGitHelperForCwd(cwd: string): Promise { if (!context) { return; } - await gitHelperPool.release(context.poolKey).catch(() => undefined); + const lease = projectGitHelperLeases.get(context.poolKey); + projectGitHelperLeases.delete(context.poolKey); + projectGitHelpersReleasing.add(context.poolKey); + try { + await drainProjectGitExecutions(context.poolKey); + lease?.releaseReservation?.(); + await gitHelperPool.release(context.poolKey).catch(() => undefined); + } finally { + projectGitHelpersReleasing.delete(context.poolKey); + projectGitExecLimits.delete(context.poolKey); + projectGitInFlight.delete(context.poolKey); + } } /** Drains the process-wide git helper pool during server shutdown. */ export async function shutdownGitHelperPool(): Promise { const pool = gitHelperPool; - gitHelperPool = null; + const keys = new Set([ + ...projectGitHelperLeases.keys(), + ...projectGitInFlight.keys(), + ]); + for (const key of keys) { + projectGitHelpersReleasing.add(key); + } + await Promise.all([...keys].map((key) => drainProjectGitExecutions(key))); + for (const lease of projectGitHelperLeases.values()) { + lease.releaseReservation?.(); + } + projectGitHelperLeases.clear(); await pool?.shutdown(); + gitHelperPool = null; + projectGitExecLimits.clear(); + projectGitInFlight.clear(); + projectGitHelpersReleasing.clear(); } export class CommandRunner { @@ -178,14 +290,33 @@ export class CommandRunner { args: string[], options: CommandOptions = {} ): Promise { - // Poolable git commands (containerized, only the working tree mounted, no stdin) are + // Poolable git commands (containerized and only the project tree mounted) are // executed inside a persistent helper container instead of a throwaway `docker run --rm`. - if (command === "git" && this.shouldRunGitInContainer(options) && !options.stdinFile) { - const cwd = this.resolveHostPath(options.cwd ?? process.cwd()); + if (command === "git" && this.shouldRunGitInContainer(options)) { + // Validate caller-controlled spawn inputs before `pool.ensure()` can create a helper. + // stdin files stay on the host and are streamed through `docker exec -i`; they do not + // require another bind mount and therefore remain eligible for the warm helper. + this.validateSpawnArgs(args); + const safeCwd = this.validateSpawnCwd(options.cwd); + const cwd = safeCwd ?? this.resolveHostPath(process.cwd()); + const safeStdinFile = options.stdinFile + ? this.validateStdinFile(options.stdinFile, safeCwd) + : undefined; const env = options.env ?? process.env; const poolContext = this.resolveGitPoolContext(cwd); - if (poolContext && this.buildGitContainerPathMappings(poolContext.mountRoot, args, env).length === 0) { - return this.runPooledGitCommand(poolContext, args, env, options); + // Once shutdown starts, the server drains the warm pool. Late Git work must stay on the + // containerized one-shot path so it cannot recreate a persistent helper behind that drain. + if ( + poolContext + && !isRuntimeShutdownInProgress() + && projectGitHelperLeases.has(poolContext.poolKey) + && !projectGitHelpersReleasing.has(poolContext.poolKey) + && this.buildGitContainerPathMappings(poolContext.mountRoot, args, env).length === 0 + ) { + const pooledOptions = safeCwd === undefined && safeStdinFile === undefined + ? options + : { ...options, cwd: safeCwd, stdinFile: safeStdinFile }; + return this.runPooledGitCommand(poolContext, args, env, pooledOptions); } } @@ -200,11 +331,26 @@ export class CommandRunner { options: CommandOptions, ): Promise { const pool = getGitHelperPool(); - const execPrefix = ["exec", "--workdir", context.containerCwd, ...this.buildGitContainerEnvArgs(env, context.mountRoot, [])]; + const projectLease = projectGitHelperLeases.get(context.poolKey); + if (projectLease && !projectLease.releaseReservation) { + projectLease.releaseReservation = pool.reserve(context.poolKey); + } + const execPrefix = [ + "exec", + ...(options.stdinFile ? ["-i"] : []), + "--workdir", + context.containerCwd, + ...this.buildGitContainerEnvArgs(env, context.mountRoot, []), + ]; const execCommand = ["git", ...this.rewriteGitArgsForContainer(context.mountRoot, args, [])]; - - const runViaExec = async (): Promise => { - const containerId = await pool.ensure(context.poolKey); + // This includes command-scoped auth/config environment values. Validate the complete exec + // argv before helper creation so malformed environment cannot cause container churn. + this.validateSpawnArgs([...execPrefix, ...execCommand]); + + const runOneShot = (): Promise => ( + this.spawnProcess(this.resolveCommand("git", args, options), options) + ); + const runViaExec = async (containerId: string): Promise => { pool.touch(context.poolKey); return this.spawnProcess( { command: "docker", args: [...execPrefix, containerId, ...execCommand], containerHostCwd: context.mountRoot }, @@ -212,23 +358,46 @@ export class CommandRunner { ); }; - let result: CommandResult; - try { - result = await runViaExec(); - } catch { - // Could not start/reach the helper — fall back to a one-shot run --rm so the op still works. - return this.spawnProcess(this.resolveCommand("git", args, options), options); - } - - if (!result.ok && pool.isContainerGone(result)) { - pool.invalidate(context.poolKey); + const runPinnedGeneration = async (): Promise<{ containerId: string; result: CommandResult }> => { + let commandStarted = false; try { - result = await runViaExec(); - } catch { - return this.spawnProcess(this.resolveCommand("git", args, options), options); + return await pool.withContainer(context.poolKey, async (containerId) => { + commandStarted = true; + return { containerId, result: await runViaExec(containerId) }; + }); + } catch (error) { + if (commandStarted) { + throw error; + } + return { containerId: "", result: await runOneShot() }; + } + }; + + const execute = async (): Promise => { + let attempt = await runPinnedGeneration(); + if (attempt.containerId && !attempt.result.ok && pool.isContainerGone(attempt.result)) { + pool.invalidate(context.poolKey, attempt.containerId); + attempt = await runPinnedGeneration(); + if (attempt.containerId && !attempt.result.ok && pool.isContainerGone(attempt.result)) { + pool.invalidate(context.poolKey, attempt.containerId); + return runOneShot(); + } + } + return attempt.result; + }; + + const operation = getProjectGitExecLimit(context.poolKey)(execute); + const inFlight = projectGitInFlight.get(context.poolKey) || new Set>(); + inFlight.add(operation); + projectGitInFlight.set(context.poolKey, inFlight); + try { + return await operation; + } finally { + inFlight.delete(operation); + if (inFlight.size === 0) { + projectGitInFlight.delete(context.poolKey); } } - return result; } /** @@ -704,6 +873,7 @@ export class CommandRunner { "run", "--rm", "-i", + ...getRuntimeOwnerDockerArgs(), "--workdir", containerCwd, "--mount", diff --git a/src/shared/subprocess/command-spawner-host.ts b/src/shared/subprocess/command-spawner-host.ts index 095e0f844b..7bdc99da41 100644 --- a/src/shared/subprocess/command-spawner-host.ts +++ b/src/shared/subprocess/command-spawner-host.ts @@ -11,6 +11,10 @@ import type { SpawnerRunMessage, SpawnerRawResult, } from "./command-spawner-protocol.js"; +import { + MAX_SPAWNER_STREAM_LINE_CHARS, + boundSpawnerStreamLine, +} from "./command-spawner-protocol.js"; import { BoundedTextBuffer } from "./bounded-text-buffer.js"; const KILL_GRACE_MS = 2_000; @@ -77,8 +81,12 @@ function runJob(message: SpawnerRunMessage): void { const stdout = new BoundedTextBuffer(options.maxStdoutChars ?? 5 * 1024 * 1024); const stderr = new BoundedTextBuffer(options.maxStderrChars ?? 4096); - const stdoutLineBuffer = new BoundedTextBuffer(options.maxStdoutChars ?? 5 * 1024 * 1024); - const stderrLineBuffer = new BoundedTextBuffer(options.maxStderrChars ?? 4096); + const stdoutLineBuffer = new BoundedTextBuffer( + Math.min(options.maxStdoutChars ?? 5 * 1024 * 1024, MAX_SPAWNER_STREAM_LINE_CHARS), + ); + const stderrLineBuffer = new BoundedTextBuffer( + Math.min(options.maxStderrChars ?? 4096, MAX_SPAWNER_STREAM_LINE_CHARS), + ); let stdoutClipped = false; let stderrClipped = false; let timedOut = false; @@ -100,7 +108,7 @@ function runJob(message: SpawnerRunMessage): void { const flushLineBuffer = (buffer: BoundedTextBuffer, stream: "stdoutLine" | "stderrLine"): void => { const trimmed = buffer.takeString().trim(); if (trimmed.length > 0) { - send({ type: stream, id, line: trimmed }); + send({ type: stream, id, line: boundSpawnerStreamLine(trimmed) }); } }; @@ -131,7 +139,7 @@ function runJob(message: SpawnerRunMessage): void { for (const line of completed.split("\n")) { const trimmed = line.trim(); if (trimmed.length > 0) { - send({ type: stream, id, line: trimmed }); + send({ type: stream, id, line: boundSpawnerStreamLine(trimmed) }); } } pending.append(text.slice(lastNewline + 1)); diff --git a/src/shared/subprocess/command-spawner-protocol.ts b/src/shared/subprocess/command-spawner-protocol.ts index 9d6cb5b6ff..4d2dabe2f0 100644 --- a/src/shared/subprocess/command-spawner-protocol.ts +++ b/src/shared/subprocess/command-spawner-protocol.ts @@ -31,6 +31,20 @@ export interface SpawnerCommandOptions { streamStderrLines?: boolean; } +/** Live line callbacks are previews; keep a single newline-free provider write from becoming a huge IPC frame. */ +export const MAX_SPAWNER_STREAM_LINE_CHARS = 64 * 1024; + +export function boundSpawnerStreamLine(line: string): string { + if (line.length <= MAX_SPAWNER_STREAM_LINE_CHARS) { + return line; + } + const marker = "\n… [stream line truncated] …\n"; + const retainedChars = MAX_SPAWNER_STREAM_LINE_CHARS - marker.length; + const headChars = Math.ceil(retainedChars / 2); + const tailChars = retainedChars - headChars; + return `${line.slice(0, headChars)}${marker}${line.slice(-tailChars)}`; +} + export interface SpawnerRunMessage { type: "run"; id: number; diff --git a/src/sprint/steps/session-sync-step.ts b/src/sprint/steps/session-sync-step.ts index 68616200c6..ee0ecc62e5 100644 --- a/src/sprint/steps/session-sync-step.ts +++ b/src/sprint/steps/session-sync-step.ts @@ -42,6 +42,14 @@ const TERMINAL_DISPATCH_STATUSES = new Set([ ]); const DISPATCH_HEARTBEAT_INTERVAL_MS = 60_000; +const MAX_PROVIDER_ACTIVITY_EVENT_TEXT_CHARS = 16 * 1024; +const MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS = 2 * 1024; +const MAX_PROVIDER_ACTIVITY_PLAN_STEPS = 64; +const WORKER_CLARIFICATION_EVENT_TYPES = [ + "worker_clarification_requested", + "worker_clarification_continued", + "worker_clarification_replied", +] as const; type WorkerClarificationSyncStatus = "none" | "pending" | "answered" | "settled" | "cancelled_run"; @@ -66,8 +74,10 @@ const resolveWorkerClarificationProjection = ( return NO_WORKER_CLARIFICATION; } - const lifecycleEvents = deps.executionRepository.listTaskRunEvents(taskRun.id, 10_000) - .filter((event) => event.eventType.startsWith("worker_clarification_")); + const lifecycleEvents = deps.executionRepository.listTaskRunEvents(taskRun.id, 500, { + eventTypes: [...WORKER_CLARIFICATION_EVENT_TYPES], + skipValidation: true, + }); const byClarificationId = new Map { + if (typeof value !== "string" || value.length <= maxChars) { + return value; + } + const marker = "\n… [provider activity truncated] …\n"; + const retainedChars = maxChars - marker.length; + const headChars = Math.ceil(retainedChars / 2); + const tailChars = retainedChars - headChars; + return `${value.slice(0, headChars)}${marker}${value.slice(-tailChars)}`; +}; + +const boundProviderActivityValue = (value: unknown): unknown => { + if (value === null || value === undefined || typeof value !== "object") { + return value; + } + try { + const serialized = JSON.stringify(value); + return serialized.length <= MAX_PROVIDER_ACTIVITY_EVENT_TEXT_CHARS + ? value + : { truncated: true, originalChars: serialized.length }; + } catch { + return { truncated: true, reason: "unserializable" }; + } +}; + const getActivityPreview = (activity: JulesActivity): string => { if (typeof activity.agentMessaged?.agentMessage === "string" && activity.agentMessaged.agentMessage.trim()) { return activity.agentMessaged.agentMessage.trim(); @@ -264,27 +302,50 @@ const getActivityKind = (activity: JulesActivity): string => { return "activity"; }; -const buildProviderActivityEventPayload = ( +export const buildProviderActivityEventPayload = ( activity: JulesActivity, sessionId: string | null, sessionName: string | null, provider: string | null, ): Record => ({ - activityId: activity.id, - activityName: activity.name, - sessionId, - sessionName, - provider, + activityId: boundProviderActivityText(activity.id, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS), + activityName: boundProviderActivityText(activity.name, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS), + sessionId: boundProviderActivityText(sessionId ?? undefined, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS) ?? null, + sessionName: boundProviderActivityText(sessionName ?? undefined, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS) ?? null, + provider: boundProviderActivityText(provider ?? undefined, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS) ?? null, kind: getActivityKind(activity), - preview: getActivityPreview(activity), - description: typeof activity.description === "string" ? activity.description : null, - agentMessaged: activity.agentMessaged || null, - userMessaged: activity.userMessaged || null, - progressUpdated: activity.progressUpdated || null, - planGenerated: activity.planGenerated || null, - planApproved: activity.planApproved || null, - sessionFailed: activity.sessionFailed || null, - sessionCompleted: activity.sessionCompleted ?? null, + preview: boundProviderActivityText(getActivityPreview(activity)), + description: boundProviderActivityText(activity.description) ?? null, + agentMessaged: activity.agentMessaged + ? { agentMessage: boundProviderActivityText(activity.agentMessaged.agentMessage) } + : null, + userMessaged: activity.userMessaged + ? { userMessage: boundProviderActivityText(activity.userMessaged.userMessage) } + : null, + progressUpdated: activity.progressUpdated + ? { + title: boundProviderActivityText(activity.progressUpdated.title), + description: boundProviderActivityText(activity.progressUpdated.description), + } + : null, + planGenerated: activity.planGenerated + ? { + plan: activity.planGenerated.plan + ? { + steps: activity.planGenerated.plan.steps + ?.slice(0, MAX_PROVIDER_ACTIVITY_PLAN_STEPS) + .map((step) => ({ title: boundProviderActivityText(step.title, 1_024) })), + } + : undefined, + } + : null, + planApproved: activity.planApproved + ? { planId: boundProviderActivityText(activity.planApproved.planId, MAX_PROVIDER_ACTIVITY_IDENTIFIER_CHARS) } + : null, + sessionFailed: activity.sessionFailed + ? { reason: boundProviderActivityText(activity.sessionFailed.reason) } + : null, + sessionCompleted: boundProviderActivityValue(activity.sessionCompleted) ?? null, }); const normalizeSessionRef = (sessionRef: string | null | undefined): string | null => { diff --git a/src/sprint/steps/start-ready-tasks-step.ts b/src/sprint/steps/start-ready-tasks-step.ts index fdda17ca2e..0c5e40601d 100644 --- a/src/sprint/steps/start-ready-tasks-step.ts +++ b/src/sprint/steps/start-ready-tasks-step.ts @@ -3,42 +3,49 @@ import type { Logger } from "../../shared/logging/logger.js"; import { getTaskDispatchDeferral } from "../../services/sprint-task-dispatch-service.js"; const PROVIDER_CAP_LOG_INTERVAL_MS = 10_000; -const providerCapLogState = new WeakMap>(); +const MAX_PROVIDER_CAP_LOG_STATE_ENTRIES = 2_048; + +export type ProviderCapLogState = Map; + +const providerCapLogState = new WeakMap(); + +const evictOldestProviderCapLogEntry = (state: ProviderCapLogState): void => { + if (state.size < MAX_PROVIDER_CAP_LOG_STATE_ENTRIES) return; + + let oldestKey: string | undefined; + let oldestLoggedAt = Number.POSITIVE_INFINITY; + for (const [key, entry] of state) { + if (entry.loggedAt < oldestLoggedAt) { + oldestKey = key; + oldestLoggedAt = entry.loggedAt; + } + } + if (oldestKey) state.delete(oldestKey); +}; const shouldLogProviderCapBlock = ( logger: Logger, + externalState: ProviderCapLogState | undefined, + scope: string | undefined, provider: string, - block: { - count: number; - limit?: number; - currentCount?: number; - source: "pre_dispatch" | "dispatch"; - taskIds: readonly string[]; - }, ): boolean => { - let state = providerCapLogState.get(logger); + let state = externalState ?? providerCapLogState.get(logger); if (!state) { state = new Map(); providerCapLogState.set(logger, state); } - const signature = [ - block.limit ?? "auto", - block.currentCount ?? "unknown", - block.count, - block.source, - ...block.taskIds, - ].join(":"); + const key = scope ? `${scope}:${provider}` : provider; const now = Date.now(); - const previous = state.get(provider); + const previous = state.get(key); if ( previous - && previous.signature === signature && now >= previous.loggedAt && now - previous.loggedAt < PROVIDER_CAP_LOG_INTERVAL_MS ) { return false; } - state.set(provider, { loggedAt: now, signature }); + if (!previous) evictOldestProviderCapLogEntry(state); + state.set(key, { loggedAt: now }); return true; }; @@ -57,6 +64,12 @@ interface StartReadyTasksOptions { getProviderForTask: (task: Subtask) => string | null; getProviderSettings: (provider: string) => { maxConcurrentTasks?: number }; getRunningCounts: () => Record; + /** Effective immediately available slots after adaptive/global admission policy. */ + getAvailableProviderCapacity?: (provider: string) => Promise; + /** Long-lived, bounded throttle state shared across orchestration cycles. */ + providerCapLogState?: ProviderCapLogState; + /** Isolates throttle windows for concurrent sprint runs that use the same provider. */ + providerCapLogScope?: string; } export const runStartReadyTasksStep = async ( @@ -76,6 +89,7 @@ export const runStartReadyTasksStep = async ( } const currentRunningCounts = options.getRunningCounts(); + const remainingAdmissionCapacity = new Map(); const readyTasks = subtasks.filter((task) => task.status === "PENDING"); const providerCapBlocks = new Map 0 && runningCount >= limit) { task.status = "PENDING"; recordProviderCapBlock({ @@ -142,6 +172,10 @@ export const runStartReadyTasksStep = async ( const session = await options.startTask(task); if (provider) { currentRunningCounts[provider] = (currentRunningCounts[provider] || 0) + 1; + const availableCapacity = remainingAdmissionCapacity.get(provider); + if (availableCapacity !== undefined && availableCapacity !== null) { + remainingAdmissionCapacity.set(provider, Math.max(0, availableCapacity - 1)); + } } task.status = "RUNNING"; task.session_name = options.resolveSessionName(session); @@ -185,7 +219,12 @@ export const runStartReadyTasksStep = async ( } for (const [provider, block] of providerCapBlocks) { - if (!shouldLogProviderCapBlock(options.logger, provider, block)) continue; + if (!shouldLogProviderCapBlock( + options.logger, + options.providerCapLogState, + options.providerCapLogScope, + provider, + )) continue; options.logger.info("Provider concurrency cap deferred ready tasks", { provider, limit: block.limit, diff --git a/tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts b/tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts index da197f5b52..71b7b06118 100644 --- a/tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts +++ b/tests/backend/app/lifecycle/dashboard-snapshot-cache.test.ts @@ -26,6 +26,7 @@ describe("DashboardSnapshotCache", () => { getOverviewTelemetrySnapshot: vi.fn().mockReturnValue({ activeProjects: [] }), getProjectExecutionSnapshot: vi.fn().mockReturnValue({ projectId: "p1" }), getProjectStatsSnapshot: vi.fn().mockReturnValue({ stats: true }), + getHeaderTokenThroughputSnapshot: vi.fn().mockReturnValue({ throughput: true }), }, connectionChatRepository: { listConnections: vi.fn().mockReturnValue([]), @@ -169,6 +170,34 @@ describe("DashboardSnapshotCache", () => { }); }); + it("bounds execution snapshots across many selected sprint scopes", () => { + for (let index = 0; index <= DashboardSnapshotCachePolicy.PROJECT_EXECUTION_CACHE_MAX_ENTRIES; index += 1) { + cache.getProjectExecutionSnapshot("p1", { selectedSprintId: `sprint-${index}` }); + } + + const callsAfterFill = mockDeps.executionRepository.getProjectExecutionSnapshot.mock.calls.length; + cache.getProjectExecutionSnapshot("p1", { selectedSprintId: "sprint-0" }); + + expect(mockDeps.executionRepository.getProjectExecutionSnapshot).toHaveBeenCalledTimes(callsAfterFill + 1); + }); + + it("bounds parameterized stats and throughput snapshots", () => { + for (let index = 0; index <= DashboardSnapshotCachePolicy.PROJECT_STATS_CACHE_MAX_ENTRIES; index += 1) { + cache.getProjectStatsSnapshot(`project-${index}`, { window: "7d" }); + } + for (let index = 0; index <= DashboardSnapshotCachePolicy.HEADER_TOKEN_THROUGHPUT_CACHE_MAX_ENTRIES; index += 1) { + cache.getHeaderTokenThroughputSnapshot({ projectId: `project-${index}`, window: "24h" }); + } + + const statsCalls = mockDeps.executionRepository.getProjectStatsSnapshot.mock.calls.length; + const throughputCalls = mockDeps.executionRepository.getHeaderTokenThroughputSnapshot.mock.calls.length; + cache.getProjectStatsSnapshot("project-0", { window: "7d" }); + cache.getHeaderTokenThroughputSnapshot({ projectId: "project-0", window: "24h" }); + + expect(mockDeps.executionRepository.getProjectStatsSnapshot).toHaveBeenCalledTimes(statsCalls + 1); + expect(mockDeps.executionRepository.getHeaderTokenThroughputSnapshot).toHaveBeenCalledTimes(throughputCalls + 1); + }); + it("scopes active attention queues to the selected sprint while project-wide mode keeps all active items", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "dashboard-snapshot-cache-")); tempDirs.push(dir); diff --git a/tests/backend/ci/workflow-health.test.ts b/tests/backend/ci/workflow-health.test.ts index 362bac19bd..af2f47de78 100644 --- a/tests/backend/ci/workflow-health.test.ts +++ b/tests/backend/ci/workflow-health.test.ts @@ -15,6 +15,7 @@ const WORKFLOWS = { const PLAYWRIGHT_CONFIG = "playwright.config.ts"; const RELEASE_INSTALL_VERIFIER = "scripts/verify-release-install.mjs"; const ELECTRON_RUNTIME_PREPARER = "scripts/prepare-electron-runtime-deps.mjs"; +const ELECTRON_INSTALL_SMOKE = "pnpm run electron:smoke-installed"; const REQUIRED_INSTALL = "pnpm install --frozen-lockfile --ignore-scripts"; const PACKAGE_MANAGER_VERSION = "11.13.0"; const MINIMUM_NODE_VERSION = "22.13"; @@ -251,6 +252,11 @@ describe("GitHub workflow health", () => { expect(releaseCandidate).toContain("node node_modules/electron/install.js"); expect(releaseCandidate).toContain("pnpm run electron:prepare-deps"); expect(releaseCandidate).toContain("pnpm exec electron-builder --config electron-builder.config.cjs ${{ matrix.electron-target }} --publish never"); + expect(releaseCandidate).toContain("sudo apt-get install --no-install-recommends -y libopenjp2-tools xvfb"); + expect(releaseCandidate).toContain("name: Install and start release candidate"); + expect(releaseCandidate).toContain(ELECTRON_INSTALL_SMOKE); + expectCommandBefore(releaseCandidate, "name: Build unsigned desktop package", "name: Install and start release candidate"); + expectCommandBefore(releaseCandidate, "name: Install and start release candidate", "name: Upload release candidate artifacts"); expect(releaseCandidate).toContain("if-no-files-found: error"); expect(releaseCandidate).not.toContain("pnpm run audit"); expect(releaseCandidate).not.toContain("pnpm run build"); @@ -272,6 +278,8 @@ describe("GitHub workflow health", () => { expect(electronRuntimePreparer).toContain('const onnxRuntimeInstallMode = "skip";'); expect(electronRuntimePreparer).toContain("onnxRuntimeInstallMode,"); expect(electronRuntimePreparer).toContain("ONNXRUNTIME_NODE_INSTALL: onnxRuntimeInstallMode"); + expect(electronRuntimePreparer).toContain('"--config.node-linker=hoisted"'); + expect(electronRuntimePreparer).toContain("validateRuntimeTree();"); }); it("keeps legacy main ruleset contexts coupled to their current validation gates", async () => { @@ -317,6 +325,7 @@ describe("GitHub workflow health", () => { expectManualOnly(releaseChecks, "Release candidate diagnostics"); expect(releaseChecks).toContain("node scripts/verify-release-install.mjs"); expect(releaseChecks).toContain("pnpm run ${{ matrix.electron-script }} -- --publish never"); + expect(releaseChecks).toContain(ELECTRON_INSTALL_SMOKE); expect(mockup).toContain("name: Mockup Sprint Diagnostics"); expectManualOnly(mockup, "Mockup sprint diagnostics"); @@ -351,6 +360,7 @@ describe("GitHub workflow health", () => { expect(desktop).toContain("node node_modules/electron/install.js"); expect(desktop).toContain("pnpm run build && pnpm run electron:prepare-deps && pnpm exec electron-builder"); expect(desktop).toContain("--publish never"); + expect(desktop).toContain(ELECTRON_INSTALL_SMOKE); expectCommandBefore(release, "run: pnpm install --frozen-lockfile --ignore-scripts", "run: pnpm run audit"); expect(desktopRelease).toContain("name: Desktop Release Diagnostics"); @@ -358,12 +368,13 @@ describe("GitHub workflow health", () => { expect(desktopRelease).toContain("permissions:\n contents: read"); expect(desktopRelease).toContain('GH_TOKEN: ""'); expect(desktopRelease).not.toContain("softprops/action-gh-release"); + expect(desktopRelease).toContain(ELECTRON_INSTALL_SMOKE); }); it("keeps Playwright config isolated, serialized, and failure-artifact friendly", async () => { const config = await readRepoFile(PLAYWRIGHT_CONFIG); - expect(config).toContain("command: 'pnpm exec vite build && node dist/index.js'"); + expect(config).toContain("command: 'node ./node_modules/vite/bin/vite.js build && node dist/index.js'"); expect(config).toContain("process.env.CODEUX_E2E_DASHBOARD_PORT || process.env.DASHBOARD_PORT || '4464'"); expect(config).toContain("baseURL: dashboardBaseUrl"); expect(config).toContain("url: `${dashboardBaseUrl}/health`"); @@ -440,7 +451,7 @@ describe("GitHub workflow health", () => { expect(scenarioScript).toContain('"mockup-sprint-qa:require-file src/qa-dag/final.js'); expect(scenarioScript).toContain('outcomes: ["changes_requested", "pass"]'); expect(scenarioScript).toContain("requireSameWorkerBranch: true"); - expect(scenarioScript.match(/injectMainCiFix:/g)).toHaveLength(2); + expect(scenarioScript.match(/injectMainCiFix:/g)).toHaveLength(3); expect(scenarioScript).toContain("minimumCompletedCiFixes: 1"); expect(scenarioScript).toContain("requireSprintLevelCiFix: true"); expect(scenarioScript).toContain('"qa-dag-follow-up": 2'); diff --git a/tests/backend/domain/sprint/ci/feature-pr-gate.test.ts b/tests/backend/domain/sprint/ci/feature-pr-gate.test.ts index 063407a52d..559af555f7 100644 --- a/tests/backend/domain/sprint/ci/feature-pr-gate.test.ts +++ b/tests/backend/domain/sprint/ci/feature-pr-gate.test.ts @@ -150,6 +150,20 @@ describe("FeaturePrGateService", () => { ); }); + it("loads latest task runs once for a wide gate evaluation", async () => { + const cachedRun = { id: "run-1", state: "COMPLETED", workerBranch: "feat/T1" }; + const listLatestTaskRuns = vi.fn().mockReturnValue(new Map([ + ["task-record-1", cachedRun], + ])); + (context.executionRepository as any).listLatestTaskRuns = listLatestTaskRuns; + + await service.evaluateCiGate(subtasks, context); + + expect(listLatestTaskRuns).toHaveBeenCalledTimes(1); + expect(listLatestTaskRuns).toHaveBeenCalledWith(["task-record-1"], "sprint-run-1"); + expect(context.executionRepository?.getLatestTaskRun).not.toHaveBeenCalled(); + }); + it("marks task as completed with PR_ONLY indicator when featurePrAutoMergeMode is CREATE_PR", async () => { // Override the autoMergeMode context.ciIntelligence.featurePrAutoMergeMode = "CREATE_PR" as any; diff --git a/tests/backend/domain/sprint/orchestrator/cycle-runner.test.ts b/tests/backend/domain/sprint/orchestrator/cycle-runner.test.ts index 3f2f6883a3..b0210af8d4 100644 --- a/tests/backend/domain/sprint/orchestrator/cycle-runner.test.ts +++ b/tests/backend/domain/sprint/orchestrator/cycle-runner.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it, vi } from "vitest"; -import { CycleRunner } from "../../../../../src/domain/sprint/orchestrator/cycle-runner.js"; +import { + CycleRunner, + resolveTaskQaReviewParallelism, +} from "../../../../../src/domain/sprint/orchestrator/cycle-runner.js"; import type { SprintOrchestratorDependencies } from "../../../../../src/sprint/sprint-orchestrator.js"; import { DEFAULT_DASHBOARD_SETTINGS } from "../../../../../src/repositories/settings-defaults.js"; @@ -91,6 +94,67 @@ function buildDeps(): SprintOrchestratorDependencies { } describe("CycleRunner attention sync", () => { + it("preserves low provider capacities and reserves host capacity above the bounded QA ceiling", () => { + const settings = structuredClone(DEFAULT_DASHBOARD_SETTINGS); + settings.aiProvider.providers["mockup-cli"] = { + ...settings.aiProvider.providers.codex, + provider: "mockup-cli", + maxConcurrentTasks: 16, + } as any; + settings.aiProvider.invocationRouting.qa_review = { + ...settings.aiProvider.invocationRouting.qa_review, + provider: "mockup-cli", + allowedProviders: ["mockup-cli"], + }; + + expect(resolveTaskQaReviewParallelism(settings)).toBe(4); + settings.aiProvider.providers["mockup-cli"]!.maxConcurrentTasks = 100; + expect(resolveTaskQaReviewParallelism(settings)).toBe(4); + settings.aiProvider.providers["mockup-cli"]!.maxConcurrentTasks = 3; + expect(resolveTaskQaReviewParallelism(settings)).toBe(3); + settings.aiProvider.providers["mockup-cli"]!.maxConcurrentTasks = 0; + settings.aiProvider.invocationRouting.qa_review.allowedProviders = []; + settings.aiProvider.invocationRouting.qa_review.provider = null; + settings.aiProvider.provider = null; + expect(resolveTaskQaReviewParallelism(settings)).toBe(4); + }); + + it("batches latest task-run lookups when collecting wide local DAG git evidence", () => { + const deps = buildDeps(); + const taskRun = { + id: "task-run-1", + taskId: "task-1", + provider: "codex", + mode: "docker_cli", + state: "COMPLETED", + } as any; + const listLatestTaskRuns = vi.fn().mockReturnValue(new Map([["task-1", taskRun]])); + const getLatestTaskRun = vi.fn(); + const listTaskRunEventsForRuns = vi.fn().mockReturnValue(new Map([ + ["task-run-1", [{ eventType: "cli_git_pushed", payload: { pushedBranch: "task/one" } }]], + ])); + (deps.executionRepository as any).listLatestTaskRuns = listLatestTaskRuns; + (deps.executionRepository as any).getLatestTaskRun = getLatestTaskRun; + (deps.executionRepository as any).listTaskRunEventsForRuns = listTaskRunEventsForRuns; + + const result = (new CycleRunner(deps) as any).collectLocalCliGitEvidence([ + { id: "T1", record_id: "task-1" }, + { id: "T2", record_id: "task-2" }, + ], { + githubMode: "LOCAL", + sprintRunId: "run-1", + }); + + expect(listLatestTaskRuns).toHaveBeenCalledOnce(); + expect(listLatestTaskRuns).toHaveBeenCalledWith(["task-1", "task-2"], "run-1"); + expect(getLatestTaskRun).not.toHaveBeenCalled(); + expect(listTaskRunEventsForRuns).toHaveBeenCalledWith(["task-run-1"], { + eventTypes: ["cli_git_pushed", "cli_git_no_changes", "ci_gate_status"], + limitPerRun: 500, + }); + expect(result.pushedTaskIds).toEqual(new Set(["task-1", "T1"])); + }); + it("never dispatches or reviews the audit task of an automatic rollback", async () => { const deps = buildDeps(); const reviewCompletedTask = vi.fn(); @@ -3040,6 +3104,86 @@ describe("CycleRunner attention sync", () => { ); }); + it("waits for CLI Git finalization before starting task QA", async () => { + const deps = buildDeps(); + const reviewCompletedTask = vi.fn().mockResolvedValue({ + reviewed: true, + reopenedTask: false, + mergeBlocked: false, + reportText: "QA passed", + }); + deps.qualityAssuranceService = { + getTaskMergeGateStatus: vi.fn().mockReturnValue({ + mergeAllowed: false, + reason: "pending_review", + summary: "QA review is required before merge.", + latestRun: null, + runsUsed: 0, + maxRuns: 1, + }), + reviewCompletedTask, + } as any; + deps.getDashboardSettings = vi.fn().mockReturnValue({ + ...DEFAULT_DASHBOARD_SETTINGS, + agents: { + ...DEFAULT_DASHBOARD_SETTINGS.agents, + qualityAssurance: { + ...DEFAULT_DASHBOARD_SETTINGS.agents.qualityAssurance, + enabled: true, + }, + }, + }); + vi.mocked(deps.executionRepository.getLatestTaskRun).mockReturnValue({ + id: "task-run-awaiting-git", + projectId: "project-1", + sprintId: "sprint-1", + taskId: "task-1", + sprintRunId: "run-1", + dispatchId: "dispatch-1", + connectionId: null, + provider: "codex", + mode: "docker_cli", + sessionId: "cli-codex-awaiting-git", + sessionName: "sessions/cli-codex-awaiting-git", + state: "COMPLETED", + workerBranch: "task/awaiting-git", + prUrl: null, + startedAt: null, + finishedAt: null, + durationMs: null, + }); + + const runner = new CycleRunner(deps); + await (runner as any).reviewCompletedTasks( + [{ + id: "T1", + record_id: "task-1", + title: "Await Git finalization", + prompt: "finish implementation", + depends_on: [], + is_independent: true, + status: "CODING_COMPLETED", + provider: "codex", + worker_branch: "task/awaiting-git", + }], + new Map([["T1", "RUNNING"]]), + { + executionContext: { + project: { id: "project-1", name: "Project 1" } as any, + sprint: { id: "sprint-1", name: "Sprint 1" } as any, + sprintNumber: 1, + }, + repoPath: "/repo/project-1", + sprintRunId: "run-1", + githubMode: "LOCAL", + } as any, + deps.getDashboardSettings(), + { pushedTaskIds: new Set(), settledTaskIds: new Set() }, + ); + + expect(reviewCompletedTask).not.toHaveBeenCalled(); + }); + it("replays a legacy failed QA fix handoff after restart even when old code changed the task state", async () => { const deps = buildDeps(); const reviewCompletedTask = vi.fn().mockResolvedValue({ @@ -3831,6 +3975,62 @@ describe("CycleRunner attention sync", () => { await reviewPromise; }); + it("starts one bounded QA wave per cycle instead of queueing the whole backlog", async () => { + const deps = buildDeps(); + let releaseReviews!: () => void; + const reviewGate = new Promise((resolve) => { + releaseReviews = resolve; + }); + deps.qualityAssuranceService = { + getTaskMergeGateStatus: vi.fn().mockReturnValue({ + mergeAllowed: false, + reason: "pending_review", + summary: "QA review is required.", + latestRun: null, + runsUsed: 0, + maxRuns: 2, + }), + reviewCompletedTask: vi.fn().mockImplementation(async () => { + await reviewGate; + return { reviewed: true }; + }), + } as any; + deps.getDashboardSettings = vi.fn().mockReturnValue({ + ...DEFAULT_DASHBOARD_SETTINGS, + agents: { + ...DEFAULT_DASHBOARD_SETTINGS.agents, + qualityAssurance: { + ...DEFAULT_DASHBOARD_SETTINGS.agents.qualityAssurance, + enabled: true, + }, + }, + }); + const runner = new CycleRunner(deps); + const tasks = Array.from({ length: 9 }, (_, index) => ({ + id: `T${index + 1}`, + record_id: `task-${index + 1}`, + status: "COMPLETED", + provider: "codex", + })); + + const reviews = (runner as any).reviewCompletedTasks( + tasks, + new Map(tasks.map((task) => [task.id, "RUNNING"])), + { + executionContext: { project: { id: "proj-1" }, sprint: { id: "sprint-1" } }, + sprintRunId: "run-1", + } as any, + deps.getDashboardSettings(), + ); + await vi.waitFor(() => { + expect(deps.qualityAssuranceService.reviewCompletedTask).toHaveBeenCalledTimes(4); + }); + + releaseReviews(); + await reviews; + expect(deps.qualityAssuranceService.reviewCompletedTask).toHaveBeenCalledTimes(4); + }); + it("passes known task PR URLs to git polling and backfills the PR head before QA", async () => { const deps = buildDeps(); const reviewCompletedTask = vi.fn().mockResolvedValue({ diff --git a/tests/backend/electron-builder-config.test.ts b/tests/backend/electron-builder-config.test.ts index f27a00ad5a..6c13a272f4 100644 --- a/tests/backend/electron-builder-config.test.ts +++ b/tests/backend/electron-builder-config.test.ts @@ -97,4 +97,47 @@ describe("electron-builder packaged defaults", () => { }), ])); }); + + it("builds a copy-safe Electron runtime with its MCP peer dependency", () => { + const config = require("../../electron-builder.config.cjs") as { + linux?: { executableName?: string }; + }; + const packageJson = JSON.parse(fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8")) as { + dependencies?: Record; + scripts?: Record; + }; + const preparer = fs.readFileSync( + path.join(process.cwd(), "scripts", "prepare-electron-runtime-deps.mjs"), + "utf8", + ); + + expect(packageJson.dependencies).toHaveProperty("zod"); + expect(packageJson.scripts?.["electron:smoke-installed"]).toBe( + "node scripts/smoke-installed-electron.mjs", + ); + expect(preparer).toContain('"--config.node-linker=hoisted"'); + expect(preparer).toContain("validateRuntimeTree();"); + expect(preparer).toContain("@modelcontextprotocol/sdk/server/index.js"); + expect(preparer).toContain('"zod"'); + expect(config.linux?.executableName).toBe("codeux"); + }); + + it("installs and launches every native package format for release smoke", () => { + const installerSmoke = fs.readFileSync( + path.join(process.cwd(), "scripts", "smoke-installed-electron.mjs"), + "utf8", + ); + const mainProcessSource = fs.readFileSync(path.join(process.cwd(), "src/electron/main.ts"), "utf8"); + + expect(installerSmoke).toContain('findArtifact(".deb")'); + expect(installerSmoke).toContain('["/S", `/D=${installDirectory}`]'); + expect(installerSmoke).toContain('findArtifact(".dmg")'); + expect(installerSmoke).toContain('run("hdiutil", ["attach"'); + expect(installerSmoke).toContain('const versionMarker = `-${packageJson.version}-`'); + expect(installerSmoke).toContain("entry.name.includes(versionMarker)"); + expect(installerSmoke).toContain("CODE_UX_ELECTRON_STARTUP_SMOKE_FILE"); + expect(installerSmoke).toContain("marker.packaged !== true"); + expect(mainProcessSource).toContain('window.webContents.once("did-finish-load"'); + expect(mainProcessSource).toContain("writeElectronStartupSmoke"); + }); }); diff --git a/tests/backend/electron/startup-smoke.test.ts b/tests/backend/electron/startup-smoke.test.ts new file mode 100644 index 0000000000..8d8cb048d2 --- /dev/null +++ b/tests/backend/electron/startup-smoke.test.ts @@ -0,0 +1,52 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { writeElectronStartupSmoke } from "../../../src/electron/startup-smoke.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => ( + rm(directory, { recursive: true, force: true }) + ))); +}); + +describe("Electron startup smoke marker", () => { + it("atomically records packaged renderer readiness", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "code-ux-electron-smoke-")); + temporaryDirectories.push(directory); + const markerPath = path.join(directory, "nested", "ready.json"); + + const record = await writeElectronStartupSmoke(markerPath, { + version: "0.9.10", + platform: process.platform, + arch: "x64", + packaged: true, + dashboardOrigin: "http://127.0.0.1:4567", + rendererUrl: "http://127.0.0.1:4567/", + pid: 42, + now: () => new Date("2026-07-15T12:00:00.000Z"), + }); + + expect(record).toEqual(expect.objectContaining({ + schemaVersion: 1, + version: "0.9.10", + packaged: true, + pid: 42, + readyAt: "2026-07-15T12:00:00.000Z", + })); + expect(JSON.parse(await readFile(markerPath, "utf8"))).toEqual(record); + }); + + it("rejects relative marker paths", async () => { + await expect(writeElectronStartupSmoke("ready.json", { + version: "0.9.10", + platform: process.platform, + arch: process.arch, + packaged: true, + dashboardOrigin: "http://127.0.0.1:4567", + rendererUrl: "http://127.0.0.1:4567/", + })).rejects.toThrow("must be absolute"); + }); +}); diff --git a/tests/backend/infrastructure/git/local-merge.test.ts b/tests/backend/infrastructure/git/local-merge.test.ts index cc6389e790..19156ee6f3 100644 --- a/tests/backend/infrastructure/git/local-merge.test.ts +++ b/tests/backend/infrastructure/git/local-merge.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { runCommandStrict } from "../../../../src/services/cli-process-runner.js"; +import { acquireProjectGitHelperForSprint } from "../../../../src/shared/subprocess/command-runner.js"; import { getCheckedOutRef, restoreCheckedOutRef, @@ -14,6 +15,7 @@ import { findRecoverableWorkerBranch, workerBranchHasMergeWork, workerBranchIsMergedIntoFeature, + resolveWorkerBranchMergeState, deleteBranchLocally, } from "../../../../src/infrastructure/git/local-merge.js"; @@ -137,6 +139,11 @@ describe("local-merge helpers", () => { expect(files).toContain("two.txt"); expect(runner.mock.calls.filter(([, args]) => args[0] === "worktree" && args[1] === "add")).toHaveLength(1); expect(runner.mock.calls.filter(([, args]) => args[0] === "worktree" && args[1] === "remove")).toHaveLength(1); + expect(runner.mock.calls.filter(([, args]) => ( + args[0] === "rev-parse" + && args[1] === "--verify" + && args[2] === "refs/heads/feature^{commit}" + ))).toHaveLength(0); }); it("retries a temporary merge when the target ref advances during publication", async () => { @@ -188,6 +195,7 @@ describe("local-merge helpers", () => { const previousGitContainerMode = process.env.CODE_UX_GIT_CONTAINER_MODE; process.env.CODE_UX_CONTAINERIZED_GIT = "1"; delete process.env.CODE_UX_GIT_CONTAINER_MODE; + const releaseGitHelper = acquireProjectGitHelperForSprint(repo); try { await git(repo, "checkout", "feature"); @@ -207,6 +215,7 @@ describe("local-merge helpers", () => { const files = (await git(repo, "ls-tree", "--name-only", "feature")).stdout; expect(files).toContain("host-worktree.txt"); } finally { + await releaseGitHelper(); if (previousContainerizedGit === undefined) { delete process.env.CODE_UX_CONTAINERIZED_GIT; } else { @@ -858,10 +867,13 @@ describe("findRecoverableWorkerBranch", () => { it("refuses to delete the currently checked-out branch", async () => { const current = (await git(repo, "symbolic-ref", "--short", "HEAD")).stdout.trim(); - const deleted = await deleteBranchLocally({ repoPath: repo, branch: current }); + const runner = vi.fn((command: string, args: string[], cwd: string) => runCommandStrict(command, args, cwd)); + const deleted = await deleteBranchLocally({ repoPath: repo, branch: current, runner }); expect(deleted).toBe(false); const list = (await git(repo, "branch", "--format=%(refname:short)")).stdout; expect(list).toContain(current); + expect(runner).toHaveBeenCalledTimes(1); + expect(runner).toHaveBeenCalledWith("git", ["branch", "-D", current], repo); }); it("returns false for a non-existent branch without throwing", async () => { @@ -950,6 +962,41 @@ describe("workerBranchHasMergeWork", () => { })).resolves.toBe(true); expect(revListRanges).toEqual(["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa..bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"]); + expect(runner).toHaveBeenCalledTimes(3); + }); + + it("resolves missing, merged, and unmerged branch state with a bounded common path", async () => { + await git(repo, "branch", "task/noop", "feature"); + await git(repo, "checkout", "-b", "task/ahead", "feature"); + await commitFile(repo, "ahead.txt", "ahead\n", "feat: ahead"); + await git(repo, "checkout", "main"); + const runner = vi.fn((command: string, args: string[], cwd: string) => runCommandStrict(command, args, cwd)); + + await expect(resolveWorkerBranchMergeState({ + repoPath: repo, + featureBranch: "feature", + workerBranch: "task/noop", + runner, + })).resolves.toMatchObject({ state: "merged", sourceCommit: expect.any(String) }); + expect(runner).toHaveBeenCalledTimes(3); + + runner.mockClear(); + await expect(resolveWorkerBranchMergeState({ + repoPath: repo, + featureBranch: "feature", + workerBranch: "task/ahead", + runner, + })).resolves.toMatchObject({ state: "unmerged", sourceCommit: expect.any(String) }); + expect(runner).toHaveBeenCalledTimes(3); + + runner.mockClear(); + await expect(resolveWorkerBranchMergeState({ + repoPath: repo, + featureBranch: "feature", + workerBranch: "task/missing", + runner, + })).resolves.toEqual({ state: "missing", sourceCommit: null, targetCommit: null }); + expect(runner).toHaveBeenCalledTimes(2); }); }); diff --git a/tests/backend/infrastructure/providers/cli/claude-code-log-parser.test.ts b/tests/backend/infrastructure/providers/cli/claude-code-log-parser.test.ts index 29dfca5283..f9145c8d90 100644 --- a/tests/backend/infrastructure/providers/cli/claude-code-log-parser.test.ts +++ b/tests/backend/infrastructure/providers/cli/claude-code-log-parser.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect } from "vitest"; -import { parseClaudeCodeSessionJsonl } from "../../../../../src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.js"; +import { + ClaudeCodeLogAccumulator, + parseClaudeCodeSessionJsonl, +} from "../../../../../src/infrastructure/providers/cli/provider-logs/claude-code-log-parser.js"; // ─── Test fixture helpers ──────────────────────────────────────────────────── @@ -686,3 +689,38 @@ describe("parseClaudeCodeSessionJsonl", () => { expect(JSON.stringify(result)).not.toContain("sk-test-secret"); }); }); + +describe("ClaudeCodeLogAccumulator", () => { + it("parses only appended chunks and reports the changed turn suffix", () => { + const first = makeUserEntry({ content: "Inspect the project." }); + const second = makeAssistantEntry({ + messageId: "msg_incremental", + content: [{ type: "text", text: "Inspection complete." }], + }); + const accumulator = new ClaudeCodeLogAccumulator(); + + const partial = accumulator.appendChunk(`${first}\n${second.slice(0, 20)}`, "session-file"); + expect(partial.conversation.map((turn) => turn.text)).toEqual(["Inspect the project."]); + + const completed = accumulator.appendChunk(`${second.slice(20)}\n`, "session-file"); + expect(completed.conversation.map((turn) => turn.text)).toEqual([ + "Inspect the project.", + "Inspection complete.", + ]); + expect(completed.conversationRevision).toBe(2); + expect(completed.conversationChangedFromIndex).toBe(1); + }); + + it("resets retained state when the transcript source changes", () => { + const accumulator = new ClaudeCodeLogAccumulator(); + accumulator.appendChunk(`${makeUserEntry({ content: "Old source" })}\n`, "old-source"); + + const result = accumulator.appendChunk( + `${makeUserEntry({ content: "New source" })}\n`, + "new-source", + ); + + expect(result.conversation.map((turn) => turn.text)).toEqual(["New source"]); + expect(result.conversationRevision).toBe(1); + }); +}); diff --git a/tests/backend/infrastructure/providers/cli/codex-log-parser.test.ts b/tests/backend/infrastructure/providers/cli/codex-log-parser.test.ts index 185ffd3c89..7f0d80c064 100644 --- a/tests/backend/infrastructure/providers/cli/codex-log-parser.test.ts +++ b/tests/backend/infrastructure/providers/cli/codex-log-parser.test.ts @@ -462,6 +462,35 @@ describe("CodexRolloutAccumulator", () => { expect(result.conversation[1]).toMatchObject({ toolOutput: "passed", toolStatus: "completed" }); }); + it("stops retaining duplicate event messages after canonical turns arrive", () => { + const accumulator = new CodexRolloutAccumulator(); + const fallback = JSON.stringify({ + type: "event_msg", + timestamp: "2026-06-01T10:00:00.000Z", + payload: { type: "agent_message", message: "fallback draft" }, + }); + const canonical = responseItem("2026-06-01T10:00:01.000Z", { + id: "msg-1", + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "canonical answer" }], + }); + const first = accumulator.update(`${fallback}\n${canonical}`); + const revision = first.conversationRevision; + + const duplicateAfterCanonical = JSON.stringify({ + type: "event_msg", + timestamp: "2026-06-01T10:00:02.000Z", + payload: { type: "agent_message", message: "duplicate canonical answer" }, + }); + const second = accumulator.update(`${fallback}\n${canonical}\n${duplicateAfterCanonical}`); + + expect(second.conversation).toEqual([ + expect.objectContaining({ kind: "assistant", text: "canonical answer" }), + ]); + expect(second.conversationRevision).toBe(revision); + }); + it("resets safely after truncation or source rotation", () => { const accumulator = new CodexRolloutAccumulator(); const first = [sessionMeta("old"), userMessage("2026-06-01T10:00:00.000Z", "old prompt")].join("\n"); diff --git a/tests/backend/infrastructure/providers/cli/docker-helper-pool.test.ts b/tests/backend/infrastructure/providers/cli/docker-helper-pool.test.ts index e62fdedd60..7333612023 100644 --- a/tests/backend/infrastructure/providers/cli/docker-helper-pool.test.ts +++ b/tests/backend/infrastructure/providers/cli/docker-helper-pool.test.ts @@ -1,10 +1,16 @@ -import { describe, expect, it, vi } from "vitest"; -import { DockerHelperContainerPool } from "../../../../../src/infrastructure/providers/cli/docker-helper-pool.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + DockerHelperContainerPool, + type HelperCommandRunner, +} from "../../../../../src/infrastructure/providers/cli/docker-helper-pool.js"; type Call = { command: string; args: string[] }; type RunnerResult = { ok: boolean; code?: number; stdout?: string; stderr?: string }; -function makePool(overrides?: (call: Call) => RunnerResult | Promise | undefined) { +function makePool( + overrides?: (call: Call) => RunnerResult | Promise | undefined, + options: { maxContainers?: number } = {}, +) { const calls: Call[] = []; const runner = vi.fn(async (command: string, args: string[]) => { calls.push({ command, args }); @@ -26,33 +32,171 @@ function makePool(overrides?: (call: Call) => RunnerResult | Promise `helper-${key}`, buildCreateArgs: (_key, name) => ["run", "-d", "--name", name, "img"], - }, runner as any); + maxContainers: options.maxContainers, + }, runner as HelperCommandRunner); return { pool, calls, runner }; } describe("DockerHelperContainerPool", () => { + const pools: DockerHelperContainerPool[] = []; + + afterEach(async () => { + await Promise.all(pools.splice(0).map((pool) => pool.shutdown())); + vi.useRealTimers(); + }); + it("creates a container once per key and reuses it", async () => { const { pool, calls } = makePool(); + pools.push(pool); const id1 = await pool.ensure("k1"); const id2 = await pool.ensure("k1"); expect(id1).toBe("cid"); expect(id2).toBe("cid"); const creates = calls.filter((c) => c.args[0] === "run" && c.args.includes("-d")); expect(creates).toHaveLength(1); + expect(calls.findIndex((call) => call.args[0] === "rm")).toBe(-1); // Created with the deterministic name from nameFor. expect(creates[0].args).toEqual(["run", "-d", "--name", "helper-k1", "img"]); }); + it("reclaims and retries once only after an explicit deterministic-name conflict", async () => { + let createCount = 0; + const { pool, calls } = makePool(({ args }) => { + if (args[0] !== "run") { + return undefined; + } + createCount += 1; + return createCount === 1 + ? { ok: false, stderr: 'Conflict. The container name "/helper-k1" is already in use by container "old".' } + : { ok: true, stdout: "replacement-cid\n" }; + }); + pools.push(pool); + + await expect(pool.ensure("k1")).resolves.toBe("replacement-cid"); + + const lifecycleCalls = calls.filter((call) => call.args[0] === "run" || call.args[0] === "rm"); + expect(lifecycleCalls.map((call) => call.args[0])).toEqual(["run", "rm", "run"]); + expect(lifecycleCalls[1].args).toEqual(["rm", "-f", "-v", "helper-k1"]); + }); + + it("does not remove or retry after a non-conflict create failure", async () => { + const { pool, calls } = makePool(({ args }) => args[0] === "run" + ? { ok: false, stderr: "daemon storage unavailable" } + : undefined); + pools.push(pool); + + await expect(pool.ensure("k1")).rejects.toThrow(/storage unavailable/); + expect(calls.filter((call) => call.args[0] === "run")).toHaveLength(1); + expect(calls.filter((call) => call.args[0] === "rm")).toHaveLength(0); + }); + it("dedupes concurrent ensures into a single create", async () => { const { pool, calls } = makePool(); + pools.push(pool); const [a, b] = await Promise.all([pool.ensure("k1"), pool.ensure("k1")]); expect(a).toBe(b); const creates = calls.filter((c) => c.args[0] === "run" && c.args.includes("-d")); expect(creates).toHaveLength(1); }); + it("bounds concurrent Docker helper lifecycle mutations across a wide start wave", async () => { + let activeCreates = 0; + let peakCreates = 0; + let releaseCreates!: () => void; + const createGate = new Promise((resolve) => { + releaseCreates = resolve; + }); + const { pool } = makePool(async ({ args }) => { + if (args[0] !== "run" || !args.includes("-d")) { + return undefined; + } + activeCreates += 1; + peakCreates = Math.max(peakCreates, activeCreates); + await createGate; + activeCreates -= 1; + return { ok: true, stdout: `cid-${peakCreates}\n` }; + }); + pools.push(pool); + + const starts = Array.from({ length: 12 }, (_, index) => pool.ensure(`k${index}`)); + await vi.waitFor(() => expect(activeCreates).toBe(4)); + releaseCreates(); + await Promise.all(starts); + + expect(peakCreates).toBe(4); + }); + + it("evicts the least-recently-used idle helper before admitting beyond capacity", async () => { + const { pool, calls } = makePool(undefined, { maxContainers: 1 }); + pools.push(pool); + + await pool.ensure("k1"); + await pool.ensure("k2"); + + const firstRemovalIndex = calls.findIndex((call) => call.args[0] === "rm" && call.args.includes("cid")); + const secondCreateIndex = calls.findIndex((call) => call.args[0] === "run" && call.args.includes("helper-k2")); + expect(firstRemovalIndex).toBeGreaterThanOrEqual(0); + expect(secondCreateIndex).toBeGreaterThan(firstRemovalIndex); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); + + it("waits for an active helper before admitting a distinct key at capacity", async () => { + let finishFirst: (() => void) | null = null; + const { pool, calls } = makePool(undefined, { maxContainers: 1 }); + pools.push(pool); + + const first = pool.withContainer("k1", async () => { + await new Promise((resolve) => { + finishFirst = resolve; + }); + }); + await vi.waitFor(() => expect(finishFirst).not.toBeNull()); + const second = pool.withContainer("k2", async () => "second"); + await Promise.resolve(); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + + finishFirst?.(); + await first; + await expect(second).resolves.toBe("second"); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); + + it("does not evict a workflow-reserved helper between commands", async () => { + const { pool, calls } = makePool(undefined, { maxContainers: 1 }); + pools.push(pool); + + const releaseReservation = pool.reserve("k1"); + await pool.ensure("k1"); + const waiting = pool.ensure("k2"); + await Promise.resolve(); + + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-k1"))).toBe(false); + + releaseReservation(); + releaseReservation(); + await expect(waiting).resolves.toBe("cid"); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); + + it("does not reap a reserved helper until its workflow lease is released", async () => { + vi.useFakeTimers(); + const { pool, calls } = makePool(); + pools.push(pool); + const releaseReservation = pool.reserve("k1"); + await pool.ensure("k1"); + + await vi.advanceTimersByTimeAsync(300_000); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-k1"))).toBe(false); + + releaseReservation(); + await vi.advanceTimersByTimeAsync(120_000); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(true); + }); + it("recreates after invalidate", async () => { const { pool, calls } = makePool(); + pools.push(pool); await pool.ensure("k1"); pool.invalidate("k1"); await pool.ensure("k1"); @@ -60,11 +204,43 @@ describe("DockerHelperContainerPool", () => { expect(creates).toHaveLength(2); }); - it("release removes the container and image-declared anonymous volumes by deterministic name", async () => { + it("does not invalidate a replacement created for an older failed generation", async () => { + let createCount = 0; + let resolveReplacement: ((value: RunnerResult) => void) | null = null; + const { pool, calls } = makePool(({ args }) => { + if (args[0] !== "run" || !args.includes("-d")) { + return undefined; + } + createCount += 1; + if (createCount === 1) { + return { ok: true, stdout: "cid-old\n" }; + } + return new Promise((resolve) => { + resolveReplacement = resolve; + }); + }); + pools.push(pool); + + const oldId = await pool.ensure("k1"); + expect(pool.invalidate("k1", oldId)).toBe(true); + const replacementPromise = pool.ensure("k1"); + await vi.waitFor(() => expect(resolveReplacement).not.toBeNull()); + + expect(pool.invalidate("k1", oldId)).toBe(false); + resolveReplacement?.({ ok: true, stdout: "cid-new\n" }); + await expect(replacementPromise).resolves.toBe("cid-new"); + await expect(pool.ensure("k1")).resolves.toBe("cid-new"); + + const creates = calls.filter((call) => call.args[0] === "run" && call.args.includes("-d")); + expect(creates).toHaveLength(2); + }); + + it("release removes the container and image-declared anonymous volumes by exact id", async () => { const { pool, calls } = makePool(); + pools.push(pool); await pool.ensure("k1"); await pool.release("k1"); - const removals = calls.filter((c) => c.args[0] === "rm" && c.args.includes("helper-k1")); + const removals = calls.filter((c) => c.args[0] === "rm" && c.args.includes("cid")); expect(removals.length).toBeGreaterThanOrEqual(1); expect(removals.every((call) => call.args.includes("-v"))).toBe(true); // After release the next ensure creates a fresh container. @@ -75,6 +251,7 @@ describe("DockerHelperContainerPool", () => { it("shutdown removes all tracked containers", async () => { const { pool, calls } = makePool(); + pools.push(pool); await pool.ensure("k1"); await pool.ensure("k2"); await pool.shutdown(); @@ -92,6 +269,7 @@ describe("DockerHelperContainerPool", () => { resolveCreate = resolve; }); }); + pools.push(pool); const ensurePromise = pool.ensure("k1"); await vi.waitFor(() => expect(resolveCreate).not.toBeNull()); @@ -109,7 +287,119 @@ describe("DockerHelperContainerPool", () => { it("isContainerGone detects missing containers", () => { const { pool } = makePool(); + pools.push(pool); expect(pool.isContainerGone({ ok: false, code: 1, stdout: "", stderr: "Error: No such container: cid" })).toBe(true); expect(pool.isContainerGone({ ok: false, code: 1, stdout: "", stderr: "fatal: something else" })).toBe(false); }); + + it("pins an active command until release completes", async () => { + let finishCommand: (() => void) | null = null; + const { pool, calls } = makePool(); + pools.push(pool); + + const command = pool.withContainer("k1", async (id) => { + expect(id).toBe("cid"); + await new Promise((resolve) => { + finishCommand = resolve; + }); + return "done"; + }); + await vi.waitFor(() => expect(finishCommand).not.toBeNull()); + + let released = false; + const release = pool.release("k1").then(() => { + released = true; + }); + await Promise.resolve(); + expect(released).toBe(false); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(false); + + finishCommand?.(); + await expect(command).resolves.toBe("done"); + await release; + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(true); + }); + + it("deduplicates concurrent releases and delays a new acquisition until draining finishes", async () => { + let finishCommand: (() => void) | null = null; + const { pool, calls } = makePool(); + pools.push(pool); + + const active = pool.withContainer("k1", async () => { + await new Promise((resolve) => { + finishCommand = resolve; + }); + }); + await vi.waitFor(() => expect(finishCommand).not.toBeNull()); + + const releaseA = pool.release("k1"); + const releaseB = pool.release("k1"); + const next = pool.withContainer("k1", async (id) => id); + await Promise.resolve(); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + + finishCommand?.(); + await active; + await Promise.all([releaseA, releaseB]); + await expect(next).resolves.toBe("cid"); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); + + it("does not reap a pinned command and reaps it after the idle TTL", async () => { + vi.useFakeTimers(); + let finishCommand: (() => void) | null = null; + const calls: Call[] = []; + const runner = vi.fn(async (command: string, args: string[]) => { + calls.push({ command, args }); + return args[0] === "run" + ? { ok: true, code: 0, stdout: "cid\n", stderr: "" } + : { ok: true, code: 0, stdout: "", stderr: "" }; + }); + const pool = new DockerHelperContainerPool({ + nameFor: (key) => `helper-${key}`, + buildCreateArgs: (_key, name) => ["run", "-d", "--name", name, "img"], + idleTtlMs: 100, + reapIntervalMs: 25, + }, runner as HelperCommandRunner); + pools.push(pool); + + const active = pool.withContainer("k1", async () => { + await new Promise((resolve) => { + finishCommand = resolve; + }); + }); + await vi.waitFor(() => expect(finishCommand).not.toBeNull()); + await vi.advanceTimersByTimeAsync(500); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(false); + + finishCommand?.(); + await active; + await vi.advanceTimersByTimeAsync(99); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("cid"))).toBe(true); + }); + + it("rejects invalid lifecycle bounds and acquisitions after shutdown", async () => { + expect(() => new DockerHelperContainerPool({ + nameFor: (key) => key, + buildCreateArgs: () => [], + idleTtlMs: -1, + })).toThrow(/idle TTL/); + expect(() => new DockerHelperContainerPool({ + nameFor: (key) => key, + buildCreateArgs: () => [], + reapIntervalMs: 0, + })).toThrow(/reap interval/); + expect(() => new DockerHelperContainerPool({ + nameFor: (key) => key, + buildCreateArgs: () => [], + maxContainers: 0, + })).toThrow(/capacity/); + + const { pool } = makePool(); + pools.push(pool); + await pool.shutdown(); + await expect(pool.ensure("late")).rejects.toThrow(/shutting down/); + }); }); diff --git a/tests/backend/infrastructure/providers/cli/docker-runner.test.ts b/tests/backend/infrastructure/providers/cli/docker-runner.test.ts index 4518191686..63ae841c99 100644 --- a/tests/backend/infrastructure/providers/cli/docker-runner.test.ts +++ b/tests/backend/infrastructure/providers/cli/docker-runner.test.ts @@ -552,9 +552,11 @@ describe("DockerRunner", () => { }); it("supports mockup-cli Docker labels, names, env files, and argv files", async () => { + const prompt = "mockup-cli:write fixture.txt :: hello"; await runner.runProviderInDocker({ command: "node", - args: ["-e", "console.log('mock')", "mockup-cli:write fixture.txt :: hello"], + args: ["-e", "console.log('mock')", prompt], + prompt, cwd: "docker-volume://workspace-1", providerEnv: { CODE_UX_MOCKUP_MODEL: "default", @@ -579,7 +581,7 @@ describe("DockerRunner", () => { "--label", "code-ux.command=node", "--label", - "code-ux.args-count=3", + "code-ux.args-count=2", ])); expect(dockerArgs.slice(-2)).toEqual(["provider-runner", "node"]); expect(dockerArgs).toContain("CODE_UX_PROVIDER_ARGV_FILE=/opt/code-ux/provider-argv.sh"); @@ -589,7 +591,13 @@ describe("DockerRunner", () => { expect(envWrite?.[1]).toContain("CODE_UX_MOCKUP_SESSION_ID=mock-session-1"); const argvWrite = vi.mocked(fs.writeFile).mock.calls.find(([file]) => String(file).endsWith("provider-argv.sh")); - expect(argvWrite?.[1]).toContain("mockup-cli:write fixture.txt :: hello"); + expect(argvWrite?.[1]).not.toContain(prompt); + const promptWrite = vi.mocked(fs.writeFile).mock.calls.find(([file]) => String(file).endsWith("provider-prompt.txt")); + expect(promptWrite?.[1]).toBe(prompt); + expect(promptWrite?.[2]).toEqual(expect.objectContaining({ mode: 0o600 })); + expect(vi.mocked(runStreamingCommand).mock.calls[0]?.[4]).toEqual(expect.objectContaining({ + stdinFile: "/tmp/code-ux-docker-123/provider-prompt.txt", + })); }); it("keeps Docker and provider execution behind mocked command runners", async () => { @@ -845,6 +853,7 @@ describe("DockerRunner", () => { await runner.runProviderInDocker({ command: "codex", args: ["exec", "--yolo", longPrompt], + prompt: longPrompt, cwd: "docker-volume://workspace-1", providerEnv: {}, sessionId: "session-1", @@ -870,10 +879,74 @@ describe("DockerRunner", () => { expect(dockerArgs.slice(-2)).toEqual(["provider-runner", "codex"]); const argvWrite = vi.mocked(fs.writeFile).mock.calls.find(([file]) => String(file).endsWith("provider-argv.sh")); - expect(argvWrite?.[1]).toContain(`plan ${"x".repeat(1024)}`); - expect(argvWrite?.[1]).toContain(" with "); - expect(argvWrite?.[1]).toContain("'\"'\"'quotes'\"'\"'"); + expect(argvWrite?.[1]).not.toContain(`plan ${"x".repeat(1024)}`); + expect(argvWrite?.[1]).toContain("'-'"); expect(argvWrite?.[2]).toEqual(expect.objectContaining({ mode: 0o600 })); + const promptWrite = vi.mocked(fs.writeFile).mock.calls.find(([file]) => String(file).endsWith("provider-prompt.txt")); + expect(promptWrite?.[1]).toBe(longPrompt); + expect(promptWrite?.[2]).toEqual(expect.objectContaining({ mode: 0o600 })); + expect(vi.mocked(runStreamingCommand).mock.calls[0]?.[4]).toEqual(expect.objectContaining({ + stdinFile: "/tmp/code-ux-docker-123/provider-prompt.txt", + })); + }); + + it.each([ + { provider: "gemini" as const, command: "gemini", args: ["--yolo", "--p", "PROMPT"], expected: ["--yolo"] }, + { provider: "qwen-code" as const, command: "qwen", args: ["--yolo", "-p", "PROMPT"], expected: ["--yolo"] }, + { provider: "claude-code" as const, command: "claude", args: ["--print", "PROMPT"], expected: ["--print"] }, + { provider: "opencode" as const, command: "opencode", args: ["run", "--format", "json", "PROMPT"], expected: ["run", "--format", "json"] }, + ])("streams oversized $provider prompts instead of reconstructing a large container argument", async ({ provider, command, args, expected }) => { + const prompt = "PROMPT".repeat(12_000); + const providerArgs = args.map((arg) => arg === "PROMPT" ? prompt : arg); + + await runner.runProviderInDocker({ + command, + args: providerArgs, + prompt, + cwd: "docker-volume://workspace-1", + providerEnv: {}, + sessionId: `large-${provider}`, + providerLabel: provider, + workflowSettings: { + executionMode: "DOCKER", + containerImage: "node:24", + containerSetupScriptPath: "", + containerCacheSetupScriptImage: false, + } as any, + repoPath: "/repo/project", + onActivity: vi.fn(), + }); + + const argvWrite = vi.mocked(fs.writeFile).mock.calls.find(([file]) => String(file).endsWith("provider-argv.sh")); + for (const expectedArg of expected) { + expect(argvWrite?.[1]).toContain(`'${expectedArg}'`); + } + expect(argvWrite?.[1]).not.toContain(prompt.slice(0, 1024)); + expect(vi.mocked(runStreamingCommand).mock.calls[0]?.[4]).toEqual(expect.objectContaining({ + stdinFile: "/tmp/code-ux-docker-123/provider-prompt.txt", + })); + }); + + it("preserves Antigravity's single prompt argument below the execve limit", () => { + const prompt = `${"é".repeat(30_000)}\n${"z".repeat(30_000)}`; + const launch = (runner as any).prepareProviderLaunch( + "antigravity", + ["--dangerously-skip-permissions", "-p", prompt], + prompt, + ) as { args: string[]; stdinPrompt: string | null }; + + expect(launch.stdinPrompt).toBeNull(); + expect(launch.args).toEqual(["--dangerously-skip-permissions", "-p", prompt]); + }); + + it("rejects an unsafe Antigravity prompt before execve can fail opaquely", () => { + const prompt = "é".repeat(70_000); + + expect(() => (runner as any).prepareProviderLaunch( + "antigravity", + ["--dangerously-skip-permissions", "-p", prompt], + prompt, + )).toThrow(/Antigravity cannot safely accept a 140000-byte prompt.*safe limit is 122880 bytes/); }); it("applies configured memory limits to provider Docker runs", async () => { diff --git a/tests/backend/infrastructure/providers/cli/mock-provider-cli-shim.test.ts b/tests/backend/infrastructure/providers/cli/mock-provider-cli-shim.test.ts new file mode 100644 index 0000000000..6aea0ad7b0 --- /dev/null +++ b/tests/backend/infrastructure/providers/cli/mock-provider-cli-shim.test.ts @@ -0,0 +1,40 @@ +import { spawn } from "node:child_process"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it } from "vitest"; + +const SHIM_PATH = path.resolve(process.cwd(), "scripts/e2e/mock-provider-cli.mjs"); + +describe("mock provider CLI shim", () => { + it("reads a large prompt from stdin when argv omits --prompt", async () => { + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "codeux-provider-shim-")); + const prompt = `[mock-provider:write=stdin-result.txt]\n${"large context ".repeat(15_000)}`; + try { + const result = await new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve, reject) => { + const child = spawn(process.execPath, [SHIM_PATH, "--provider", "codex", "--model", "default"], { + cwd, + stdio: ["pipe", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + child.on("close", (code) => resolve({ + code, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + })); + child.stdin.end(prompt, "utf8"); + }); + + expect(result.code).toBe(0); + expect(result.stderr).toContain("provider=codex model=default"); + expect(result.stdout).toContain('"type":"turn.completed"'); + expect(await fs.readFile(path.join(cwd, "stdin-result.txt"), "utf8")).toContain("Code UX mock provider output"); + } finally { + await fs.rm(cwd, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/backend/infrastructure/providers/cli/provider-execution-loop.test.ts b/tests/backend/infrastructure/providers/cli/provider-execution-loop.test.ts index e7b1289837..4e8bd49bb4 100644 --- a/tests/backend/infrastructure/providers/cli/provider-execution-loop.test.ts +++ b/tests/backend/infrastructure/providers/cli/provider-execution-loop.test.ts @@ -106,6 +106,24 @@ describe("ProviderExecutionLoop", () => { expect(opts.trackingOnActivity).toHaveBeenCalledWith("Claude Code could not resume the previous conversation (no conversation found). Retrying once with a fresh session...", "provider"); }); + it("does not replace a required Claude continuation with a fresh session", async () => { + const runCmd = vi.fn().mockResolvedValue({ ok: false, stdout: "", stderr: "No conversation found" }); + const opts: ProviderExecutionLoopOptions = { + ...getDefaultOptions(), + provider: "claude-code", + continueSession: true, + allowFreshSessionFallback: false, + runCmd, + isClaudeConversationNotFoundError: vi.fn().mockReturnValue(true), + }; + + const result = await runProviderExecutionLoop(opts); + + expect(result.ok).toBe(false); + expect(runCmd).toHaveBeenCalledTimes(1); + expect(opts.buildFreshClaudeSpec).not.toHaveBeenCalled(); + }); + it("retries OpenCode with a fresh session when the native session is not found", async () => { const runCmd = vi.fn() .mockResolvedValueOnce({ ok: false, stdout: "", stderr: "Error: Session not found" }) @@ -127,6 +145,24 @@ describe("ProviderExecutionLoop", () => { expect(opts.trackingOnActivity).toHaveBeenCalledWith("OpenCode could not resume the previous session (session not found). Retrying once with a fresh session...", "provider"); }); + it("does not replace a required OpenCode continuation with a fresh session", async () => { + const runCmd = vi.fn().mockResolvedValue({ ok: false, stdout: "", stderr: "Session not found" }); + const opts: ProviderExecutionLoopOptions = { + ...getDefaultOptions(), + provider: "opencode", + continueSession: true, + allowFreshSessionFallback: false, + runCmd, + isOpenCodeSessionNotFoundError: vi.fn().mockReturnValue(true), + }; + + const result = await runProviderExecutionLoop(opts); + + expect(result.ok).toBe(false); + expect(runCmd).toHaveBeenCalledTimes(1); + expect(opts.buildFreshOpenCodeSpec).not.toHaveBeenCalled(); + }); + it("demotes Antigravity run to failure when diagnostics indicate an error", async () => { const runCmd = vi.fn().mockResolvedValue({ ok: true, stdout: "", stderr: "" }); const readAntigravityDiagnostics = vi.fn().mockResolvedValue("Executor error: INTERNAL_ERROR"); diff --git a/tests/backend/infrastructure/providers/cli/provider-telemetry-watcher.test.ts b/tests/backend/infrastructure/providers/cli/provider-telemetry-watcher.test.ts index 89d63575ba..36b760169e 100644 --- a/tests/backend/infrastructure/providers/cli/provider-telemetry-watcher.test.ts +++ b/tests/backend/infrastructure/providers/cli/provider-telemetry-watcher.test.ts @@ -200,6 +200,83 @@ describe("ProviderTelemetryWatcher", () => { await watcher.stop(); }); + it("passes append-parsed Claude deltas without retaining or joining the raw JSONL", async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + const jsonl = `${JSON.stringify({ + type: "assistant", + sessionId: "claude-chunk-session", + timestamp: "2026-07-14T00:00:01.000Z", + message: { + id: "claude-message-1", + role: "assistant", + content: [{ type: "text", text: "incremental Claude output" }], + usage: { input_tokens: 10, output_tokens: 5 }, + }, + })}\n`; + const bytes = Buffer.from(jsonl); + const readChunk = vi.fn(async ( + _nativeSessionId: string, + cursor: { sourceId: string | null; offset: number }, + ) => cursor.offset === 0 + ? { + sourceId: "claude-file", + startOffset: 0, + nextOffset: bytes.length, + totalBytes: bytes.length, + contentBase64: bytes.toString("base64"), + reset: true, + } + : { + sourceId: "claude-file", + startOffset: bytes.length, + nextOffset: bytes.length, + totalBytes: bytes.length, + contentBase64: "", + reset: false, + }); + const opts = { + provider: "claude-code" as const, + model: "test-model", + prompt: "test", + cwd: "/cwd", + startedMs: Date.parse("2026-07-14T00:00:00.000Z"), + workflowSettings: { executionMode: "DOCKER" as const }, + signal: controller.signal, + getAccumulatedRawStdout: () => "", + getAccumulatedStderr: () => "", + nativeSessionId: "claude-chunk-session", + sessionId: "sess-1", + antigravityLogPath: null, + readClaudeSessionJsonl: vi.fn(), + readClaudeSessionJsonlChunk: readChunk, + readCodexLatestSessionJson: vi.fn(), + readQwenLogData: vi.fn(), + parseAntigravityConversationId: vi.fn(), + readAntigravityTranscript: vi.fn(), + resolveAntigravityDatabase: vi.fn(), + onTelemetry: vi.fn(), + }; + const watcher = new ProviderTelemetryWatcher(opts as any); + watcher.start(); + + await vi.advanceTimersByTimeAsync(1000); + + expect(readChunk).toHaveBeenNthCalledWith(1, "claude-chunk-session", { sourceId: null, offset: 0 }); + expect(opts.readClaudeSessionJsonl).not.toHaveBeenCalled(); + expect(collectProviderUsageTelemetry).toHaveBeenCalledTimes(1); + expect(collectProviderUsageTelemetry).toHaveBeenCalledWith(expect.objectContaining({ + claudeSessionJsonl: null, + claudeSessionLog: expect.objectContaining({ + conversation: [expect.objectContaining({ text: "incremental Claude output" })], + conversationRevision: 1, + }), + })); + + controller.abort(); + await watcher.stop(); + }); + it.each([ { provider: "claude-code" as const, diff --git a/tests/backend/infrastructure/providers/cli/workspace-manager.test.ts b/tests/backend/infrastructure/providers/cli/workspace-manager.test.ts index 81709fd5a0..68d6c9b329 100644 --- a/tests/backend/infrastructure/providers/cli/workspace-manager.test.ts +++ b/tests/backend/infrastructure/providers/cli/workspace-manager.test.ts @@ -17,8 +17,18 @@ vi.mock("../../../../../src/services/cli-workflow-text-utils.js", () => ({ vi.mock("../../../../../src/services/cli-process-runner.js", () => ({ runCommandStrict: vi.fn(), })); +vi.mock("../../../../../src/infrastructure/providers/cli/workspace-volume-helper.js", () => ({ + workspaceVolumeHelperPool: { + exec: vi.fn(), + reserve: vi.fn(() => vi.fn()), + releaseVolume: vi.fn(), + }, +})); import { runCommandStrict } from "../../../../../src/services/cli-process-runner.js"; +import { workspaceVolumeHelperPool } from "../../../../../src/infrastructure/providers/cli/workspace-volume-helper.js"; + +const commandOk = (stdout = "") => ({ ok: true, stdout, stderr: "", code: 0, signal: null }); describe("WorkspaceManager", () => { let manager: WorkspaceManager; @@ -30,6 +40,8 @@ describe("WorkspaceManager", () => { vi.mocked(fs.rm).mockResolvedValue(undefined); vi.mocked(fs.realpath).mockImplementation(async (candidate) => String(candidate)); vi.mocked(fs.writeFile).mockResolvedValue(undefined); + vi.mocked(workspaceVolumeHelperPool.exec).mockResolvedValue(commandOk()); + vi.mocked(workspaceVolumeHelperPool.releaseVolume).mockResolvedValue(undefined); }); it("builds Docker volume handles for isolated workspaces", () => { @@ -110,6 +122,7 @@ describe("WorkspaceManager", () => { }); it("resolves current branch for a Docker workspace", async () => { + vi.mocked(workspaceVolumeHelperPool.exec).mockResolvedValue(commandOk("feature/task-2\n")); vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (command === "docker" && args[0] === "volume" && args[1] === "inspect") { return { ok: true, stdout: "[]", stderr: "", code: 0, signal: null } as any; @@ -117,24 +130,18 @@ describe("WorkspaceManager", () => { if (command === "docker" && args[0] === "image" && args[1] === "inspect") { return { ok: true, stdout: "[]", stderr: "", code: 0, signal: null } as any; } - if (command === "docker" && args[0] === "run" && args.includes("git")) { - return { ok: true, stdout: "feature/task-2\n", stderr: "", code: 0, signal: null } as any; - } return { ok: true, stdout: "", stderr: "", code: 0, signal: null } as any; }); const result = await manager.resolveCurrentBranch("docker-volume://workspace-1"); expect(result).toBe("feature/task-2"); - expect(runCommandStrict).toHaveBeenCalledWith("docker", expect.arrayContaining([ - "run", - "--entrypoint", - "git", - "alpine/git", - "rev-parse", - "--abbrev-ref", - "HEAD", - ]), expect.any(String), expect.anything(), expect.anything()); + expect(workspaceVolumeHelperPool.exec).toHaveBeenCalledWith( + "workspace-1", + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + "workspace-1-runtime", + expect.objectContaining({ workdir: "/workspace" }), + ); }); it("returns null when current branch cannot be resolved", async () => { @@ -175,24 +182,17 @@ describe("WorkspaceManager", () => { expect.arrayContaining(["volume", "create", "--label", "code-ux.workspace-runtime=true"]), expect.any(String), ); - const bootstrapCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") - && call[4] - && typeof call[4] === "object" - && "stdinFile" in call[4] + const bootstrapCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" + && call[3] + && "stdinFile" in call[3] ); - expect(bootstrapCall?.[1]).toEqual(expect.arrayContaining([ - "run", - "--rm", - "-i", - "--entrypoint", - "sh", - "alpine/git", - "-lc", - ])); - expect(bootstrapCall?.[4]).toEqual(expect.objectContaining({ + expect(bootstrapCall?.slice(0, 3)).toEqual([ + expect.stringMatching(/^code-ux-project-[a-f0-9]{12}-session-1-snapshot$/), + ["sh", "-lc", expect.any(String)], + expect.stringMatching(/^code-ux-project-[a-f0-9]{12}-session-1-snapshot-runtime$/), + ]); + expect(bootstrapCall?.[3]).toEqual(expect.objectContaining({ stdinFile: bundlePath, })); const bootstrapCommand = String(bootstrapCall?.[1]?.at(-1) || ""); @@ -205,6 +205,8 @@ describe("WorkspaceManager", () => { expect(bootstrapCommand).toContain("+refs/*:refs/*"); expect(bootstrapCommand).toContain("git -C /workspace config user.name"); expect(bootstrapCommand).toContain("git -C /workspace config user.email"); + expect(bootstrapCommand).toContain("(rm -rf /workspace/"); + expect(bootstrapCommand).toContain("(git -C /workspace remote remove origin"); expect(bootstrapCommand).not.toContain("git clone"); expect(vi.mocked(runCommandStrict).mock.calls.some((call) => call[0] === "bash")).toBe(false); if (typeof process.getuid === "function" && typeof process.getgid === "function") { @@ -214,6 +216,7 @@ describe("WorkspaceManager", () => { }); it("reuses a snapshot workspace only when it has a valid Git HEAD", async () => { + vi.mocked(workspaceVolumeHelperPool.exec).mockResolvedValue(commandOk("existing-head\n")); vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (command === "git" && args[0] === "rev-parse" && args[1] === "--show-toplevel") { return { ok: true, stdout: "/repo/project\n", stderr: "" } as any; @@ -232,12 +235,11 @@ describe("WorkspaceManager", () => { }); expect(workspace).toMatch(/^docker-volume:\/\/code-ux-project-[a-f0-9]{12}-session-1-snapshot$/); - expect(runCommandStrict).toHaveBeenCalledWith( - "docker", - expect.arrayContaining(["git", "rev-parse", "--verify", "HEAD"]), - expect.any(String), - expect.anything(), - expect.anything(), + expect(workspaceVolumeHelperPool.exec).toHaveBeenCalledWith( + expect.stringMatching(/^code-ux-project-[a-f0-9]{12}-session-1-snapshot$/), + ["git", "rev-parse", "--verify", "HEAD"], + expect.stringMatching(/^code-ux-project-[a-f0-9]{12}-session-1-snapshot-runtime$/), + expect.any(Object), ); expect(runCommandStrict).not.toHaveBeenCalledWith( "docker", @@ -247,6 +249,12 @@ describe("WorkspaceManager", () => { }); it("rebuilds an interrupted snapshot volume that has no Git HEAD", async () => { + vi.mocked(workspaceVolumeHelperPool.exec).mockImplementation(async (_volumeName, commandArgs) => { + if (commandArgs[0] === "git" && commandArgs.includes("--verify") && commandArgs.includes("HEAD")) { + return { ok: false, stdout: "", stderr: "fatal: Needed a single revision", code: 128, signal: null }; + } + return commandOk(); + }); vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (command === "git" && args[0] === "rev-parse" && args[1] === "--show-toplevel") { return { ok: true, stdout: "/repo/project\n", stderr: "" } as any; @@ -260,15 +268,6 @@ describe("WorkspaceManager", () => { if (command === "docker" && args[0] === "image" && args[1] === "inspect") { return { ok: true, stdout: "[]", stderr: "" } as any; } - if ( - command === "docker" - && args.includes("git") - && args.includes("rev-parse") - && args.includes("--verify") - && args.includes("HEAD") - ) { - throw new Error("fatal: Needed a single revision"); - } if (command === "git" && args[0] === "remote") { return { ok: true, stdout: "git@github.com:example/repo.git\n", stderr: "" } as any; } @@ -383,10 +382,8 @@ describe("WorkspaceManager", () => { fallbackBranch: "main", }); - const checkoutCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const checkoutCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && String(call[1].at(-1)).includes("git -C /workspace checkout") ); expect(String(checkoutCall?.[1].at(-1))).toContain( @@ -431,6 +428,15 @@ describe("WorkspaceManager", () => { it("falls back to a full seed when the targeted checkout fails", async () => { let checkoutAttempts = 0; + vi.mocked(workspaceVolumeHelperPool.exec).mockImplementation(async (_volumeName, commandArgs) => { + if (commandArgs.includes("checkout") || String(commandArgs.at(-1)).includes("git -C /workspace checkout")) { + checkoutAttempts += 1; + if (checkoutAttempts <= 2) { + return { ok: false, stdout: "", stderr: "checkout failed: missing object", code: 1, signal: null }; + } + } + return commandOk(); + }); vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (args[0] === "rev-parse" && args[1] === "--show-toplevel") { return { ok: true, stdout: "/repo/project\n", stderr: "" } as any; @@ -444,19 +450,6 @@ describe("WorkspaceManager", () => { if (args[0] === "show-ref") { throw new Error("missing ref"); } - // The in-volume checkout (docker run --entrypoint git ... checkout) fails the first time, - // which should trigger a full re-seed + retry. - if ( - command === "docker" - && args.includes("--entrypoint") - && (args.includes("checkout") || String(args.at(-1)).includes("git -C /workspace checkout")) - ) { - checkoutAttempts += 1; - if (checkoutAttempts <= 2) { - throw new Error("checkout failed: missing object"); - } - return { ok: true, stdout: "", stderr: "" } as any; - } return { ok: true, stdout: "", stderr: "" } as any; }); @@ -502,10 +495,8 @@ describe("WorkspaceManager", () => { await manager.createSnapshotWorkspace("/repo/project", "session-1"); - const checkoutCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const checkoutCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && String(call[1].at(-1)).includes("git -C /workspace checkout") ); expect(String(checkoutCall?.[1].at(-1))).toContain( @@ -568,15 +559,12 @@ describe("WorkspaceManager", () => { await manager.createSnapshotWorkspace("/repo/project", "session-1"); expect(vi.mocked(runCommandStrict).mock.calls.some((call) => call[0] === "bash")).toBe(false); - const bootstrapCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") - && call[4] - && typeof call[4] === "object" - && "stdinFile" in call[4] + const bootstrapCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" + && call[3] + && "stdinFile" in call[3] ); - expect(bootstrapCall?.[4]).toEqual(expect.objectContaining({ + expect(bootstrapCall?.[3]).toEqual(expect.objectContaining({ stdinFile: expect.stringContaining("C:\\Users\\pierr\\AppData\\Local\\Temp\\code-ux-bundle-k9Efgd"), })); expect(bootstrapCall?.[1].join(" ")).not.toContain("C:\\Users\\pierr\\AppData\\Local\\Temp"); @@ -637,10 +625,8 @@ describe("WorkspaceManager", () => { ], "/repo/project", ); - const seedCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const seedCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && call[1].some((arg) => typeof arg === "string" && arg.includes("git -C /workspace checkout -B 'feature/task-1' 'origin/feature/task-1'")) ); expect(seedCall).toBeDefined(); @@ -670,6 +656,21 @@ describe("WorkspaceManager", () => { it("reseeds a Docker prepare worktree when the prepared volume has no HEAD", async () => { let headChecks = 0; + vi.mocked(workspaceVolumeHelperPool.exec).mockImplementation(async (_volumeName, commandArgs) => { + if ( + commandArgs[0] === "git" + && commandArgs.includes("rev-parse") + && commandArgs.includes("--verify") + && commandArgs.includes("HEAD") + ) { + headChecks += 1; + if (headChecks === 1) { + return { ok: false, stdout: "", stderr: "not a git repository", code: 128, signal: null }; + } + return commandOk("abc123\n"); + } + return commandOk(); + }); vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (command === "git" && args[0] === "rev-parse" && args[1] === "--show-toplevel") { return { ok: true, stdout: "/repo/project\n", stderr: "" } as any; @@ -690,20 +691,6 @@ describe("WorkspaceManager", () => { if (command === "docker" && args[0] === "volume" && args[1] === "inspect") { throw new Error("missing"); } - if ( - command === "docker" - && args.includes("--entrypoint") - && args.includes("git") - && args.includes("rev-parse") - && args.includes("--verify") - && args.includes("HEAD") - ) { - headChecks += 1; - if (headChecks === 1) { - throw new Error("not a git repository"); - } - return { ok: true, stdout: "abc123\n", stderr: "", code: 0, signal: null } as any; - } return { ok: true, stdout: "", stderr: "", code: 0, signal: null } as any; }); @@ -715,13 +702,10 @@ describe("WorkspaceManager", () => { ); expect(headChecks).toBe(2); - const seedCalls = vi.mocked(runCommandStrict).mock.calls.filter((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") - && call[4] - && typeof call[4] === "object" - && "stdinFile" in call[4] + const seedCalls = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.filter((call) => + call[1][0] === "sh" + && call[3] + && "stdinFile" in call[3] ); expect(seedCalls.length).toBeGreaterThanOrEqual(2); }); @@ -735,6 +719,21 @@ describe("WorkspaceManager", () => { const secondSeedStartedPromise = new Promise((resolve) => { secondSeedStarted = resolve; }); let secondSeedDidStart = false; + vi.mocked(workspaceVolumeHelperPool.exec).mockImplementation(async (volumeName, commandArgs) => { + if (commandArgs[0] !== "sh" || !String(commandArgs.at(-1)).includes("git init /workspace")) { + return commandOk(); + } + if (volumeName.includes("session-1")) { + firstSeedStarted(); + await releaseFirstSeedPromise; + } + if (volumeName.includes("session-2")) { + secondSeedDidStart = true; + secondSeedStarted(); + } + return commandOk(); + }); + vi.mocked(runCommandStrict).mockImplementation(async (command, args) => { if (command === "git" && args[0] === "rev-parse" && args[1] === "--show-toplevel") { return { ok: true, stdout: "/repo/project\n", stderr: "" } as any; @@ -756,20 +755,6 @@ describe("WorkspaceManager", () => { if (command === "docker" && args[0] === "volume" && args[1] === "inspect") { throw new Error("missing"); } - if (command === "docker" && args.includes("--entrypoint") && args.includes("sh")) { - const mount = args.find((arg) => typeof arg === "string" && arg.startsWith("type=volume,source=")) || ""; - if (!String(mount).includes("target=/workspace")) { - return { ok: true, stdout: "", stderr: "", code: 0, signal: null } as any; - } - if (String(mount).includes("session-1")) { - firstSeedStarted(); - await releaseFirstSeedPromise; - } - if (String(mount).includes("session-2")) { - secondSeedDidStart = true; - secondSeedStarted(); - } - } return { ok: true, stdout: "", stderr: "", code: 0, signal: null } as any; }); @@ -938,10 +923,8 @@ describe("WorkspaceManager", () => { "dev", ); - const seedCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const seedCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && call[1].some((arg) => typeof arg === "string" && arg.includes("update-ref 'refs/heads/dev' 'refs/remotes/origin/dev'")) ); expect(seedCall).toBeDefined(); @@ -979,10 +962,8 @@ describe("WorkspaceManager", () => { "feature/sprint-1", ); - const seedCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const seedCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && call[1].some((arg) => typeof arg === "string" && arg.includes("git -C /workspace checkout -B 'feature/task-1' 'origin/feature/task-1'")) ); expect(seedCall).toBeDefined(); @@ -1022,10 +1003,8 @@ describe("WorkspaceManager", () => { { remoteOnly: true }, ); - const seedCall = vi.mocked(runCommandStrict).mock.calls.find((call) => - call[0] === "docker" - && call[1].includes("--entrypoint") - && call[1].includes("sh") + const seedCall = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.find((call) => + call[1][0] === "sh" && call[1].some((arg) => typeof arg === "string" && arg.includes("git -C /workspace checkout -B 'feature/task-1' 'origin/feature/sprint-1'")) ); expect(seedCall).toBeDefined(); @@ -1140,6 +1119,7 @@ describe("WorkspaceManager", () => { it("builds workspace guidance with in-volume path checks", async () => { vi.mocked(runCommandStrict).mockResolvedValue({ ok: true, stdout: "exists\n", stderr: "" } as any); + vi.mocked(workspaceVolumeHelperPool.exec).mockResolvedValue(commandOk("exists\n")); const guidance = await manager.buildWorkspaceGuidance("Check src/index.ts and ../outside", "docker-volume://workspace-1"); @@ -1164,7 +1144,7 @@ describe("WorkspaceManager", () => { expect(fs.readFile).not.toHaveBeenCalled(); }); - it("runs workspace commands with an explicit container entrypoint", async () => { + it("runs ordinary workspace commands through the reusable sidecar with filtered environment", async () => { vi.mocked(runCommandStrict).mockResolvedValue({ ok: true, stdout: "", stderr: "" } as any); await manager.runWorkspaceCommand("docker-volume://workspace-1", "git", ["status", "--short"], { @@ -1174,43 +1154,81 @@ describe("WorkspaceManager", () => { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "http.https://github.com/.extraheader", GIT_CONFIG_VALUE_0: "Authorization: Basic redacted", + GIT_PROVIDER_API_KEY: "git-prefixed-provider-secret", + OPENAI_API_KEY: "provider-secret", APP_SECRET_SHOULD_NOT_LEAK: "secret", }, }); - const call = vi.mocked(runCommandStrict).mock.calls.find((candidate) => - candidate[0] === "docker" && candidate[1].includes("run") + expect(workspaceVolumeHelperPool.exec).toHaveBeenCalledWith( + "workspace-1", + ["git", "status", "--short"], + "workspace-1-runtime", + expect.objectContaining({ + environment: expect.objectContaining({ + HOME: "/tmp/code-ux-home", + GIT_AUTHOR_NAME: "Code UX", + GIT_AUTHOR_EMAIL: "agents@codeux.ai", + GIT_COMMITTER_NAME: "Code UX", + GIT_COMMITTER_EMAIL: "agents@codeux.ai", + GIT_INDEX_FILE: ".code-ux-export.index", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "http.https://github.com/.extraheader", + GIT_CONFIG_VALUE_0: "Authorization: Basic redacted", + }), + workdir: "/workspace", + }), ); - expect(call?.[0]).toBe("docker"); - expect(call?.[1]).toEqual(expect.arrayContaining([ - "run", - "--entrypoint", + const environment = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls[0]?.[3]?.environment; + expect(environment).not.toHaveProperty("OPENAI_API_KEY"); + expect(environment).not.toHaveProperty("GIT_PROVIDER_API_KEY"); + expect(environment).not.toHaveProperty("APP_SECRET_SHOULD_NOT_LEAK"); + }); + + it("throws when a reusable sidecar command fails", async () => { + vi.mocked(runCommandStrict).mockResolvedValue(commandOk()); + vi.mocked(workspaceVolumeHelperPool.exec).mockResolvedValue({ + ok: false, + stdout: "", + stderr: "fatal: invalid workspace", + code: 128, + signal: null, + }); + + await expect(manager.runWorkspaceCommand( + "docker-volume://workspace-1", "git", - "alpine/git", - "status", - "--short", - ])); - expect(call?.[1]).toEqual(expect.arrayContaining([ - "-e", - "GIT_AUTHOR_NAME=Code UX", - "-e", - "GIT_AUTHOR_EMAIL=agents@codeux.ai", - "-e", - "GIT_COMMITTER_NAME=Code UX", - "-e", - "GIT_COMMITTER_EMAIL=agents@codeux.ai", - "-e", - "GIT_INDEX_FILE=.code-ux-export.index", - "-e", - "GIT_CONFIG_COUNT=1", - "-e", - "GIT_CONFIG_KEY_0=http.https://github.com/.extraheader", - "-e", - "GIT_CONFIG_VALUE_0=Authorization: Basic redacted", - ])); - expect(call?.[1]).not.toContain("APP_SECRET_SHOULD_NOT_LEAK=secret"); + ["status", "--short"], + )).rejects.toThrow("git status --short failed: fatal: invalid workspace"); }); + it.each(["fetch", "push", "pull", "ls-remote", "submodule"])( + "keeps networked git %s commands on a one-shot container", + async (gitCommand) => { + vi.mocked(runCommandStrict).mockResolvedValue(commandOk()); + + await manager.runWorkspaceCommand( + "docker-volume://workspace-1", + "git", + [gitCommand, "origin"], + { env: { ...process.env, OPENAI_API_KEY: "provider-secret" } }, + ); + + expect(workspaceVolumeHelperPool.exec).not.toHaveBeenCalled(); + const runCall = vi.mocked(runCommandStrict).mock.calls.find((call) => ( + call[0] === "docker" && call[1][0] === "run" + )); + expect(runCall?.[1]).toEqual(expect.arrayContaining([ + "--entrypoint", + "git", + "alpine/git", + gitCommand, + "origin", + ])); + expect(runCall?.[1].join(" ")).not.toContain("provider-secret"); + }, + ); + it("reuses successful public helper image checks across Docker workspace commands", async () => { vi.mocked(runCommandStrict).mockResolvedValue({ ok: true, stdout: "", stderr: "" } as any); @@ -1223,11 +1241,13 @@ describe("WorkspaceManager", () => { const inspectCalls = vi.mocked(runCommandStrict).mock.calls.filter((call) => call[0] === "docker" && call[1][0] === "image" && call[1][1] === "inspect" ); - const runCalls = vi.mocked(runCommandStrict).mock.calls.filter((call) => - call[0] === "docker" && call[1][0] === "run" - ); expect(inspectCalls).toHaveLength(1); - expect(runCalls).toHaveLength(3); + expect(workspaceVolumeHelperPool.exec).toHaveBeenCalledTimes(3); + expect(vi.mocked(workspaceVolumeHelperPool.exec).mock.calls.map((call) => call[0])).toEqual([ + "workspace-1", + "workspace-2", + "workspace-3", + ]); }); it("allows callers to override Docker workspace Git identity env", async () => { @@ -1243,26 +1263,16 @@ describe("WorkspaceManager", () => { }, }); - const call = vi.mocked(runCommandStrict).mock.calls.find((candidate) => - candidate[0] === "docker" && candidate[1].includes("run") - ); - expect(call?.[1]).toEqual(expect.arrayContaining([ - "-e", - "GIT_AUTHOR_NAME=Custom Author", - "-e", - "GIT_AUTHOR_EMAIL=author@example.com", - "-e", - "GIT_COMMITTER_NAME=Custom Committer", - "-e", - "GIT_COMMITTER_EMAIL=committer@example.com", - ])); - expect(call?.[1]).not.toContain("GIT_COMMITTER_EMAIL=agents@codeux.ai"); + const options = vi.mocked(workspaceVolumeHelperPool.exec).mock.calls[0]?.[3]; + expect(options?.environment).toEqual(expect.objectContaining({ + GIT_AUTHOR_NAME: "Custom Author", + GIT_AUTHOR_EMAIL: "author@example.com", + GIT_COMMITTER_NAME: "Custom Committer", + GIT_COMMITTER_EMAIL: "committer@example.com", + })); if (typeof process.getuid === "function" && typeof process.getgid === "function") { - expect(call?.[1]).toEqual(expect.arrayContaining([ - "--user", - `${process.getuid()}:${process.getgid()}`, - ])); + expect(options?.user).toBe(`${process.getuid()}:${process.getgid()}`); } }); @@ -1291,11 +1301,10 @@ describe("WorkspaceManager", () => { "{}\n", "utf8", ); - expect(runCommandStrict).toHaveBeenCalledWith( - "docker", - expect.arrayContaining(["run", "alpine/git", "status", "--short"]), - expect.any(String), - expect.any(Object), + expect(workspaceVolumeHelperPool.exec).toHaveBeenCalledWith( + "workspace-1", + ["git", "status", "--short"], + "workspace-1-runtime", expect.any(Object), ); }); @@ -1374,6 +1383,10 @@ describe("WorkspaceManager", () => { await manager.removeWorktree("/repo/project", "docker-volume://code-ux-project-abcd1234ef56-session-1"); + expect(workspaceVolumeHelperPool.releaseVolume).toHaveBeenCalledWith( + "code-ux-project-abcd1234ef56-session-1", + ); + expect(runCommandStrict).toHaveBeenCalledWith( "docker", ["volume", "rm", "-f", "code-ux-project-abcd1234ef56-session-1"], @@ -1386,6 +1399,37 @@ describe("WorkspaceManager", () => { ); }); + it("releases a preserved Docker workspace helper without deleting its volumes", async () => { + await manager.releaseWorkspaceHelper("docker-volume://workspace-1"); + await manager.releaseWorkspaceHelper("/repo/project/.worktrees/session-1"); + + expect(workspaceVolumeHelperPool.releaseVolume).toHaveBeenCalledTimes(1); + expect(workspaceVolumeHelperPool.releaseVolume).toHaveBeenCalledWith("workspace-1"); + expect(runCommandStrict).not.toHaveBeenCalledWith( + "docker", + expect.arrayContaining(["volume", "rm"]), + expect.anything(), + ); + }); + + it("reserves the exact workspace and runtime-volume helper generation", () => { + const releaseReservation = vi.fn(); + vi.mocked(workspaceVolumeHelperPool.reserve).mockReturnValueOnce(releaseReservation); + + const release = manager.reserveWorkspaceHelper("docker-volume://workspace-1"); + + expect(workspaceVolumeHelperPool.reserve).toHaveBeenCalledWith( + "workspace-1", + "workspace-1-runtime", + ); + release(); + expect(releaseReservation).toHaveBeenCalledOnce(); + + const releaseHost = manager.reserveWorkspaceHelper("/repo/project/.worktrees/session-1"); + releaseHost(); + expect(workspaceVolumeHelperPool.reserve).toHaveBeenCalledTimes(1); + }); + describe("fastForwardResumedWorkspace", () => { const WORKTREE = "/repo/project/.worktrees/session-1"; const ok = (stdout = "") => ({ ok: true, stdout, stderr: "", code: 0, signal: null } as any); diff --git a/tests/backend/infrastructure/providers/cli/workspace-volume-helper.test.ts b/tests/backend/infrastructure/providers/cli/workspace-volume-helper.test.ts index 9de17953a1..8960c332c9 100644 --- a/tests/backend/infrastructure/providers/cli/workspace-volume-helper.test.ts +++ b/tests/backend/infrastructure/providers/cli/workspace-volume-helper.test.ts @@ -1,15 +1,33 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { WorkspaceVolumeHelperPool } from "../../../../../src/infrastructure/providers/cli/workspace-volume-helper.js"; +import { + WorkspaceVolumeHelperPool, + type WorkspaceSidecarExecOptions, +} from "../../../../../src/infrastructure/providers/cli/workspace-volume-helper.js"; +import type { + HelperCommandRunner, + HelperRunnerOptions, +} from "../../../../../src/infrastructure/providers/cli/docker-helper-pool.js"; +import { getRuntimeOwnerLabel } from "../../../../../src/shared/config/runtime-owner.js"; -type Call = { command: string; args: string[] }; +type Call = { command: string; args: string[]; options?: HelperRunnerOptions }; +type RunnerResult = { ok: boolean; code?: number; stdout?: string; stderr?: string }; -function makeRunner(overrides?: (call: Call) => { ok: boolean; stdout?: string; stderr?: string } | undefined) { +function makeRunner( + overrides?: (call: Call) => RunnerResult | Promise | undefined, +): { runner: HelperCommandRunner; calls: Call[] } { const calls: Call[] = []; - const runner = vi.fn(async (command: string, args: string[]) => { - calls.push({ command, args }); - const custom = overrides?.({ command, args }); + const runner: HelperCommandRunner = vi.fn(async (command, args, options) => { + const call = { command, args, options }; + calls.push(call); + const custom = overrides?.(call); if (custom) { - return { ok: custom.ok, code: custom.ok ? 0 : 1, stdout: custom.stdout ?? "", stderr: custom.stderr ?? "" }; + const resolved = await custom; + return { + ok: resolved.ok, + code: resolved.code ?? (resolved.ok ? 0 : 1), + stdout: resolved.stdout ?? "", + stderr: resolved.stderr ?? "", + }; } if (args[0] === "run" && args.includes("-d")) { return { ok: true, code: 0, stdout: "helper-container-id\n", stderr: "" }; @@ -17,7 +35,6 @@ function makeRunner(overrides?: (call: Call) => { ok: boolean; stdout?: string; if (args[0] === "exec") { return { ok: true, code: 0, stdout: "file-contents", stderr: "" }; } - // rm -f and anything else return { ok: true, code: 0, stdout: "", stderr: "" }; }); return { runner, calls }; @@ -27,25 +44,25 @@ describe("WorkspaceVolumeHelperPool", () => { const pools: WorkspaceVolumeHelperPool[] = []; afterEach(async () => { - await Promise.all(pools.splice(0).map((p) => p.shutdown())); + await Promise.all(pools.splice(0).map((pool) => pool.shutdown())); + vi.useRealTimers(); + vi.unstubAllEnvs(); }); - it("starts one persistent helper per volume and reuses it via docker exec", async () => { + it("starts one secure Git-capable sidecar per volume and reuses it", async () => { const { runner, calls } = makeRunner(); - const pool = new WorkspaceVolumeHelperPool(runner as any); + const pool = new WorkspaceVolumeHelperPool(runner); pools.push(pool); const a = await pool.exec("vol-1", ["cat", "/workspace/a.txt"]); - const b = await pool.exec("vol-1", ["sh", "-c", "echo hi"]); + const b = await pool.exec("vol-1", ["git", "status", "--short"]); expect(a.stdout).toBe("file-contents"); expect(b.stdout).toBe("file-contents"); - - // Exactly one container was created (docker run -d) despite two operations. - const createCalls = calls.filter((c) => c.args[0] === "run" && c.args.includes("-d")); + const createCalls = calls.filter((call) => call.args[0] === "run" && call.args.includes("-d")); expect(createCalls).toHaveLength(1); - expect(createCalls[0].args).toContain("alpine:3.20"); - expect(createCalls[0].args.join(" ")).toContain("code-ux.helper=volume"); + expect(createCalls[0].args).toContain("alpine/git"); + expect(createCalls[0].args).toContain(getRuntimeOwnerLabel()); expect(createCalls[0].args).toEqual(expect.arrayContaining([ "--network", "none", @@ -53,54 +70,193 @@ describe("WorkspaceVolumeHelperPool", () => { "no-new-privileges", "--label", "code-ux.managed=true", + "--mount", + "type=tmpfs,target=/git", + "--mount", + "type=tmpfs,target=/tmp/code-ux-home,tmpfs-mode=1777,tmpfs-size=1048576", ])); expect(createCalls[0].args).not.toContain("-p"); expect(createCalls[0].args).not.toContain("--publish"); - // Both operations ran via `docker exec` into the same helper container id. - const execCalls = calls.filter((c) => c.args[0] === "exec"); + const execCalls = calls.filter((call) => call.args[0] === "exec"); expect(execCalls).toHaveLength(2); expect(execCalls[0].args).toEqual(["exec", "helper-container-id", "cat", "/workspace/a.txt"]); - expect(execCalls[1].args).toEqual(["exec", "helper-container-id", "sh", "-c", "echo hi"]); + expect(execCalls[1].args).toEqual(["exec", "helper-container-id", "git", "status", "--short"]); + }); + + it("keeps separate helpers per workspace volume", async () => { + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + await Promise.all([ + pool.exec("vol-a", ["cat", "x"]), + pool.exec("vol-b", ["cat", "x"]), + ]); + + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); }); - it("keeps separate helpers per volume", async () => { + it("evicts an idle workspace sidecar before exceeding its configured capacity", async () => { const { runner, calls } = makeRunner(); - const pool = new WorkspaceVolumeHelperPool(runner as any); + const pool = new WorkspaceVolumeHelperPool(runner, "alpine/git", { maxContainers: 1 }); pools.push(pool); await pool.exec("vol-a", ["cat", "x"]); await pool.exec("vol-b", ["cat", "x"]); - const createCalls = calls.filter((c) => c.args[0] === "run" && c.args.includes("-d")); - expect(createCalls).toHaveLength(2); + const createIndexes = calls + .map((call, index) => ({ call, index })) + .filter(({ call }) => call.args[0] === "run" && call.args.includes("-d")) + .map(({ index }) => index); + expect(createIndexes).toHaveLength(2); + const firstRemovalIndex = calls.findIndex((call, index) => ( + index > createIndexes[0] && call.args[0] === "rm" && call.args.includes("helper-container-id") + )); + expect(firstRemovalIndex).toBeGreaterThan(createIndexes[0]); + expect(firstRemovalIndex).toBeLessThan(createIndexes[1]); }); - it("recreates the helper transparently when it has disappeared", async () => { - let execCount = 0; + it("keeps a reserved workspace helper across commands while excess work waits", async () => { + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner, "alpine/git", { maxContainers: 1 }); + pools.push(pool); + const releaseReservation = pool.reserve("vol-a"); + + await pool.exec("vol-a", ["git", "status"]); + const waiting = pool.exec("vol-b", ["git", "status"]); + await Promise.resolve(); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + + await pool.exec("vol-a", ["git", "rev-parse", "HEAD"]); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + releaseReservation(); + + await expect(waiting).resolves.toMatchObject({ ok: true }); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); + + it("applies stdin, process controls, identity, workdir, and filtered environment per command", async () => { + vi.stubEnv("OPENAI_API_KEY", "ambient-provider-secret"); + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + const abortController = new AbortController(); + const onStdoutLine = vi.fn(); + const onStderrLine = vi.fn(); + const options: WorkspaceSidecarExecOptions = { + stdinFile: "/tmp/paths.list", + signal: abortController.signal, + trimOutput: false, + maxStdoutChars: 1234, + onStdoutLine, + onStderrLine, + user: "1000:1001", + workdir: "/workspace/subdir", + environment: { + GIT_CONFIG_COUNT: "1", + GITHUB_TOKEN: "project-a-token", + EMPTY_VALUE: "", + OMITTED_VALUE: undefined, + }, + }; + + await pool.exec("vol-1", ["git", "hash-object", "--stdin"], undefined, options); + await pool.exec("vol-1", ["git", "status"], undefined, { + environment: { GITHUB_TOKEN: "project-b-token" }, + }); + + const [first, second] = calls.filter((call) => call.args[0] === "exec"); + expect(first.args).toEqual([ + "exec", + "-i", + "--workdir", + "/workspace/subdir", + "--user", + "1000:1001", + "--env", + "GIT_CONFIG_COUNT=1", + "--env", + "GITHUB_TOKEN=project-a-token", + "helper-container-id", + "git", + "hash-object", + "--stdin", + ]); + expect(first.args.join(" ")).not.toContain("ambient-provider-secret"); + expect(first.args).not.toContain("EMPTY_VALUE="); + expect(first.options).toEqual({ + stdinFile: "/tmp/paths.list", + signal: abortController.signal, + trimOutput: false, + maxStdoutChars: 1234, + onStdoutLine, + onStderrLine, + }); + expect(second.args).toContain("GITHUB_TOKEN=project-b-token"); + expect(second.args).not.toContain("GITHUB_TOKEN=project-a-token"); + expect(second.args).not.toContain("--user"); + expect(second.args).not.toContain("--workdir"); + }); + + it("rejects unsafe command, mount, environment, user, and workdir inputs before Docker", async () => { + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + await expect(pool.exec("vol,readonly", ["cat", "x"])).rejects.toThrow(/volume name/); + await expect(pool.exec("vol-1", [])).rejects.toThrow(/executable/); + await expect(pool.exec("vol-1", ["cat", "bad\0arg"])).rejects.toThrow(/null bytes/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + environment: { "BAD-NAME": "value" }, + })).rejects.toThrow(/environment name/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + environment: { GOOD_NAME: "bad\0value" }, + })).rejects.toThrow(/environment value/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + user: "--privileged", + })).rejects.toThrow(/sidecar user/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + workdir: "relative/path", + })).rejects.toThrow(/absolute container path/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + workdir: "/workspace/../../etc", + })).rejects.toThrow(/mounted workspace/); + await expect(pool.exec("vol-1", ["git", "status"], undefined, { + workdir: "/code-ux-runtime-home", + })).rejects.toThrow(/mounted workspace/); + expect(calls).toHaveLength(0); + }); + + it("recreates one shared replacement when concurrent commands observe a missing generation", async () => { + let createCount = 0; const { runner, calls } = makeRunner(({ args }) => { + if (args[0] === "run" && args.includes("-d")) { + createCount += 1; + return { ok: true, stdout: `cid-${createCount}\n` }; + } + if (args[0] === "exec" && args.includes("cid-1")) { + return { ok: false, stderr: "Error: No such container: cid-1" }; + } if (args[0] === "exec") { - execCount += 1; - if (execCount === 1) { - return { ok: false, stderr: "Error: No such container: helper-container-id" }; - } + return { ok: true, stdout: "recovered" }; } return undefined; }); - const pool = new WorkspaceVolumeHelperPool(runner as any); + const pool = new WorkspaceVolumeHelperPool(runner); pools.push(pool); - await pool.exec("vol-1", ["cat", "x"]); // primes the helper - const result = await pool.exec("vol-1", ["cat", "x"]); // helper vanished -> recreate + retry + const [a, b] = await Promise.all([ + pool.exec("vol-1", ["git", "status"]), + pool.exec("vol-1", ["cat", "x"]), + ]); - expect(result.ok).toBe(true); - expect(result.stdout).toBe("file-contents"); - // Two creates: initial + recreate after the container went missing. - const createCalls = calls.filter((c) => c.args[0] === "run" && c.args.includes("-d")); - expect(createCalls.length).toBeGreaterThanOrEqual(2); + expect(a).toMatchObject({ ok: true, stdout: "recovered" }); + expect(b).toMatchObject({ ok: true, stdout: "recovered" }); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); }); - it("falls back to docker run --rm when the helper cannot be created", async () => { + it("falls back to an equivalent secure one-shot command when creation fails", async () => { const { runner, calls } = makeRunner(({ args }) => { if (args[0] === "run" && args.includes("-d")) { return { ok: false, stderr: "cannot create container" }; @@ -110,45 +266,176 @@ describe("WorkspaceVolumeHelperPool", () => { } return undefined; }); - const pool = new WorkspaceVolumeHelperPool(runner as any); + const pool = new WorkspaceVolumeHelperPool(runner); pools.push(pool); + const controller = new AbortController(); - const result = await pool.exec("vol-1", ["cat", "x"]); + const result = await pool.exec("vol-1", ["git", "hash-object", "--stdin"], "vol-1-runtime", { + stdinFile: "/tmp/input", + signal: controller.signal, + trimOutput: false, + user: "1000:1000", + workdir: "/code-ux-runtime-home", + environment: { GIT_CONFIG_COUNT: "1" }, + }); - expect(result.ok).toBe(true); - expect(result.stdout).toBe("fallback-output"); - const fallbackCalls = calls.filter((c) => c.args[0] === "run" && c.args.includes("--rm")); - expect(fallbackCalls).toHaveLength(1); - expect(fallbackCalls[0].args).toEqual(expect.arrayContaining(["alpine:3.20", "cat", "x"])); - expect(fallbackCalls[0].args).toEqual(expect.arrayContaining([ + expect(result).toMatchObject({ ok: true, stdout: "fallback-output" }); + const fallback = calls.find((call) => call.args[0] === "run" && call.args.includes("--rm")); + expect(fallback).toBeDefined(); + expect(fallback?.args).toEqual(expect.arrayContaining([ "--network", "none", "--security-opt", "no-new-privileges", - "--label", - "code-ux.managed=true", - "--label", - "code-ux.helper=volume", + "--mount", + "type=tmpfs,target=/git", + "--mount", + "type=tmpfs,target=/tmp/code-ux-home,tmpfs-mode=1777,tmpfs-size=1048576", + "--entrypoint", + "git", + "alpine/git", + "hash-object", + "--stdin", + "-i", + "--user", + "1000:1000", + "--workdir", + "/code-ux-runtime-home", + "--env", + "GIT_CONFIG_COUNT=1", + ])); + expect(fallback?.args.join(" ")).toContain("source=vol-1-runtime,target=/code-ux-runtime-home"); + expect(fallback?.args).not.toContain("--publish"); + expect(fallback?.options).toMatchObject({ + stdinFile: "/tmp/input", + signal: controller.signal, + trimOutput: false, + }); + }); + + it("falls back once when the replacement helper also stops", async () => { + const { runner, calls } = makeRunner(({ args }) => { + if (args[0] === "exec") { + return { ok: false, stderr: `Error response from daemon: container ${args[1]} is not running` }; + } + if (args[0] === "run" && args.includes("--rm")) { + return { ok: true, stdout: "fallback-after-retry" }; + } + return undefined; + }); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + const result = await pool.exec("vol-1", ["cat", "x"]); + + expect(result).toMatchObject({ ok: true, stdout: "fallback-after-retry" }); + expect(calls.filter((call) => call.args[0] === "exec")).toHaveLength(2); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("--rm"))).toHaveLength(1); + }); + + it("does not repeat a command when the host runner throws", async () => { + const { runner, calls } = makeRunner(({ args }) => { + if (args[0] === "exec") { + return Promise.reject(new Error("host runner disconnected")); + } + return undefined; + }); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + await expect(pool.exec("vol-1", ["git", "update-ref", "refs/heads/x", "abc"])).rejects.toThrow( + /host runner disconnected/, + ); + expect(calls.filter((call) => call.args[0] === "exec")).toHaveLength(1); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("--rm"))).toHaveLength(0); + }); + + it("mounts the runtime volume and allows a runtime-scoped workdir", async () => { + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + await pool.exec( + "vol-1", + ["cat", "session.jsonl"], + "vol-1-runtime", + { workdir: "/code-ux-runtime-home/.codex" }, + ); + + const createCall = calls.find((call) => call.args[0] === "run" && call.args.includes("-d")); + expect(createCall?.args.join(" ")).toContain("source=vol-1,target=/workspace"); + expect(createCall?.args.join(" ")).toContain("source=vol-1-runtime,target=/code-ux-runtime-home"); + expect(calls.find((call) => call.args[0] === "exec")?.args).toEqual(expect.arrayContaining([ + "--workdir", + "/code-ux-runtime-home/.codex", ])); - expect(fallbackCalls[0].args).not.toContain("-p"); - expect(fallbackCalls[0].args).not.toContain("--publish"); }); - it("mounts the runtime volume when provided and releases helpers for the workspace", async () => { + it("drains an in-flight command before release, preserves volumes, and fences new commands", async () => { + let finishFirst: ((result: RunnerResult) => void) | null = null; + let execCount = 0; + const { runner, calls } = makeRunner(({ args }) => { + if (args[0] === "exec") { + execCount += 1; + if (execCount === 1) { + return new Promise((resolve) => { + finishFirst = resolve; + }); + } + } + return undefined; + }); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + + const first = pool.exec("vol-1", ["git", "status"], "vol-1-runtime"); + await vi.waitFor(() => expect(finishFirst).not.toBeNull()); + const releaseA = pool.releaseVolume("vol-1"); + const releaseB = pool.releaseVolume("vol-1"); + const next = pool.exec("vol-1", ["cat", "x"], "vol-1-runtime"); + await Promise.resolve(); + + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(1); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-container-id"))).toBe(false); + + finishFirst?.({ ok: true, stdout: "first" }); + await expect(first).resolves.toMatchObject({ ok: true, stdout: "first" }); + await Promise.all([releaseA, releaseB]); + await expect(next).resolves.toMatchObject({ ok: true }); + + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-container-id"))).toBe(true); + expect(calls.some((call) => call.args[0] === "volume")).toBe(false); + }); + + it("reaps an idle sidecar after the configured bounded lifetime", async () => { + vi.useFakeTimers(); const { runner, calls } = makeRunner(); - const pool = new WorkspaceVolumeHelperPool(runner as any); + const pool = new WorkspaceVolumeHelperPool(runner, "alpine/git", { + idleTtlMs: 100, + reapIntervalMs: 25, + }); pools.push(pool); - await pool.exec("vol-1", ["cat", "/code-ux-runtime-home/.codex/session.jsonl"], "vol-1-runtime"); - await pool.releaseVolume("vol-1"); + await pool.exec("vol-1", ["cat", "x"]); + await vi.advanceTimersByTimeAsync(99); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-container-id"))).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(calls.some((call) => call.args[0] === "rm" && call.args.includes("helper-container-id"))).toBe(true); + + await pool.exec("vol-1", ["cat", "x"]); + expect(calls.filter((call) => call.args[0] === "run" && call.args.includes("-d"))).toHaveLength(2); + }); - const createCall = calls.find((c) => c.args[0] === "run" && c.args.includes("-d")); - expect(createCall?.args.join(" ")).toContain("source=vol-1"); - expect(createCall?.args.join(" ")).toContain("target=/workspace"); - expect(createCall?.args.join(" ")).toContain("source=vol-1-runtime"); - expect(createCall?.args.join(" ")).toContain("target=/code-ux-runtime-home"); + it("makes shutdown idempotent and rejects later commands without falling back", async () => { + const { runner, calls } = makeRunner(); + const pool = new WorkspaceVolumeHelperPool(runner); + pools.push(pool); + await pool.exec("vol-1", ["cat", "x"]); - const removeCall = calls.find((c) => c.args[0] === "rm" && c.args.includes("-f")); - expect(removeCall).toBeDefined(); + await Promise.all([pool.shutdown(), pool.shutdown()]); + const callCount = calls.length; + await expect(pool.exec("vol-1", ["cat", "x"])).rejects.toThrow(/shutting down/); + expect(calls).toHaveLength(callCount); }); }); diff --git a/tests/backend/integrations/jules-api-client.test.ts b/tests/backend/integrations/jules-api-client.test.ts index a642363117..60494c97bb 100644 --- a/tests/backend/integrations/jules-api-client.test.ts +++ b/tests/backend/integrations/jules-api-client.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it, vi } from "vitest"; -import { JulesApiClient, JulesNotFoundError } from "../../../src/integrations/jules-api-client.js"; +import { + isJulesSessionCapacityError, + isJulesSessionConsumingConcurrentTask, + JulesApiClient, + JulesApiRequestError, + JulesNotFoundError, +} from "../../../src/integrations/jules-api-client.js"; import axios from "axios"; const mockInstance = Object.assign( @@ -33,6 +39,18 @@ vi.mock("axios", () => { }); describe("JulesApiClient coverage", () => { + it("counts only executing Jules states against concurrent-task capacity", () => { + expect(isJulesSessionConsumingConcurrentTask({ state: "QUEUED" })).toBe(true); + expect(isJulesSessionConsumingConcurrentTask({ state: "PLANNING" })).toBe(true); + expect(isJulesSessionConsumingConcurrentTask({ state: "IN_PROGRESS" })).toBe(true); + expect(isJulesSessionConsumingConcurrentTask({ state: "AWAITING_PLAN_APPROVAL" })).toBe(false); + expect(isJulesSessionConsumingConcurrentTask({ state: "AWAITING_USER_FEEDBACK" })).toBe(false); + expect(isJulesSessionConsumingConcurrentTask({ state: "PAUSED" })).toBe(false); + expect(isJulesSessionConsumingConcurrentTask({ state: "COMPLETED" })).toBe(false); + expect(isJulesSessionConsumingConcurrentTask({ state: "FAILED" })).toBe(false); + expect(isJulesSessionConsumingConcurrentTask({ state: undefined })).toBe(true); + }); + it("handles listAllSources pagination", async () => { vi.mocked(mockInstance.get) .mockResolvedValueOnce({ data: { sources: [{ id: "1" }], nextPageToken: "token" } }) @@ -80,6 +98,47 @@ describe("JulesApiClient coverage", () => { expect(client.resolveSessionName({ name: "sessions/2" })).toBe("sessions/2"); }); + it("preserves bounded provider detail for create-session capacity errors", async () => { + vi.mocked(mockInstance.post).mockRejectedValueOnce(Object.assign(new Error("Request failed with status code 400"), { + response: { + status: 400, + data: { + error: { + status: "INVALID_ARGUMENT", + message: "Maximum 10 active sessions reached; api_key=do-not-persist", + }, + }, + }, + })); + const client = new JulesApiClient({ baseUrl: "http://url", apiKey: "key", minRequestIntervalMs: 0 }); + + const error = await client.createSession({ + prompt: "full task prompt", + sourceContext: { source: "sources/1" }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(JulesApiRequestError); + expect(error).toMatchObject({ status: 400, apiStatus: "INVALID_ARGUMENT" }); + expect((error as Error).message).toContain("Maximum 10 active sessions reached"); + expect((error as Error).message).not.toContain("do-not-persist"); + expect(isJulesSessionCapacityError(error)).toBe(true); + }); + + it("does not classify unrelated create-session validation errors as capacity", async () => { + vi.mocked(mockInstance.post).mockRejectedValueOnce(Object.assign(new Error("Request failed with status code 400"), { + response: { status: 400, data: { error: { message: "startingBranch is invalid" } } }, + })); + const client = new JulesApiClient({ baseUrl: "http://url", apiKey: "key", minRequestIntervalMs: 0 }); + + const error = await client.createSession({ + prompt: "p", + sourceContext: { source: "sources/1" }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(JulesApiRequestError); + expect(isJulesSessionCapacityError(error)).toBe(false); + }); + it("interceptor adds api key if present", async () => { const client = new JulesApiClient({ baseUrl: "http://url", apiKey: "key" }); const cb = (mockInstance.interceptors.request as any)._cb; @@ -188,6 +247,22 @@ describe("JulesApiClient coverage", () => { expect(r1.map((s) => s.id)).toEqual(["1"]); }); + it("capacity checks coalesce and use one bounded first-page API request", async () => { + vi.mocked(mockInstance.get).mockReset(); + let resolveGet: (v: unknown) => void = () => {}; + vi.mocked(mockInstance.get).mockReturnValueOnce(new Promise((resolve) => { resolveGet = resolve; })); + + const client = new JulesApiClient({ baseUrl: "http://url", apiKey: "key", minRequestIntervalMs: 0, now: () => 0 }); + const first = client.getSessionsForCapacityCheck(); + const second = client.getSessionsForCapacityCheck(); + resolveGet({ data: { sessions: [{ id: "active-1" }], nextPageToken: "more-history" } }); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockInstance.get).toHaveBeenCalledTimes(1); + expect(firstResult).toBe(secondResult); + expect(firstResult.map((session) => session.id)).toEqual(["active-1"]); + }); + it("getCachedSessions serves the cached snapshot within the TTL and re-fetches after expiry", async () => { vi.mocked(mockInstance.get).mockReset(); vi.mocked(mockInstance.get) @@ -245,6 +320,28 @@ describe("JulesApiClient coverage", () => { expect((await client.getCachedSessions()).map((s) => s.id)).toEqual(["1"]); }); + it("capacity checks fail closed instead of serving a stale session snapshot", async () => { + vi.mocked(mockInstance.get).mockReset(); + vi.mocked(mockInstance.get) + .mockResolvedValueOnce({ data: { sessions: [{ id: "1" }] } }) + .mockRejectedValueOnce(Object.assign(new Error("connect ETIMEDOUT"), { code: "ETIMEDOUT" })); + + let t = 0; + const client = new JulesApiClient({ + baseUrl: "http://url", + apiKey: "key", + minRequestIntervalMs: 0, + maxTransientRetries: 0, + sessionsCacheTtlMs: 10_000, + sessionsCapacityCacheTtlMs: 1_000, + now: () => t, + }); + expect((await client.getCachedSessions()).map((s) => s.id)).toEqual(["1"]); + t = 2_000; + + await expect(client.getSessionsForCapacityCheck()).rejects.toThrow("ETIMEDOUT"); + }); + it("retries transient network errors (ETIMEDOUT) then resolves", async () => { vi.useFakeTimers(); new JulesApiClient({ baseUrl: "http://url", apiKey: "key", minRequestIntervalMs: 0, maxTransientRetries: 2 }); diff --git a/tests/backend/repositories/app-db-storage.test.ts b/tests/backend/repositories/app-db-storage.test.ts index 65673f9f44..ece3727a9a 100644 --- a/tests/backend/repositories/app-db-storage.test.ts +++ b/tests/backend/repositories/app-db-storage.test.ts @@ -133,8 +133,10 @@ describe("AppDbStorage", () => { const taskRunEventIndexes = db.prepare("PRAGMA index_list('task_run_events')").all() as Array<{ name: string }>; expect(taskRunEventIndexes.some((idx) => idx.name === "idx_task_run_events_project_created")).toBe(true); expect(taskRunEventIndexes.some((idx) => idx.name === "idx_task_run_events_task_run_created_id")).toBe(true); + expect(taskRunEventIndexes.some((idx) => idx.name === "idx_task_run_events_task_run_type_created_id")).toBe(true); expect(getIndexColumns(db, "idx_task_run_events_project_created")).toEqual(["project_id", "created_at", "id"]); expect(getIndexColumns(db, "idx_task_run_events_task_run_created_id")).toEqual(["task_run_id", "created_at", "id"]); + expect(getIndexColumns(db, "idx_task_run_events_task_run_type_created_id")).toEqual(["task_run_id", "event_type", "created_at", "id"]); }); it("uses the explicit dbPath when provided", async () => { diff --git a/tests/backend/repositories/db/app-db-schema.test.ts b/tests/backend/repositories/db/app-db-schema.test.ts index 194a3e4f2d..5343ece64e 100644 --- a/tests/backend/repositories/db/app-db-schema.test.ts +++ b/tests/backend/repositories/db/app-db-schema.test.ts @@ -29,6 +29,7 @@ const liveSnapshotIndexNames = [ "idx_task_runs_project_sprint_run_lookup", "idx_task_run_events_project_created", "idx_task_run_events_task_run_created_id", + "idx_task_run_events_task_run_type_created_id", "idx_sprint_run_events_sprint_run_created_id", "idx_sprint_runs_project_lookup", "idx_project_attention_items_project_status_updated", @@ -65,6 +66,7 @@ const liveSnapshotIndexColumns: Record<(typeof liveSnapshotIndexNames)[number], idx_task_runs_project_sprint_run_lookup: ["project_id", "sprint_run_id", "id"], idx_task_run_events_project_created: ["project_id", "created_at", "id"], idx_task_run_events_task_run_created_id: ["task_run_id", "created_at", "id"], + idx_task_run_events_task_run_type_created_id: ["task_run_id", "event_type", "created_at", "id"], idx_sprint_run_events_sprint_run_created_id: ["sprint_run_id", "created_at", "id"], idx_sprint_runs_project_lookup: ["project_id", "id", "sprint_id", "status"], idx_project_attention_items_project_status_updated: ["project_id", "status", "updated_at"], diff --git a/tests/backend/repositories/execution-repository.test.ts b/tests/backend/repositories/execution-repository.test.ts index 95acc3d16f..3b8729225d 100644 --- a/tests/backend/repositories/execution-repository.test.ts +++ b/tests/backend/repositories/execution-repository.test.ts @@ -2181,6 +2181,51 @@ describe("ExecutionRepository", () => { }); }); + it("filters and batches bounded task-run event slices for wide DAG reads", async () => { + const { projectRepository, executionRepository } = await createRepositories(); + const project = projectRepository.createProject({ + name: "Batched Event Project", + sourceType: "local", + sourceRef: "/workspace/batched-event-project", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Batched Event Sprint", + number: 4, + }); + const taskRuns = ["one", "two"].map((title) => { + const task = projectRepository.createTask(project.id, { + sprintId: sprint.id, + title, + }); + return executionRepository.createTaskRun({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + state: "RUNNING", + }); + }); + for (const taskRun of taskRuns) { + executionRepository.appendTaskRunEvent(taskRun.id, "provider_activity", "agent", { ignored: true }); + executionRepository.appendTaskRunEvent(taskRun.id, "cli_git_pushed", "system", { pushedBranch: "task/branch" }); + executionRepository.appendTaskRunEvent(taskRun.id, "ci_gate_status", "system", { state: "merged_branch" }); + } + + expect(executionRepository.listTaskRunEvents(taskRuns[0]!.id, 10, { + eventTypes: ["cli_git_pushed"], + }).map((event) => event.eventType)).toEqual(["cli_git_pushed"]); + + const batched = executionRepository.listTaskRunEventsForRuns( + taskRuns.map((taskRun) => taskRun.id), + { + eventTypes: ["cli_git_pushed", "ci_gate_status"], + limitPerRun: 1, + }, + ); + expect([...batched.keys()]).toEqual(taskRuns.map((taskRun) => taskRun.id)); + expect([...batched.values()].map((events) => events.length)).toEqual([1, 1]); + expect([...batched.values()].flat().every((event) => event.eventType !== "provider_activity")).toBe(true); + }); + it("projects sprint-run events into the unified runtime timeline", async () => { const { projectRepository, executionRepository } = await createRepositories(); const project = projectRepository.createProject({ diff --git a/tests/backend/repositories/guardrail-repository.test.ts b/tests/backend/repositories/guardrail-repository.test.ts index 31d358876f..63f10c731c 100644 --- a/tests/backend/repositories/guardrail-repository.test.ts +++ b/tests/backend/repositories/guardrail-repository.test.ts @@ -57,6 +57,38 @@ describe("GuardrailRepository", () => { expect(repo.getTotal(taskId)).toBe(4); }); + it("refunds an operational interruption exactly once without going below zero", async () => { + const { repo, projectId, taskId } = await createFixture(); + + repo.record({ projectId, taskId, purpose: "task_coding" }); + repo.record({ projectId, taskId, purpose: "task_coding" }); + expect(repo.refund({ + projectId, + taskId, + purpose: "task_coding", + sourceKey: "runtime-restart:run-1", + reason: "runtime_restart_interrupted", + })).toEqual({ applied: true, count: 1 }); + expect(repo.refund({ + projectId, + taskId, + purpose: "task_coding", + sourceKey: "runtime-restart:run-1", + })).toEqual({ applied: false, count: 1 }); + expect(repo.refund({ + projectId, + taskId, + purpose: "task_coding", + sourceKey: "runtime-restart:run-2", + })).toEqual({ applied: true, count: 0 }); + expect(repo.refund({ + projectId, + taskId, + purpose: "task_coding", + sourceKey: "runtime-restart:run-3", + })).toEqual({ applied: true, count: 0 }); + }); + it("resets all counters for a task", async () => { const { repo, projectId, taskId } = await createFixture(); diff --git a/tests/backend/repositories/project-runtime/runtime-status-projection.test.ts b/tests/backend/repositories/project-runtime/runtime-status-projection.test.ts index a0d77e5e18..dc3e6ff1ed 100644 --- a/tests/backend/repositories/project-runtime/runtime-status-projection.test.ts +++ b/tests/backend/repositories/project-runtime/runtime-status-projection.test.ts @@ -247,6 +247,36 @@ describe("RuntimeStatusProjection", () => { expect(refreshedStatus.subtasks[0]?.activities?.map((activity) => activity.id)).toEqual(["act-1", "act-2"]); }); + it("bounds oversized legacy activity fields before retaining live projections", async () => { + const { projection } = await createProjection(); + const huge = "x".repeat(1_000_000); + + const activity = projection.mapTaskActivityRow({ + task_id: "task-1", + session_id: "session-1", + session_name: "session-1", + provider: "codex", + activity_id: "activity-1", + activity_name: "activity-1", + created_at: "2024-01-01T10:05:00Z", + originator: "agent", + payload_json: JSON.stringify({ + description: huge, + agentMessaged: { agentMessage: huge }, + progressUpdated: { title: huge, description: huge }, + planGenerated: { plan: { steps: Array.from({ length: 100 }, () => ({ title: huge })) } }, + sessionCompleted: { transcript: huge }, + }), + }); + + expect(activity?.description?.length).toBeLessThanOrEqual(8 * 1024); + expect(activity?.agentMessaged?.agentMessage?.length).toBeLessThanOrEqual(8 * 1024); + expect(activity?.progressUpdated?.description?.length).toBeLessThanOrEqual(8 * 1024); + expect(activity?.planGenerated?.plan?.steps).toHaveLength(32); + expect(activity?.sessionCompleted).toEqual({ truncated: true, originalChars: 1_000_017 }); + expect(JSON.stringify(activity).length).toBeLessThan(64 * 1024); + }); + it("projects latest task self-reflection ratings for live status and omits unrated tasks", async () => { const { projection, projectRepository, executionRepository, ratingRepository } = await createProjection(); diff --git a/tests/backend/repositories/qa-review-repository.test.ts b/tests/backend/repositories/qa-review-repository.test.ts index 88f0b9f6b3..1e8e2e37e9 100644 --- a/tests/backend/repositories/qa-review-repository.test.ts +++ b/tests/backend/repositories/qa-review-repository.test.ts @@ -130,6 +130,19 @@ describe("QaReviewRepository", () => { }); expect(repository.countTaskRuns(task.id)).toBe(2); expect(repository.countDecisiveTaskRuns(task.id)).toBe(1); + const snapshots = repository.listTaskReviewSnapshots([task.id, "task-without-reviews", task.id]); + expect(snapshots.get(task.id)).toMatchObject({ + latestRun: expect.objectContaining({ id: cancelledTaskRun.id }), + latestCycleRuns: [expect.objectContaining({ id: cancelledTaskRun.id })], + runsUsed: 2, + decisiveRuns: 1, + }); + expect(snapshots.get("task-without-reviews")).toEqual({ + latestRun: null, + latestCycleRuns: [], + runsUsed: 0, + decisiveRuns: 0, + }); expect(repository.hasSprintReviewRun(sprint.id)).toBe(false); diff --git a/tests/backend/repositories/session-tracking-repository.test.ts b/tests/backend/repositories/session-tracking-repository.test.ts index cd2bed1a16..c5715e9933 100644 --- a/tests/backend/repositories/session-tracking-repository.test.ts +++ b/tests/backend/repositories/session-tracking-repository.test.ts @@ -32,6 +32,39 @@ describe("SessionTrackingRepository", () => { expect(repo.getSession("cli-codex-running")?.outputs).toEqual([ { pullRequest: { url: undefined, workerBranch: "task/feature-t01-codex" } }, ]); + expect(repo.getSession("cli-codex-running")?.prompt).toBe(""); + }); + + it("stores prompts only for Jules because CLI invocation messages are already durable", async () => { + const repo = await createRepo(); + const oversizedPrompt = "wide-dag-context".repeat(100_000); + + repo.createSession({ id: "cli-large", provider: "codex", prompt: oversizedPrompt }); + repo.createSession({ id: "jules-large", provider: "jules", prompt: oversizedPrompt }); + + expect(repo.getSession("cli-large")?.prompt).toBe(""); + expect(repo.getSession("jules-large")?.prompt).toBe(oversizedPrompt); + }); + + it("removes legacy local CLI prompt copies once when opening an upgraded database", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-session-prompt-migration-")); + tempDirs.push(dir); + const databasePath = path.join(dir, "session-tracking.db"); + const initial = new SessionTrackingRepository(databasePath); + initial.createSession({ id: "legacy-cli", provider: "codex" }); + initial.createSession({ id: "legacy-jules", provider: "jules", prompt: "hosted prompt" }); + initial.getDatabase().prepare("UPDATE provider_sessions SET prompt = ? WHERE id = ?") + .run("legacy duplicated prompt", "legacy-cli"); + initial.getDatabase().exec("PRAGMA user_version = 0"); + initial.close(); + + const upgraded = new SessionTrackingRepository(databasePath); + + expect(upgraded.getSession("legacy-cli")?.prompt).toBe(""); + expect(upgraded.getSession("legacy-jules")?.prompt).toBe("hosted prompt"); + const version = upgraded.getDatabase().prepare("PRAGMA user_version").get() as { user_version: number }; + expect(version.user_version).toBe(1); + upgraded.close(); }); it("recovers interrupted running cli sessions and leaves other sessions untouched", async () => { @@ -365,13 +398,18 @@ describe("SessionTrackingRepository", () => { it("lists sessions", async () => { const repo = await createRepo(); - repo.createSession({ id: "s1", provider: "jules", title: "T1" }); + repo.createSession({ id: "s1", provider: "jules", title: "T1", prompt: "large prompt" }); repo.createSession({ id: "s2", provider: "gemini", title: "T2" }); const list = repo.listSessions(10); expect(list.sessions).toHaveLength(2); expect(list.sessions.map(s => s.id)).toContain("s1"); expect(list.sessions.map(s => s.id)).toContain("s2"); + expect(list.sessions.find(s => s.id === "s1")?.prompt).toBe("large prompt"); + + const syncProjection = repo.listSessions(10, { includePrompt: false }); + expect(syncProjection.sessions.find(s => s.id === "s1")?.prompt).toBe(""); + expect(syncProjection.sessions.find(s => s.id === "s1")?.title).toBe("T1"); }); it("fetches recent activities", async () => { diff --git a/tests/backend/scripts/mockup-sprint-pentest-runner.test.ts b/tests/backend/scripts/mockup-sprint-pentest-runner.test.ts index 63c3e9599f..d98c5bbb64 100644 --- a/tests/backend/scripts/mockup-sprint-pentest-runner.test.ts +++ b/tests/backend/scripts/mockup-sprint-pentest-runner.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import * as fs from "fs/promises"; import * as os from "os"; import * as path from "path"; +import { DatabaseSync } from "node:sqlite"; import { runCommandStrict } from "../../../src/services/cli-process-runner.js"; import { getScenario } from "../../../scripts/e2e/mockup-sprint-pentest-scenarios.mjs"; @@ -10,6 +11,7 @@ type RunnerModule = { progressChanged: boolean; expectedOutputReadinessChanged: boolean; providerAdmissionHeartbeatChanged?: boolean; + runtimeRestartCompleted?: boolean; }) => boolean; summarizeMockupProgressTasks: ( tasks: Array<{ key: string; status: string; merged: boolean; mergeIndicator: string | null }>, @@ -86,6 +88,39 @@ type RunnerModule = { }, expected: Record, ) => Record; + resolveScenarios: (scenarioArg: string) => Array<{ id: string; heavy?: boolean; localOnly?: boolean }>; + summarizeMockupRuntimeResources: (samples: Array>) => { + sampleCount: number; + observedPids: number[]; + peakRuntimeRssBytes: number; + peakWalBytes: number; + peakTaskRuns: number; + peakRunningTaskRuns: number; + peakFailedTaskRuns: number; + peakActiveTaskRuns: number; + peakTaskRunEvents: number; + peakProviderInvocations: number; + }; + assertExpectedRuntimeResources: ( + summary: Record, + expected: Record, + ) => Record; + readScopedResourceCounts: ( + databasePath: string, + projectId: string, + sprintId: string, + ) => { + taskRuns: number; + runningTaskRuns: number; + activeTaskRuns: number; + taskRunEvents: number; + providerInvocations: number; + } | null; + readScopedWorkflowTimings: ( + databasePath: string, + projectId: string, + sprintId: string, + ) => Record | null; }; const runner = await import("../../../scripts/e2e/run-mockup-sprint-pentest.mjs") as RunnerModule; @@ -132,6 +167,296 @@ describe("mockup sprint pentest runner polling", () => { }); }); + it("keeps the 400-task restart pentest local-only and validates its adversarial DAG shape", () => { + const scenario = getScenario("extreme-dag-recovery"); + const projectRun = scenario?.projectRuns[0]; + expect(scenario).toMatchObject({ heavy: true, localOnly: true }); + expect(projectRun?.tasks).toHaveLength(400); + expect(new Set(projectRun?.tasks.map((task: { key: string }) => task.key)).size).toBe(400); + expect(projectRun?.tasks.reduce( + (total: number, task: { promptMarkdown: string }) => total + task.promptMarkdown.length, + 0, + )).toBeGreaterThan(220_000); + expect(projectRun?.tasks.every((task: { promptMarkdown: string }) => ( + task.promptMarkdown.includes("Local-only QA context pressure:") + ))).toBe(true); + expect(projectRun?.tasks.filter((task: { dependsOn?: string[] }) => (task.dependsOn?.length || 0) >= 3).length) + .toBeGreaterThan(80); + expect(projectRun?.tasks.find((task: { key: string }) => task.key === "extreme-no-change-gate")) + .toMatchObject({ dependsOn: [ + "extreme-region-01", + "extreme-region-02", + "extreme-region-03", + "extreme-region-04", + ] }); + const finalManifest = projectRun?.tasks.find( + (task: { key: string }) => task.key === "extreme-final-manifest", + ); + const finalWriteDirective = finalManifest?.promptMarkdown + .split("\n") + .find((line: string) => line.startsWith("mockup-cli:write src/extreme/final.js")); + expect(finalWriteDirective).toContain("\\nimport { extremeRegion02 }"); + expect(finalWriteDirective).toContain("extremeDagTaskCount = 400"); + expect(finalWriteDirective).toContain("extremeQaFollowUp"); + expect(projectRun).toMatchObject({ + duringOrchestration: { + injectMainCiFix: { + markerPath: "src/extreme/ci-fix.js", + }, + }, + expected: { + qa: { + tasks: { + "extreme-layer-10": { + outcomes: ["changes_requested", "pass"], + requireFollowUp: true, + requireSameWorkerBranch: true, + }, + }, + sprintOutcomes: ["pass"], + }, + invocations: { + minimumCompletedCiFixes: 1, + requireSprintLevelCiFix: true, + }, + statusReporting: { + followUpTaskKeys: ["extreme-layer-10"], + requireSprintQa: true, + }, + }, + }); + expect(projectRun?.expected.resources).toMatchObject({ + minTaskQaPromptChars: 5_000, + maxTaskQaPromptChars: 50_000, + minSprintQaPromptChars: 250_000, + maxSprintQaPromptChars: 400_000, + maxProjectGitHelpers: 1, + maxWorkspaceHelpers: 16, + maxProjectGitHelperGenerations: 10, + }); + expect(Object.keys(projectRun?.expected.qa.tasks || {})).toHaveLength(399); + expect(projectRun?.expected.qa.tasks).not.toHaveProperty("extreme-no-change-gate"); + expect(runner.resolveScenarios("all").map((item) => item.id)).not.toContain("extreme-dag-recovery"); + expect(runner.resolveScenarios("pentest").map((item) => item.id)).not.toContain("extreme-dag-recovery"); + expect(runner.resolveScenarios("extreme-dag-recovery").map((item) => item.id)) + .toEqual(["extreme-dag-recovery"]); + }); + + it("summarizes and enforces extreme-DAG resource ceilings", () => { + const summary = runner.summarizeMockupRuntimeResources([ + { + pid: 10, + runtimeRssBytes: 120, + databaseBytes: 50, + walBytes: 40, + sessionDatabaseBytes: 30, + sessionWalBytes: 20, + counts: { + taskRuns: 20, + runningTaskRuns: 4, + completedTaskRuns: 15, + failedTaskRuns: 1, + cancelledTaskRuns: 0, + activeTaskRuns: 5, + taskRunEvents: 100, + providerInvocations: 18, + maxTaskQaPromptChars: 90, + maxSprintQaPromptChars: 80, + }, + dockerHelpers: { + projectGitHelperIds: ["git-a"], + workspaceHelperIds: ["workspace-a", "workspace-b"], + nonRunningContainerIds: ["created-a"], + }, + }, + { + pid: 11, + runtimeRssBytes: 150, + databaseBytes: 60, + walBytes: 30, + sessionDatabaseBytes: 35, + sessionWalBytes: 25, + counts: { + taskRuns: 21, + runningTaskRuns: 6, + completedTaskRuns: 20, + failedTaskRuns: 1, + cancelledTaskRuns: 0, + activeTaskRuns: 8, + taskRunEvents: 120, + providerInvocations: 20, + maxTaskQaPromptChars: 100, + maxSprintQaPromptChars: 110, + }, + dockerHelpers: { + projectGitHelperIds: ["git-b"], + workspaceHelperIds: ["workspace-b"], + nonRunningContainerIds: [], + }, + }, + ]); + expect(summary).toMatchObject({ + sampleCount: 2, + observedPids: [10, 11], + peakRuntimeRssBytes: 150, + peakWalBytes: 40, + peakSessionDatabaseBytes: 35, + peakSessionWalBytes: 25, + peakTaskRuns: 21, + peakRunningTaskRuns: 6, + peakFailedTaskRuns: 1, + peakActiveTaskRuns: 8, + peakTaskRunEvents: 120, + peakProviderInvocations: 20, + peakTaskQaPromptChars: 100, + peakSprintQaPromptChars: 110, + peakProjectGitHelpers: 1, + peakWorkspaceHelpers: 2, + projectGitHelperGenerations: 2, + workspaceHelperGenerations: 2, + peakNonRunningContainers: 1, + finalNonRunningContainers: 0, + finalTaskAttemptAmplification: 1.05, + }); + expect(() => runner.assertExpectedRuntimeResources(summary, { + maxRuntimeRssBytes: 150, + maxDatabaseBytes: 60, + maxWalBytes: 40, + maxSessionDatabaseBytes: 35, + maxSessionWalBytes: 25, + maxTaskRuns: 21, + maxFailedTaskRuns: 1, + maxTaskAttemptAmplification: 1.05, + maxActiveTaskRuns: 8, + maxTaskRunEvents: 120, + maxProviderInvocations: 20, + minTaskQaPromptChars: 90, + maxTaskQaPromptChars: 100, + minSprintQaPromptChars: 100, + maxSprintQaPromptChars: 110, + maxProjectGitHelpers: 1, + maxWorkspaceHelpers: 2, + maxProjectGitHelperGenerations: 2, + maxFinalNonRunningContainers: 0, + })).not.toThrow(); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxRuntimeRssBytes: 149 })) + .toThrow("expected runtime RSS to stay at or below 149, received 150"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxActiveTaskRuns: 7 })) + .toThrow("expected active task runs to stay at or below 7, received 8"); + expect(() => runner.assertExpectedRuntimeResources(summary, { minTaskQaPromptChars: 101 })) + .toThrow("expected the largest task QA prompt to contain at least 101 characters, received 100"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxTaskQaPromptChars: 99 })) + .toThrow("expected task QA prompt to stay at or below 99, received 100"); + expect(() => runner.assertExpectedRuntimeResources(summary, { minSprintQaPromptChars: 111 })) + .toThrow("expected the largest sprint QA prompt to contain at least 111 characters, received 110"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxSprintQaPromptChars: 109 })) + .toThrow("expected sprint QA prompt to stay at or below 109, received 110"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxWorkspaceHelpers: 1 })) + .toThrow("expected concurrent workspace helpers to stay at or below 1, received 2"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxProjectGitHelperGenerations: 1 })) + .toThrow("expected project Git helper generations to stay at or below 1, received 2"); + expect(() => runner.assertExpectedRuntimeResources( + { ...summary, finalNonRunningContainers: 1 }, + { maxFinalNonRunningContainers: 0 }, + )).toThrow("expected final non-running containers to stay at or below 0, received 1"); + expect(() => runner.assertExpectedRuntimeResources(summary, { maxTaskAttemptAmplification: 1.04 })) + .toThrow("expected task-attempt amplification to stay at or below 1.04, received 1.05"); + }); + + it("counts only provider-slot-reserving task runs as active resources", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-resource-counts-")); + const databasePath = path.join(directory, "app.db"); + try { + const database = new DatabaseSync(databasePath); + database.exec(` + CREATE TABLE task_runs ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + sprint_id TEXT NOT NULL, + dispatch_id TEXT, + state TEXT NOT NULL + ); + CREATE TABLE task_dispatches (id TEXT PRIMARY KEY, status TEXT NOT NULL); + CREATE TABLE task_run_events (task_run_id TEXT NOT NULL, event_type TEXT NOT NULL); + CREATE TABLE provider_invocations ( + project_id TEXT NOT NULL, + sprint_id TEXT NOT NULL, + task_id TEXT, + task_run_id TEXT, + status TEXT NOT NULL, + purpose TEXT, + prompt_chars INTEGER + ); + INSERT INTO task_dispatches VALUES ('terminal-dispatch', 'running'), ('active-dispatch', 'running'); + INSERT INTO task_runs VALUES + ('terminal-run', 'project', 'sprint', 'terminal-dispatch', 'RUNNING'), + ('active-run', 'project', 'sprint', 'active-dispatch', 'RUNNING'); + INSERT INTO provider_invocations VALUES + ('project', 'sprint', 'task-1', 'terminal-run', 'completed', 'qa_review', 120), + ('project', 'sprint', 'task-2', 'active-run', 'completed', 'qa_review', 110), + ('project', 'sprint', 'task-2', 'active-run', 'running', 'task_coding', 80), + ('project', 'sprint', NULL, NULL, 'completed', 'qa_review', 130); + `); + database.close(); + + expect(runner.readScopedResourceCounts(databasePath, "project", "sprint")).toMatchObject({ + taskRuns: 2, + runningTaskRuns: 2, + activeTaskRuns: 1, + providerInvocations: 4, + maxTaskQaPromptChars: 120, + maxSprintQaPromptChars: 130, + }); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("summarizes completed CLI workflow phase timings without loading event payloads", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-workflow-timings-")); + const databasePath = path.join(directory, "app.db"); + try { + const database = new DatabaseSync(databasePath); + database.exec(` + CREATE TABLE task_runs ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + sprint_id TEXT NOT NULL, + state TEXT NOT NULL, + duration_ms INTEGER + ); + CREATE TABLE task_run_events ( + task_run_id TEXT NOT NULL, + event_type TEXT NOT NULL, + created_at TEXT NOT NULL + ); + INSERT INTO task_runs VALUES ('run-1', 'project', 'sprint', 'COMPLETED', 10000); + INSERT INTO task_run_events VALUES + ('run-1', 'cli_prepare_started', '2026-07-15T00:00:00.000Z'), + ('run-1', 'cli_prepare_completed', '2026-07-15T00:00:02.000Z'), + ('run-1', 'cli_provider_started', '2026-07-15T00:00:02.000Z'), + ('run-1', 'cli_provider_completed', '2026-07-15T00:00:07.000Z'), + ('run-1', 'cli_memory_capture_started', '2026-07-15T00:00:07.000Z'), + ('run-1', 'cli_memory_capture_completed', '2026-07-15T00:00:07.500Z'), + ('run-1', 'cli_git_finalize_started', '2026-07-15T00:00:07.500Z'), + ('run-1', 'cli_git_pushed', '2026-07-15T00:00:09.000Z'), + ('run-1', 'cli_workflow_completed', '2026-07-15T00:00:10.000Z'); + `); + database.close(); + + expect(runner.readScopedWorkflowTimings(databasePath, "project", "sprint")).toMatchObject({ + prepare: { count: 1, p50Ms: 2_000, maxMs: 2_000 }, + provider: { count: 1, p50Ms: 5_000, maxMs: 5_000 }, + memoryCapture: { count: 1, p50Ms: 500, maxMs: 500 }, + gitFinalize: { count: 1, p50Ms: 1_500, maxMs: 1_500 }, + workflow: { count: 1, p50Ms: 10_000, maxMs: 10_000 }, + taskRun: { count: 1, p50Ms: 10_000, maxMs: 10_000 }, + }); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + it("synchronizes the checked-out default worktree after fixture mutation", async () => { const repoDir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-mockup-mutation-")); try { @@ -181,6 +506,11 @@ describe("mockup sprint pentest runner polling", () => { expectedOutputReadinessChanged: false, providerAdmissionHeartbeatChanged: true, })).toBe(false); + expect(runner.isMockupPollStateProgress({ + progressChanged: false, + expectedOutputReadinessChanged: false, + runtimeRestartCompleted: true, + })).toBe(true); }); it("logs bounded task deltas instead of repeating the full wide-DAG snapshot", () => { diff --git a/tests/backend/server/activity-cache-service.test.ts b/tests/backend/server/activity-cache-service.test.ts index 2c6b64bf19..1266b786ca 100644 --- a/tests/backend/server/activity-cache-service.test.ts +++ b/tests/backend/server/activity-cache-service.test.ts @@ -172,6 +172,35 @@ describe('ActivityCacheService', () => { }); }); + it('evicts inactive sessions and bounds oversized live activity previews', async () => { + const oversizedActivity = { + ...mockActivity, + description: `${'a'.repeat(70_000)}TAIL`, + }; + mockDeps.getSubtasks.mockReturnValue([mockTask]); + mockDeps.resolveSessionNameFromTask.mockReturnValue('session-1'); + mockDeps.fetchRecentActivities.mockResolvedValue([oversizedActivity]); + + const first = await service.getLiveActivitiesForActiveTasks(); + expect(first['session-1'][0].description).toHaveLength(64 * 1024); + expect(first['session-1'][0].description).toContain('[activity preview truncated]'); + expect(first['session-1'][0].description?.endsWith('TAIL')).toBe(true); + + const secondTask = { ...mockTask, id: 'task-2' }; + mockDeps.getSubtasks.mockReturnValue([secondTask]); + mockDeps.resolveSessionNameFromTask.mockReturnValue('session-2'); + mockDeps.fetchRecentActivities.mockResolvedValue([{ ...mockActivity, id: 'act-2' }]); + await service.getLiveActivitiesForActiveTasks(); + + mockDeps.getSubtasks.mockReturnValue([mockTask]); + mockDeps.resolveSessionNameFromTask.mockReturnValue('session-1'); + mockDeps.fetchRecentActivities.mockClear(); + mockDeps.fetchRecentActivities.mockResolvedValue([mockActivity]); + await service.getLiveActivitiesForActiveTasks(); + + expect(mockDeps.fetchRecentActivities).toHaveBeenCalledWith('session-1', PAGE_SIZE); + }); + it('should return empty object if no active tasks', async () => { const inactiveTask = { ...mockTask, status: 'COMPLETED' as const }; mockDeps.getSubtasks.mockReturnValue([inactiveTask]); diff --git a/tests/backend/server/dashboard-realtime-websocket-server.test.ts b/tests/backend/server/dashboard-realtime-websocket-server.test.ts index 4eda5a0042..1459963bc3 100644 --- a/tests/backend/server/dashboard-realtime-websocket-server.test.ts +++ b/tests/backend/server/dashboard-realtime-websocket-server.test.ts @@ -657,16 +657,16 @@ describe("DashboardRealtimeWebSocketServer", () => { it("disconnects slow websocket clients before buffering unbounded realtime frames", () => { const { sendClientMessage, socket } = setupClient(); - Object.defineProperty(socket, "writable", { value: true, configurable: true }); - Object.defineProperty(socket, "destroyed", { value: false, configurable: true }); - Object.defineProperty(socket, "writableLength", { value: 20 * 1024 * 1024, configurable: true }); - sendClientMessage({ type: "set_subscriptions", scopes: ["project:p1:live"], lastSequence: 0, }); + Object.defineProperty(socket, "writable", { value: true, configurable: true }); + Object.defineProperty(socket, "destroyed", { value: false, configurable: true }); + Object.defineProperty(socket, "writableLength", { value: 20 * 1024 * 1024, configurable: true }); + const subscribeCallback = realtimeService.subscribe.mock.calls[0][0]; subscribeCallback({ sequence: 12, diff --git a/tests/backend/server/jules-agent-server.test.ts b/tests/backend/server/jules-agent-server.test.ts index 684e299810..91dc9f8197 100644 --- a/tests/backend/server/jules-agent-server.test.ts +++ b/tests/backend/server/jules-agent-server.test.ts @@ -34,12 +34,7 @@ import { DEFAULT_DASHBOARD_SETTINGS } from "../../../src/repositories/settings-d import { DefaultRuntimeContext } from "../../../src/app/runtime-context.js"; const stopServer = async (server: CodeUxServer): Promise => { - const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as never); - try { - await (server as any).handleSigint?.().catch?.(() => undefined); - } finally { - exitSpy.mockRestore(); - } + await server.close(); }; describe("CodeUxServer", () => { @@ -89,6 +84,12 @@ describe("CodeUxServer", () => { "cux_test_abcdefghijklmnopqrstuvwxyz123456", ], projectRoot); const serverModeServer = new CodeUxServer({ projectRoot, appConfig: serverModeConfig }); + const projectManagementRepository = ( + serverModeServer as unknown as { + projectManagementRepository: { getSelectedProjectId(): string | null }; + } + ).projectManagementRepository; + const selectedProjectSpy = vi.spyOn(projectManagementRepository, "getSelectedProjectId"); try { await serverModeServer.run(); @@ -103,6 +104,10 @@ describe("CodeUxServer", () => { } finally { await serverModeServer.close(); } + + const callsAfterClose = selectedProjectSpy.mock.calls.length; + await new Promise((resolve) => setTimeout(resolve, 300)); + expect(selectedProjectSpy).toHaveBeenCalledTimes(callsAfterClose); }); describe("getEffectiveJulesApiKey", () => { @@ -356,7 +361,7 @@ describe("CodeUxServer", () => { trackedSession, { ...remoteSession, provider: "jules" } ]); - expect((server as any).sessionTracking.listSessions).toHaveBeenCalledWith(300); + expect((server as any).sessionTracking.listSessions).toHaveBeenCalledWith(300, { includePrompt: false }); expect(getCachedSessionsSpy).toHaveBeenCalled(); isJulesApiConfiguredSpy.mockRestore(); @@ -929,8 +934,10 @@ describe("CodeUxServer", () => { bootDashboardArgs.syncGitSettingsFromDashboard(); + const originalLogger = (runServer as any).logger; bootDashboardArgs.setLogger("newLogger" as any); - // expect removed + expect((runServer as any).logger).toBe("newLogger"); + bootDashboardArgs.setLogger(originalLogger); expect(bootMcpTransport).toHaveBeenCalled(); expect(bootMcpHttpTransport).toHaveBeenCalled(); diff --git a/tests/backend/services/__snapshots__/sprint-preview-docker-plan.test.ts.snap b/tests/backend/services/__snapshots__/sprint-preview-docker-plan.test.ts.snap index f1582992d7..12a0218b5d 100644 --- a/tests/backend/services/__snapshots__/sprint-preview-docker-plan.test.ts.snap +++ b/tests/backend/services/__snapshots__/sprint-preview-docker-plan.test.ts.snap @@ -20,6 +20,8 @@ exports[`SprintPreviewDockerPlanBuilder > matches snapshot 1`] = ` "--label", "code-ux.managed=true", "--label", + "code-ux.runtime-owner=", + "--label", "code-ux.preview=true", "--label", "code-ux.project-id=proj-1", diff --git a/tests/backend/services/activity-write-coalescer.test.ts b/tests/backend/services/activity-write-coalescer.test.ts index 772efab906..54f12c254f 100644 --- a/tests/backend/services/activity-write-coalescer.test.ts +++ b/tests/backend/services/activity-write-coalescer.test.ts @@ -78,6 +78,22 @@ describe("ActivityWriteCoalescer", () => { expect(sink.batches[0].items.map((item) => item.originator)).toEqual(["agent", "agent", "provider"]); }); + it("clips a single oversized activity before buffering it", () => { + const sink = makeSink(); + const coalescer = new ActivityWriteCoalescer(sink, "s1", { + flushIntervalMs: 250, + maxChunkChars: 256, + }); + + coalescer.push(`${"a".repeat(400)}TAIL`, "agent"); + coalescer.stop(); + + const description = sink.batches[0].items[0].description; + expect(description).toHaveLength(256); + expect(description).toContain("[activity truncated]"); + expect(description.endsWith("TAIL")).toBe(true); + }); + it("stop() flushes the tail and a subsequent timer does not double-write", () => { const sink = makeSink(); const coalescer = new ActivityWriteCoalescer(sink, "s1", { flushIntervalMs: 250 }); diff --git a/tests/backend/services/cli-process-runner.test.ts b/tests/backend/services/cli-process-runner.test.ts index 723e64d015..14563ffaee 100644 --- a/tests/backend/services/cli-process-runner.test.ts +++ b/tests/backend/services/cli-process-runner.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import * as fs from "fs/promises"; import * as os from "os"; import * as path from "path"; -import { runCommandStrict } from "../../../src/services/cli-process-runner.js"; +import { runCommandStrict, runStreamingCommand } from "../../../src/services/cli-process-runner.js"; describe("runCommandStrict", () => { it("forwards stdin files to the shared command runner", async () => { @@ -25,3 +25,27 @@ describe("runCommandStrict", () => { } }); }); + +describe("runStreamingCommand", () => { + it("forwards stdin files while preserving streaming callbacks", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-streaming-process-runner-")); + const inputPath = path.join(tempDir, "input.txt"); + try { + await fs.writeFile(inputPath, "streamed-stdin\n", "utf8"); + const lines: string[] = []; + + const result = await runStreamingCommand( + "node", + ["-e", "process.stdin.pipe(process.stdout)"], + tempDir, + process.env, + { stdinFile: inputPath, onStdoutLine: (line) => lines.push(line) }, + ); + + expect(result.stdout).toBe("streamed-stdin"); + expect(lines).toEqual(["streamed-stdin"]); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/backend/services/cli-workflow-service.test.ts b/tests/backend/services/cli-workflow-service.test.ts index e72d20ec89..122894d23a 100644 --- a/tests/backend/services/cli-workflow-service.test.ts +++ b/tests/backend/services/cli-workflow-service.test.ts @@ -694,6 +694,9 @@ describe("CliWorkflowService unpushed commit detection", () => { logger: { error: vi.fn() }, }; const service = new CliWorkflowService(deps as any); + const releaseWorkspaceReservation = vi.fn(); + const reserveWorkspaceHelper = vi.spyOn((service as any).workspaceManager, "reserveWorkspaceHelper") + .mockReturnValue(releaseWorkspaceReservation); // Mock the external stages const { executePrepareStage } = await import("../../../src/services/cli-workflow/pipeline/prepare-stage.js"); @@ -733,6 +736,8 @@ describe("CliWorkflowService unpushed commit detection", () => { expect.objectContaining({ completionTimestamp: expect.any(String) }), ); expect(executeCleanupStage).toHaveBeenCalled(); + expect(reserveWorkspaceHelper).toHaveBeenCalledWith(expect.stringMatching(/^docker-volume:\/\//)); + expect(releaseWorkspaceReservation).toHaveBeenCalledOnce(); expect(executionRepository.appendTaskRunEvent).toHaveBeenCalledWith( "run-1", "cli_prepare_started", @@ -777,7 +782,10 @@ describe("CliWorkflowService unpushed commit detection", () => { ); }); - it("resumes Git finalization without invoking the provider twice after a restart crash window", async () => { + it.each([ + "terminal_provider_active_dispatch_mismatch", + "shutdown_interrupted_after_provider_completion", + ])("resumes Git finalization without invoking the provider twice after a restart crash window marked by %s", async (recoveryReason) => { let storedInvocation: Record | null = null; const executionRepository = { getTaskRun: vi.fn().mockReturnValue({ @@ -812,7 +820,7 @@ describe("CliWorkflowService unpushed commit detection", () => { { eventType: "task_dispatch_reconciled", payload: { - reason: "terminal_provider_active_dispatch_mismatch", + reason: recoveryReason, providerStatus: "completed", }, }, diff --git a/tests/backend/services/cli-workflow/pipeline/pipeline-stages.test.ts b/tests/backend/services/cli-workflow/pipeline/pipeline-stages.test.ts index 2a09eceefb..fac24a3ae5 100644 --- a/tests/backend/services/cli-workflow/pipeline/pipeline-stages.test.ts +++ b/tests/backend/services/cli-workflow/pipeline/pipeline-stages.test.ts @@ -169,6 +169,7 @@ const createMockContext = (): PipelineContext => { resolveResumeWorktreePath: vi.fn(), prepareWorktree: vi.fn(), removeWorktree: vi.fn(), + releaseWorkspaceHelper: vi.fn(), buildWorkspaceGuidance: vi.fn(), } as any, invocationWorkspacePreparer: { @@ -913,6 +914,7 @@ describe("executeCleanupStage", () => { await executeCleanupStage(ctx); expect(ctx.workspaceManager.removeWorktree).toHaveBeenCalledWith("/repo", "/repo/worktree"); + expect(ctx.workspaceManager.releaseWorkspaceHelper).not.toHaveBeenCalled(); }); it("preserves the worktree if cleanupWorktreeOnSuccess is false and workflow succeeded", async () => { @@ -923,6 +925,7 @@ describe("executeCleanupStage", () => { await executeCleanupStage(ctx); expect(ctx.workspaceManager.removeWorktree).not.toHaveBeenCalled(); + expect(ctx.workspaceManager.releaseWorkspaceHelper).toHaveBeenCalledWith("/repo/worktree"); expect(ctx.deps.sessionTracking.appendActivity).toHaveBeenCalledWith(ctx.sessionId, expect.objectContaining({ description: expect.stringContaining("Preserving worktree") })); @@ -937,6 +940,7 @@ describe("executeCleanupStage", () => { await executeCleanupStage(ctx); expect(ctx.workspaceManager.removeWorktree).not.toHaveBeenCalled(); + expect(ctx.workspaceManager.releaseWorkspaceHelper).toHaveBeenCalledWith("/repo/worktree"); expect(ctx.deps.sessionTracking.appendActivity).toHaveBeenCalledWith(ctx.sessionId, expect.objectContaining({ description: expect.stringContaining("Preserving worktree") })); diff --git a/tests/backend/services/dashboard-realtime-service.test.ts b/tests/backend/services/dashboard-realtime-service.test.ts index d4decb6ee6..82c744d072 100644 --- a/tests/backend/services/dashboard-realtime-service.test.ts +++ b/tests/backend/services/dashboard-realtime-service.test.ts @@ -1213,10 +1213,50 @@ describe("DashboardRealtimeService backpressure and metrics", () => { await vi.advanceTimersByTimeAsync(100); expect(getProjectLiveSnapshot).not.toHaveBeenCalled(); - expect(eventRepoMock.appendEvent).not.toHaveBeenCalled(); + expect(eventRepoMock.appendEvent).toHaveBeenCalledWith(expect.objectContaining({ + eventType: "project.live.updated", + replayable: false, + })); expect(service.getMetrics("project.live.updated").skipped).toBe(1); }); + it("skips all snapshot loaders when no websocket scope has subscribers", async () => { + const loggerMock = { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn(), child: vi.fn() }; + let sequence = 1; + const eventRepoMock = { + getLatestSequence: () => sequence, + appendEvent: vi.fn().mockImplementation((event) => ({ sequence: ++sequence, ...event })), + }; + const loaders = { + getProjectLiveSnapshot: vi.fn(() => ({} as any)), + getProjectsSnapshot: vi.fn(() => ({} as any)), + getProjectExecutionSnapshot: vi.fn(() => ({} as any)), + getProjectStatusSnapshot: vi.fn(() => ({} as any)), + getOverviewTelemetrySnapshot: vi.fn(() => ({} as any)), + }; + const service = new DashboardRealtimeService(eventRepoMock as any, loggerMock as any); + service.setScopeInterestResolver(() => false); + service.setSnapshotLoaders(loaders); + + service.scheduleProjectExecutionRefresh("proj-1"); + service.scheduleProjectRuntimeStatusRefresh("proj-1"); + service.scheduleProjectStructureRefresh("proj-1"); + service.scheduleProjectsRefresh(); + await service.drain(); + + expect(loaders.getProjectLiveSnapshot).not.toHaveBeenCalled(); + expect(loaders.getProjectsSnapshot).not.toHaveBeenCalled(); + expect(loaders.getProjectExecutionSnapshot).not.toHaveBeenCalled(); + expect(loaders.getProjectStatusSnapshot).not.toHaveBeenCalled(); + expect(loaders.getOverviewTelemetrySnapshot).not.toHaveBeenCalled(); + // The lightweight non-replayable watermark is still recorded so a disconnected client can + // detect that it missed invalidations. Only the expensive snapshot loaders are interest-gated. + expect(eventRepoMock.appendEvent).toHaveBeenCalled(); + expect(eventRepoMock.appendEvent.mock.calls.every(([event]) => event.replayable === false)).toBe(true); + expect(service.getMetrics("execution_refresh").published).toBe(1); + expect(service.getMetrics("project.structure.updated").skipped).toBe(1); + }); + it("bounds redundant burst snapshot writes to one publish per coalesced event type", async () => { const loggerMock = { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn(), child: vi.fn() }; let sequence = 1; diff --git a/tests/backend/services/database-maintenance-service.test.ts b/tests/backend/services/database-maintenance-service.test.ts index 5e03631bbc..597ea2eef7 100644 --- a/tests/backend/services/database-maintenance-service.test.ts +++ b/tests/backend/services/database-maintenance-service.test.ts @@ -158,6 +158,23 @@ describe("DatabaseMaintenanceService", () => { expect(mockSessionDb.prepare).toHaveBeenCalledWith(expect.stringContaining("DELETE FROM provider_activities")); expect(mockAppDb.exec).toHaveBeenCalledWith("PRAGMA wal_checkpoint(PASSIVE);"); }); + + it("checkpoints WAL files while provider work is active without running retention writes", () => { + mockAppDb.prepare.mockImplementation((sql: string) => ({ + all: vi.fn(() => []), + run: vi.fn(() => ({ changes: 0 })), + get: vi.fn(() => sql.includes("FROM provider_invocations") ? { active: 1 } : undefined), + })); + + createService().runPeriodicMaintenance(); + + expect(mockAppDb.prepare).not.toHaveBeenCalledWith(expect.stringContaining("DELETE FROM task_runs")); + expect(mockSessionDb.prepare).not.toHaveBeenCalled(); + expect(mockAppDb.exec).toHaveBeenCalledWith("PRAGMA wal_checkpoint(PASSIVE);"); + expect(mockLogger.debug).toHaveBeenCalledWith( + "Skipping periodic database pruning while provider invocations are active.", + ); + }); }); describe("DatabaseMaintenanceService SQLite retention", () => { diff --git a/tests/backend/services/docker-asset-prune-service.test.ts b/tests/backend/services/docker-asset-prune-service.test.ts index c1326f0e15..9eec9f8753 100644 --- a/tests/backend/services/docker-asset-prune-service.test.ts +++ b/tests/backend/services/docker-asset-prune-service.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { SessionTrackingRepository } from "../../../src/repositories/session-tracking-repository.js"; import { DockerAssetPruneService } from "../../../src/services/docker-asset-prune-service.js"; +import { getRuntimeOwnerLabel } from "../../../src/shared/config/runtime-owner.js"; import * as fs from "fs/promises"; import { runCommandStrict } from "../../../src/services/cli-process-runner.js"; @@ -21,6 +22,68 @@ describe("DockerAssetPruneService", () => { vi.mocked(fs.readFile).mockRejectedValue(new Error("missing")); }); + it("scopes every startup Docker scan to the current runtime state home", async () => { + const sessionTracking = { + listTrackedCliSessions: vi.fn(() => []), + } as unknown as SessionTrackingRepository; + vi.mocked(runCommandStrict).mockResolvedValue({ + ok: true, + stdout: "", + stderr: "", + code: 0, + } as any); + + await new DockerAssetPruneService(sessionTracking).cleanupOnStartup(); + + const dockerCalls = vi.mocked(runCommandStrict).mock.calls.map((call) => call[1]); + const ownerFilter = `label=${getRuntimeOwnerLabel()}`; + const assetScans = dockerCalls.filter((args) => ( + (args[0] === "ps" && args.includes("label=code-ux.helper")) + || (args[0] === "ps" && args.includes("label=code-ux.login=true")) + || (args[0] === "ps" && args.includes("label=code-ux.command")) + || (args[0] === "volume" && args[1] === "ls") + )); + expect(assetScans.length).toBeGreaterThanOrEqual(6); + expect(assetScans.every((args) => args.includes(ownerFilter))).toBe(true); + }); + + it("removes owner-scoped provider containers left in running or created state", async () => { + const sessionTracking = { + listTrackedCliSessions: vi.fn(() => []), + } as unknown as SessionTrackingRepository; + vi.mocked(runCommandStrict).mockImplementation(async (_command, args) => ({ + ok: true, + stdout: args[0] === "ps" && args.includes("label=code-ux.command") + ? "provider-running\nprovider-created\n" + : "", + stderr: "", + code: 0, + } as any)); + + const result = await new DockerAssetPruneService(sessionTracking).cleanupOnStartup(); + + expect(result.prunedProviderContainers).toEqual(["provider-running", "provider-created"]); + expect(runCommandStrict).toHaveBeenCalledWith( + "docker", + ["rm", "-f", "-v", "provider-running", "provider-created"], + expect.any(String), + process.env, + { timeout: 10_000 }, + ); + expect(vi.mocked(runCommandStrict).mock.calls).toEqual(expect.arrayContaining([ + expect.arrayContaining([ + "docker", + expect.arrayContaining([ + "ps", + "-aq", + "label=code-ux.managed=true", + "label=code-ux.command", + `label=${getRuntimeOwnerLabel()}`, + ]), + ]), + ])); + }); + it("prunes stale workspace volumes while preserving cached setup images on startup", async () => { const sessionTracking = { listTrackedCliSessions: vi.fn(() => [ diff --git a/tests/backend/services/git-status-service.test.ts b/tests/backend/services/git-status-service.test.ts index b7da78619d..6a08997697 100644 --- a/tests/backend/services/git-status-service.test.ts +++ b/tests/backend/services/git-status-service.test.ts @@ -266,6 +266,55 @@ describe("GitStatusService", () => { } }); + it("bounds cached status payloads and never retains raw access tokens in cache keys", async () => { + GitStatusService.invalidateCache(); + runner.mockImplementation(async (cmd: string, args: string[]) => { + if (args.includes("--is-inside-work-tree")) return { ok: true, stdout: "true\n", stderr: "" }; + if (args.includes("--show-toplevel")) return { ok: true, stdout: "/repo\n", stderr: "" }; + if (args.includes("--show-current")) return { ok: true, stdout: "main\n", stderr: "" }; + if (cmd === "git" && args[0] === "remote") return { ok: true, stdout: "origin\n", stderr: "" }; + return { ok: true, stdout: "", stderr: "" }; + }); + + for (let index = 0; index < 140; index += 1) { + await service.getStatus( + "LOCAL", + { githubToken: `raw-secret-${index}` }, + { scope: "FEATURE_PR_CI", featureBranch: `feature/cache-${index}` }, + 10_000, + ); + } + + const cache = (GitStatusService as unknown as { + statusCache: Map; + }).statusCache; + expect(cache.size).toBeLessThanOrEqual(128); + expect([...cache.keys()].join("\n")).not.toContain("raw-secret-"); + GitStatusService.invalidateCache(); + }); + + it("invalidates status keys for Windows repository paths without relying on JSON escaping", () => { + GitStatusService.invalidateCache(); + const cache = (GitStatusService as unknown as { + statusCache: Map }>; + }).statusCache; + const windowsRepo = "C:\\Users\\developer\\Code UX"; + const otherRepo = "C:\\Users\\developer\\Other"; + cache.set(JSON.stringify({ repoPath: windowsRepo, mode: "LOCAL" }), { + timestamp: Date.now(), + promise: Promise.resolve({}), + }); + cache.set(JSON.stringify({ repoPath: otherRepo, mode: "LOCAL" }), { + timestamp: Date.now(), + promise: Promise.resolve({}), + }); + + GitStatusService.invalidateCache(windowsRepo); + + expect([...cache.keys()].map((key) => JSON.parse(key).repoPath)).toEqual([otherRepo]); + GitStatusService.invalidateCache(); + }); + it("returns unavailable if not inside git worktree", async () => { runner.mockResolvedValue({ ok: false, stdout: "false\n", stderr: "not a git repo" }); const status = await service.getStatus("REMOTE"); diff --git a/tests/backend/services/guardrail-service.test.ts b/tests/backend/services/guardrail-service.test.ts index 77c9bf099f..90de9eac4d 100644 --- a/tests/backend/services/guardrail-service.test.ts +++ b/tests/backend/services/guardrail-service.test.ts @@ -16,6 +16,10 @@ function makeRepo(initial: Record = {}) { counts.set(k, next); return next; }), + refund: vi.fn((input: { taskId: string; purpose: string }) => ({ + applied: true, + count: Math.max(0, (counts.get(key(input.taskId, input.purpose)) ?? 0) - 1), + })), getCount: vi.fn((taskId: string, purpose: string) => counts.get(key(taskId, purpose)) ?? 0), getCounts: vi.fn(() => ({})), getTotal: vi.fn((taskId: string) => { @@ -92,6 +96,14 @@ describe("GuardrailService.record / reset", () => { const service = new GuardrailService(repo, () => settings()); expect(service.record(scope, "t1", "task_coding")).toBe(1); expect(repo.record).toHaveBeenCalledWith({ projectId: "proj-1", taskId: "t1", purpose: "task_coding" }); + expect(service.refund(scope, "t1", "task_coding", "runtime-restart:run-1", "restart")).toBe(0); + expect(repo.refund).toHaveBeenCalledWith({ + projectId: "proj-1", + taskId: "t1", + purpose: "task_coding", + sourceKey: "runtime-restart:run-1", + reason: "restart", + }); service.reset("t1"); expect(repo.reset).toHaveBeenCalledWith("t1"); service.resetPurpose("t1", "merge_conflict"); diff --git a/tests/backend/services/jules-api-client.test.ts b/tests/backend/services/jules-api-client.test.ts index 16c373dfa1..6c2e622259 100644 --- a/tests/backend/services/jules-api-client.test.ts +++ b/tests/backend/services/jules-api-client.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import { JulesApiClient } from "../../../src/integrations/jules-api-client.js"; +import { JulesApiClient, JulesApiRequestError } from "../../../src/integrations/jules-api-client.js"; import axios from "axios"; const mockAxiosInstance = { @@ -73,6 +73,25 @@ describe("JulesApiClient", () => { expect(res.id).toBe("s1"); }); + it("surfaces the Jules response message when session creation is rejected", async () => { + vi.mocked(mockAxios().post).mockRejectedValue({ + message: "Request failed with status code 400", + response: { + status: 400, + data: { error: { status: "INVALID_ARGUMENT", message: "Session request is invalid" } }, + }, + }); + + await expect(client.createSession({ + prompt: "p", + sourceContext: { source: "src" }, + })).rejects.toEqual(expect.objectContaining({ + name: "JulesApiRequestError", + status: 400, + message: "Jules API create session failed (HTTP 400 INVALID_ARGUMENT): Session request is invalid", + } satisfies Partial)); + }); + it("gets session", async () => { vi.mocked(mockAxios().get).mockResolvedValue({ data: { id: "s1" } }); await client.getSession("s1"); diff --git a/tests/backend/services/planning-agent-service.test.ts b/tests/backend/services/planning-agent-service.test.ts index edeee9c7b6..255d93b71f 100644 --- a/tests/backend/services/planning-agent-service.test.ts +++ b/tests/backend/services/planning-agent-service.test.ts @@ -429,6 +429,24 @@ describe("PlanningAgentService", () => { .find((record) => record.sprintId === sprint.id); expect(planningInvocation).toBeDefined(); const messages = executionRepository.listExecutionInvocationMessages(planningInvocation!.id); + expect(messages).toEqual(expect.arrayContaining([ + expect.objectContaining({ + role: "user", + metadata: expect.objectContaining({ + planningRequest: expect.objectContaining({ + kind: "plan_sprint", + autoStart: true, + replan: false, + overrides: expect.objectContaining({ + designGuidance: expect.objectContaining({ + selectedTechStackId: "code-ux-product-stack", + selectedStyleguideId: "game-experience", + }), + }), + }), + }), + }), + ])); const executionPlanMessage = messages.find((message) => { const widgetMetadata = message.metadata?.widget_metadata as Record | undefined; return widgetMetadata?.type === "planning_request" && widgetMetadata.status === "completed"; @@ -795,14 +813,15 @@ describe("PlanningAgentService", () => { expect(call?.continueSessionId).toBe("native-original"); expect(call?.sessionId).toBe("planning-claude-code-old"); expect(call?.prompt).toContain("Continue the previous planning attempt"); - expect(call?.prompt).toContain("If the previous provider conversation cannot be resumed"); + expect(call?.prompt).toContain("Use the original planning instructions below as the complete source of truth while continuing this conversation"); + expect(call?.allowFreshSessionFallback).toBe(false); expect(call?.prompt).toContain("## Original Planning Instructions"); expect(call?.prompt).toContain("Plan with context"); expect(call?.prompt).toContain("Output the complete valid JSON sprint definition now"); expect(executionRepository.getExecutionInvocation(failedInvocation.id)?.preservedAt).toEqual(expect.any(String)); }); - it("continues a cancelled planning invocation by reusing the preserved workspace", async () => { + it("automatically continues a restart-cancelled planning invocation with its durable options", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-planning-cancel-continue-")); tempDirs.push(dir); @@ -838,13 +857,14 @@ describe("PlanningAgentService", () => { }), }), }; + const executionControlService = { orchestrateSprint: vi.fn().mockResolvedValue(undefined) }; const service = new PlanningAgentService({ projectManagementRepository: projectRepository, connectionChatRepository: connectionRepository, executionRepository, settingsRepository, agentPresetSyncService: syncService, - executionControlService: { orchestrateSprint: vi.fn() } as any, + executionControlService: executionControlService as any, providerRunner, }); @@ -876,10 +896,23 @@ describe("PlanningAgentService", () => { model: "claude-fable-5", errorMessage: "Cancelled from dashboard", }); + executionRepository.appendExecutionInvocationMessage(cancelledInvocation.id, { + role: "user", + contentMarkdown: "Original planning prompt", + metadata: { + planningRequest: { + kind: "plan_sprint", + autoStart: true, + replan: false, + }, + }, + }); - const continued = await service.restartInvocation(cancelledInvocation.id, "continue_session"); + const continued = await service.recoverInterruptedInvocation(cancelledInvocation.id); expect(continued.createdTaskIds).toHaveLength(1); + expect(continued.started).toBe(true); + expect(executionControlService.orchestrateSprint).toHaveBeenCalledWith(project.id, sprint.id); expect(WorkspaceManager.prototype.createOrReuseSnapshotWorkspace).toHaveBeenCalledWith(repoPath, expect.stringContaining(sprint.id), expect.objectContaining({ branch: "main", remoteOnly: true, @@ -887,8 +920,77 @@ describe("PlanningAgentService", () => { expect(WorkspaceManager.prototype.createSnapshotWorkspace).not.toHaveBeenCalled(); const call = vi.mocked(providerRunner.runProviderForText).mock.calls[0]?.[0]; expect(call?.continueSessionId).toBe("native-cancelled"); + expect(call?.allowFreshSessionFallback).toBe(false); expect(call?.prompt).toContain("Continue the previous planning attempt"); expect(executionRepository.getExecutionInvocation(cancelledInvocation.id)?.preservedAt).toEqual(expect.any(String)); + expect(executionRepository.listExecutionInvocationMessages(cancelledInvocation.id)).toEqual(expect.arrayContaining([ + expect.objectContaining({ + role: "system", + metadata: expect.objectContaining({ + recovery: "startup_planning_request_resumed", + continuationMode: "continue_session", + }), + }), + ])); + + const preProviderSprint = projectRepository.createSprint(project.id, { + name: "Pre-provider Restart Sprint", + goal: "Recover the full planning request", + }); + const preProviderInvocation = executionRepository.createExecutionInvocation({ + projectId: project.id, + sprintId: preProviderSprint.id, + type: "planning", + status: "failed", + provider: "claude-code", + errorMessage: "Restarted before provider linkage", + }); + executionRepository.appendExecutionInvocationMessage(preProviderInvocation.id, { + role: "user", + contentMarkdown: "Original pre-provider planning prompt", + metadata: { + planningRequest: { + kind: "plan_sprint", + autoStart: false, + replan: false, + }, + }, + }); + + const retried = await service.recoverInterruptedInvocation(preProviderInvocation.id); + + expect(retried.createdTaskIds).toHaveLength(1); + expect(retried.started).toBe(false); + const retryCall = vi.mocked(providerRunner.runProviderForText).mock.calls[1]?.[0]; + expect(retryCall?.continueSessionId).toBeNull(); + expect(retryCall?.prompt).toContain("Recover the full planning request"); + expect(retryCall?.prompt).not.toContain("Continue the previous planning attempt"); + + const missingNativeSprint = projectRepository.createSprint(project.id, { + name: "Missing Native Session Sprint", + goal: "Do not silently replace the provider conversation", + }); + const missingNativeUsage = executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: missingNativeSprint.id, + sessionId: "planning-claude-code-missing-native", + provider: "claude-code", + purpose: "planning", + status: "cancelled", + }); + const missingNativeInvocation = executionRepository.createExecutionInvocation({ + projectId: project.id, + sprintId: missingNativeSprint.id, + providerInvocationId: missingNativeUsage.id, + type: "planning", + status: "cancelled", + provider: "claude-code", + }); + + await expect(service.recoverInterruptedInvocation(missingNativeInvocation.id)).rejects.toThrow( + "Refusing to start a fresh session", + ); + expect(providerRunner.runProviderForText).toHaveBeenCalledTimes(2); }); it("stops virtual planning rate-limit retries after the configured max", async () => { diff --git a/tests/backend/services/quality-assurance-service.test.ts b/tests/backend/services/quality-assurance-service.test.ts index a3b47cdf6e..a1753f43e7 100644 --- a/tests/backend/services/quality-assurance-service.test.ts +++ b/tests/backend/services/quality-assurance-service.test.ts @@ -106,6 +106,9 @@ describe("QualityAssuranceService", () => { .mockResolvedValue("docker-volume://qa-snapshot"); const removeWorktree = vi.spyOn((service as any).workspaceManager, "removeWorktree") .mockResolvedValue(undefined); + const releaseSnapshotReservation = vi.fn(); + const reserveWorkspaceHelper = vi.spyOn((service as any).workspaceManager, "reserveWorkspaceHelper") + .mockReturnValue(releaseSnapshotReservation); const result = await (service as any).runReview({ triggerType: "sprint_completion", @@ -127,6 +130,8 @@ describe("QualityAssuranceService", () => { cwd: "docker-volume://qa-snapshot", })); expect(removeWorktree).toHaveBeenCalledWith("/repo/project", "docker-volume://qa-snapshot"); + expect(reserveWorkspaceHelper).toHaveBeenCalledWith(expect.stringMatching(/^docker-volume:\/\//)); + expect(releaseSnapshotReservation).toHaveBeenCalledOnce(); }); it("runs HOST-mode QA against a detached review-branch snapshot", async () => { @@ -419,7 +424,7 @@ describe("QualityAssuranceService", () => { provider: "qwen-code", worker_branch: "task/update-alpha", pr_url: "https://example.test/pull/1", - activities: [], + activities: [{ description: "Current task activity remains complete." }], }; const prompt = (service as any).buildReviewPrompt({ triggerType: "task_completion", @@ -435,9 +440,18 @@ describe("QualityAssuranceService", () => { depends_on: [], is_independent: true, status: "COMPLETED", - provider: "qwen-code", + provider: "codex", worker_branch: "task/update-beta", pr_url: "https://example.test/pull/2", + activities: [{ description: "Sibling activity must not be included." }], + }, + { + id: "T03", + title: "Pending gamma task", + prompt: "This task has not run yet.", + depends_on: [], + is_independent: true, + status: "pending", activities: [], }, ], @@ -446,14 +460,124 @@ describe("QualityAssuranceService", () => { expect(prompt).toContain("## REVIEW SCOPE"); expect(prompt).toContain("This is a single-task QA review. The only task under review is T01."); - expect(prompt).toContain("## FULL TASK INSTRUCTIONS (SPRINT CONTEXT; ONLY CURRENT TASK IS UNDER REVIEW)"); + expect(prompt).toContain("## PREVIOUSLY COMPLETED SPRINT TASKS (TITLES ONLY)"); + expect(prompt).toContain("- Update beta.md"); + expect(prompt).not.toContain("T02"); + expect(prompt).not.toContain("Pending gamma task"); + expect(prompt).not.toContain("Write exactly one line to beta.md."); + expect(prompt).not.toContain("Sibling activity must not be included."); + expect(prompt).not.toContain("task/update-beta"); + expect(prompt).not.toContain("https://example.test/pull/2"); + expect(prompt).not.toContain("Provider: codex"); expect(prompt).toContain("## CURRENT TASK UNDER REVIEW"); + expect(prompt).toContain("Write exactly one line to alpha.md."); + expect(prompt).toContain("Current task activity remains complete."); + expect(prompt).toContain("Depends on: none"); expect(prompt).toContain("Assume the current workspace/branch contains only the current task's changes on top of its base branch."); expect(prompt).toContain("A task-level review must pass when the current task satisfies its own prompt"); expect(prompt).toContain("Do not request changes because files, commits, PRs, or behavior from other completed sibling tasks are missing from this branch."); expect(prompt).toContain("Do not tell the coding session to implement, restore, or modify another task's scope."); expect(prompt).toContain("For task-level reviews, review only the current task and return `targetTaskKey` as the current task key when changes are required."); - expect(prompt).toContain("Write exactly one line to beta.md."); + }); + + it("shows the first half of every task instruction when sprint QA context exceeds 100k tokens", () => { + const service = new QualityAssuranceService({ + projectManagementRepository: {} as any, + executionRepository: {} as any, + guardrailService: qaGuardrailStub(), + sessionTracking: {} as any, + qaReviewRepository: {} as any, + taskService: {} as any, + agentPresetSyncService: {} as any, + providerRunner: {} as any, + getDashboardSettings: () => DEFAULT_DASHBOARD_SETTINGS, + getGithubToken: () => undefined, + sendSessionMessage: async () => ({}), + }); + const subtasks = Array.from({ length: 400 }, (_, index) => { + const taskNumber = index + 1; + return { + id: `T${taskNumber}`, + title: `Task ${taskNumber}`, + prompt: [ + `Instruction ${taskNumber} first-half: ${"x".repeat(600)}`, + `Instruction ${taskNumber} second-half: ${"z".repeat(600)}`, + ].join("\n"), + depends_on: [], + is_independent: true, + status: "COMPLETED", + activities: [{ description: `Full activity ${taskNumber}: ${"y".repeat(100)}` }], + }; + }); + const prompt = (service as any).buildReviewPrompt({ + triggerType: "sprint_completion", + projectName: "Extreme QA Project", + sprintGoal: "Validate a wide DAG.", + agentInstructions: "Review critically.", + subtasks, + currentTask: null, + }); + + expect(prompt).toContain("exceeding the 100,000-token threshold"); + expect(prompt.match(/Every task remains listed in order, but each task instruction below contains only its first half\./g)) + .toHaveLength(1); + expect(prompt).toContain("Instruction 1 first-half:"); + expect(prompt).not.toContain("Instruction 1 second-half:"); + expect(prompt).toContain("Instruction 399 first-half:"); + expect(prompt).not.toContain("Instruction 399 second-half:"); + expect(prompt).toContain("Instruction 400 first-half:"); + expect(prompt).not.toContain("Instruction 400 second-half:"); + expect(prompt).toContain("Full activity 1:"); + expect(prompt).toContain("Full activity 400:"); + expect(prompt).toContain("Recent activity excerpts are not shortened."); + }); + + it("keeps extreme-DAG task QA title-only for completed siblings and full for the current task", () => { + const service = new QualityAssuranceService({ + projectManagementRepository: {} as any, + executionRepository: {} as any, + guardrailService: qaGuardrailStub(), + sessionTracking: {} as any, + qaReviewRepository: {} as any, + taskService: {} as any, + agentPresetSyncService: {} as any, + providerRunner: {} as any, + getDashboardSettings: () => DEFAULT_DASHBOARD_SETTINGS, + getGithubToken: () => undefined, + sendSessionMessage: async () => ({}), + }); + const subtasks = Array.from({ length: 400 }, (_, index) => { + const taskNumber = index + 1; + return { + id: `T${taskNumber}`, + title: `Completed sibling title ${taskNumber}`, + prompt: `Full instruction ${taskNumber}: ${"x".repeat(1_000)}`, + depends_on: taskNumber === 400 ? ["T399"] : [], + is_independent: taskNumber !== 400, + status: taskNumber === 400 ? "CODING_COMPLETED" : "completed", + activities: [{ description: `Full activity ${taskNumber}: ${"y".repeat(100)}` }], + }; + }); + + const prompt = (service as any).buildReviewPrompt({ + triggerType: "task_completion", + projectName: "Extreme QA Project", + sprintGoal: "Validate a wide DAG.", + agentInstructions: "Review critically.", + subtasks, + currentTask: subtasks[399], + }); + + expect(prompt.length).toBeGreaterThan(5_000); + expect(prompt.length).toBeLessThan(50_000); + expect(prompt).toContain("- Completed sibling title 1"); + expect(prompt).toContain("- Completed sibling title 399"); + expect(prompt).not.toContain("- Completed sibling title 400"); + expect(prompt).not.toContain("Full instruction 1:"); + expect(prompt).not.toContain("Full activity 1:"); + expect(prompt).toContain("Full instruction 400:"); + expect(prompt).toContain("Full activity 400:"); + expect(prompt).toContain("Depends on: T399"); }); it("creates sprint follow-up tasks from QA output", async () => { @@ -1437,6 +1561,34 @@ describe("QualityAssuranceService", () => { status: "COMPLETED" as const, }; + it("computes wide-DAG QA merge gates from one repository snapshot", () => { + const latestRun = { + id: "qa-run-1", + taskId: "task-1", + status: "completed", + outcome: "pass", + summaryMarkdown: "Passed.", + runIndex: 1, + }; + const listTaskReviewSnapshots = vi.fn().mockReturnValue(new Map([ + ["task-1", { latestRun, latestCycleRuns: [latestRun], runsUsed: 1, decisiveRuns: 1 }], + ["task-2", { latestRun: null, latestCycleRuns: [], runsUsed: 0, decisiveRuns: 0 }], + ])); + const service = buildGateService({ listTaskReviewSnapshots }); + const secondTask = { ...noPrCompletedTask, id: "T2", record_id: "task-2" }; + + const gates = service.getTaskMergeGateStatuses({ + projectId: "project-1", + sprintId: "sprint-1", + tasks: [noPrCompletedTask, secondTask], + }); + + expect(listTaskReviewSnapshots).toHaveBeenCalledOnce(); + expect(listTaskReviewSnapshots).toHaveBeenCalledWith(["task-1", "task-2"]); + expect(gates.get("task-1")).toMatchObject({ mergeAllowed: true, reason: "passed" }); + expect(gates.get("task-2")).toMatchObject({ mergeAllowed: false, reason: "pending_review" }); + }); + it("fails closed (no merge) when the verdict budget is exhausted without a pass", () => { // A single decisive QA verdict that did not pass, at the cap of 1. This is // the exact hole that used to silently complete no-PR tasks. diff --git a/tests/backend/services/runtime-startup-recovery-service.test.ts b/tests/backend/services/runtime-startup-recovery-service.test.ts index 88222e3d21..570615defe 100644 --- a/tests/backend/services/runtime-startup-recovery-service.test.ts +++ b/tests/backend/services/runtime-startup-recovery-service.test.ts @@ -29,13 +29,18 @@ const tempDirs: string[] = []; async function createFixture(options?: { recoverSprintRun?: SprintOrchestrator["recoverSprintRun"]; - logger?: Pick; + logger?: Pick; dockerService?: { listContainers: () => Promise }>>; removeContainers?: (containerIds: string[], options?: { removeVolumes?: boolean }) => Promise; }; isProcessAlive?: (pid: number) => boolean; getDashboardSettings?: () => DashboardSettings; + listDurableRemoteSessions?: () => Promise>; + resumeInterruptedPlanningInvocation?: ( + invocationId: string, + mode: "continue_session" | "retry_full_prompt", + ) => Promise; }) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-startup-recovery-")); tempDirs.push(dir); @@ -75,6 +80,8 @@ async function createFixture(options?: { } as SprintOrchestrator, dockerService: options?.dockerService as any, getDashboardSettings: options?.getDashboardSettings ?? (() => DEFAULT_DASHBOARD_SETTINGS), + listDurableRemoteSessions: options?.listDurableRemoteSessions, + resumeInterruptedPlanningInvocation: options?.resumeInterruptedPlanningInvocation, isProcessAlive: options?.isProcessAlive, logger: options?.logger, }); @@ -992,13 +999,14 @@ describe("RuntimeStartupRecoveryService", () => { }); }); - it("fails stale running planning invocation audit rows without provider runtime linkage on startup", async () => { + it("resumes even a freshly-created planning request interrupted before provider linkage", async () => { + const resumeInterruptedPlanningInvocation = vi.fn().mockResolvedValue(undefined); const { projectRepository, executionRepository, sessionTracking, service, - } = await createFixture(); + } = await createFixture({ resumeInterruptedPlanningInvocation }); const project = projectRepository.createProject({ name: "Planning Audit Recovery Project", @@ -1016,7 +1024,7 @@ describe("RuntimeStartupRecoveryService", () => { type: "planning", provider: "qwen-code", status: "running", - startedAt: "2026-03-29T10:00:00.000Z", + startedAt: new Date().toISOString(), }); const result = await service.recover(); @@ -1026,12 +1034,15 @@ describe("RuntimeStartupRecoveryService", () => { status: "failed", errorMessage: expect.stringContaining("without provider runtime linkage"), }); + expect(result.resumedPlanningInvocationIds).toEqual([invocation.id]); + expect(resumeInterruptedPlanningInvocation).toHaveBeenCalledWith(invocation.id, "continue_session"); }); it("fails a stale pre-provider CLI coding row with useful recovery evidence", async () => { const { projectRepository, executionRepository, + guardrailRepository, service, } = await createFixture(); @@ -1068,6 +1079,57 @@ describe("RuntimeStartupRecoveryService", () => { ])); }); + it("schedules one continuation for a planning request interrupted mid-provider", async () => { + const resumeInterruptedPlanningInvocation = vi.fn().mockResolvedValue(undefined); + const { + projectRepository, + executionRepository, + service, + } = await createFixture({ + resumeInterruptedPlanningInvocation, + dockerService: { listContainers: vi.fn().mockResolvedValue([]) }, + }); + const project = projectRepository.createProject({ + name: "Planning Provider Recovery Project", + sourceType: "local", + sourceRef: "/workspace/planning-provider-recovery-project", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Planning Provider Recovery Sprint", + number: 22, + status: "planning", + }); + const providerUsage = executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: sprint.id, + sessionId: "planning-qwen-restart", + nativeSessionId: "native-qwen-restart", + provider: "qwen-code", + purpose: "planning", + status: "running", + executionMode: "DOCKER", + startedAt: "2026-03-29T10:00:00.000Z", + }); + const invocation = executionRepository.createExecutionInvocation({ + projectId: project.id, + sprintId: sprint.id, + providerInvocationId: providerUsage.id, + type: "planning", + provider: "qwen-code", + status: "running", + startedAt: "2026-03-29T10:00:01.000Z", + }); + + const result = await service.recover(); + + expect(result.reconciledContainerInvocationIds).toContain(providerUsage.id); + expect(result.resumedPlanningInvocationIds).toEqual([invocation.id]); + expect(resumeInterruptedPlanningInvocation).toHaveBeenCalledTimes(1); + expect(resumeInterruptedPlanningInvocation).toHaveBeenCalledWith(invocation.id, "continue_session"); + expect(executionRepository.getExecutionInvocation(invocation.id)).toMatchObject({ status: "cancelled" }); + expect(executionRepository.getProviderInvocationUsage(providerUsage.id)).toMatchObject({ status: "cancelled" }); + }); + it("settles an interrupted CLI workflow from terminal provider and dispatch evidence", async () => { const { projectRepository, @@ -1193,6 +1255,230 @@ describe("RuntimeStartupRecoveryService", () => { ])); }); + it("requeues recovered provider completion when restart interrupts before Git finalization", async () => { + const { + projectRepository, + executionRepository, + guardrailRepository, + sessionTracking, + service, + } = await createFixture(); + + const project = projectRepository.createProject({ + name: "Post-provider Git recovery project", + sourceType: "local", + sourceRef: "/workspace/post-provider-git-recovery", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Post-provider Git recovery sprint", + number: 91, + status: "running", + }); + const task = projectRepository.createTask(project.id, { + sprintId: sprint.id, + taskKey: "T01", + title: "Finalize recovered branch", + executorType: "docker_cli", + status: "coding_completed", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + executorMode: "docker_cli", + status: "running", + }); + const dispatch = executionRepository.createTaskDispatch({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + executorType: "docker_cli", + status: "completed", + startedAt: "2026-07-15T10:00:00.000Z", + finishedAt: "2026-07-15T10:01:00.000Z", + }); + const sessionId = "cli-codex-post-provider-git"; + const workerBranch = "task/post-provider-git"; + sessionTracking.createSession({ + id: sessionId, + provider: "codex", + state: "RUNNING", + taskId: buildTaskRunKey(project.baseDir, sprint.number, "T01"), + title: "Finalize recovered branch", + featureBranch: "feature/post-provider-git", + workerBranch, + repoPath: project.baseDir, + }); + const taskRun = executionRepository.createTaskRun({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + provider: "codex", + mode: "docker_cli", + sessionId, + sessionName: `sessions/${sessionId}`, + state: "COMPLETED", + workerBranch, + startedAt: "2026-07-15T10:00:00.000Z", + finishedAt: "2026-07-15T10:01:00.000Z", + }); + executionRepository.appendTaskRunEvent(taskRun.id, "cli_workspace_bound", "system", { + workspaceSessionId: sessionId, + worktreePath: `docker-volume://${sessionId}`, + }); + executionRepository.appendTaskRunEvent(taskRun.id, "cli_provider_completed", "system", { + provider: "codex", + }); + const providerInvocation = executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + taskRunId: taskRun.id, + sessionId, + provider: "codex", + purpose: "task_coding", + executionMode: "DOCKER", + status: "completed", + startedAt: "2026-07-15T10:00:05.000Z", + finishedAt: "2026-07-15T10:00:55.000Z", + }); + guardrailRepository.record({ projectId: project.id, taskId: task.id, purpose: "task_coding" }); + + const result = await service.recover(); + + expect(result.recoveredCliSessionIds).toEqual([sessionId]); + expect(result.reconciledPostProviderTaskRunIds).toEqual([taskRun.id]); + expect(executionRepository.getTaskRun(taskRun.id)).toMatchObject({ state: "FAILED" }); + expect(executionRepository.getTaskDispatch(dispatch.id)).toMatchObject({ + status: "cancelled", + errorMessage: null, + }); + expect(projectRepository.getTask(task.id)).toMatchObject({ + status: "pending", + isMerged: false, + }); + expect(executionRepository.getProviderInvocationUsage(providerInvocation.id)).toMatchObject({ + status: "completed", + }); + expect(guardrailRepository.getCount(task.id, "task_coding")).toBe(0); + expect(executionRepository.listTaskRunEvents(taskRun.id)).toEqual(expect.arrayContaining([ + expect.objectContaining({ + eventType: "task_dispatch_reconciled", + payload: expect.objectContaining({ + reason: "shutdown_interrupted_after_provider_completion", + providerStatus: "completed", + }), + }), + ])); + expect(executionRepository.getLatestTaskWorkspaceResumeTarget(task.id, sprintRun.id)).toMatchObject({ + taskRunId: taskRun.id, + sessionId, + workerBranch, + worktreePath: `docker-volume://${sessionId}`, + }); + }); + + it("does not requeue finalized or non-recovered terminal CLI workflows", async () => { + const { + projectRepository, + executionRepository, + sessionTracking, + service, + } = await createFixture(); + const project = projectRepository.createProject({ + name: "Terminal CLI recovery exclusions", + sourceType: "local", + sourceRef: "/workspace/terminal-cli-recovery-exclusions", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Terminal CLI recovery exclusions sprint", + number: 92, + status: "running", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + executorMode: "docker_cli", + status: "running", + }); + const fixtures = [ + { key: "T01", finalEvent: "cli_git_pushed", recovered: true }, + { key: "T02", finalEvent: "cli_git_no_changes", recovered: true }, + { key: "T03", finalEvent: "cli_workflow_completed", recovered: true }, + { key: "T04", finalEvent: null, recovered: false }, + ] as const; + const runIds: string[] = []; + + for (const fixture of fixtures) { + const task = projectRepository.createTask(project.id, { + sprintId: sprint.id, + taskKey: fixture.key, + title: `Terminal exclusion ${fixture.key}`, + executorType: "docker_cli", + status: "coding_completed", + }); + const dispatch = executionRepository.createTaskDispatch({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + executorType: "docker_cli", + status: "completed", + }); + const sessionId = `cli-codex-terminal-${fixture.key.toLowerCase()}`; + sessionTracking.createSession({ + id: sessionId, + provider: "codex", + state: fixture.recovered ? "RUNNING" : "COMPLETED", + taskId: buildTaskRunKey(project.baseDir, sprint.number, fixture.key), + title: fixture.key, + featureBranch: "feature/terminal-exclusions", + workerBranch: `task/${fixture.key.toLowerCase()}`, + repoPath: project.baseDir, + }); + const taskRun = executionRepository.createTaskRun({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + provider: "codex", + mode: "docker_cli", + sessionId, + state: "COMPLETED", + workerBranch: `task/${fixture.key.toLowerCase()}`, + }); + runIds.push(taskRun.id); + executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + taskRunId: taskRun.id, + sessionId, + provider: "codex", + purpose: "task_coding", + executionMode: "DOCKER", + status: "completed", + }); + if (fixture.finalEvent) { + executionRepository.appendTaskRunEvent(taskRun.id, fixture.finalEvent, "system", {}); + } + } + + const result = await service.recover(); + + expect(result.reconciledPostProviderTaskRunIds).toEqual([]); + for (const runId of runIds) { + expect(executionRepository.getTaskRun(runId)).toMatchObject({ state: "COMPLETED" }); + } + }); + it("reconciles stale non-task execution audit rows when the provider invocation already failed", async () => { const { projectRepository, @@ -1975,6 +2261,224 @@ describe("RuntimeStartupRecoveryService", () => { }); }); + it("reactivates local projections when a durable Jules session is still active remotely", async () => { + const listDurableRemoteSessions = vi.fn().mockResolvedValue([{ + id: "jules-active-after-restart", + name: "sessions/jules-active-after-restart", + state: "IN_PROGRESS", + prompt: "Continue the hosted task.", + }]); + const { + projectRepository, + executionRepository, + service, + } = await createFixture({ listDurableRemoteSessions }); + + const project = projectRepository.createProject({ + name: "Durable Remote Recovery Project", + sourceType: "local", + sourceRef: "/workspace/durable-remote-recovery-project", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Durable Remote Recovery Sprint", + number: 20, + status: "failed", + }); + const task = projectRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Preserve hosted work", + executorType: "jules", + status: "pending", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + executorMode: "jules", + status: "failed", + }); + const dispatch = executionRepository.createTaskDispatch({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + executorType: "jules", + status: "failed", + }); + executionRepository.updateTaskDispatch(dispatch.id, { + status: "failed", + startedAt: "2026-03-29T10:00:00.000Z", + finishedAt: "2026-03-29T10:05:00.000Z", + errorMessage: "Provider session failed before dispatch reconciliation.", + }); + const taskRun = executionRepository.createTaskRun({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + provider: "jules", + mode: "jules", + sessionId: "jules-active-after-restart", + sessionName: "sessions/jules-active-after-restart", + state: "FAILED", + startedAt: "2026-03-29T10:00:00.000Z", + finishedAt: "2026-03-29T10:05:00.000Z", + }); + const usage = executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + taskRunId: taskRun.id, + sessionId: "jules-active-after-restart", + nativeSessionId: "jules-active-after-restart", + provider: "jules", + purpose: "task_coding", + status: "failed", + startedAt: "2026-03-29T10:00:00.000Z", + finishedAt: "2026-03-29T10:05:00.000Z", + invocationSource: "EXTERNAL_API", + }); + const invocation = executionRepository.createExecutionInvocation({ + projectId: project.id, + sprintId: sprint.id, + taskId: task.id, + sprintRunId: sprintRun.id, + dispatchId: dispatch.id, + taskRunId: taskRun.id, + providerInvocationId: usage.id, + type: "task_coding", + provider: "jules", + status: "failed", + startedAt: "2026-03-29T10:00:00.000Z", + finishedAt: "2026-03-29T10:05:00.000Z", + errorMessage: "Recovered stale task coding invocation after the linked sprint run was already failed.", + invocationSource: "EXTERNAL_API", + }); + + const result = await service.recover(); + + expect(listDurableRemoteSessions).toHaveBeenCalledTimes(1); + expect(result.reactivatedDurableRemoteTaskRunIds).toEqual([taskRun.id]); + expect(result.reactivatedDurableRemoteSprintRunIds).toEqual([sprintRun.id]); + expect(projectRepository.getRawSprintStatus(sprint.id)).toBe("running"); + expect(projectRepository.getTask(task.id)).toMatchObject({ status: "in_progress" }); + expect(executionRepository.getSprintRun(sprintRun.id)).toMatchObject({ status: "running", finishedAt: null }); + expect(executionRepository.getTaskDispatch(dispatch.id)).toMatchObject({ status: "running", finishedAt: null, errorMessage: null }); + expect(executionRepository.getTaskRun(taskRun.id)).toMatchObject({ state: "RUNNING", finishedAt: null }); + expect(executionRepository.getProviderInvocationUsage(usage.id)).toMatchObject({ status: "running", finishedAt: null }); + expect(executionRepository.getExecutionInvocation(invocation.id)).toMatchObject({ status: "running", finishedAt: null, errorMessage: null }); + }); + + it("bounds durable remote reconciliation so provider latency cannot block readiness", async () => { + vi.useFakeTimers(); + try { + const logger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; + const { + service, + projectRepository, + executionRepository, + } = await createFixture({ + logger, + listDurableRemoteSessions: () => new Promise(() => undefined), + }); + const project = projectRepository.createProject({ + name: "Timeout Fail-safe Project", + sourceType: "local", + sourceRef: "/workspace/timeout-fail-safe-project", + }); + const sprint = projectRepository.createSprint(project.id, { + name: "Timeout Fail-safe Sprint", + number: 25, + status: "failed", + }); + const task = projectRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Preserve unverifiable hosted work", + executorType: "jules", + status: "pending", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + executorMode: "jules", + status: "failed", + }); + const dispatch = executionRepository.createTaskDispatch({ + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + taskId: task.id, + executorType: "jules", + status: "failed", + }); + executionRepository.updateTaskDispatch(dispatch.id, { + status: "failed", + finishedAt: "2026-03-29T10:05:00.000Z", + errorMessage: "Earlier scheduler projection failed.", + }); + const taskRun = executionRepository.createTaskRun({ + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + taskId: task.id, + dispatchId: dispatch.id, + provider: "jules", + mode: "jules", + sessionId: "jules-timeout-fail-safe", + sessionName: "sessions/jules-timeout-fail-safe", + state: "FAILED", + finishedAt: "2026-03-29T10:05:00.000Z", + }); + executionRepository.createProviderInvocationUsage({ + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + taskId: task.id, + taskRunId: taskRun.id, + dispatchId: dispatch.id, + sessionId: "jules-timeout-fail-safe", + nativeSessionId: "jules-timeout-fail-safe", + provider: "jules", + purpose: "task_coding", + status: "running", + invocationSource: "EXTERNAL_API", + }); + + const recovery = (service as unknown as { + reconcileDurableRemoteSessions: () => Promise<{ + reactivatedTaskRunIds: string[]; + reactivatedSprintRunIds: string[]; + }>; + }).reconcileDurableRemoteSessions(); + vi.advanceTimersByTime(5_000); + await Promise.resolve(); + const result = await recovery; + + expect(result.reactivatedTaskRunIds).toEqual([taskRun.id]); + expect(result.reactivatedSprintRunIds).toEqual([sprintRun.id]); + expect(projectRepository.getRawSprintStatus(sprint.id)).toBe("running"); + expect(projectRepository.getTask(task.id)).toMatchObject({ status: "in_progress" }); + expect(executionRepository.getSprintRun(sprintRun.id)).toMatchObject({ status: "running", finishedAt: null }); + expect(executionRepository.getTaskRun(taskRun.id)).toMatchObject({ state: "RUNNING", finishedAt: null }); + expect(executionRepository.getTaskDispatch(dispatch.id)).toMatchObject({ status: "running", finishedAt: null }); + expect(logger.warn).toHaveBeenCalledWith( + "Could not reconcile durable remote sessions during startup", + expect.objectContaining({ + provider: "jules", + error: expect.stringContaining("timed out after 5000ms"), + }), + ); + } finally { + vi.useRealTimers(); + } + }); + it("rehydrates active Jules sessions from terminal sprint runs and resumes one recovered run", async () => { const { projectRepository, @@ -2338,6 +2842,7 @@ describe("RuntimeStartupRecoveryService", () => { const { projectRepository, executionRepository, + guardrailRepository, service, } = await createFixture(); @@ -2379,9 +2884,11 @@ describe("RuntimeStartupRecoveryService", () => { dispatchId: dispatch.id, provider: "codex", mode: "docker_cli", + sessionId: "cli-codex-restart-refund", state: "RUNNING", durationMs: 321, }); + guardrailRepository.record({ projectId: project.id, taskId: task.id, purpose: "task_coding" }); const result = await service.recover(); @@ -2405,6 +2912,10 @@ describe("RuntimeStartupRecoveryService", () => { }), }); expect(projectRepository.getTask(task.id)?.status).toBe("pending"); + expect(guardrailRepository.getCount(task.id, "task_coding")).toBe(0); + + await service.recover(); + expect(guardrailRepository.getCount(task.id, "task_coding")).toBe(0); }); it("cancels interrupted local dispatches even when their Docker session container survived restart", async () => { diff --git a/tests/backend/services/shutdown-container-service.test.ts b/tests/backend/services/shutdown-container-service.test.ts index f2fafc5a71..696c927a2b 100644 --- a/tests/backend/services/shutdown-container-service.test.ts +++ b/tests/backend/services/shutdown-container-service.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { ActiveDispatchRegistry } from "../../../src/services/active-dispatch-registry.js"; import { ShutdownContainerService } from "../../../src/services/shutdown-container-service.js"; +import { getRuntimeOwnerLabel } from "../../../src/shared/config/runtime-owner.js"; const dockerPsLine = (input: { id: string; names: string; labels: string }) => JSON.stringify({ ID: input.id, @@ -15,7 +16,7 @@ describe("ShutdownContainerService", () => { commandRunner = vi.fn(); }); - it("requests active dispatch stops and kills running Code UX containers", async () => { + it("requests active dispatch stops and removes Code UX containers in every state", async () => { const activeDispatchRegistry = new ActiveDispatchRegistry(); const requestStop = vi.fn().mockResolvedValue({ accepted: true }); activeDispatchRegistry.register({ @@ -29,11 +30,13 @@ describe("ShutdownContainerService", () => { if (args[0] === "ps") { return { stdout: [ - dockerPsLine({ id: "container-1", names: "code-ux-codex-session-1", labels: "code-ux.session-id=session-1" }), + dockerPsLine({ id: "container-1", names: "code-ux-codex-session-1", labels: `code-ux.session-id=session-1,${getRuntimeOwnerLabel()}` }), dockerPsLine({ id: "container-2", names: "unrelated", labels: "com.example.owner=test" }), - dockerPsLine({ id: "container-3", names: "code-ux-login", labels: "code-ux.login=true" }), - dockerPsLine({ id: "container-4", names: "code-ux-vol-helper-workspace", labels: "code-ux.helper=volume" }), - dockerPsLine({ id: "container-5", names: "code-ux-git-helper-project", labels: "" }), + dockerPsLine({ id: "container-3", names: "code-ux-login", labels: `code-ux.login=true,${getRuntimeOwnerLabel()}` }), + dockerPsLine({ id: "container-4", names: "code-ux-vol-helper-workspace", labels: `code-ux.helper=volume,${getRuntimeOwnerLabel()}` }), + dockerPsLine({ id: "container-5", names: "code-ux-git-helper-project", labels: getRuntimeOwnerLabel() }), + dockerPsLine({ id: "container-foreign", names: "code-ux-git-helper-foreign", labels: "code-ux.helper=git,code-ux.runtime-owner=another-runtime" }), + dockerPsLine({ id: "container-legacy", names: "code-ux-git-helper-legacy", labels: "code-ux.helper=git" }), ].join("\n"), }; } @@ -48,11 +51,13 @@ describe("ShutdownContainerService", () => { killedContainerIds: ["container-1", "container-3", "container-4", "container-5"], }); expect(requestStop).toHaveBeenCalledWith("test shutdown"); - expect(commandRunner).toHaveBeenCalledWith("docker", ["kill", "container-1"], process.cwd()); - expect(commandRunner).toHaveBeenCalledWith("docker", ["kill", "container-3"], process.cwd()); - expect(commandRunner).toHaveBeenCalledWith("docker", ["kill", "container-4"], process.cwd()); - expect(commandRunner).toHaveBeenCalledWith("docker", ["kill", "container-5"], process.cwd()); - expect(commandRunner).not.toHaveBeenCalledWith("docker", ["kill", "container-2"], process.cwd()); + expect(commandRunner).toHaveBeenCalledWith("docker", ["ps", "-a", "--format", "{{json .}}"], process.cwd()); + expect(commandRunner).toHaveBeenCalledWith("docker", [ + "rm", "-f", "-v", "container-1", "container-3", "container-4", "container-5", + ], process.cwd()); + expect(commandRunner.mock.calls.flatMap((call) => call[1])).not.toContain("container-2"); + expect(commandRunner.mock.calls.flatMap((call) => call[1])).not.toContain("container-foreign"); + expect(commandRunner.mock.calls.flatMap((call) => call[1])).not.toContain("container-legacy"); }); it("continues killing containers when a dispatch stop hook rejects", async () => { @@ -66,7 +71,7 @@ describe("ShutdownContainerService", () => { commandRunner.mockImplementation(async (_command, args) => { if (args[0] === "ps") { return { - stdout: dockerPsLine({ id: "container-1", names: "code-ux-codex-session-1", labels: "code-ux.session-id=session-1" }), + stdout: dockerPsLine({ id: "container-1", names: "code-ux-codex-session-1", labels: `code-ux.session-id=session-1,${getRuntimeOwnerLabel()}` }), }; } return { stdout: "" }; @@ -81,4 +86,83 @@ describe("ShutdownContainerService", () => { dispatchId: "dispatch-1", })); }); + + it("can cancel dispatches before draining helpers without requesting duplicate stops", async () => { + const activeDispatchRegistry = new ActiveDispatchRegistry(); + const requestStop = vi.fn().mockResolvedValue({ accepted: true }); + activeDispatchRegistry.register({ + dispatchId: "dispatch-1", + sessionId: "session-1", + executorType: "docker_cli", + requestStop, + }); + commandRunner.mockImplementation(async (_command, args) => ({ + stdout: args[0] === "ps" + ? dockerPsLine({ + id: "container-1", + names: "code-ux-codex-session-1", + labels: `code-ux.session-id=session-1,${getRuntimeOwnerLabel()}`, + }) + : "", + })); + const service = new ShutdownContainerService({ activeDispatchRegistry, commandRunner }); + + const requestedDispatchStops = await service.requestActiveDispatchStops("ordered shutdown"); + const result = await service.stopRemainingContainers(requestedDispatchStops); + + expect(requestStop).toHaveBeenCalledTimes(1); + expect(requestStop).toHaveBeenCalledWith("ordered shutdown"); + expect(result).toEqual({ requestedDispatchStops: 1, killedContainerIds: ["container-1"] }); + }); + + it("bounds large shutdown waves into parallel-safe Docker removal batches", async () => { + const activeDispatchRegistry = new ActiveDispatchRegistry(); + const containers = Array.from({ length: 19 }, (_, index) => ({ + id: `container-${index + 1}`, + names: `code-ux-provider-${index + 1}`, + labels: `code-ux.managed=true,${getRuntimeOwnerLabel()}`, + })); + commandRunner.mockImplementation(async (_command, args) => ({ + stdout: args[0] === "ps" ? containers.map(dockerPsLine).join("\n") : "", + })); + + const result = await new ShutdownContainerService({ + activeDispatchRegistry, + commandRunner, + }).stopRunningContainers(); + + const removalCalls = commandRunner.mock.calls.filter((call) => call[1][0] === "rm"); + expect(removalCalls).toHaveLength(3); + expect(removalCalls.every((call) => call[1].slice(3).length <= 8)).toBe(true); + expect(result.killedContainerIds).toEqual(containers.map((container) => container.id)); + }); + + it("treats concurrent container disappearance as idempotent cleanup", async () => { + const activeDispatchRegistry = new ActiveDispatchRegistry(); + commandRunner.mockImplementation(async (_command, args) => { + if (args[0] === "ps") { + return { + stdout: dockerPsLine({ + id: "container-raced", + names: "code-ux-codex-raced", + labels: `code-ux.managed=true,${getRuntimeOwnerLabel()}`, + }), + }; + } + throw new Error("Error response from daemon: No such container: container-raced"); + }); + const logger = { warn: vi.fn(), info: vi.fn() }; + + const result = await new ShutdownContainerService({ + activeDispatchRegistry, + logger: logger as any, + commandRunner, + }).stopRunningContainers(); + + expect(result.killedContainerIds).toEqual(["container-raced"]); + expect(logger.warn).not.toHaveBeenCalledWith( + "Failed to kill Code UX container during shutdown", + expect.anything(), + ); + }); }); diff --git a/tests/backend/services/sprint-preview-docker-plan.test.ts b/tests/backend/services/sprint-preview-docker-plan.test.ts index 0a8c0010f0..d3a1c16bb7 100644 --- a/tests/backend/services/sprint-preview-docker-plan.test.ts +++ b/tests/backend/services/sprint-preview-docker-plan.test.ts @@ -184,6 +184,8 @@ describe("SprintPreviewDockerPlanBuilder", () => { bootstrapScript: "echo 'bootstrap'", }); - expect(args).toMatchSnapshot(); + expect(args.map((arg) => ( + arg.startsWith("code-ux.runtime-owner=") ? "code-ux.runtime-owner=" : arg + ))).toMatchSnapshot(); }); }); diff --git a/tests/backend/services/sprint-task-dispatch-service.test.ts b/tests/backend/services/sprint-task-dispatch-service.test.ts index 0e69ad7c82..4fad2ff0e4 100644 --- a/tests/backend/services/sprint-task-dispatch-service.test.ts +++ b/tests/backend/services/sprint-task-dispatch-service.test.ts @@ -8,6 +8,7 @@ import { ExecutionRepository } from "../../../src/repositories/execution-reposit import { SprintTaskDispatchService, ProviderCapReachedError } from "../../../src/services/sprint-task-dispatch-service.js"; import { ProviderConcurrencyService } from "../../../src/services/provider-concurrency-service.js"; import { DEFAULT_DASHBOARD_SETTINGS } from "../../../src/repositories/settings-defaults.js"; +import { JulesApiRequestError } from "../../../src/integrations/jules-api-client.js"; const tempDirs: string[] = []; @@ -648,6 +649,298 @@ describe("SprintTaskDispatchService", () => { expect(messages.some((message) => message.contentMarkdown.includes("Jules dispatch failed: Jules API unavailable"))).toBe(true); }); + it("defers Jules capacity responses without failing the task or leaking the claimed slot", async () => { + const { projectManagementRepository, executionRepository, taskService, service } = await createFixture(); + const project = projectManagementRepository.createProject({ + name: "Jules Capacity Project", + sourceType: "local", + sourceRef: "/workspace/jules-capacity-project", + }); + const sprint = projectManagementRepository.createSprint(project.id, { + name: "Jules Capacity Sprint", + number: 21, + }); + const taskRecord = projectManagementRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Wait for hosted capacity", + promptMarkdown: "Start when a hosted session slot becomes available.", + executorType: "jules", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + status: "running", + executorMode: "jules", + }); + taskService.resolveTaskProvider.mockReturnValue("jules"); + taskService.startSprintTask.mockRejectedValue(new JulesApiRequestError( + "Jules API create session failed (HTTP 400 INVALID_ARGUMENT): Maximum active sessions reached", + 400, + "INVALID_ARGUMENT", + )); + const startArgs = { + task: { + id: taskRecord.taskKey, + record_id: taskRecord.id, + title: taskRecord.title, + prompt: taskRecord.promptMarkdown, + depends_on: [], + is_independent: true, + status: "PENDING" as const, + }, + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + featureBranch: "feature/sprint-21", + repoPath: "/workspace/jules-capacity-project", + sprintNumber: 21, + }; + + await expect(service.startTask(startArgs)).rejects.toBeInstanceOf(ProviderCapReachedError); + await expect(service.startTask(startArgs)).rejects.toBeInstanceOf(ProviderCapReachedError); + + expect(taskService.startSprintTask).toHaveBeenCalledTimes(1); + const [dispatch] = executionRepository.listTaskDispatches({ + projectId: project.id, + sprintRunId: sprintRun.id, + taskId: taskRecord.id, + }); + expect(dispatch).toMatchObject({ status: "queued", finishedAt: null, errorMessage: null }); + expect(executionRepository.getLatestTaskRun(taskRecord.id, sprintRun.id)).toMatchObject({ + state: "PENDING", + finishedAt: null, + }); + expect(projectManagementRepository.getTask(taskRecord.id)).toMatchObject({ status: "pending" }); + const [usage] = executionRepository.listProviderInvocationsForTask(project.id, taskRecord.id); + expect(usage).toMatchObject({ provider: "jules", status: "cancelled" }); + const [invocation] = executionRepository.listExecutionInvocations({ + projectId: project.id, + sprintRunId: sprintRun.id, + }); + expect(invocation).toMatchObject({ provider: "jules", status: "cancelled", errorMessage: null }); + }); + + it("reserves Jules slots for remotely active API sessions outside local accounting", async () => { + const { projectManagementRepository, executionRepository, taskService, service } = await createFixture(); + const project = projectManagementRepository.createProject({ + name: "Remote Jules Capacity Project", + sourceType: "local", + sourceRef: "/workspace/remote-jules-capacity-project", + }); + const sprint = projectManagementRepository.createSprint(project.id, { + name: "Remote Jules Capacity Sprint", + number: 23, + }); + const firstTask = projectManagementRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Use the final remote slot", + promptMarkdown: "Start only if the API reports one slot.", + executorType: "jules", + }); + const secondTask = projectManagementRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Wait behind the remote capacity cap", + promptMarkdown: "Do not create a sixteenth session.", + executorType: "jules", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + status: "running", + executorMode: "jules", + }); + const remoteSessions = Array.from({ length: 14 }, (_, index) => ({ + id: `remote-active-${index}`, + name: `sessions/remote-active-${index}`, + prompt: "Existing remote work", + state: "IN_PROGRESS", + })); + const listJulesSessionsForCapacity = vi.fn().mockResolvedValue(remoteSessions); + const capacityAwareService = new SprintTaskDispatchService( + executionRepository, + projectManagementRepository, + taskService as any, + (service as any).guardrailService, + (service as any).providerConcurrencyService, + () => DEFAULT_DASHBOARD_SETTINGS, + (service as any).logger, + listJulesSessionsForCapacity, + ); + taskService.resolveTaskProvider.mockReturnValue("jules"); + taskService.startSprintTask.mockResolvedValue({ + id: "jules-final-slot", + name: "sessions/jules-final-slot", + provider: "jules", + }); + const buildArgs = (taskRecord: typeof firstTask) => ({ + task: { + id: taskRecord.taskKey, + record_id: taskRecord.id, + title: taskRecord.title, + prompt: taskRecord.promptMarkdown, + depends_on: [], + is_independent: true, + status: "PENDING" as const, + }, + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + featureBranch: "feature/sprint-23", + repoPath: "/workspace/remote-jules-capacity-project", + sprintNumber: 23, + }); + + await expect(capacityAwareService.startTask(buildArgs(firstTask))).resolves.toMatchObject({ + id: "jules-final-slot", + }); + await expect(capacityAwareService.startTask(buildArgs(secondTask))).rejects.toMatchObject({ + provider: "jules", + limit: 15, + currentCount: 15, + }); + + expect(listJulesSessionsForCapacity).toHaveBeenCalledTimes(2); + expect(taskService.startSprintTask).toHaveBeenCalledTimes(1); + expect(projectManagementRepository.getTask(secondTask.id)).toMatchObject({ status: "pending" }); + expect(executionRepository.getLatestTaskRun(secondTask.id, sprintRun.id)).toMatchObject({ + state: "PENDING", + finishedAt: null, + }); + }); + + it("defers Jules dispatch when the API capacity check is unavailable", async () => { + const { projectManagementRepository, executionRepository, taskService, service } = await createFixture(); + const project = projectManagementRepository.createProject({ + name: "Jules Capacity Check Project", + sourceType: "local", + sourceRef: "/workspace/jules-capacity-check-project", + }); + const sprint = projectManagementRepository.createSprint(project.id, { + name: "Jules Capacity Check Sprint", + number: 24, + }); + const taskRecord = projectManagementRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Wait for a verified provider slot", + promptMarkdown: "Fail closed when Jules cannot report active sessions.", + executorType: "jules", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + status: "running", + executorMode: "jules", + }); + const capacityAwareService = new SprintTaskDispatchService( + executionRepository, + projectManagementRepository, + taskService as any, + (service as any).guardrailService, + (service as any).providerConcurrencyService, + () => DEFAULT_DASHBOARD_SETTINGS, + (service as any).logger, + vi.fn().mockRejectedValue(new Error("Jules list sessions unavailable")), + ); + taskService.resolveTaskProvider.mockReturnValue("jules"); + + await expect(capacityAwareService.startTask({ + task: { + id: taskRecord.taskKey, + record_id: taskRecord.id, + title: taskRecord.title, + prompt: taskRecord.promptMarkdown, + depends_on: [], + is_independent: true, + status: "PENDING", + }, + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + featureBranch: "feature/sprint-24", + repoPath: "/workspace/jules-capacity-check-project", + sprintNumber: 24, + })).rejects.toThrow("Provider concurrency cap reached for jules"); + + expect(taskService.startSprintTask).not.toHaveBeenCalled(); + expect(projectManagementRepository.getTask(taskRecord.id)).toMatchObject({ status: "pending" }); + }); + + it("defers a generic Jules failed precondition even when the bounded snapshot is under capacity", async () => { + const { projectManagementRepository, executionRepository, taskService, service } = await createFixture(); + const project = projectManagementRepository.createProject({ + name: "Jules Precondition Capacity Project", + sourceType: "local", + sourceRef: "/workspace/jules-precondition-capacity-project", + }); + const sprint = projectManagementRepository.createSprint(project.id, { + name: "Jules Precondition Capacity Sprint", + number: 26, + }); + const taskRecord = projectManagementRepository.createTask(project.id, { + sprintId: sprint.id, + title: "Recheck ambiguous provider capacity", + promptMarkdown: "Treat the provider rejection as authoritative when pagination hides older running work.", + executorType: "jules", + }); + const sprintRun = executionRepository.createSprintRun({ + projectId: project.id, + sprintId: sprint.id, + status: "running", + executorMode: "jules", + }); + const makeRemoteSessions = (count: number) => Array.from({ length: count }, (_, index) => ({ + id: `precondition-active-${index}`, + name: `sessions/precondition-active-${index}`, + prompt: "Existing remote work", + state: "IN_PROGRESS", + })); + const listJulesSessionsForCapacity = vi.fn() + .mockResolvedValueOnce(makeRemoteSessions(14)) + .mockResolvedValueOnce(makeRemoteSessions(14)); + const capacityAwareService = new SprintTaskDispatchService( + executionRepository, + projectManagementRepository, + taskService as any, + (service as any).guardrailService, + (service as any).providerConcurrencyService, + () => DEFAULT_DASHBOARD_SETTINGS, + (service as any).logger, + listJulesSessionsForCapacity, + ); + taskService.resolveTaskProvider.mockReturnValue("jules"); + taskService.startSprintTask.mockRejectedValue(new JulesApiRequestError( + "Jules API create session failed (HTTP 400 FAILED_PRECONDITION): Precondition check failed.", + 400, + "FAILED_PRECONDITION", + )); + + await expect(capacityAwareService.startTask({ + task: { + id: taskRecord.taskKey, + record_id: taskRecord.id, + title: taskRecord.title, + prompt: taskRecord.promptMarkdown, + depends_on: [], + is_independent: true, + status: "PENDING", + }, + projectId: project.id, + sprintId: sprint.id, + sprintRunId: sprintRun.id, + featureBranch: "feature/sprint-26", + repoPath: "/workspace/jules-precondition-capacity-project", + sprintNumber: 26, + })).rejects.toMatchObject({ provider: "jules", currentCount: 15 }); + + expect(listJulesSessionsForCapacity).toHaveBeenCalledTimes(2); + expect(taskService.startSprintTask).toHaveBeenCalledTimes(1); + expect(projectManagementRepository.getTask(taskRecord.id)).toMatchObject({ status: "pending" }); + expect(executionRepository.getLatestTaskRun(taskRecord.id, sprintRun.id)).toMatchObject({ + state: "PENDING", + finishedAt: null, + }); + }); + it("defers task start and records wait event when concurrency cap is reached", async () => { const { projectManagementRepository, executionRepository, taskService, service } = await createFixture(); const project = projectManagementRepository.createProject({ diff --git a/tests/backend/shared/logging/logger.test.ts b/tests/backend/shared/logging/logger.test.ts index 622ebe75cc..8f5868f83c 100644 --- a/tests/backend/shared/logging/logger.test.ts +++ b/tests/backend/shared/logging/logger.test.ts @@ -200,6 +200,20 @@ describe("createLogger", () => { expect(output).toContain("keep this"); }); + it("drops console records while stderr is over the pending-write bound", () => { + const stderrSpy = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const writableLengthSpy = vi.spyOn(process.stderr, "writableLength", "get").mockReturnValue(9 * 1024 * 1024); + try { + const logger = createLogger({ environment: "development", consoleLogLevel: "debug" }); + + logger.error("do not queue another record"); + + expect(stderrSpy).not.toHaveBeenCalled(); + } finally { + writableLengthSpy.mockRestore(); + } + }); + it("filters console and debug file output independently", async () => { const stderrSpy = vi.spyOn(process.stderr, "write").mockImplementation(() => true); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "code-ux-logger-")); diff --git a/tests/backend/shared/subprocess/command-runner.test.ts b/tests/backend/shared/subprocess/command-runner.test.ts index e1f59308a4..682bc90cea 100644 --- a/tests/backend/shared/subprocess/command-runner.test.ts +++ b/tests/backend/shared/subprocess/command-runner.test.ts @@ -5,6 +5,101 @@ import * as path from "path"; import { CommandRunner } from "../../../../src/shared/subprocess/command-runner.js"; import { beginRuntimeShutdown, resetRuntimeShutdownForTests } from "../../../../src/services/shutdown-state.js"; +const DOCKER_HELPER_POOL_MODULE = "../../../../src/infrastructure/providers/cli/docker-helper-pool.js"; + +async function createGitRepositoryFixture(prefix: string): Promise<{ tempDir: string; repoDir: string }> { + const tempDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), prefix)); + const repoDir = path.join(tempDir, "repo"); + await fsPromises.mkdir(path.join(repoDir, ".git"), { recursive: true }); + await fsPromises.writeFile(path.join(repoDir, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8"); + return { tempDir, repoDir }; +} + +async function loadCommandRunnerWithMockedGitPool() { + vi.resetModules(); + let capturedSpec: unknown; + const pool = { + ensure: vi.fn(async (_key: string) => "git-helper-1"), + reserve: vi.fn((_key: string) => vi.fn()), + withContainer: vi.fn(async (key: string, operation: (containerId: string) => Promise) => ( + operation(await pool.ensure(key)) + )), + touch: vi.fn((_key: string) => undefined), + invalidate: vi.fn((_key: string, _expectedId?: string) => true), + release: vi.fn(async (_key: string) => undefined), + shutdown: vi.fn(async () => undefined), + isContainerGone: vi.fn((result: { stdout?: string; stderr?: string }) => ( + `${result.stderr ?? ""} ${result.stdout ?? ""}`.toLowerCase().includes("no such container") + )), + }; + + vi.doMock(DOCKER_HELPER_POOL_MODULE, () => ({ + HELPER_LABEL: "code-ux.helper", + HELPER_OWNER_NAME_SUFFIX: "test-owner", + DockerHelperContainerPool: class MockDockerHelperContainerPool { + constructor(spec: unknown) { + capturedSpec = spec; + } + + ensure(key: string): Promise { + return pool.ensure(key); + } + + reserve(key: string): () => void { + return pool.reserve(key); + } + + withContainer(key: string, operation: (containerId: string) => Promise): Promise { + return pool.withContainer(key, operation) as Promise; + } + + touch(key: string): void { + pool.touch(key); + } + + invalidate(key: string, expectedId?: string): boolean { + return pool.invalidate(key, expectedId); + } + + release(key: string): Promise { + return pool.release(key); + } + + shutdown(): Promise { + return pool.shutdown(); + } + + isContainerGone(result: { stdout?: string; stderr?: string }): boolean { + return pool.isContainerGone(result); + } + }, + })); + + const commandRunnerModule = await import("../../../../src/shared/subprocess/command-runner.js"); + const isolatedRunner = new commandRunnerModule.CommandRunner(); + const spawnProcess = vi.fn(async () => ({ + ok: true, + code: 0, + stdout: "", + stderr: "", + })); + (isolatedRunner as unknown as { spawnProcess: typeof spawnProcess }).spawnProcess = spawnProcess; + + return { + commandRunnerModule, + isolatedRunner, + pool, + spawnProcess, + activate: (repoDir: string) => commandRunnerModule.acquireProjectGitHelperForSprint(repoDir), + getCapturedSpec: () => capturedSpec, + cleanup: async () => { + await commandRunnerModule.shutdownGitHelperPool(); + vi.doUnmock(DOCKER_HELPER_POOL_MODULE); + vi.resetModules(); + }, + }; +} + describe("CommandRunner", () => { const runner = new CommandRunner(); const node = process.execPath; @@ -582,6 +677,445 @@ describe("CommandRunner", () => { } }); + it("uses distinct git helper pool keys for separate repositories", async () => { + const first = await createGitRepositoryFixture("code-ux-git-pool-first-"); + const second = await createGitRepositoryFixture("code-ux-git-pool-second-"); + try { + const resolveContext = (CommandRunner as unknown as { + resolveGitPoolContextForPath: (cwd: string) => { poolKey: string; mountRoot: string } | null; + }).resolveGitPoolContextForPath; + const firstContext = resolveContext(first.repoDir); + const secondContext = resolveContext(second.repoDir); + + expect(firstContext?.mountRoot).toBe(first.repoDir); + expect(secondContext?.mountRoot).toBe(second.repoDir); + expect(firstContext?.poolKey).not.toBe(secondContext?.poolKey); + } finally { + await Promise.all([ + fsPromises.rm(first.tempDir, { recursive: true, force: true }), + fsPromises.rm(second.tempDir, { recursive: true, force: true }), + ]); + } + }); + + it("streams Git stdin through the warm helper and keeps auth environment command-scoped", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-stdin-"); + const stdinFile = path.join(fixture.tempDir, "paths"); + await fsPromises.writeFile(stdinFile, "first.txt\0second.txt\0", "utf8"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + try { + const firstResult = await mocked.isolatedRunner.run( + "git", + ["add", "--pathspec-from-file=-", "--pathspec-file-nul"], + { + cwd: fixture.repoDir, + stdinFile, + env: { CODE_UX_CONTAINERIZED_GIT: "1", GH_TOKEN: "project-one-token" }, + }, + ); + const secondResult = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1", GH_TOKEN: "project-two-token" }, + }); + + expect(firstResult.ok).toBe(true); + expect(secondResult.ok).toBe(true); + expect(mocked.pool.ensure).toHaveBeenCalledTimes(2); + expect(mocked.pool.ensure.mock.calls[0]?.[0]).toBe(mocked.pool.ensure.mock.calls[1]?.[0]); + expect(mocked.spawnProcess).toHaveBeenCalledTimes(2); + + const [firstCommand, firstOptions] = mocked.spawnProcess.mock.calls[0] ?? []; + expect(firstCommand).toMatchObject({ command: "docker", containerHostCwd: fixture.repoDir }); + expect(firstCommand?.args).toEqual(expect.arrayContaining([ + "exec", + "-i", + "--workdir", + "/workspace", + "-e", + "GH_TOKEN=project-one-token", + "git-helper-1", + "git", + "add", + "--pathspec-from-file=-", + ])); + expect(firstCommand?.args).not.toContain("GH_TOKEN=project-two-token"); + expect(firstOptions).toMatchObject({ cwd: fixture.repoDir, stdinFile }); + + const [secondCommand] = mocked.spawnProcess.mock.calls[1] ?? []; + expect(secondCommand?.args).toEqual(expect.arrayContaining([ + "exec", + "-e", + "GH_TOKEN=project-two-token", + "git-helper-1", + "git", + "status", + ])); + expect(secondCommand?.args).not.toContain("-i"); + expect(secondCommand?.args).not.toContain("GH_TOKEN=project-one-token"); + expect(mocked.getCapturedSpec()).toBeDefined(); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("falls back to a stdin-capable one-shot Git helper when warm helper startup fails", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-fallback-"); + const stdinFile = path.join(fixture.tempDir, "paths"); + await fsPromises.writeFile(stdinFile, "first.txt\0", "utf8"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + mocked.pool.ensure.mockRejectedValueOnce(new Error("Docker daemon unavailable")); + try { + const result = await mocked.isolatedRunner.run( + "git", + ["add", "--pathspec-from-file=-", "--pathspec-file-nul"], + { + cwd: fixture.repoDir, + stdinFile, + env: { CODE_UX_CONTAINERIZED_GIT: "1", GH_TOKEN: "fallback-token" }, + }, + ); + + expect(result.ok).toBe(true); + expect(mocked.spawnProcess).toHaveBeenCalledOnce(); + const [fallbackCommand, fallbackOptions] = mocked.spawnProcess.mock.calls[0] ?? []; + expect(fallbackCommand?.command).toBe("docker"); + expect(fallbackCommand?.args).toEqual(expect.arrayContaining([ + "run", + "--rm", + "-i", + "GH_TOKEN=fallback-token", + "--entrypoint", + "git", + "alpine/git", + ])); + expect(fallbackCommand?.args).not.toContain("exec"); + expect(fallbackOptions).toMatchObject({ stdinFile }); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("rejects an invalid Git stdin file before creating a warm helper", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-invalid-stdin-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + try { + await expect(mocked.isolatedRunner.run("git", ["hash-object", "--stdin"], { + cwd: fixture.repoDir, + stdinFile: path.join(fixture.tempDir, "missing-input"), + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + })).rejects.toThrow(/stdinFile is not a readable file/); + + expect(mocked.getCapturedSpec()).toBeUndefined(); + expect(mocked.pool.ensure).not.toHaveBeenCalled(); + expect(mocked.spawnProcess).not.toHaveBeenCalled(); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("keeps commands needing an external Git path mount on the one-shot helper", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-external-path-"); + const indexDir = path.join(fixture.tempDir, "indexes"); + await fsPromises.mkdir(indexDir); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + try { + const result = await mocked.isolatedRunner.run("git", ["read-tree", "HEAD"], { + cwd: fixture.repoDir, + env: { + CODE_UX_CONTAINERIZED_GIT: "1", + GIT_INDEX_FILE: path.join(indexDir, "temporary.index"), + }, + }); + + expect(result.ok).toBe(true); + expect(mocked.pool.ensure).not.toHaveBeenCalled(); + expect(mocked.spawnProcess).toHaveBeenCalledOnce(); + expect(mocked.spawnProcess.mock.calls[0]?.[0]?.args).toEqual(expect.arrayContaining([ + "run", + "--rm", + "--mount", + `type=bind,source=${indexDir},target=/mnt/code-ux/git-paths/0`, + "GIT_INDEX_FILE=/mnt/code-ux/git-paths/0/temporary.index", + ])); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("invalidates only a vanished helper generation and retries with its replacement", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-retry-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + mocked.pool.ensure + .mockResolvedValueOnce("git-helper-old") + .mockResolvedValueOnce("git-helper-new"); + mocked.spawnProcess + .mockResolvedValueOnce({ + ok: false, + code: 1, + stdout: "", + stderr: "Error: No such container: git-helper-old", + }) + .mockResolvedValueOnce({ ok: true, code: 0, stdout: "clean", stderr: "" }); + try { + const result = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + + const poolKey = mocked.pool.ensure.mock.calls[0]?.[0]; + expect(result).toMatchObject({ ok: true, stdout: "clean" }); + expect(mocked.pool.invalidate).toHaveBeenCalledWith(poolKey, "git-helper-old"); + expect(mocked.pool.ensure).toHaveBeenCalledTimes(2); + expect(mocked.spawnProcess.mock.calls[0]?.[0]?.args).toContain("git-helper-old"); + expect(mocked.spawnProcess.mock.calls[1]?.[0]?.args).toContain("git-helper-new"); + expect(mocked.spawnProcess.mock.calls.some(([command]) => command.args?.includes("run"))).toBe(false); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("uses one-shot fallback after two vanished warm-helper generations", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-double-gone-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + mocked.pool.ensure + .mockResolvedValueOnce("git-helper-old") + .mockResolvedValueOnce("git-helper-replacement"); + mocked.spawnProcess + .mockResolvedValueOnce({ ok: false, code: 1, stdout: "", stderr: "No such container" }) + .mockResolvedValueOnce({ ok: false, code: 1, stdout: "", stderr: "container is not running" }) + .mockResolvedValueOnce({ ok: true, code: 0, stdout: "fallback", stderr: "" }); + mocked.pool.isContainerGone.mockImplementation((result) => ( + /no such container|not running/i.test(`${result.stderr ?? ""} ${result.stdout ?? ""}`) + )); + try { + const result = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + + const poolKey = mocked.pool.ensure.mock.calls[0]?.[0]; + expect(result).toMatchObject({ ok: true, stdout: "fallback" }); + expect(mocked.pool.invalidate).toHaveBeenNthCalledWith(1, poolKey, "git-helper-old"); + expect(mocked.pool.invalidate).toHaveBeenNthCalledWith(2, poolKey, "git-helper-replacement"); + expect(mocked.spawnProcess).toHaveBeenCalledTimes(3); + expect(mocked.spawnProcess.mock.calls[2]?.[0]?.args).toEqual(expect.arrayContaining([ + "run", + "--rm", + "--entrypoint", + "git", + ])); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("does not fall back for ordinary Git failures from a live warm helper", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-git-failure-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + mocked.spawnProcess.mockResolvedValueOnce({ + ok: false, + code: 128, + stdout: "", + stderr: "fatal: invalid reference", + }); + try { + const result = await mocked.isolatedRunner.run("git", ["show", "missing-ref"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + + expect(result).toMatchObject({ ok: false, code: 128 }); + expect(mocked.pool.invalidate).not.toHaveBeenCalled(); + expect(mocked.pool.ensure).toHaveBeenCalledOnce(); + expect(mocked.spawnProcess).toHaveBeenCalledOnce(); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("does not replay a Git command when warm-helper execution throws unexpectedly", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-exec-error-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + mocked.spawnProcess.mockRejectedValueOnce(new Error("unexpected execution transport failure")); + try { + await expect(mocked.isolatedRunner.run("git", ["commit", "-m", "test"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + })).rejects.toThrow("unexpected execution transport failure"); + + expect(mocked.pool.ensure).toHaveBeenCalledOnce(); + expect(mocked.spawnProcess).toHaveBeenCalledOnce(); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("keeps one project helper for active sprint leases and bounds concurrent Git execs", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-active-sprint-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLeaseA = mocked.activate(fixture.repoDir); + const releaseLeaseB = mocked.activate(fixture.repoDir); + let activeExecs = 0; + let peakExecs = 0; + let releaseExecs!: () => void; + const execGate = new Promise((resolve) => { + releaseExecs = resolve; + }); + mocked.spawnProcess.mockImplementation(async (command) => { + if (command.args?.includes("exec")) { + activeExecs += 1; + peakExecs = Math.max(peakExecs, activeExecs); + await execGate; + activeExecs -= 1; + } + return { ok: true, code: 0, stdout: "", stderr: "" }; + }); + + try { + const commands = Array.from({ length: 8 }, (_, index) => mocked.isolatedRunner.run( + "git", + ["show", `ref-${index}`], + { cwd: fixture.repoDir, env: { CODE_UX_CONTAINERIZED_GIT: "1" } }, + )); + await vi.waitFor(() => expect(activeExecs).toBe(4)); + expect(mocked.pool.reserve).toHaveBeenCalledOnce(); + expect(mocked.getCapturedSpec()).toBeDefined(); + + releaseExecs(); + await Promise.all(commands); + expect(peakExecs).toBe(4); + expect(mocked.pool.ensure).toHaveBeenCalledTimes(8); + + await releaseLeaseA(); + expect(mocked.pool.release).not.toHaveBeenCalled(); + await releaseLeaseB(); + expect(mocked.pool.release).toHaveBeenCalledOnce(); + } finally { + releaseExecs(); + await releaseLeaseA(); + await releaseLeaseB(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("uses one-shot Git outside an active sprint and leaves no persistent helper", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-no-active-sprint-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + try { + const result = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + + expect(result.ok).toBe(true); + expect(mocked.getCapturedSpec()).toBeUndefined(); + expect(mocked.pool.ensure).not.toHaveBeenCalled(); + expect(mocked.spawnProcess).toHaveBeenCalledOnce(); + expect(mocked.spawnProcess.mock.calls[0]?.[0]?.args).toEqual(expect.arrayContaining([ + "run", + "--rm", + "--entrypoint", + "git", + ])); + } finally { + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("releases a repo-local worktree through its shared project helper key and drains the pool", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-release-"); + const worktreeDir = path.join(fixture.repoDir, ".worktrees", "session-1"); + const worktreeGitDir = path.join(fixture.repoDir, ".git", "worktrees", "session-1"); + await fsPromises.mkdir(worktreeDir, { recursive: true }); + await fsPromises.mkdir(worktreeGitDir, { recursive: true }); + await fsPromises.writeFile(path.join(worktreeGitDir, "HEAD"), "ref: refs/heads/session-1\n", "utf8"); + await fsPromises.writeFile(path.join(worktreeGitDir, "commondir"), "../..\n", "utf8"); + await fsPromises.writeFile(path.join(worktreeDir, ".git"), `gitdir: ${worktreeGitDir}\n`, "utf8"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + try { + await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + const projectPoolKey = mocked.pool.ensure.mock.calls[0]?.[0]; + + await mocked.commandRunnerModule.releaseGitHelperForCwd(worktreeDir); + await mocked.commandRunnerModule.shutdownGitHelperPool(); + + expect(mocked.pool.release).toHaveBeenCalledWith(projectPoolKey); + expect(mocked.pool.shutdown).toHaveBeenCalledOnce(); + } finally { + await releaseLease(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("does not recreate the warm Git helper pool after runtime shutdown begins", async () => { + const fixture = await createGitRepositoryFixture("code-ux-git-pool-shutdown-"); + const mocked = await loadCommandRunnerWithMockedGitPool(); + const releaseLease = mocked.activate(fixture.repoDir); + const shutdownState = await import("../../../../src/services/shutdown-state.js"); + try { + const warmResult = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + shutdownState.beginRuntimeShutdown(); + await mocked.commandRunnerModule.shutdownGitHelperPool(); + const lateResult = await mocked.isolatedRunner.run("git", ["status", "--porcelain"], { + cwd: fixture.repoDir, + env: { CODE_UX_CONTAINERIZED_GIT: "1" }, + }); + + expect(warmResult.ok).toBe(true); + expect(lateResult.ok).toBe(true); + expect(mocked.pool.ensure).toHaveBeenCalledOnce(); + expect(mocked.pool.shutdown).toHaveBeenCalledOnce(); + expect(mocked.spawnProcess).toHaveBeenCalledTimes(2); + expect(mocked.spawnProcess.mock.calls[0]?.[0]?.args).toContain("exec"); + expect(mocked.spawnProcess.mock.calls[1]?.[0]?.args).toEqual(expect.arrayContaining([ + "run", + "--rm", + "--entrypoint", + "git", + ])); + expect(mocked.spawnProcess.mock.calls[1]?.[0]?.args).not.toContain("exec"); + } finally { + await releaseLease(); + shutdownState.resetRuntimeShutdownForTests(); + await mocked.cleanup(); + await fsPromises.rm(fixture.tempDir, { recursive: true, force: true }); + } + }); + it("mounts the project root for one-shot git commands started from repo-local worktrees", async () => { const tempDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "code-ux-git-one-shot-root-")); const repoDir = path.join(tempDir, "repo"); diff --git a/tests/backend/shared/subprocess/command-spawner-client.test.ts b/tests/backend/shared/subprocess/command-spawner-client.test.ts index 56b4106c93..0ddadbaa2c 100644 --- a/tests/backend/shared/subprocess/command-spawner-client.test.ts +++ b/tests/backend/shared/subprocess/command-spawner-client.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; import { computeEnvDiff } from "../../../../src/shared/subprocess/command-spawner-client.js"; +import { + MAX_SPAWNER_STREAM_LINE_CHARS, + boundSpawnerStreamLine, +} from "../../../../src/shared/subprocess/command-spawner-protocol.js"; describe("computeEnvDiff", () => { it("returns useBaseEnv when the effective env matches the base", () => { @@ -39,3 +43,15 @@ describe("computeEnvDiff", () => { expect(computeEnvDiff(base, effective)).toEqual({ useBaseEnv: true }); }); }); + +describe("boundSpawnerStreamLine", () => { + it("bounds oversized live IPC lines while retaining diagnostic head and tail context", () => { + const line = `${"head".repeat(20_000)}${"tail".repeat(20_000)}`; + const bounded = boundSpawnerStreamLine(line); + + expect(bounded).toHaveLength(MAX_SPAWNER_STREAM_LINE_CHARS); + expect(bounded.startsWith("head")).toBe(true); + expect(bounded.endsWith("tail")).toBe(true); + expect(bounded).toContain("stream line truncated"); + }); +}); diff --git a/tests/backend/sprint/session-sync-step.test.ts b/tests/backend/sprint/session-sync-step.test.ts index 833a0df90a..f0835977ed 100644 --- a/tests/backend/sprint/session-sync-step.test.ts +++ b/tests/backend/sprint/session-sync-step.test.ts @@ -2,8 +2,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import * as fs from "fs/promises"; import * as os from "os"; import * as path from "path"; -import { runSessionSyncStep } from "../../../src/sprint/steps/session-sync-step.js"; -import type { Subtask } from "../../../src/contracts/app-types.js"; +import { + buildProviderActivityEventPayload, + runSessionSyncStep, +} from "../../../src/sprint/steps/session-sync-step.js"; +import type { JulesActivity, Subtask } from "../../../src/contracts/app-types.js"; import { AppDbStorage } from "../../../src/repositories/app-db-storage.js"; import { ProjectManagementRepository } from "../../../src/repositories/project-management-repository.js"; import { ExecutionRepository } from "../../../src/repositories/execution-repository.js"; @@ -16,6 +19,30 @@ afterEach(async () => { }); describe("runSessionSyncStep", () => { + it("bounds oversized provider activity payloads before durable event persistence", () => { + const oversized = `${"head".repeat(10_000)}${"tail".repeat(10_000)}`; + const activity: JulesActivity = { + id: oversized, + name: oversized, + createTime: "2026-07-15T00:00:00.000Z", + description: oversized, + agentMessaged: { agentMessage: oversized }, + progressUpdated: { title: "Progress", description: oversized }, + planApproved: { planId: oversized }, + sessionCompleted: { output: oversized }, + }; + + const payload = buildProviderActivityEventPayload(activity, "session-1", "sessions/session-1", "codex"); + expect((payload.activityId as string).length).toBeLessThanOrEqual(2 * 1024); + expect((payload.activityName as string).length).toBeLessThanOrEqual(2 * 1024); + expect((payload.description as string).length).toBeLessThanOrEqual(16 * 1024); + expect((payload.agentMessaged as { agentMessage: string }).agentMessage.length).toBeLessThanOrEqual(16 * 1024); + expect((payload.progressUpdated as { description: string }).description.length).toBeLessThanOrEqual(16 * 1024); + expect((payload.planApproved as { planId: string }).planId.length).toBeLessThanOrEqual(2 * 1024); + expect(payload.sessionCompleted).toEqual(expect.objectContaining({ truncated: true })); + expect(JSON.stringify(payload).length).toBeLessThan(80 * 1024); + }); + it("skips session polling for terminal local CLI tasks that already have merge evidence", async () => { const listSessions = vi.fn().mockResolvedValue({ sessions: [] }); const subtasks: Subtask[] = [ diff --git a/tests/backend/sprint/steps/start-ready-tasks-step.test.ts b/tests/backend/sprint/steps/start-ready-tasks-step.test.ts index 6e196baffa..e008d39f82 100644 --- a/tests/backend/sprint/steps/start-ready-tasks-step.test.ts +++ b/tests/backend/sprint/steps/start-ready-tasks-step.test.ts @@ -189,7 +189,51 @@ describe("start-ready-tasks-step", () => { })); }); + it("starts only the effective adaptive capacity even when the configured limit is higher", async () => { + const subtasks: Subtask[] = Array.from({ length: 5 }, (_, index) => ({ + id: String(index + 1), + title: `t${index + 1}`, + prompt: "p", + depends_on: [], + is_independent: true, + status: "PENDING" as const, + })); + const startTask = vi.fn().mockImplementation(async (task: Subtask) => ({ + id: `session-${task.id}`, + provider: "codex", + })); + const getAvailableProviderCapacity = vi.fn().mockResolvedValue(2); + const info = vi.fn(); + + const result = await runStartReadyTasksStep(subtasks, { + action: "orchestrate", + getConsecutiveFailures: () => 0, + setConsecutiveFailures: vi.fn(), + maxFailures: 3, + startTask, + resolveSessionName: (session) => session.id, + extractSessionId: (session) => session.id, + logger: { info, error: vi.fn() } as any, + getProviderForTask: () => "codex", + getProviderSettings: () => ({ maxConcurrentTasks: 16 }), + getRunningCounts: () => ({ codex: 0 }), + getAvailableProviderCapacity, + }); + + expect(getAvailableProviderCapacity).toHaveBeenCalledOnce(); + expect(startTask).toHaveBeenCalledTimes(2); + expect(result.subtasks.map((task) => task.status)).toEqual([ + "RUNNING", "RUNNING", "PENDING", "PENDING", "PENDING", + ]); + expect(info).toHaveBeenCalledWith( + "Provider concurrency cap deferred ready tasks", + expect.objectContaining({ blockedTaskCount: 3, source: "pre_dispatch" }), + ); + }); + it("coalesces unchanged provider-cap diagnostics across rapid orchestration cycles", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-15T00:00:00.000Z")); const subtasks: Subtask[] = [ { id: "1", title: "t1", prompt: "p1", depends_on: [], is_independent: false, status: "PENDING" }, { id: "2", title: "t2", prompt: "p2", depends_on: [], is_independent: false, status: "PENDING" }, @@ -219,6 +263,42 @@ describe("start-ready-tasks-step", () => { { id: "3", title: "t3", prompt: "p3", depends_on: [], is_independent: false, status: "PENDING" }, ], options); + expect(infoSpy).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(10_000); + await runStartReadyTasksStep(subtasks, options); + expect(infoSpy).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("coalesces provider-cap diagnostics when each cycle creates a new child logger", async () => { + const providerCapLogState = new Map(); + const infoSpy = vi.fn(); + const subtasks: Subtask[] = [ + { id: "1", title: "t1", prompt: "p1", depends_on: [], is_independent: false, status: "PENDING" }, + ]; + const runCycle = async (): Promise => { + await runStartReadyTasksStep(subtasks, { + action: "orchestrate", + getConsecutiveFailures: () => 0, + setConsecutiveFailures: vi.fn(), + maxFailures: 3, + startTask: vi.fn(), + resolveSessionName: (session: any) => session.id, + extractSessionId: (session: any) => session.id, + logger: { info: infoSpy, error: vi.fn() } as any, + getProviderForTask: () => "codex", + getProviderSettings: () => ({ maxConcurrentTasks: 2 }), + getRunningCounts: () => ({ codex: 2 }), + providerCapLogState, + providerCapLogScope: "project:sprint:run", + }); + }; + + await runCycle(); + await runCycle(); + + expect(infoSpy).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/dashboard/v2/sprint-menu-positioning.test.ts b/tests/dashboard/v2/sprint-menu-positioning.test.ts index a8bd9b369e..618db91871 100644 --- a/tests/dashboard/v2/sprint-menu-positioning.test.ts +++ b/tests/dashboard/v2/sprint-menu-positioning.test.ts @@ -26,6 +26,18 @@ describe("computeSprintActionMenuPosition", () => { expect(result.left).toBe(280); }); + it("uses the larger region when an early menu measurement fits on both sides", () => { + const result = computeSprintActionMenuPosition( + { top: 432, left: 1054, right: 1130, bottom: 478, width: 76, height: 46 }, + { width: 1216, height: 720 }, + { width: 184, height: 180 }, + ); + + expect(result.placement).toBe("top"); + expect(result.top).toBe(244); + expect(result.left).toBe(946); + }); + it("clamps to viewport right edge padding for large menus", () => { const result = computeSprintActionMenuPosition( { top: 120, left: 890, right: 920, bottom: 150, width: 30, height: 30 }, diff --git a/tests/dashboard/v2/sprints-page-integration.test.tsx b/tests/dashboard/v2/sprints-page-integration.test.tsx index 1fa155344b..ae8412e1d8 100644 --- a/tests/dashboard/v2/sprints-page-integration.test.tsx +++ b/tests/dashboard/v2/sprints-page-integration.test.tsx @@ -300,7 +300,7 @@ describe("SprintsPage Integration Regressions", () => { const moreTrigger = screen.getAllByRole("button", { name: /Open actions menu for sprint/i })[0]; await userEvent.click(moreTrigger); - const editBtn = await screen.findByRole('button', { name: /Edit/i }); + const editBtn = await screen.findByRole('menuitem', { name: /Edit/i }); expect(editBtn.className).toContain("focus-visible:ring-offset-2"); }); diff --git a/tests/dashboard/v2/ui-components.test.tsx b/tests/dashboard/v2/ui-components.test.tsx index 31229ed60c..2552cf0767 100644 --- a/tests/dashboard/v2/ui-components.test.tsx +++ b/tests/dashboard/v2/ui-components.test.tsx @@ -578,6 +578,65 @@ describe("UI Components Coverage", () => { expect(onOpenChange).not.toHaveBeenCalled(); }); + it("passes DropdownMenu intrinsic height to custom positioning near the viewport edge", async () => { + const rect = (values: Partial): DOMRect => ({ + x: values.left ?? 0, + y: values.top ?? 0, + top: 0, + right: 0, + bottom: 0, + left: 0, + width: 0, + height: 0, + toJSON: () => ({}), + ...values, + }); + const originalInnerHeight = window.innerHeight; + const boundsSpy = vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockImplementation(function () { + if (this.getAttribute("role") === "menu") { + return rect({ top: 0, right: 240, bottom: 40, left: 0, width: 240, height: 40 }); + } + if (this.getAttribute("aria-haspopup") === "menu") { + return rect({ top: 700, right: 140, bottom: 732, left: 100, width: 40, height: 32 }); + } + return rect({}); + }); + const scrollHeightSpy = vi.spyOn(HTMLElement.prototype, "scrollHeight", "get").mockImplementation(function () { + return this.getAttribute("role") === "menu" ? 360 : 0; + }); + const computePosition = vi.fn(({ triggerRect, menuRect }: { triggerRect: DOMRect; menuRect: DOMRect }) => ({ + top: triggerRect.top - menuRect.height - 8, + left: triggerRect.left, + })); + Object.defineProperty(window, "innerHeight", { configurable: true, value: 768 }); + + try { + render( + {}} + menuAriaLabel="Long actions" + computePosition={computePosition} + content={Delete} + > + + + ); + + const menu = await screen.findByRole("menu", { name: "Long actions" }); + await waitFor(() => { + expect(menu).toHaveStyle({ top: "332px", left: "100px", maxHeight: "428px" }); + }); + expect(computePosition).toHaveBeenLastCalledWith(expect.objectContaining({ + menuRect: expect.objectContaining({ height: 360 }), + })); + } finally { + boundsSpy.mockRestore(); + scrollHeightSpy.mockRestore(); + Object.defineProperty(window, "innerHeight", { configurable: true, value: originalInnerHeight }); + } + }); + it("uses instant Dialog transitions when reduced motion is enabled", () => { vi.mocked(useReducedMotion).mockReturnValue(true); diff --git a/tests/e2e/agents/agent-avatar-scene.spec.ts b/tests/e2e/agents/agent-avatar-scene.spec.ts index 67c4a9414b..7893cc5f15 100644 --- a/tests/e2e/agents/agent-avatar-scene.spec.ts +++ b/tests/e2e/agents/agent-avatar-scene.spec.ts @@ -17,18 +17,21 @@ test.describe('AgentAvatarScene E2E Tests', () => { agentName = agent.name; }); - test('should render the WebGL canvas when WebGL is supported', async ({ page }) => { + test('should render the WebGL canvas or the accessible fallback when the context pool is unavailable', async ({ page }) => { await page.goto('/agents'); await page.getByRole('button', { name: new RegExp(escapeRegExp(agentName)) }).click(); await expect(page.locator('h2').filter({ hasText: agentName })).toBeVisible(); - // Assert that the 3D scene container is rendered and contains a canvas const avatarScene = page.locator('[data-testid="agent-avatar-scene"]'); - await expect(avatarScene).toBeVisible(); - - const canvas = avatarScene.locator('canvas'); - await expect(canvas).toBeVisible(); + const fallback = page.locator('[data-testid="agent-avatar-fallback"]'); + await expect(avatarScene.or(fallback).first()).toBeVisible(); + if (await avatarScene.isVisible()) { + await expect(avatarScene.locator('canvas')).toBeVisible(); + } else { + await expect(fallback).toHaveRole('img'); + await expect(fallback).toHaveAccessibleName(/Agent avatar preview/i); + } }); test('should render fallback UI (SVG) when WebGL is unsupported or fails', async ({ page }) => { @@ -59,8 +62,10 @@ test.describe('AgentAvatarScene E2E Tests', () => { await page.goto('/chat?stageTool=wrench'); const avatarScene = page.locator('[data-testid="agent-avatar-scene"]'); - await expect(avatarScene).toBeVisible(); - await expect(avatarScene).toHaveAttribute('data-tool', 'wrench'); + const initialFallback = page.locator('[data-testid="agent-avatar-fallback"]'); + const initialAvatar = avatarScene.or(initialFallback).first(); + await expect(initialAvatar).toBeVisible(); + await expect(initialAvatar).toHaveAttribute('data-tool', 'wrench'); await page.emulateMedia({ reducedMotion: 'reduce' }); const fallback = page.locator('[data-testid="agent-avatar-fallback"]'); @@ -70,7 +75,9 @@ test.describe('AgentAvatarScene E2E Tests', () => { await page.emulateMedia({ reducedMotion: 'no-preference' }); await page.goto('/chat?stageTool=torch'); - await expect(page.locator('[data-testid="agent-avatar-scene"]')).toHaveAttribute('data-tool', 'torch'); + const torchScene = page.locator('[data-testid="agent-avatar-scene"]'); + const torchFallback = page.locator('[data-testid="agent-avatar-fallback"]'); + await expect(torchScene.or(torchFallback).first()).toHaveAttribute('data-tool', 'torch'); await page.goto('/agents'); await expect(page.locator('[data-testid="agent-avatar-scene"][data-tool]')).toHaveCount(0);