diff --git a/docs/conductor-build-plan.md b/docs/conductor-build-plan.md new file mode 100644 index 0000000..13a17b0 --- /dev/null +++ b/docs/conductor-build-plan.md @@ -0,0 +1,417 @@ +# Conductor — Phased Build Plan + +> Companion to [conductor-design.md](./conductor-design.md) (architecture + +> decisions) and [conductor-session-resolution.md](./conductor-session-resolution.md) +> (the retrieval deep-dive). Those two docs ARE the spec; this is the sequencing. + +## Branch strategy + +- **One feature branch off `main`: `feat/conductor`.** `main` is ruleset-protected + (PR + CI required), so every phase lands as its own reviewable PR onto + `feat/conductor` (or PRs straight to `main` if you prefer trunk-ish); the design + docs are the first commit. +- Phases are **vertical slices** — each ends in a working, shippable daemon. No + phase leaves the tree broken. Prefer merging a phase before starting the next. +- **P1 (retrieval) and P2 (identity) are independent subsystems** (memory engine vs + ZeroID) — they can be built in parallel (two people / two worktrees) and only + meet at P3. + +## Sequencing principles (codeoid-specific) + +1. **Measure before you tune.** The retrieval work is only trustworthy against a + labeled eval set — build it first (P0). +2. **Contracts before consumers.** Card/fact schemas + the fleet protocol land + before the conductor LLM or any client consumes them. +3. **Read before write.** Fleet *read* tools (list/find/summary) ship and bake + before any *act* tool (send/spawn) — the blast-radius rule. +4. **The daemon owns state; clients are pure renderers.** Every capability is a + daemon/protocol feature first; web, Telegram, and mobile are thin views over it. + +--- + +## Phase overview + +| # | Phase | Ships | Depends on | +|---|---|---|---| +| **P0** | Branch + eval harness + schemas | precision@1 baseline; card/fact schema; labeled set | — | +| **P1** | Session-resolution retrieval upgrade *(linchpin)* | fuzzy ref → right session, cross-workspace, sub-2s | P0 | +| **P2** | Conductor identity foundation | durable owner-delegated conductor identity | P0 | +| **P3** | Conductor session + read-only fleet tools | conductor can list/find/summarize the fleet | P1, P2 | +| **P4** | Dispatch + routing (durable queue + roles + hardening) | confirm → act; **restart-proof** workers; child digests | P3 | +| **P4.5** | Routines (scheduled + triggered autonomy) | unattended cron/webhook fleet tasks; `[SILENT]` monitors | P4 | +| **P5** | Front doors: web + Telegram | conductor chat + separate switchable session list | P4 | +| **P6** | Act on behalf (email/web/calendar) | approval-gated egress via delegated children | P4 | +| **P7** | Mobile app contract + conductor screen | mobile renders conductor + session switcher | P5 | +| **P8** | Governance *(later)* | Shield egress, Cedar policy, budgets, loop cap | P4/P6 | + +--- + +## P0 — Branch, eval harness, schemas + +**Goal:** a fast deterministic feedback loop for the linchpin, plus the data +contracts everything else consumes. + +**Slices** +- Create `feat/conductor`; commit the three design docs. +- **Labeled eval set**: ~40–60 `(NL reference → correct session_id)` pairs mined + from your own codeoid history across repos (mix identifier-heavy + fuzzy). Store + as a fixture. +- **Eval harness**: runs a resolver over the fixture, reports **precision@1 / + MRR / recall@k** + p50/p95 latency. First target: current `searchSessions()`. +- **Schemas** (types + SQLite DDL, no behavior yet): `session_card` + `{session_id, workspace, repo, branch, task, state, last_action, open_threads, + entities, updated_at}`; `fact(subject, predicate, object, valid_at, invalid_at, + created_at, expired_at, embedding)`. + +**Exit:** `bun test` runs the eval; you have a **baseline precision@1 number** to +beat, and the card/fact tables migrate cleanly (verify against a real +`bun:sqlite`, not DDL strings). + +**Baseline (MEASURED, on current-main base)** — against the real Hetzner corpus +(16 sessions / 11 workspaces / 11,938 episodes), 37 genericized fuzzy references. +The naive cross-workspace baseline scores **~22% P@1** — a *ranking/fusion* problem, +not recall (R@5 is far higher, so the right session is in the top few, just not #1; +a small workspace's batch-relative BM25 scores dominate the naive merge). Full +write-up, per-stage numbers, and repro: +[../src/daemon/eval/BASELINE.md](../src/daemon/eval/BASELINE.md). **That ~22% is the +number P1 must beat** — P1 slices 1–2 (global fusion + cross-encoder rerank) take it +to **~73%** on pure-fuzzy references at <100 ms. + +--- + +## P1 — Session-resolution retrieval upgrade (the linchpin) + +**Goal:** resolve a fuzzy reference to the right session, across all workspaces, +sub-2s — the capability the whole product hinges on. Pure memory-engine work; +benefits every client immediately, no conductor/identity risk. + +**Slices (each independently testable against the P0 harness)** +- **1a — Embedder → BGE-M3.** New `Embedder` impl (ONNX/int8) behind the existing + interface; persist **dense + learned-sparse** vectors (BGE-M3 emits both). Keep + `bge-small-en` as the degraded fallback. +- **1b — Identifier-aware lexical.** Rebuild `episodes_fts` (+ new `cards_fts`) + with a code/trigram analyzer + boosted id columns (ticket/branch/path). Add + Stage-0 regex query-typing, Stage-3 exact-id override, and a deterministic + entity boost. +- **1c — Session cards + facts.** Generate a card per session via + **LFM2-350M-Extract** (grammar-constrained JSON) on `IndexScheduler` checkpoints; + keep canonical with mem0-style ADD/UPDATE/invalidate reconciliation; record state + changes as **bi-temporal facts** (invalidate-don't-delete). +- **1d — Cross-workspace mode.** `ALL` scope on `recall` / `loadVectorMatrix` / + `ftsSearch` (today all `workspaceId`-bound). +- **1e — Cross-encoder rerank.** `bge-reranker-v2-m3` (ONNX-int8) over top-k cards. +- **1f — Fusion.** RRF (k=60) now; swap to TMM-normalized convex + α-by-query-type + once the P0 labels justify it. Add recency/reinforcement/pinned priors. + +**Exit:** precision@1 on the P0 set **beats baseline by the target margin**; p95 +resolution **< 2s** on the Hetzner box; identifier queries (`studio#870`) and fuzzy +queries (`the version-scoping bug`) both land. This is the go/no-go gate for the +whole feature. + +**Risk:** BGE-M3 CPU latency — mitigate with ONNX-int8 + offline batch embedding +(only the query embeds at request time). Verify tok/s on the actual box. + +--- + +## P2 — Conductor identity foundation + +**Goal:** an owner-delegated, durable, revocable conductor identity (design R1+R2). +Independent of P1 — build in parallel. + +**Slices** +- Add a **conductor scope profile** granting `session:*` to an agent identity + (extend `AGENT_TOOL_SCOPES`); register with a policy `max_delegation_depth ≥ 3`. +- **Delegate `session:*` from the human owner** to the conductor (not the owner's + own token); verify the `human → conductor → child → sub-agent` chain mints at + depth 3 against `zeroid` (cae_test confirms depth-3 works). +- **Durable identity**: persist the conductor's `identityId` + credential to the + store; on `resumeSessions()`, reload rather than re-register (survives restart → + one stable WIMSE URI). Children stay disposable. +- Verify **cascading revocation**: deactivating the conductor kills its subtree. + +**Exit:** conductor identity persists across a daemon restart (same URI), holds +`session:*` via owner delegation, every fleet action audits under its URI, and one +`agents.deactivate` kills the whole subtree. + +--- + +## P3 — Conductor session + read-only fleet tools + +**Goal:** a single global `role:"conductor"` session that can *see* the fleet. +Read-only — no act/spawn risk. + +**Slices** +- `role: "conductor"` on `Session`; conductor is discoverable + attachable like any + session, and self-persists (durable identity from P2). +- **`codeoid_fleet` in-process MCP server** (mirror `buildMemoryMcpServer` at + `session.ts:810`), injected only for the conductor: `fleet_list`, `fleet_find` + (calls the P1 pipeline), `fleet_summary` (compressed, not raw scrollback), + `fleet_recall`, `machine_map` (workspaces + git/running state). +- *(Optional, later)* A protocol-level `fleet.find`/`fleet.list` message would give + clients an **instant fleet-search box** that skips an LLM turn — but per the mobile + plan (§8) the conductor needs **zero new wire types**; clients light it up by + attaching to the conductor session. Defer unless a client wants LLM-free search. + +**Exit:** via CLI (`codeoid attach conductor`), "which session was the authz fix?" +resolves correctly across workspaces; conductor holds only an index, never raw +child transcripts. + +--- + +## P4 — Dispatch + routing (send-class) + +**Goal:** direct existing sessions and spawn disposable workers, safely. + +**Slices** +- `fleet_send` (fire-and-forget to an existing session), `fleet_interrupt`, + `fleet_spawn` (disposable child with a delegated identity from P2). Dispatch + carries a **`shape`: ship vs scout** (deliver-a-change vs investigate-and-report). +- **Durable work-queue backbone** *(hermes Kanban)*: back dispatch with a + SQLite task board + a dispatcher loop that atomically claims, reclaims stale + claims, and **auto-blocks a task after N consecutive failures** (anti-spin, + complements the stuck-loop guard below). Restart-proof — a spawned worker survives + a daemon restart because its claim + state live in the DB, not the turn. +- **Delegate role model** *(hermes leaf/orchestrator)*: a spawned child is a + **`leaf`** (focused worker — no `fleet_spawn`/`send`/act-on-behalf) unless granted + **`orchestrator`** (may spawn, bounded `max_spawn_depth`/`max_concurrent`). + Enforced via **ZeroID scopes**, not a config flag — this *is* R1 delegation-depth + + read-only-by-construction, made cryptographic. +- **Routing safety (R3):** send-class to an existing user-owned session first + proposes with **repo + branch + content shown**, acts only on confirm (reuse the + `approvalId` correlation flow); reads stay silent. Per-workspace autonomy mode + (`no-mistakes`/`direct-PR`/`local-only` + `+yolo`) sets how much confirm is needed. +- **Zero-token event-driven supervision** *(firstmate)*: the daemon absorbs benign + events and wakes a conductor turn only on *actionable* ones; heartbeat backstop + with exponential backoff; a durable event queue for crash recovery. +- **Session-lifecycle hardening** *(hermes)*: `resume_pending` (soft — continue the + transcript) vs `suspended` (hard wipe); **stuck-loop escalation** (auto-suspend a + worker active across 3+ restarts); a `.clean_shutdown` marker; a **burst-collapse + message queue** (single next-up slot + FIFO overflow) so message bursts never + process out of order. +- **Event-driven digests:** child completion emits an event → conductor turn + receives a *compressed* result (never raw transcript) — the never-OOC guarantee. + +**Exit:** "continue the authz `latest_only` fix in that session" → resolves → +confirms (right repo/branch) → sends; a spawned child's result returns as a digest; +a worker survives a daemon restart and resumes; conductor context stays +O(active threads). + +--- + +## P4.5 — Routines (scheduled + triggered autonomy) + +**Goal:** turn the conductor from a fleet *supervisor* into a personal *assistant* — +it runs tasks unattended, on a schedule or an external trigger. + +**Slices** *(hermes cron / webhooks)* +- **Scheduled jobs:** cron expressions + human intervals ("every 2h", "0 2 * * *", + one-shot ISO timestamps). Per-job `skills` / `model` / `workdir` / delivery-target. +- **Triggered jobs:** webhook subscriptions (GitHub events, generic API POST with + HMAC auth) that dispatch a fleet task from the payload. +- **Script-injection + `[SILENT]` pattern:** a pre-run script does the mechanical + work (fetch/diff/compute), its stdout becomes the prompt context, and the job + emits nothing unless something changed — zero-spam, near-zero-token monitors. + (`no_agent=true` makes the script the whole job.) +- **Cron hardening:** hard per-run interrupt (a runaway loop can't monopolize the + scheduler), a tick file-lock (no duplicate ticks across processes), catchup/grace + windows, and routine output lands in its **own** session frame — never corrupting + the conductor's main-conversation role alternation. +- Delivery reuses the P5 front doors; a routine can also deliver to a file or an + existing session with no front door at all. + +**Exit:** "every night, triage the backlog and open a draft PR" runs unattended and +delivers a digest; a monitor stays silent until it fires; a GitHub PR event triggers +a scoped review — all under the conductor's identity, audited, and cost-metered. + +**Depends on:** P4 (a routine fires dispatch). Delivery breadth grows with P5. + +--- + +## P5 — Front doors: web UI + Telegram + +**Goal:** talk to the conductor from web + Telegram; browse/switch any session +separately. + +**Slices** +- **Web (SolidJS):** a **conductor pane** (attaches to the conductor session) plus a + **session list** view. NL session search is served by the conductor itself; an + optional LLM-free search box would use the deferred `fleet.find` message (P3). +- **Telegram:** route DMs to the conductor session; `/sessions` lists + lets you + switch/attach to any session (reuse the embedded `SessionManager` access). +- Both are thin `Frontend` plugins over the same daemon — no new state owner. + +**Exit:** from web and Telegram you can (a) converse with the conductor and (b) +list + switch into any individual session; both stay in sync (daemon-owned state). + +--- + +## P6 — Act on behalf (email / web / calendar) + +**Goal:** the "master of my machine" surface, approval-gated (design R4). + +**Slices** +- Integrations as **narrowly-scoped delegated children** (e.g. a Gmail MCP child + holding only `email.send`), never on the conductor directly. +- **Owner approval gate** showing recipient + subject + body preview (informed + approval, not theater). +- Web lookup delegated to children (they already have `WebSearch`/`WebFetch`); + conductor gets digests back. + +**Exit:** "email X the summary" → conductor drafts → approval with full preview → +send; research tasks delegated and returned as digests. **No Shield yet** (P8). + +--- + +## P7 — Mobile app: conductor screen + session switcher + +**Goal:** a codeoid mobile client — because clients are pure renderers, this is +mostly a new front end over the *existing* protocol. + +**Slices** +- **Conductor screen** = attach to the conductor session (same protocol as web). +- **Session list** = `session.list` + the P3 protocol `fleet_find` search box, + rendered as a separate, switchable list; tapping one attaches to it. +- Confirm-before-act (R3) and approval prompts (R4) render as native mobile + confirmations over the `approvalId` flow. +- Contract check: confirmed by the mobile plan (§8) — the conductor needs **no new + wire types**; the app attaches to the conductor session like any other. The one + *optional* addition is the deferred protocol-level `fleet.find` search box (P3). + +**Exit:** mobile app shows the conductor chat + a separate switchable session list; +switching sessions and approving actions work identically to web. + +--- + +## P8 — Governance (later) + +**Goal:** the deferred safety layer (design R4/R5 "later"). + +**Slices:** Shield on egress (fail-open to approval if down); Cedar policy per +identity / `delegation_depth`; per-conductor token budget + alert; cheap +per-instruction bounce cap (loop guard). + +**Exit:** injected child cannot exfiltrate past Shield; runaway loop is capped; +spend has a ceiling. Great Highflame dogfood story. + +--- + +## Informed by firstmate (prior art) + +See [conductor-prior-art-firstmate.md](./conductor-prior-art-firstmate.md) — a +shipped conductor built as codeoid's architectural inverse. It validates our +daemon + identity + structured-memory foundation and yields refinements folded +into the phases: + +- **P3/P4 — conductor is read-only over targets *by construction*.** The + `codeoid_fleet` surface is read + dispatch only; no file/git/shell-write tool on + target repos. All mutation flows through crewmates behind approval — and we + *enforce* it by denying write scopes to the conductor identity (firstmate can + only ask via prompt). Turns R3/R4 into an architectural invariant. +- **P4 — supervision is zero-token + event-driven.** Conductor LLM turns fire only + on *actionable* daemon events; benign ones are absorbed with no turn; heartbeat + backstop with exponential backoff; actionable events hit a durable queue for + crash recovery. Daemon push beats firstmate's bash pane-scraping. +- **P4 — dispatch carries a `shape`: ship vs scout.** ship → PR/merge → teardown; + scout → report, never pushes, scratch worktree. +- **P4 — per-workspace autonomy modes** (`no-mistakes`/`direct-PR`/`local-only` + + `+yolo`) replace blanket confirm, and map directly onto the Cedar layer (P8). +- **P4 — sentinel/out-of-band marker** so daemon-injected event digests are never + confused with real user messages. +- **P5 — `/afk` batched-digest away-mode + `/stow` knowledge sweep** as conductor UX. +- **P5/meta — harness dispatch profiles** (NL rules → per-task harness/model/effort) + feed the meta-harness direction. +- **Topology (future) — secondmates:** domain sub-conductors via the *same* + delegation-depth identity chain; keeps the single global conductor as v1 default. + +## Informed by hermes (prior art) + +See [conductor-prior-art-hermes.md](./conductor-prior-art-hermes.md) — Nous's +hermes-agent, the most complete personal-assistant prior art. It occupies the +personal-assistant niche firstmate doesn't, and drove two changes above: + +- **P4 upgraded** — durable Kanban-style work-queue (atomic claim, stale reclaim, + failure-limit auto-block → restart-proof workers) + the `leaf`/`orchestrator` + delegate role model (enforced via ZeroID scopes, not config flags) + + session-lifecycle hardening (resume_pending / stuck-loop escalation / + clean-shutdown / burst-collapse message queue). +- **New P4.5 — Routines** — scheduled + webhook/event-triggered autonomy with the + script-injection `[SILENT]` monitor pattern and cron hardening. This is the piece + that makes the conductor a personal *assistant*, not only a fleet supervisor. +- **Mined for later:** the multi-platform gateway shape (`SessionSource` + + deterministic session-key) for P5 beyond web+Telegram; zero-context-cost tool-RPC + scripts; the Curator safe-autonomy invariants (archive-not-delete, pinned-exempt, + agent-provenance-scoped) if we add agent-authored skills; ACP interop; and + serverless-persistence (Modal/Daytona) for a cheap always-on cloud conductor. +- **Where codeoid stays ahead:** cryptographic identity (hermes is allowlist + + DM-pairing), rerank + bi-temporal retrieval (hermes is FTS5 + LLM-summary), and a + typed modular daemon (hermes is a Python monolith with 250–738 KB god-files). + +## Reconciliation with the mobile app plan + +[mobile-app-design.md](./mobile-app-design.md) (Expo/React Native, separate +`codeoid-mobile` repo) is conductor-aware and mostly *agrees* with this plan. + +**Aligned:** +- **IA matches our topology.** Mobile makes "the conductor" a pinnable **home + surface** and the session list a secondary **fleet view** — exactly our + single-global-conductor + separately-listed/switchable-sessions decision. +- **Approvals are the shared crown jewel.** Our confirm-before-send (R3/P4) rides + the `approvalId` mechanism; mobile turns that same mechanism into **native push + + voice approvals** ("approve / deny / show me the diff first"). The conductor is + the backend; the phone is its highest-value front door. +- **Voice-approval convergence.** Mobile borrows `iris`'s `hermesGate` (propose → + read-back → user must actually *speak* → only then act) — the same invariant as + firstmate's read-only-by-construction and our R3: **enforce confirm-before- + side-effect in code, not by trusting the model.** Three independent sources agree. + +**Correction — the conductor needs ZERO new client↔daemon wire types.** The mobile +doc's §8 is right: the conductor is just a session, and its fleet actions render as +ordinary tool calls in the transcript. So the earlier P3/P7 idea of "expose +`fleet_find` over the WS protocol so clients get a search box" is **not required** +and is downgraded to an optional later enhancement (a client-side instant +fleet-search that skips an LLM turn). v1 clients — web, Telegram, mobile — light up +the conductor purely by attaching to the conductor session. This *unblocks* mobile: +it builds its whole P0–P4 on today's attach/scrollback/session-list surface and +surfaces the conductor the moment our P3+ lands. + +**Shared prerequisite — extract `@codeoid/protocol` + `@codeoid/core`** (wire types ++ WS client + reducers) from `codeoid` once, up front. Mobile P0 requires it; the +conductor and `web/` benefit too (ends today's type triplication). Do it regardless +of which track leads. + +**Two workstreams, one contract:** +- *Conductor* (this plan, P0–P8) — daemon-side: session resolution, identity, fleet + tools, dispatch, act-on-behalf. Protocol-complete for clients at **P3**. +- *Mobile* (mobile-design P0–P5, separate `codeoid-mobile` repo) — the RN client; + only its P5 (conductor surface) depends on this plan's **P1 + P3**. Everything + before P5 ships on today's protocol. + +**Sequencing recommendation — conductor backend first.** Do the shared core +extraction, then **P1 (session resolution)** as the risk-retiring go/no-go gate: +it's the moat, it has **no prior art** (firstmate and Happy both lack it), it +improves every existing client, and it's the riskiest unknown — prove it before +building a front-end around it. The mobile app proceeds **in parallel** on today's +protocol (it is not blocked), and the two converge at mobile-P5. Shipping mobile +*first* gives you "control one session from your phone" (which Happy already does); +shipping the conductor *first* is what makes the eventual mobile app "supervise a +fleet by voice" — the position no competitor holds. + +## Dependency graph + +``` +P0 ──┬── P1 (retrieval) ──┐ + └── P2 (identity) ───┴── P3 (conductor + read) ── P4 (dispatch) + ├── P4.5 (routines) + ├── P5 (web + telegram) ── P7 (mobile) + ├── P6 (act on behalf) + └── P8 (governance, later) +``` + +## Verification philosophy + +Every phase states how you *know* it's done (exit criteria above). Two hard gates: +- **P1 exit is the go/no-go for the whole feature** — if fuzzy resolution isn't + reliably sub-2s and precision@1-strong on your own history, nothing downstream + matters. +- **Read-tools (P3) bake before act-tools (P4)** — never ship `fleet_send` before + `fleet_find` is trustworthy. diff --git a/docs/conductor-design.md b/docs/conductor-design.md new file mode 100644 index 0000000..af8d620 --- /dev/null +++ b/docs/conductor-design.md @@ -0,0 +1,339 @@ +# Codeoid Conductor — Design Proposal + +> Status: **DRAFT for grilling** · Author: design session 2026-06-30 +> Goal: turn codeoid into a locally-running "master of my machine" personal +> assistant — one that takes natural-language instructions, controls and inspects +> every Claude/Gemini/codeoid session, remembers all threads of work, browses the +> web, and acts on the owner's behalf (email, etc.) — **without going out of +> context** and **without leaving the identity-native model.** + +--- + +## 1. Goal & non-goals + +**Goal.** A persistent *conductor* — a special codeoid session whose job is not to +do work but to **route** work: receive an instruction in natural language, decide +which existing session should handle it (or spawn a new one), dispatch, collect a +*compressed* result, remember it, and report back. It is the single front door to +the whole fleet. + +**Non-goals.** +- Not a rewrite. The conductor is a thin layer over the existing + `SessionManager` / `Session` / `AgentIdentityManager` primitives. +- **No OpenClaw runtime integration.** We borrow *ideas and code* from OpenClaw + (channel adapters, skill/file conventions, scheduling) and reimplement them + native to codeoid. We do not run OpenClaw as a sidecar. (See §10.) +- Not identity-optional. Every conductor action, and every action it delegates, + is attributed to a ZeroID identity with a verifiable delegation chain. This is + the whole point of building it *in codeoid* rather than in OpenClaw. + +--- + +## Decisions locked (from grilling) + +- **Authority = owner-delegated privileged agent.** The conductor is its own + ZeroID agent identity; the owner delegates `session:*` to it; children are + further delegations. One verifiable tree, one revocation root. Verified against + `zeroid`: `delegation_depth` is a JWT claim, graph cap = 10, per-identity + `max_delegation_depth` policy must be ≥3, and `handleMessage` gates on scope + *membership* regardless of subject type — so an agent token carrying `session:*` + passes the same gate a human does. (Grill R1) +- **Identity durability = durable conductor, disposable children.** Persist only + the conductor's identity (survives restarts → one stable WIMSE URI for weeks). + Conductor-spawned children are per-task workers that die with the turn. One + credential at rest. (Grill R2) +- **Primary mode = coordinate the EXISTING session population.** The owner talks + only to the conductor and directs it to act *in existing sessions*. Two session + classes: (a) long-lived, user-owned sessions the conductor observes + directs; + (b) disposable conductor-spawned workers. This makes cross-workspace **session + tracking** (§6) the linchpin capability. (User refinement) +- **Routing safety = confirm before send-class acts.** `find`/`summary`/`recall` + run silently; any send-class action to an existing user-owned session first + proposes it with **repo + branch + content shown** and acts only on confirm. + Near-zero wrong-repo risk. (Grill R3) +- **Egress trust (v1) = owner approval only, no Shield.** Send-class egress + (`email.send`, external HTTP, outbound shell) is gated by owner approval via the + existing `approvalId` flow, showing recipient + subject + body preview so + approval is informed. Shield-like inspection is a deliberate *later* integration + — the assistant must not depend on the local stack being up. (Grill R4 + owner + follow-up) +- **Cost guard (v1) = metrics only.** No hard token budget or loop cap in v1; the + event-driven idle model keeps it cheap and the existing metrics UI + (tokens/cost/turns) gives visibility. A cheap per-instruction bounce cap is a + low-cost later toggle; hard budgets ship with the Shield-era governance. + (Grill R5) + +--- + +## 2. The core principle — why it never goes out of context + +A "master agent" fails the moment it tries to *do everything in one growing +context*. The conductor is built on three rules, all of which codeoid already +supports: + +1. **The conductor holds an index, not transcripts.** It never ingests a child + session's full output. Children return a *summary* (the existing memory / + saliency-compression path — `buildMemoryMcpServer`, saar/extraction work in + codeoid#39). The conductor's own context is: current instruction + a thin + fleet index (session names, states, last-summary-per-thread) pulled on demand + via tools. + +2. **State lives in the daemon, not the conversation.** "Remember every thread" + is a *query against durable state* (SQLite `store.ts` + the memory engine), + not memory held in a chat. Codeoid already owns this: "sessions are + daemon-owned; clients are stateless" (`CLAUDE.md`). The conductor is just + another daemon-owned, resumable session. + +3. **Long-running = resumable + event-driven, not one infinite turn.** The + conductor wakes on an event (a child finished, a Telegram message, a cron + tick), rehydrates from durable state, acts, and goes idle. Codeoid's + `resumeSessions()` + transcript persistence make the conductor itself + crash-proof and restartable. When its own context approaches the window, it + self-summarizes into the fleet index and continues — the same compaction + codeoid already does per session. + +The net: the conductor's working set is **O(active threads)**, not +O(total history). That is the structural answer to "doesn't go out of context." + +--- + +## 3. Where it sits in codeoid + +Three additions, no architectural change: + +| Addition | What it is | Mirrors existing | +| --- | --- | --- | +| **Conductor session role** | A `Session` created with `role: "conductor"` — same `query()` loop, but a different system prompt and an extra in-process MCP server bound to it. | normal `Session` | +| **`codeoid_fleet` MCP server** | In-process Agent-SDK MCP server exposing fleet tools (list / spawn / send / watch / summarize / interrupt sessions, recall across threads). Bound to the conductor session only. | `buildMemoryMcpServer` at `session.ts:810` | +| **Conductor identity grant** | The conductor's ZeroID agent identity additionally holds `session:*` scopes, so it can drive the fleet *as a first-class delegated authority* (see §4). | `AgentIdentityManager.registerSessionAgent` | + +Injection point is already there: `session.ts:810` merges `codeoid_memory` into the +`mcpServers` passed to `query()`. The conductor adds `codeoid_fleet` the same way, +gated on `role === "conductor"`. One P3 gotcha: the Claude provider's +`allowedTools` currently allowlists only `mcp__codeoid_memory__*` +(`providers/claude/index.ts`) — it must be widened to admit +`mcp__codeoid_fleet__*` for the conductor session, or the mounted server's tools +stay unreachable. + +--- + +## 4. Identity model — the crux + +Today there are **two disjoint scope namespaces**: + +- **Protocol scopes** (`protocol/scopes.ts`): `session:create|list|send|attach| + watch|interrupt|approve|destroy`, `fs:read`. Held by **human/client** tokens. + Enforced per inbound message in `SessionManager.handleMessage`. +- **Agent tool scopes** (`agent-identity.ts`): `tools:read|write|execute|agent`. + Held by **agent** identities, delegated to sub-agents via RFC 8693 + (`tokens.delegate`, actor assertion, `act` chain, `delegation_depth`, scope + intersection enforced by ZeroID). + +The conductor blurs these: it is an **agent** that must exercise **`session:*`** +(a capability the model reserves for human clients). The proposed resolution — +which keeps everything identity-native: + +**The conductor is a privileged agent whose authority is *delegated from the human +owner*, and every session it spawns is a further delegation.** The chain becomes: + +``` +human owner (ZeroID sub, IdP-verified) + └─ delegate session:list,create,send,watch,interrupt → CONDUCTOR agent + └─ delegate (per spawn) → child session agent + └─ delegate (attenuated) → child sub-agents +``` + +Concretely: +- The conductor gets its own identity scope profile (`CONDUCTOR_SCOPES` in + `agent-identity.ts`: `session:read` + `session:dispatch`, both protocol + scopes) — deliberately **separate from** `AGENT_TOOL_SCOPES`, keeping the two + namespaces disjoint, and deliberately excluding `tools:write`/`tools:execute` + so the conductor's whole delegation subtree is read-only on targets. The + conductor's token is minted by delegation from the owner, not handed the + owner's own token. *(Implemented in P2.)* +- When the conductor spawns a child, the child's `created_by` is the + **conductor's WIMSE URI**, and the child's token is `tokens.delegate`-d from the + conductor — so `delegation_depth` increments (human=0 → conductor=1 → child=2 → + sub-agent=3) and the `act` chain is fully verifiable. +- **Cascading revocation already does the right thing**: deactivate the conductor + → every child + sub-agent token it minted dies by construction + (`deactivateSessionAgent` cascades). Kill-switch for the whole fleet = revoke + one identity. +- Each fleet tool call is audited under the conductor's WIMSE URI + (`store.audit`), so "what did my assistant do at 3am" is a SQL query. + +This is the identity-native payoff OpenClaw structurally cannot match: the master +agent and everything it touches sit on one verifiable delegation tree with one +revocation root. + +--- + +## 5. The `codeoid_fleet` tool surface + +In-process MCP tools (closure-bound to the conductor's auth + the +`SessionManager`), each gated on a `session:*` scope the conductor holds: + +| Tool | Maps to | Scope | +| --- | --- | --- | +| `fleet_list` | `SessionManager` list path | `session:list` | +| `fleet_spawn(name, workdir, backend, brief)` | session create + delegated identity | `session:create` | +| `fleet_send(name, message)` | session send (async) | `session:send` | +| `fleet_summary(name)` | pull *compressed* latest state, not raw scrollback | `session:watch` | +| `fleet_interrupt(name)` | interrupt | `session:interrupt` | +| `fleet_find(query)` | resolve an NL reference → ranked session card(s) (see §6) | `session:list` | +| `fleet_recall(query)` | **cross-workspace** episode recall (see §6) | `session:list` | +| `machine_map()` | enumerate workspaces + git/running state (see §6) | `session:list` | +| `fleet_destroy(name)` | destroy | `session:destroy` (off by default) | + +Key discipline: `fleet_send` is **fire-and-forget**; results come back as *events* +(§7), and `fleet_summary` returns a compressed digest. The conductor never slurps +a child transcript into its own window. + +**Multi-backend.** `backend` selects Claude (native `query()`) or, via the +existing `anyagent` adapter (`.claude/` → Codex/Gemini/Hermes), a Gemini/Codex +child. The child is still a codeoid `Session` with its own ZeroID identity — so +"control a Gemini session" stays identity-native too. + +--- + +## 6. Session tracking & recall — the core capability + +This is the linchpin, not a side feature. Because the owner only ever talks to the +conductor, the conductor must resolve a *fuzzy natural-language reference* ("the +session where I was fixing the authz `latest_only` bug", "studio#870", "the durga +extraction eval") to the *right* session across every workspace on the machine — +and it must be right, because routing a command to the wrong repo's session is +harmful. + +**Codeoid already has the right foundation.** The memory engine +(`memory/engine.ts`) is a hybrid retriever — its `RecallHit` carries +`components.{vector, fts, recency, pathOverlap}`, i.e. semantic (embeddings) **and** +keyword (SQLite FTS5) **and** recency **and** path-overlap, already blended. And +`IndexScheduler` + `buildWorkspaceIndex` already refresh indexes on a schedule. +Three extensions turn this into fleet-wide *session* tracking: + +1. **Session-granular "cards" (new).** Per session, a durable card: + `{ name, workspace/repo, workdir, branch, created, last_active, status, rolling + summary of current work, salient entities (files, symbols, ticket ids, branch + names) }`. Embed the card (semantic) and FTS-index its text (keyword). Retrieval + returns *sessions*, not raw episodes. +2. **Cross-workspace scope.** Today `recall()` is bound to one `workspaceId` and + excludes the current session. Add an `ALL`-workspace mode so the conductor + searches the whole machine. +3. **Continuous cheap summarization.** Cards stay fresh via the saliency/extraction + path (saar / codeoid#39) riding on `IndexScheduler` — the same investment that + keeps the conductor's own context small keeps the cards current. + +**Why keyword matters as much as semantic (your point).** Embeddings are weak on +exact identifiers — `studio#870`, `latest_only`, a branch name, a file path. FTS +nails those; embeddings nail "the session where I was frustrated with flaky auth". +The hybrid is what makes *both* queries land — which is exactly why we extend +codeoid's existing blended scorer rather than bolt on a pure vector store. + +**Machine awareness.** A `machine_map` tool enumerates workspaces (repos under the +root), each session's workdir + git branch/status + running state — so the +conductor has "knowledge of the machine", not just of sessions. + +**Cross-ownership.** Because the conductor's authority is *delegated from the owner* +(§4), it operates within the owner's tenancy and can therefore see + drive the +owner's own pre-existing sessions — not only ones it spawned. `getOwnedSession` +resolves against the owner's tenancy, so no ownership hack is needed. + +--- + +## 7. Front door & wake model + +- **Front door:** reuse the existing **Telegram frontend** (`frontends/telegram/`, + embedded, direct `SessionManager` access). The owner DMs the bot; the message is + routed to the conductor session. No new channel needed for v1. (Web UI cockpit + remains the visual view.) +- **Wake model:** the conductor is event-driven. Wake sources: + 1. owner message (Telegram/Web), + 2. child-session completion (daemon emits an event → conductor turn), + 3. scheduled tick (a native cron, borrowed from OpenClaw's scheduler concept). +- **No busy-poll.** Between events the conductor session is idle (no tokens + burned). This is both the cost story and the never-OOC story. + +--- + +## 8. Acting on the owner's behalf (email, web, etc.) + +- **Web lookup:** children already have `WebSearch`/`WebFetch`. The conductor + delegates research to a child; gets a digest back. +- **Email / calendar / Slack:** native MCP servers (or borrowed adapters) + registered as in-process MCP tools — **but never on the conductor directly for + send-class actions.** Egress (`email.send`, shell, external POST) is delegated + to a child whose token carries only that scope, and is gated (§9). The conductor + *decides*; a narrowly-scoped child *acts*. + +--- + +## 9. Security boundary — dogfood Highflame + +"Master of my machine" = maximal blast radius. The owner runs an AI-agent +*security* platform; the conductor should be the flagship dogfood: + +- **Per-identity Cedar policy**: what may the conductor do vs. a child vs. a + sub-agent? Policy keyed on the WIMSE URI / `delegation_depth`. +- **Shield on egress** *(later phase — NOT v1)*: v1 egress is gated by owner + approval only, per the R4 decision above. Once the assistant no longer needs + to work with the local stack down, route `email.send` / shell / external HTTP + through Shield so a prompt-injected child can't exfiltrate. Codeoid is already + a `@highflame/sdk` consumer — a natural extension, not a new dependency. +- **Fail-closed defaults**: `fleet_destroy` and any send-class egress off unless + explicitly granted; approvals surface to the owner via the existing + permission-correlation (`approvalId`) flow. + +The demo writes itself: *"I let an autonomous agent run my machine, and here is the +policy boundary + audit tree that makes that safe."* + +--- + +## 10. What to borrow from OpenClaw (reimplemented native) + +| OpenClaw concept | Borrow as | Why native | +| --- | --- | --- | +| Multi-channel adapters | optional extra `Frontend` plugins | codeoid's `Frontend` interface already exists; keep direct-`SessionManager` access | +| Skills/memory as plain files | a skills loader for the conductor | stays inside codeoid's `~/.codeoid/` data model | +| Scheduler / cron | native wake source (§7) | must mint identity-scoped tokens per run — can't outsource | +| "Orchestrate Codex workers" | the `backend` param (§5) via `anyagent` | every worker must get a ZeroID identity; OpenClaw workers don't | + +The throughline: every borrowed capability must hang off a ZeroID identity. That +constraint is *why* we don't just run OpenClaw. + +--- + +## 11. Build phases + +1. **P0 — Conductor session + fleet read tools.** `role: "conductor"`, + `codeoid_fleet` MCP with `fleet_list` / `fleet_summary` / `fleet_recall` + (read-only). Identity grant = owner-delegated `session:list|watch`. Telegram + routes to it. *Proves the loop without any spawn/act risk.* +2. **P1 — Spawn + dispatch.** `fleet_spawn` / `fleet_send` with full delegation + chain + event-driven result digests. Cross-workspace recall (§6). +3. **P2 — Act on behalf.** Email/web/calendar via delegated, Shield-gated + children (§8, §9). Cedar policy per identity. +4. **P3 — Multi-backend + scheduler.** `anyagent` backends; native cron wake. + +--- + +## 12. Open questions (grill seeds) + +1. **Identity authority (the crux).** Conductor as owner-delegated privileged + agent holding `session:*` (§4) — or a different model (e.g. conductor *is* a + client using a human-style token, no agent identity)? Does ZeroID's + `delegation_depth` / `act` chain support human→conductor→child→sub-agent (depth + 3) cleanly today, or does that need work first? +2. **One conductor or many?** Single global conductor vs. one per context/project + (isolation vs. cross-thread reasoning). Affects memory scoping and revocation. +3. **Memory: widen vs. new journal.** Add a global recall mode to the existing + engine, or a separate `FleetJournal`? (§6) +4. **Spawn ownership semantics.** Is a conductor-spawned child "owned by" the + conductor (dies with it) or re-parented to the human (survives conductor + restart)? Revocation vs. durability tension. +5. **Egress trust.** Is Shield required in the loop for v1, or a P2 hardening? + What's the minimum gate before `email.send` is allowed at all? +6. **Cost ceiling & runaway guard.** What stops a conductor↔child feedback loop or + a forever-burning idle agent? Per-conductor token budget? Loop-detection? +7. **Front door scope.** Telegram-only for v1, or does the Web UI cockpit need a + conductor pane too? diff --git a/docs/conductor-prior-art-firstmate.md b/docs/conductor-prior-art-firstmate.md new file mode 100644 index 0000000..435fe34 --- /dev/null +++ b/docs/conductor-prior-art-firstmate.md @@ -0,0 +1,153 @@ +# Prior Art: firstmate — what to borrow, where codeoid is better + +> [firstmate](https://github.com/kunchenguid/firstmate) (by @kunchenguid) is the +> single most relevant prior art for the conductor: a **shipped** implementation +> of the exact "talk to one agent, it runs a crew" model, dogfooding the author's +> own tools (treehouse worktrees, herdr, Orca). It is the **architectural inverse** +> of codeoid — which is precisely why it's worth studying. Analysis based on its +> `docs/architecture.md`, `AGENTS.md` §1 (identity) + §8 (supervision), the 40+ +> `bin/` scripts, and its skills. + +## The mirror + +Both systems are the same idea: one liaison you talk to, a crew of autonomous +workers, worktree isolation, approval-gated merges, restart-proof state. The +substrate is opposite: + +| Axis | firstmate | codeoid conductor | +|---|---|---| +| Orchestrator | **`AGENTS.md` (122 KB prompt) + bash helpers** — "a directory that turns any agent into your firstmate" | **TS/Bun daemon** — logic in code, thin prompts | +| Worker isolation | git worktree (treehouse) per task, visible tmux/herdr/zellij/Orca window | daemon-owned session per task | +| State owner | disk (`data/`, `state/`) + the session backend | the daemon (clients are pure renderers) | +| Session/task resolution | **human names it; conductor greps `data/backlog.md`** — no semantic search | **hybrid retrieval + rerank over cards** (the P1 linchpin) | +| Identity/safety | "guarded by construction" + git auth + merge approval — **no crypto identity** | **ZeroID/WIMSE per session + delegation chain + (later) Cedar/Shield** | +| Supervision | zero-token bash watcher; absorbs benign wakes, wakes LLM only on actionable events | (to build — see P4) | +| Harness | **any** (claude/codex/opencode/pi/grok) via adapters | Claude-first, going meta-harness | + +The headline: firstmate **needs** a 122 KB prompt precisely *because* it has no +daemon to enforce invariants — every rule ("never write to a project", "keep one +live watcher cycle") is a fragile instruction the model must remember every turn. +Codeoid enforces those in code. So firstmate is both a **feature catalog to mine** +and a **proof of what you pay if you skip the daemon.** + +## STEAL — ranked by leverage + +### 1. The conductor is read-only over targets *by construction* +firstmate Hard Rule #1: "**Never write to a project.** You read projects to +understand them; crewmates change them." Only 6 narrow, all-fast-forward/guarded +write exceptions exist. This is a **stronger invariant than our R3** ("confirm +before send-class acts") because it's architectural, not a prompt discipline. +→ **Adopt:** the conductor's `codeoid_fleet` tool surface is **read + dispatch +only** — no file/git/shell-write tool on target repos ever. All mutation flows +through spawned crewmates behind approval. The blast-radius bound becomes a +property of the tool surface, and codeoid can *enforce* it (deny those tools to +the conductor identity via scopes), where firstmate can only *ask* for it. + +### 2. Zero-token, event-driven supervision (their crown jewel) +A cheap watcher (`fm-watch.sh` + `fm-classify-lib.sh` + `fm-crew-state.sh`) +classifies every wake in bash, **absorbs the benign majority** (`working:` notes, +no-change heartbeats, provably-working stale panes) without ever spending an LLM +turn, and wakes the conductor **only on actionable events** (`needs-decision` / +`blocked` / `failed` / `done` / `PR ready` / `merged`). Idle fleet = zero tokens. +Heartbeats **back off exponentially** (600 s → 2 h cap). Actionable wakes hit a +**durable queue** before detector state advances, so a missed exit is recoverable. +→ **Adopt (and do it better):** this is the answer to our "never OOC + cost" goal +and our P4 "event-driven digests." Codeoid's daemon already owns session state, so +it can **push** real events — no bash polling, no pane-tail regex. Conductor LLM +turns fire only on actionable daemon events; a "provably working" predicate gates +absorption; heartbeat backstop with exponential backoff; durable event queue for +crash recovery. + +### 3. Two task shapes: **ship** vs **scout** +ship = deliver a change (PR / local-merge → teardown); scout = investigate / plan / +reproduce / audit → report at `data//report.md`, never pushes, worktree is +scratch from the start. +→ **Adopt:** dispatch carries a `shape`. Scout results are reports (great for "go +find out X" without touching code); ship results are PRs/merges; teardown rules +differ. Clean taxonomy our P4 lacked. + +### 4. Per-project autonomy modes (not blanket confirm) +`data/projects.md` gives each project a mode — `no-mistakes` / `direct-PR` / +`local-only` — plus optional **`+yolo`** ("make routine approval decisions +yourself; destructive/irreversible/security-sensitive still escalates"). +→ **Adopt:** replace our blanket R4 "owner approval only" with **per-workspace +autonomy policy**. This maps *perfectly* onto codeoid's identity/Cedar future — a +project's mode literally *is* a policy bound to the conductor's identity. Low-risk +repos run hands-off; sensitive repos require approval. + +### 5. `/afk` batched-digest away-mode + `/stow` knowledge sweep +`/afk` hands supervision to a daemon that self-handles routine wakes and escalates +**only captain-relevant events as one batched, single-line digest** — cutting cost +while you step away. `/stow` sweeps the session for durable knowledge and routes +each finding to its disk home (prefs → `captain.md`, gotchas → `learnings.md`, +project knowledge → project `AGENTS.md`, task notes → backlog). +→ **Adopt** both as conductor UX: an away-mode that batches escalations, and a +handoff/knowledge-capture that routes findings to durable homes (codeoid already +has the memory engine to route into). + +### 6. Harness dispatch profiles +`config/crew-dispatch.json` — natural-language rules the conductor reads at intake +to pick `--harness/--model/--effort` per task; the shell validates the shape, the +LLM matches intent. +→ **Adopt:** directly feeds codeoid's meta-harness direction — the conductor routes +each task to the best backend (claude/codex/gemini) per NL rules + a validated +config. + +### 7. Sentinel marker for system-injected messages +Daemon escalations injected into the conductor's chat are prefixed with +`FM_INJECT_MARK` (ASCII unit-separator `0x1f`) so the conductor can tell an +internal escalation from a real captain message. +→ **Adopt:** our conductor faces the same ambiguity (daemon event-digests vs. real +messages from web/Telegram/mobile). Use an out-of-band field or sentinel so +injected events are never confused with user input. + +### 8. Secondmates = the nested-conductor scaling path (future) +Persistent **domain supervisors** that are "ordinary direct reports run from +isolated homes" — "there is no second architecture; a secondmate is a crewmate +whose workspace is an isolated home and whose brief is a charter." +→ **Note:** validates that our single-global-conductor choice can grow *domain +sub-conductors* later using the **same delegation-depth identity chain** we already +designed (human → conductor → sub-conductor → crewmate). No new architecture. + +Also worth borrowing: **restart-proof reconcile + durable wake-queue**, and the +**status-vs-current-state discipline** (`fm-crew-state.sh` reconciles an +authoritative run-step over a possibly-stale status line — a worker that reported +`done:` before a long validation isn't actually done). + +## Where codeoid is already better — keep these + +1. **Identity-native.** firstmate has *no* cryptographic identity; safety is prompt + discipline + git auth + merge approval. For the "master of my machine" ambition + (email, arbitrary tools, many agents), codeoid's per-session ZeroID + delegation + + (later) Cedar/Shield is a real moat firstmate can't match. Its worker + isolation is filesystem homes; ours is cryptographic delegation with cascading + revocation. +2. **Semantic session resolution.** firstmate resolves "which session" via the + human naming it + the conductor grepping a markdown backlog. It has **no** + embedding/rerank/hybrid recall — exactly the P1 linchpin we're building. This is + a genuine advance over the most mature conductor in the wild. +3. **Determinism — logic in code, not a 122 KB prompt.** Orchestration invariants + live in testable TS, not re-read-every-turn prose. Cheaper context, deterministic + behavior, unit-testable (see our P0 tests). +4. **Daemon-native events beat bash polling.** firstmate scrapes tmux panes with + regex because tmux "has no native primitive and always reports unknown." Our + daemon owns lifecycle → real push events, no scraping. + +## Anti-patterns to avoid + +1. **Prompt-as-program.** Don't drift toward encoding conductor logic in a giant + `AGENTS.md`. Keep it in code; prompts stay thin. +2. **Pane-tail-regex liveness.** Never guess worker health by scraping terminal + output — use daemon-authoritative state. +3. **Status side-channels that go stale.** firstmate spends pages reconciling + "status log says done but a run is active." Don't reintroduce a lossy status + stream parallel to the daemon's authoritative state. + +## Net + +The comparison **validates codeoid's architecture** (daemon + identity + structured +memory) as the right foundation, and firstmate proves the conductor concept ships. +The highest-leverage borrows are **behavioral/design, not code** (opposite +substrate): read-only-by-construction (#1), zero-token event supervision (#2), +ship/scout shapes (#3), per-project modes (#4). Fold #1–#4 into the design now; +#5–#8 are UX/scaling adds for P4–P7. diff --git a/docs/conductor-prior-art-hermes.md b/docs/conductor-prior-art-hermes.md new file mode 100644 index 0000000..6dedefe --- /dev/null +++ b/docs/conductor-prior-art-hermes.md @@ -0,0 +1,153 @@ +# Prior Art: hermes-agent — what to borrow, where codeoid is better + +> [hermes-agent](https://github.com/NousResearch/hermes-agent) (Nous Research, +> MIT) is the most *complete* personal-assistant prior art we've studied — a +> multi-platform gateway (Telegram/Discord/Slack/WhatsApp/Signal/email), cron +> **routines**, autonomous skill creation, cross-session memory + user modeling, +> `delegate_task` + a durable **Kanban** work-queue, all running on a $5 VPS. It is +> the closest thing to the original "master of my machine" ask. Analysis from its +> `README`, `docs/session-lifecycle.md`, `hermes-already-has-routines.md`, +> `AGENTS.md` (Delegation / Curator / Cron / Kanban), and the subsystem layout. + +## Triangulation — where codeoid sits + +Three prior-arts, three niches: + +| System | Niche | Security model | Substrate | +|---|---|---|---| +| **OpenClaw** | channel-gateway breadth (front door + life-admin) | allowlist | files + bash | +| **hermes** | **most complete personal assistant** (gateway + routines + skills + memory + user model) | allowlist + DM-pairing + command-approval | Python monolith (`cli.py` 738 KB) | +| **firstmate** | *coding-fleet* conductor (crew + worktrees + PRs) | guarded-by-construction + merge approval | 122 KB prompt + bash | +| **codeoid** | **identity-native + retrieval-first + typed multi-client daemon** | **ZeroID/WIMSE per-session + delegation** | TS daemon, clients-are-renderers | + +hermes and firstmate each hold one or two legs; **codeoid is the only one with all +three** (crypto identity + semantic session resolution + typed multi-client daemon). +That triad is the defensible position. hermes serves the *personal-assistant +breadth* better than firstmate — which is exactly why it's worth mining for the +"master of my machine" surface our plan was thin on. + +## STEAL — ranked by leverage + +### 1. Routines — scheduled + triggered autonomy (the biggest gap in our plan) +hermes has cron **and** webhook/event triggers, in natural language: +`hermes cron create "0 2 * * *" "triage the backlog" --deliver telegram`; +`hermes webhook subscribe pr-review --events pull_request --prompt "…"` (HMAC-auth). +Per-job fields: `skills`, `model`/`provider` override, `script` (pre-run +data-collection whose stdout is injected — or `no_agent=True` to make the script +the *whole* job), `context_from` (chain job A's output into job B), `workdir` (run +in a repo with its `AGENTS.md` loaded), multi-platform delivery. +`cron/jobs.py` (store) + `cron/scheduler.py` (tick loop). +**Two hardening patterns to steal whole:** +- **Script-injection + `[SILENT]` pattern** — a script does the mechanical work + (fetch/diff/compute), the agent only *reasons*, and the job emits nothing unless + something changed (`respond with [SILENT]`). Zero-spam, near-zero-token monitors. +- **Cron hardening** — 3-minute hard interrupt (runaway loops can't monopolize the + scheduler), file-lock (`.tick.lock`) against duplicate ticks across processes, + catchup/grace windows, `skip_memory=True` by default on cron sessions, and cron + output lands in its *own* session (header/footer frame) so it never corrupts the + main conversation's role alternation. +→ **New conductor phase (P4.5).** Neither codeoid nor firstmate has this, and it's +core to "master of my machine" (nightly triage, monitors, digests, event triggers). + +### 2. Durable Kanban work-queue as the dispatch backbone +`AGENTS.md` §Kanban + `tools/kanban_tools.py`: a **SQLite-backed board** with a +**dispatcher loop** (default 60 s) that reclaims stale claims, promotes ready tasks, +**atomically claims**, and spawns the assigned worker. **Board = hard boundary** +(workers get `HERMES_KANBAN_BOARD` pinned in env, can't see other boards); tenant = +soft namespace within a board. After `failure_limit` consecutive failures (default +2) the dispatcher **auto-blocks the task** to prevent spin loops. +→ Far more restart-proof than firstmate's markdown backlog. **Upgrade P4's dispatch +to sit on a durable queue like this** (codeoid already has SQLite): atomic claim, +stale-claim reclaim, failure-limit auto-block (complements firstmate's stuck-loop). + +### 3. `delegate_task` role model — leaf vs orchestrator +`tools/delegate_tool.py`: a subagent gets an isolated context + terminal. +`role="leaf"` (default) is a focused worker that **cannot** call `delegate_task`, +`memory`, `send_message`, `execute_code`; `role="orchestrator"` **can** spawn, +bounded by `max_spawn_depth` (default 2) and `max_concurrent_children` (default 3). +Background delegation returns an id immediately and re-enters via an async +completion queue. Durability rule: background delegate is process-local — for +restart-survival use a cronjob or `terminal(background, notify_on_complete)`. +→ This *is* our read-only-by-construction + delegation-depth decisions, with the +concrete knobs — except **codeoid enforces the role's capability restriction +cryptographically via ZeroID scopes**, where hermes uses config flags. Adopt the +leaf/orchestrator split; enforce it at the scope layer, not by prompt or flag. + +### 4. Session-lifecycle hardening (`docs/session-lifecycle.md`) +A mature, battle-tested state machine worth mining for reliability: +- **Restart recovery:** `resume_pending` (soft — preserve `session_id`, continue the + transcript) vs `suspended` (hard wipe); `suspend_recently_active(120s)` on a crash + (no `.clean_shutdown` marker); a `.clean_shutdown` marker skips resurrection after + a clean restart. +- **Stuck-loop escalation:** a restart-count file auto-suspends a session active + across 3+ consecutive restarts (terminal escalation, complements Kanban's + failure-limit). +- **Agent LRU cache** (128 entries, 1 h idle TTL) that **preserves prompt-cache** + across turns; background expiry watcher (5 min) finalizes + evicts. +- **Burst-collapse message queue:** single "next-up" slot per session (repeat sends + overwrite) + FIFO overflow for explicit `/queue`, so multi-message bursts during a + turn never process out of order. +- **Per-session token/cost tracking** baked into the session record. +→ codeoid's daemon has some of this; the `resume_pending`/stuck-loop/clean-shutdown +state machine + burst-collapse queue are concrete P4 hardening. + +### 5. Multi-platform gateway +One gateway process → Telegram/Discord/Slack/WhatsApp/Signal/email, with a clean +`SessionSource` (message-origin descriptor) → deterministic session-key +(`agent:main:{platform}:{chat_type}:{chat_id}:{thread}:{participant}`) → home +channels + delivery routing + multi-user isolation + PII-redaction-in-prompt. +→ How our P5 adds platforms cheaply beyond web+Telegram. Copy the SessionSource + +session-key shape; codeoid's daemon already owns the session store this plugs into. + +### 6. Zero-context-cost tool-RPC scripts +"Write a Python script that calls tools via RPC, collapsing multi-step pipelines +into one zero-context turn." Complements codeoid's saar/extraction work and +firstmate's zero-token supervision — a script surface for mechanical multi-step +work that never floods the conductor's context. + +### 7. Self-improving loop (Curator) — aspirational, clean invariants +`agent/curator.py`: autonomous skill creation after complex tasks + a maintenance +loop that tracks per-skill usage and **archives (never deletes)** stale skills, +exempts pinned, and **only touches `created_by: agent` skills** (bundled/hub skills +off-limits). Plus Honcho dialectic user modeling ("who you are across sessions"). +→ The direction codeoid's memory engine could grow (autonomous skills + a user +model); the safe-autonomy invariants (archive-not-delete, pinned-exempt, +provenance-scoped) are worth copying if/when we add agent-authored skills. + +### 8. Notes +- **ACP** (`acp_adapter`, `acp_registry`, `agent/copilot_acp_client.py`) — hermes + speaks the Agent-Client-Protocol. An interop/meta-harness play: codeoid speaking + ACP would let editors (Zed/Copilot) drive it. +- **Serverless-persistence backends** (Modal/Daytona) — hibernate-when-idle so a + cloud conductor costs ~nothing between sessions. The cheap-VPS story. + +## Where codeoid is already better — keep + +1. **Identity.** hermes is allowlist + DM-pairing + command-approval — no + cryptographic per-session identity or delegation chain. For "master of my + machine" with email + arbitrary tools + a fleet, codeoid's ZeroID is the moat; + and it lets us enforce the delegate role model (#3) *cryptographically* rather + than by config flag. +2. **Retrieval.** hermes cross-session recall = FTS5 + LLM summarization (better + than firstmate's markdown grep, but no rerank or bi-temporal cards). Our P1 + (BGE-M3 hybrid + cross-encoder rerank + bi-temporal cards) is more sophisticated. +3. **Typed daemon + modularity.** hermes is a Python monolith with 250–738 KB + god-files. codeoid's typed TS daemon + clients-are-renderers keeps the + multi-client (web/TUI/mobile) story crisp and the code testable. + +## Anti-patterns to avoid +1. **God-files** (`cli.py` 738 KB, `run_agent.py` 268 KB, `hermes_state.py` 255 KB) + — keep the daemon in small typed modules. +2. **Allowlist-only security** — don't regress from ZeroID to DM-pairing/allowlists. +3. **Prompt-heavy `AGENTS.md`** (71 KB) — same trap as firstmate, less extreme; keep + orchestration in code. + +## Net — refinements to the plan +- **Upgrade P4** — durable Kanban-style work-queue (atomic claim, stale reclaim, + failure-limit auto-block) + the `leaf`/`orchestrator` role model (enforced via + scopes) + session-lifecycle hardening (resume_pending / stuck-loop / clean-shutdown + / burst-collapse queue). +- **Add P4.5 — Routines** — scheduled + webhook/event-triggered autonomy, with the + script-injection `[SILENT]` pattern and cron hardening (hard interrupt, tick lock, + own-session output). This is what turns the conductor from a fleet *supervisor* + into a personal *assistant*. diff --git a/docs/conductor-session-resolution.md b/docs/conductor-session-resolution.md new file mode 100644 index 0000000..eba4136 --- /dev/null +++ b/docs/conductor-session-resolution.md @@ -0,0 +1,281 @@ +# Session Resolution — SOTA Architecture + +> Companion to [conductor-design.md](./conductor-design.md) §6. This is the deep +> design for the conductor's linchpin capability: resolve a fuzzy natural-language +> reference — "the session where I was fixing the authz `latest_only` bug", +> "studio#870", "the durga extraction eval" — to the RIGHT AI-coding session among +> hundreds, across every workspace on the machine. +> +> Backed by a 9-agent research fan-out (retrieval methods; local models — +> embedders / rerankers / LFM2 / MTEB+CPU-latency; OSS prior art — mem0, +> Letta/MemGPT, Zep/Graphiti, Cognee, LlamaIndex, Haystack, txtai, OpenClaw/ +> SwarmClaw, Claude ecosystem). Model picks are **final** (CPU latencies are +> calibrated estimates — verify on the box). Sources at the bottom. + +--- + +## 0. Baseline — what codeoid already has + +Reading the source ([engine.ts](../src/daemon/memory/engine.ts), +[store.ts](../src/daemon/memory/store.ts), +[embedder.ts](../src/daemon/memory/embedder.ts)): codeoid already ships a credible +session resolver — `engine.ts:searchSessions()` runs hybrid recall, groups episode +hits by session, and scores `topEpisodeScore + log(matchCount)·bonus + recency + +nameMatch`. **This is an upgrade of an existing primitive, not greenfield** — and +notably codeoid's memory store is already ~80% of the convergent SOTA stack below +(SQLite + FTS5 + same-row embeddings + weighted hybrid). The gaps are specific: + +| Layer | As-built | SOTA gap | +| --- | --- | --- | +| Embedder | `Xenova/bge-small-en-v1.5` (384-d, English-only, 512-tok, WASM) | no sparse leg → misses identifiers; weak + monolingual | +| Vector | brute-force cosine, in-memory, **per-workspace** | fine at scale; needs cross-workspace + eventually ANN | +| Sparse | FTS5 + `bm25()`, **default tokenizer** | splits `studio#870` → `studio`+`870`; no id handling | +| Fusion | ad-hoc weighted linear | normalize + tune (convex); RRF as cold-start | +| Rerank | **none** | the single biggest precision@1 lever, missing | +| Cards | `sessionSummaries()` = first user_turn line | no rich, self-maintained, embedded digest | +| Scope | `workspaceId`-bound everywhere | cross-workspace is the §6 gap | + +--- + +## 1. The retrieval pipeline (fuzzy reference → the one right session) + +Six stages. Precision@1 ("pick the ONE right session") is the metric — a wrong +route runs a command in the wrong repo. **Hybrid is mandatory, not optional**: on +exact-identifier queries BM25 gets ~70% recall vs dense ~5%; on paraphrases the +reverse. The discriminating signal here is *exact tokens* (paths, branch/repo/fn +names), so the lexical leg is load-bearing. + +**Stage 0 — Query typing (regex, no LLM).** Classify `has_identifier` (matches +`\w+#\d+`, `path/like/this`, dotted/snake symbols, branch patterns) vs +`pure_semantic`. Sets the fusion weight. ~10 lines, zero latency. + +**Stage 1 — Candidate generation (two channels, always both).** +- *Dense*: embed query → top 50–100 over session cards + episodes. +- *Lexical*: FTS5/BM25 with **field boosting** (ticket/branch/path fields ≫ body) + and a **code/trigram analyzer** so `studio#870`, `feat/agent-action-timeline`, + `torch.nn.x` tokenize as whole matchable units. (Today `store.ftsSearch` uses + the default tokenizer — this is the fix.) + +**Stage 2 — Fusion.** **RRF (k=60) as the day-1 cold-start default** (rank-only, +no score calibration needed); **move to a convex combination of theoretical-min-max +normalized scores once ~40 labeled query→session pairs exist** (an afternoon's +work). Tuned convex then beats RRF *and* preserves score margins (how much #1 beats +#2), which RRF discards. Set **α by query type** from Stage 0 (static 2-value table: +≈0.3 favor-lexical for identifier queries, ≈0.7 favor-dense for semantic). *This is +exactly what the two closest prior-art twins already do* — SwarmClaw ships +`relevance = sem·0.50 + lex·0.35 + fts·0.15`, txtai a `[w, 1-w]` convex fusion — so +convex-once-tuned is the field-validated choice. + +**Stage 3 — Exact-identifier override (deterministic, pre-rerank).** If Stage 0 +found an identifier AND a candidate matches it literally, pin it to the top. Exact +ids are a **correctness guarantee, not a soft weight**. Cheaply reinforced by an +**entity boost** (mem0's pattern): extract repo/branch/file/symbol tokens +deterministically and additively boost sessions that share them. + +**Stage 4 — Cross-encoder rerank of top-k (~20–50) session cards.** The single +biggest precision@1 lever (documented +5–15 NDCG@10, precision@1 → ~1.0), and +codeoid has none. Rerank the **cards**, not raw episodes. **bge-reranker-v2-m3** +(ONNX-int8) does ~50 short pairs in ~0.5–1s on a 12-core CPU — inside the sub-2s +budget. Keep the candidate set ≤50. + +**Stage 5 — Additive priors as tie-breakers (never co-equal rankers).** Recency +(forgetting-curve / ~30-day half-life decay, from SwarmClaw), log usage-frequency / +reinforcement (bump on access), currently-active-session boost, pinned boost. +Applied *after* semantic scoring; optionally an MMR diversity pass (λ≈0.7) if +returning a list. + +> `searchSessions()` already implements Stages 1–2 + crude priors at episode +> granularity. The SOTA delta = **card** granularity + identifier handling +> (0 / boosted-lexical / 3 + entity boost) + a real **rerank** (4) + **cross-workspace**. + +--- + +## 2. The session-card data model (convergent: mem0 + Letta + Zep + Cognee) + +Multiple independent memory frameworks point at the same design. **Steal the data +model + self-editing loop; do NOT adopt any runtime** (all assume they own the loop ++ a heavy store, competing with codeoid's daemon). + +**(a) Card = an addressable, self-maintained digest** *(Letta memory-block; +SwarmClaw `session_archive` row)*. One per session: +`{ session_id/slug, repo, branch, task-in-NL, current-state, last-action, +open-threads, entities }` with a hard size *limit* forcing digest-not-transcript. +Both the conductor and the session agent attach to it; edits are live — codeoid's +"daemon owns state, clients render." Embed only the *fuzzy* fields (task, goal, +branch — Cognee's `index_fields` idea); keep ids/paths/timestamps as filterable +columns, out of the embedded text. + +**(b) State as bi-temporal facts, never mutable fields** *(Zep/Graphiti)*. Record +state changes ("WIP" → "merged") as facts with four timestamps +(`valid_at, invalid_at, created_at, expired_at`). On a change: set the old fact's +`invalid_at = new.valid_at`, stamp `expired_at`, insert the new — **never +UPDATE-in-place, never DELETE**. Free time-travel ("what was this session's status +last Tuesday?") + lossless audit + correct "was WIP, now merged" so "continue the X +fix" resolves to the *right iteration*. Plain SQLite: `facts(subject, predicate, +object, valid_at, invalid_at, created_at, expired_at, embedding)`. The four-timestamp +discipline is the pattern — **not Neo4j** (mem0 *removed* its graph after their own +benchmark showed it slower/costlier with no accuracy win; Graphiti's embedded +backend is vaporware). + +**(c) Keep cards canonical via write-time reconciliation** *(mem0 paper)*. On an +update, fetch top-s similar existing cards → LLM decides **ADD/UPDATE/invalidate** → +one evolving record per thread, not 40 near-dups. That same "here are the 10 closest +threads — which one?" prompt shape *is* reference resolution, reusable at query time. +**Avoid mem0 v3's ADD-only default** — a trap for long-lived mutating threads. + +**(d) Deterministic entity resolution before any LLM** *(Zep MinHash/LSH; mem0 +spaCy)*. Merge/boost on repo/branch/file/symbol tokens with a deterministic fast +path; reserve the LLM for genuine ambiguity. Cheap and fast on a laptop. + +**(e) Extract at checkpoints, not per turn** *(Zep cost lesson)*. LLM-per-turn +extraction (4+ calls/episode) would dominate a many-session daemon. Regenerate a +card on session start / status-change / merge via `IndexScheduler`; use cheap rules +for structured signals. + +--- + +## 3. Model choices (FINAL) + +| Role | Model | Size | CPU latency (12-core, ONNX-int8/GGUF-Q4) | Why | +| --- | --- | --- | --- | --- | +| **Embedder** | **BGE-M3** | 568M | query embed ~tens of ms; corpus embedded once, offline | **Only model emitting dense + learned-sparse + ColBERT in one pass** → semantics *and* the identifier-matching sparse leg from one model, no second index. MIT, 8192 ctx. Team already runs it. | +| Embedder (dense-only alt) | Qwen3-Embedding-0.6B | 0.6B | similar | Tops MTEB-multilingual small tier (64.33 vs BGE-M3 59.56) — but **dense-only**, so you'd bolt on BM25 for identifiers. Switch only if a dense eval on *your* data shows a real win. | +| **Reranker** | **bge-reranker-v2-m3** (ONNX-int8) | 568M | ~50 short pairs ≈ **0.5–1.0s** (~8 ms/pair) | Encoder cross-encoder → far cheaper on CPU than decoder rerankers; same family/tokenizer as BGE-M3; Apache-2.0; mature CPU tooling (FastEmbed). | +| Reranker (tiny fast fallback) | Ettin-150M (MIT) or ms-marco-MiniLM-L6-v2 (22M) | 150M / 22M | ~0.3–0.5s / ~0.15–0.4s for 50 pairs | Ettin-150M ≈ mxbai-large accuracy at a fraction of the cost; MiniLM is the English-only floor. | +| **Card extractor (small LM)** | **LFM2-350M-Extract** (or LFM2.5-350M) | 350M | ~313 tok/s decode — **offline at ingest, off the hot path** | Purpose-built transcript→JSON extractor (Liquid ships it; beats Gemma-3-4B 11× at extraction, first-party). Pair with **GBNF/JSON grammar** for guaranteed-valid output; field semantics in the prompt (your durga finding). | + +**Upgrade the embedder** from `bge-small-en-v1.5` (33M, English-only, dense-only, +512-tok): you specifically need the **sparse/lexical leg for identifiers** (the +single biggest quality lever), longer context, and better semantics. The 568M vs +33M size gap is irrelevant — thousands of docs embedded once offline; only the +query embeds at request time (tens of ms). + +**The LFM2-230M question, answered:** it belongs in exactly **one** slot — **offline +structured card extraction/summarization** — where its speed is free and grammar +constraints backstop its limits. Per subtask: (a) query rewrite — viable ~230–350M +but *skip by default*, rewriting terse references adds little; (b) HyDE — **skip** +(hallucinates identifiers on exact-match corpora, adds latency); (c) **listwise +reranking — NOT viable at 230–350M**, the whole literature is 7B+ and even 7B emits +malformed rankings — **use the cross-encoder**; (d) card extraction — **its genuine +strength**. Net: prefer **LFM2.5-350M / LFM2-350M-Extract** over 230M (negligibly +slower on 12 cores, meaningfully better instruction-following). Don't let Liquid's +"on-device RAG" framing tempt you into using it as a reranker — that's the trap. + +--- + +## 4. Deliberately skipped (keep it lean) + +Overkill at single-user, hundreds-to-thousands-of-docs, one-machine scale: +**SPLADE/learned-sparse** (BGE-M3's sparse output already covers it), +**ColBERT/PLAID late-interaction** (built for millions of passages; a top-k +cross-encoder wins here), **HyDE** (hurts on identifier corpora; keep only as a +narrow low-confidence fallback), **multi-query/RAG-fusion/decomposition** (session +refs are single-intent), **LLM-per-query fusion & listwise LLM reranking** +(latency-fatal: 50-doc GPT-4 rerank ≈ 1 min/H100), and **any external memory +runtime or graph DB** (steal the patterns onto codeoid's SQLite + memory engine). + +--- + +## 5. Convergent synthesis — the 6 patterns + 3 pitfalls + +Endorsed independently across the frameworks: + +1. **SQLite as the single spine** — content + metadata + FTS5(BM25) + vectors + (sqlite-vec). txtai, SwarmClaw, and every Claude-session tool converge here. + Codeoid already has this (minus sqlite-vec — it uses in-memory brute-force). +2. **Hybrid retrieval + fusion + light rerank** — BM25 for exact tokens, dense for + paraphrase; RRF cold-start → tuned convex; cross-encoder only on top-k. +3. **Index a compacted "gist" + metadata, not raw transcripts** — repo/cwd, branch, + first+last prompt, one-line gist, entities, status, timestamps. +4. **Deterministic entity extraction + entity boost, LLM only as fallback** — mem0 + spaCy entities, Zep MinHash/LSH. Cheap, fast, high-signal. +5. **Temporal thread state first-class, invalidate-don't-delete** — Zep's bi-temporal + edges; resolve to the *right iteration* + answer "what was I doing last week." +6. **Incremental byte-offset watcher over the source files** — tail growing logs from + last-read offset; re-index only new bytes (the `claude-code-sessions` pattern). + +**Pitfalls to avoid:** (1) **graph DB as the default store** — mem0 removed it after +its own benchmark; borrow the temporal-edge *pattern*, not Neo4j/Kùzu. (2) +**Agent-self-managed / LLM-in-the-loop memory as the primary resolver** (Letta-style) +— non-deterministic and latency-variable; resolution must be a deterministic ranked +index, LLM only disambiguates the top few. (3) **Files-as-memory-only** (base +OpenClaw, Cline Memory Bank, Cursor rules) — no ranking, doesn't scale to many +sessions; fine as a human-readable per-session status doc, not the retrieval layer. + +--- + +## 6. Prior-art verdicts + +| System | STEAL | AVOID | +| --- | --- | --- | +| **SwarmClaw** *(closest twin)* | single-table hybrid recall (FTS5 + BLOB embeddings + `sem·0.5+lex·0.35+fts·0.15`); session-archive-as-memory row; reinforcement+decay+pinned salience; per-session single-run lock + preempt/steer + restart-recoverable runs | brute-force cosine as it scales (add ANN); its resume = static `backend→id` map with **no NL→session matcher** (our hard problem is unsolved there — no shortcut) | +| **txtai** *(stack twin)* | the whole storage design: SQLite content+metadata + **BM25-in-SQLite** + ANN, `similar()`-then-SQL-filter, weighted hybrid fusion, 9.0 rerank pipeline | — (build directly on it if Python; else replicate with FTS5 + sqlite-vec) | +| **Claude ecosystem** | parse `~/.claude/projects/**.jsonl` incrementally → per-session metadata+gist → SQLite FTS5 + sqlite-vec → hybrid RRF (`claude-code-sessions` is purpose-built prior art for our exact input) | — | +| **mem0** | entity-boosted hybrid scoring (spaCy entities + damped additive boost); paper's reconciliation loop; SQLite change-log | v3 ADD-only/overwrite-on-update (loses thread history); LLM-only index | +| **Zep / Graphiti** | bi-temporal 4-timestamp invalidate-don't-delete; deterministic MinHash/LSH entity resolution before LLM; node-distance rerank | 4-LLM-calls/episode in hot path; server graph-DB dependency | +| **Letta / MemGPT** | tiered core(always-loaded digest)/archival(vector)/recall(history) split; sleep-time compaction | running Letta as engine; self-managed memory as the *resolver*; FIFO-summarization as index | +| **Cognee** | ECL framing (Extract→Cognify→Load); typed nodes + `index_fields` (embed only fuzzy fields) | full ontology + 3-DB stack; its weak temporal model | +| **LlamaIndex / Haystack** | the retrieve→fuse(RRF)→rerank + metadata-pre-filter *pattern*; filter grammar (`{field,op,value}` + AND/OR/NOT) | adopting the framework as a dependency (too heavy for one machine) | + +**"claude tag" resolution:** not a memory/retrieval system. It's either Anthropic's +Slack "tag Claude into a thread" product, or a `claude plugin tag` git-tag +subcommand — plus some open feature-requests to *label* Claude Code sessions. Only +that last strand is relevant, and only as inspiration for **manual session tags as +one routing signal**. + +--- + +## 7. Grafting onto codeoid (concrete) + +1. **Cards table + facts table** in the memory SQLite (alongside `episodes`). Cards + embedded on write (fuzzy fields only); facts bi-temporal. +2. **Embedder swap** `bge-small-en` → **BGE-M3**; persist dense **and** sparse + vectors (BGE-M3 emits both). This alone fixes the identifier-recall gap. +3. **sqlite-vec + tokenizer fix**: move brute-force cosine to sqlite-vec as it + grows; rebuild `episodes_fts` (+ new `cards_fts`) with a code/trigram analyzer + + boosted identifier columns. +4. **Fusion**: RRF now; swap to TMM-normalized convex + Stage-0 α table + Stage-3 + exact-override + entity boost once ~40 labels exist (generate them from your own + usage). +5. **Rerank stage**: add **bge-reranker-v2-m3** (ONNX-int8) over top-k cards. +6. **Card maintenance**: **LFM2-350M-Extract** (grammar-constrained) on + `IndexScheduler` checkpoints + mem0-style reconciliation on write; salience + decay + reinforcement on access (SwarmClaw). +7. **Cross-workspace mode**: add an `ALL` scope to `recall` / `loadVectorMatrix` / + `ftsSearch` (today all `workspaceId`-bound). +8. **`fleet_find` MCP tool** (conductor-design §5) calls this pipeline; returns + ranked cards with evidence snippets (`SessionSearchHit` already carries snippets). + +--- + +## 8. Bonus — index Claude sessions started *outside* codeoid + +Your original ask includes controlling *all* Claude sessions, not just codeoid's. +Claude Code persists every session as JSONL at +`~/.claude/projects//.jsonl` (session id, cwd, git branch, +one line per message). So the conductor can run a **byte-offset watcher** over that +directory (the `claude-code-sessions` pattern) and index those sessions into the +same cards table — giving it visibility into raw `claude` sessions the user ran +without codeoid. Same pipeline, second source. (Codeoid's own sessions already have +richer episode data via the memory engine.) + +--- + +## 9. Sources (top, by track) + +**Retrieval methods:** Bruch et al. "Analysis of Fusion Functions for Hybrid +Retrieval" (ACM TOIS, convex>RRF once tuned); tianpan.co 2026 hybrid-search-in-prod +(BM25 wins on identifiers, α≈0.3 technical); DAT (arXiv 2503.23013, per-query α); +cross-encoder rerank lifts (bigdataboutique); Re3 (arXiv 2509.01306, recency as +prior). +**Models:** BGE-M3 (arXiv 2402.03216, dense+sparse+ColBERT one pass); +bge-reranker-v2-m3 + Ettin reranker cards (CPU pairs/sec); Qwen3-Embedding report +(arXiv 2506.05176, MTEB table); LFM2 report (arXiv 2511.23404) + LFM2.5-230M/350M + +LFM2-*-Extract cards; RankZephyr/RankLLM (listwise needs 7B+); GOLFer (arXiv +2506.04762, small-LM query expansion); Intel/HF CPU embedding latency. +**Prior art:** SwarmClaw `memory-db.ts`; txtai (neuml/txtai); Graphiti (arXiv +2501.13956) + getzep/graphiti; mem0 (arXiv 2504.19413) + v3 migration notes; Letta +(MemGPT arXiv 2310.08560); Cognee (topoteretes/cognee); LlamaIndex +`fusion_retriever.py`; Haystack `document_joiner.py`; Anthropic Claude Code memory ++ JSONL session storage; `claude-code-sessions` (FTS5 + sqlite-vec + RRF + JSONL +byte-offset watcher). diff --git a/docs/session-resolution.md b/docs/session-resolution.md new file mode 100644 index 0000000..7a3181e --- /dev/null +++ b/docs/session-resolution.md @@ -0,0 +1,126 @@ +# Session Resolution — how codeoid finds the right session + +> How the shipped capability works. For the research/design rationale behind it +> see [conductor-session-resolution.md](./conductor-session-resolution.md); for the +> eval methodology + numbers see +> [../src/daemon/eval/BASELINE.md](../src/daemon/eval/BASELINE.md). + +You say — in natural language — *"continue the auth token-refresh fix"* or *"the +session where I was comparing the two caching strategies"*, and codeoid resolves it +to the **right session among all your workspaces**, in under 100 ms. This is the +linchpin of the conductor (talk to one agent, it runs your fleet): you don't +remember which repo or which of a dozen sessions it was — you describe it, and it's +found. + +## The problem + +codeoid already indexes every session's episodes with hybrid retrieval — dense +embeddings (semantic) + FTS5 BM25 (keyword/identifier) + recency + path overlap — +but **scoped to one workspace** (`recall(workspaceId, …)`). That's right for +in-session memory ("what did I try earlier *here*"), but the conductor needs the +opposite: find a session *without* knowing its workspace. + +The naive fix — run the per-workspace search everywhere and merge the results — +scores only **22% precision@1**. The reason is subtle and instructive: BM25 scores +are normalized **batch-relative, per workspace**. A workspace with few episodes +produces inflated normalized scores, so it dominates the merged ranking regardless +of semantic relevance. **Scores from different workspaces aren't comparable.** + +## The pipeline (as built) + +Resolution runs as **two stages** — the classic bi-encoder-recall → cross-encoder- +rerank shape: + +**Stage 1 — global fusion (`engine.recallGlobal`).** Embed the query once, then +gather candidates across *all* workspaces: +- **Dense**: cosine of the query vector against every workspace's embedding matrix. +- **Lexical**: FTS5 BM25 across all workspaces (`store.ftsSearchGlobal`). + +Union them into **one candidate set** and rank that single batch with the hybrid +ranker (vector + BM25 + recency + path). Because it's one batch, **BM25 +normalization is global** — scores are finally comparable across workspaces. +Episodes are grouped into sessions and scored (`searchSessions` with no +`workspaceId`). + +**Stage 2 — cross-encoder rerank.** Take the top ~8 candidate sessions and rerank +them with a cross-encoder that reads the `(query, session-evidence)` pair *jointly* +and scores relevance for *this* query (`Reranker` → `searchSessions({ rerank })`). + +## Why two stages + +Stage 1 alone gets the right session into the **top-5 ~86–92% of the time** — but +only to **~35% precision@1**. A bi-encoder compresses each session into one +query-agnostic vector, and broad-match session scoring lets big, verbose sessions +fill the #1 slot. So the right answer is *there*, just not first. + +That's the tell that **this is a ranking problem, not a recall problem** — and the +cross-encoder is the fix. Reading query and evidence together, it discriminates the +near-ties the first stage can't, pulling the right session to #1. It runs on only +~8 candidates, so it's cheap. + +## Results (measured on a real corpus) + +16 sessions / 11 workspaces / ~12k episodes, 37 hand-labeled fuzzy references, via +the re-runnable harness in `src/daemon/eval/`: + +| Cross-workspace resolution | P@1 | R@5 | p95 latency | +|---|---|---|---| +| Naive per-workspace merge | 21.6% | 73% | 4966 ms | +| + global fusion (stage 1) | 35.1% | 86% | 28 ms | +| **+ cross-encoder rerank (stage 2)** | **73.0%** | **92%** | **88 ms** | + +73% precision@1 on these pure-conceptual references (identifier-bearing references — +a PR number, a branch name — resolve higher still) approaches the 97.3% same-workspace +ceiling. Global fusion also made it **~200× faster** (one search + one query +embed, vs eleven per-workspace searches). Everything runs **locally** — no cloud, +no API. + +## Models (both local, both swappable) + +- **Embedder**: `Xenova/bge-small-en-v1.5` (384-d, ~50 MB WASM). Behind an `Embedder` + interface; BGE-M3 (dense + sparse in one pass) is the planned upgrade. +- **Reranker**: `Xenova/ms-marco-MiniLM-L-6-v2` cross-encoder (~22 MB WASM, English, + fast). Behind a `Reranker` interface; `bge-reranker-v2-m3` (multilingual) drops in + as a config swap. + +Both run via `@xenova/transformers` (pure WASM) — no native deps, no network at +query time after the one-time model download. + +## Code map + +| Piece | Where | +|---|---| +| Global recall + rerank orchestration | `src/daemon/memory/engine.ts` — `recallGlobal`, `searchSessions` | +| Cross-workspace store primitives | `src/daemon/memory/store.ts` — `ftsSearchGlobal`, `listWorkspaceIds`, `episodesByIds` | +| Hybrid ranker (vector+BM25+recency+path) | `src/daemon/memory/ranker.ts` | +| Reranker interface + cross-encoder | `src/daemon/memory/reranker.ts`, `reranker-transformersjs.ts` | +| Eval harness (precision@1 / MRR / recall@k) | `src/daemon/eval/{metrics,baseline}.ts`, `fixtures/` | + +## Reproduce / measure + +```sh +MEMORY_DB=~/.codeoid/memory.db bun run src/daemon/eval/baseline.ts +``` + +Prints within-workspace, naive-merge, global-fusion, and global+rerank regimes side +by side against the labeled fixture. + +## Design principles + +- **Evaluation-driven.** Every change is measured against a labeled fixture; the + precision@1 number is the gate. (The rerank was chosen *because* the eval showed + R@5 was already high — recall wasn't the problem.) +- **Degrade, don't die.** No embedder → FTS-only recall. No reranker → fusion-only + ranking. The daemon keeps working; only quality drops. +- **Local-first.** All models run on-device; nothing about your sessions leaves the + machine to resolve a reference. + +## Limitations & what's next + +- The measured corpus is small (16 sessions) — re-run the harness as usage grows. +- MiniLM is English-only; swap to `bge-reranker-v2-m3` + BGE-M3 for multilingual. +- Cross-workspace search is single-tenant today (all *your* workspaces); the + conductor's identity will scope it per tenant. +- Next retrieval upgrade: per-session **cards** (compact self-maintained digests) + + bi-temporal state, so ranking and rerank operate on clean session summaries rather + than raw episode text. diff --git a/package.json b/package.json index 2773f20..4e1e4bf 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "format": "biome format --write src/ packages/protocol/src/ packages/core/src/", "test": "bun test src/tests src/daemon packages/protocol/src packages/core/src", "test:coverage": "bun test src/tests src/daemon packages/protocol/src packages/core/src --coverage --coverage-reporter=lcov --reporter=junit --reporter-outfile=junit.xml", + "test:integration": "bun test src/integration", "test:web": "cd web && bun run test", "typecheck": "bun x tsc --noEmit && bun x tsc --noEmit -p packages/protocol && bun x tsc --noEmit -p packages/core", "prepublishOnly": "bun run build:web" diff --git a/packages/protocol/src/scopes.ts b/packages/protocol/src/scopes.ts index 5f0c199..7a339ce 100644 --- a/packages/protocol/src/scopes.ts +++ b/packages/protocol/src/scopes.ts @@ -22,6 +22,16 @@ export const SCOPES = { SESSION_DESTROY: "session:destroy", /** List sessions and their status */ SESSION_LIST: "session:list", + /** + * Read-class fleet visibility for the conductor — list, find, and + * summarize sessions across the fleet without the ability to act in them. + */ + SESSION_READ: "session:read", + /** + * Send-class fleet routing for the conductor — direct an existing session, + * interrupt it, or spawn a disposable worker on the owner's behalf. + */ + SESSION_DISPATCH: "session:dispatch", /** Read files and list directories under a session's workdir */ FS_READ: "fs:read", } as const; diff --git a/src/daemon/agent-identity.ts b/src/daemon/agent-identity.ts index ebdf0ab..b16109d 100644 --- a/src/daemon/agent-identity.ts +++ b/src/daemon/agent-identity.ts @@ -17,6 +17,7 @@ import { type RegisterAgentRequest, ZeroIDClient } from "@highflame/sdk"; import { generateAgentKeypair, signActorAssertion } from "./agent-assertion.js"; import type { AuthConfig } from "./auth.js"; +import { SCOPES } from "../protocol/scopes.js"; import type { Store } from "./store.js"; export interface AgentIdentityConfig { @@ -24,6 +25,12 @@ export interface AgentIdentityConfig { /** Account + project for ZeroID tenant scoping */ accountId: string; projectId: string; + /** + * Prefix for the conductor's ZeroID external_id. Defaults to + * "codeoid-conductor"; integration tests override it so their throwaway + * identities are unmistakable (codeoid-conductor-test-*) and sweepable. + */ + conductorExternalIdPrefix?: string; } interface RegisteredAgent { @@ -48,6 +55,20 @@ const AGENT_TOOL_SCOPES = [ "tools:agent", // Spawn sub-agents ] as const; +/** + * Conductor scope profile (design R1) — the conductor routes work, it never + * does work in a target session. Fleet visibility + dispatch only; the + * deliberate absence of `tools:write` / `tools:execute` means that even a + * fully-delegated chain rooted at the conductor can never mint a token that + * edits or runs anything in a target (ZeroID grants the *intersection* of the + * subject's scopes on every hop, so what the conductor lacks its whole + * subtree lacks — read-only-by-construction, made cryptographic). + */ +export const CONDUCTOR_SCOPES = [ + SCOPES.SESSION_READ, // list / find / summarize the fleet + SCOPES.SESSION_DISPATCH, // direct, interrupt, or spawn sessions +] as const; + /** Sub-agents get read-only by default unless explicitly promoted. */ const SUBAGENT_DEFAULT_SCOPES = ["tools:read"] as const; @@ -58,11 +79,41 @@ const SUBAGENT_SCOPE_MAP: Record = { Plan: ["tools:read"], }; +/** + * The durable conductor identity (design R2). Unlike session agents — which + * are disposable and die with their session — the conductor's identity is + * persisted to the Store and reloaded on daemon restart, so it keeps ONE + * stable WIMSE URI for its whole lifetime. Only the api_key rests on disk; + * the actor keypair is regenerated per process and re-registered with ZeroID. + */ +interface RegisteredConductor { + identityId: string; + wimseUri: string; + apiKey: string; + /** + * ZeroID client authed as the conductor — the delegation *subject* when + * the conductor later delegates to disposable child workers (P4). + */ + client: ZeroIDClient; + /** + * Signs the conductor's actor assertions so the *owner* can delegate + * `session:read session:dispatch` to it via RFC 8693 token exchange. + * Process-local: regenerated (and re-registered) on every boot. + */ + privateKey: CryptoKey; +} + +export interface ConductorIdentity { + identityId: string; + wimseUri: string; +} + export class AgentIdentityManager { #client: ZeroIDClient; #store: Store; #config: AgentIdentityConfig; #agents = new Map(); + #conductor?: RegisteredConductor; constructor(config: AgentIdentityConfig, store: Store) { this.#config = config; @@ -336,4 +387,251 @@ export class AgentIdentityManager { getAgentUri(sessionId: string): string | undefined { return this.#agents.get(sessionId)?.wimseUri; } + + // ── Conductor identity (durable, owner-delegated — design R1/R2) ────── + + /** The conductor's stable WIMSE URI, when one is registered/resumed. */ + get conductorUri(): string | undefined { + return this.#conductor?.wimseUri; + } + + /** + * Ensure the durable conductor identity exists: reuse the in-memory one, + * else reload the persisted one from the Store, else register a fresh + * identity in ZeroID (scope ceiling = CONDUCTOR_SCOPES) and persist it. + * Best-effort like the rest of the identity layer — returns null on + * failure so the daemon keeps working without a conductor identity. + */ + async registerConductor(ownerSub: string): Promise { + const resumed = await this.resumeConductor(); + if (resumed) return resumed; + + const prefix = this.#config.conductorExternalIdPrefix ?? "codeoid-conductor"; + // Unique per registration — durability comes from the Store row, not from + // external_id reuse (a deactivated identity keeps its external_id, so a + // stable one would collide on legitimate re-registration). + const externalId = `${prefix}-${crypto.randomUUID().slice(0, 8)}`; + + try { + // The conductor is both a delegation *actor* (the owner delegates + // session:read/session:dispatch TO it) and a future *subject* (it + // delegates onward to disposable child workers) — so it registers a + // public key for actor assertions AND gets an api_key for the + // orchestrator client. (`allowed_scopes` / `public_key_pem` are + // server-accepted but missing from the SDK type through 0.3.17.) + const keypair = await generateAgentKeypair(); + const registerReq = { + name: "codeoid/conductor", + external_id: externalId, + sub_type: "orchestrator" as const, + trust_level: "first_party" as const, + framework: "claude-agent-sdk", + publisher: "codeoid", + created_by: ownerSub, + allowed_scopes: [...CONDUCTOR_SCOPES], + public_key_pem: keypair.publicKeyPem, + metadata: JSON.stringify({ role: "conductor", owner: ownerSub }), + }; + const resp = await this.#client.agents.register( + registerReq as RegisterAgentRequest, + ); + + // Persist BEFORE exposing the in-memory identity: if the Store write + // fails, callers must see the registration as failed rather than a + // "durable" conductor that would vanish on the next restart. + this.#store.saveConductorIdentity({ + accountId: this.#config.accountId, + projectId: this.#config.projectId, + identityId: resp.identity.id, + wimseUri: resp.identity.wimse_uri, + apiKey: resp.api_key, + }); + this.#conductor = { + identityId: resp.identity.id, + wimseUri: resp.identity.wimse_uri, + apiKey: resp.api_key, + client: this.#clientForAgent(resp.api_key), + privateKey: keypair.privateKey, + }; + + this.#store.audit( + resp.identity.wimse_uri, + "conductor.identity.registered", + undefined, + `external_id=${externalId} owner=${ownerSub}`, + ); + + return { identityId: resp.identity.id, wimseUri: resp.identity.wimse_uri }; + } catch (err) { + console.error( + "[codeoid] failed to register conductor identity:", + err instanceof Error ? err.message : err, + ); + return null; + } + } + + /** + * Reload the persisted conductor identity on daemon restart (called from + * SessionManager.resumeSessions). Reuses the stored identity — same + * identityId, same WIMSE URI — instead of re-registering, and rotates the + * process-local actor keypair with ZeroID so owner→conductor delegation + * keeps working. Returns null when nothing is persisted or the stored + * identity is no longer usable (deactivated / api_key revoked), in which + * case the stale row is dropped so the next registerConductor starts clean. + */ + async resumeConductor(): Promise { + if (this.#conductor) { + return { + identityId: this.#conductor.identityId, + wimseUri: this.#conductor.wimseUri, + }; + } + + const row = this.#store.getConductorIdentity( + this.#config.accountId, + this.#config.projectId, + ); + if (!row) return null; + + try { + // Liveness probe: minting from the api_key fails iff the identity was + // deactivated or the key revoked — exactly the cases where the stored + // row is dead and a fresh registration is the right call. + await this.#client.tokens.issueApiKey(row.apiKey); + } catch (err) { + console.error( + "[codeoid] persisted conductor identity is no longer usable, dropping:", + err instanceof Error ? err.message : err, + ); + this.#store.deleteConductorIdentity( + this.#config.accountId, + this.#config.projectId, + ); + this.#store.audit( + row.wimseUri, + "conductor.identity.dropped_stale", + undefined, + `identity_id=${row.identityId}`, + ); + return null; + } + + // The previous process's actor private key died with it — register a + // fresh public key under the SAME identity so assertions keep verifying. + // Durable identity, ephemeral keys. + const keypair = await generateAgentKeypair(); + try { + await this.#client.identities.update(row.identityId, { + public_key_pem: keypair.publicKeyPem, + }); + } catch (err) { + // Keep the resumed identity (URI stability wins) — delegation will + // surface a clear assertion-verification error if this mattered. + console.error( + "[codeoid] failed to rotate conductor actor key (delegation may fail):", + err instanceof Error ? err.message : err, + ); + } + + this.#conductor = { + identityId: row.identityId, + wimseUri: row.wimseUri, + apiKey: row.apiKey, + client: this.#clientForAgent(row.apiKey), + privateKey: keypair.privateKey, + }; + + this.#store.audit( + row.wimseUri, + "conductor.identity.resumed", + undefined, + `identity_id=${row.identityId}`, + ); + + return { identityId: row.identityId, wimseUri: row.wimseUri }; + } + + /** + * Mint the conductor's working token by OWNER delegation (RFC 8693): the + * owner's subject token grants, the conductor's self-signed assertion + * acts. The result carries `delegation_depth: 1`, an `act` chain rooted at + * the owner, and at most CONDUCTOR_SCOPES (three-way scope intersection) — + * so the conductor acts on the owner's behalf, never on its own authority, + * and deactivating either end kills the token. + */ + async mintConductorToken(ownerSubjectToken: string): Promise { + const conductor = this.#conductor; + if (!conductor) return null; + + try { + const assertion = await signActorAssertion( + conductor.privateKey, + conductor.wimseUri, + this.#config.auth.baseUrl, + ); + const resp = await this.#client.tokens.issueTokenExchange( + ownerSubjectToken, + assertion, + { scope: CONDUCTOR_SCOPES.join(" ") }, + ); + this.#store.audit( + conductor.wimseUri, + "conductor.token.delegated", + undefined, + `scope=${resp.scope ?? ""}`, + ); + return resp.access_token; + } catch (err) { + console.error( + "[codeoid] owner->conductor delegation failed:", + err instanceof Error ? err.message : err, + ); + return null; + } + } + + /** + * Deactivate the conductor identity. ZeroID cascade-revokes every active + * credential in its delegation subtree (owner-delegated conductor token, + * child-worker tokens, their sub-agent tokens) via the parent_jti chain — + * one call kills the whole tree. On success the persisted row is cleared + * so the next registerConductor starts a fresh identity; on failure the + * row is KEPT — it's the only durable record of an identity that is still + * live in ZeroID, and a later deactivateConductor() retries against it. + */ + async deactivateConductor(): Promise { + const conductor = this.#conductor; + const row = + conductor ?? + this.#store.getConductorIdentity( + this.#config.accountId, + this.#config.projectId, + ); + if (!row) return; + + // Stop using the identity locally regardless of the remote outcome — + // the caller's intent is deactivation. + this.#conductor = undefined; + + try { + await this.#client.agents.deactivate(row.identityId); + } catch (err) { + console.error( + "[codeoid] failed to deactivate conductor identity (row kept for retry):", + err instanceof Error ? err.message : err, + ); + return; + } + this.#store.audit( + row.wimseUri, + "conductor.identity.deactivated", + undefined, + `identity_id=${row.identityId}`, + ); + this.#store.deleteConductorIdentity( + this.#config.accountId, + this.#config.projectId, + ); + } } diff --git a/src/daemon/eval/BASELINE.md b/src/daemon/eval/BASELINE.md new file mode 100644 index 0000000..c681c7d --- /dev/null +++ b/src/daemon/eval/BASELINE.md @@ -0,0 +1,69 @@ +# P0 Baseline — session resolution + +Measured against the **real corpus** (16 sessions / 11 workspaces / 11,938 +episodes, pulled from the Hetzner daemon's `memory.db`) with 37 hand-labeled fuzzy +references (`fixtures/session-resolution.json`), using the resolver +(`MemoryEngine.searchSessions`, `Xenova/bge-small-en-v1.5` embedder + +`Xenova/ms-marco-MiniLM-L-6-v2` reranker), on the current-main base. Reproduce: + +```sh +MEMORY_DB=/path/to/memory.db bun run src/daemon/eval/baseline.ts +``` + +> The references are deliberately **pure-conceptual** (no exact identifiers — no PR +> numbers, branch names, or error strings). That is the *conservative* case: +> exact identifiers are strong lexical signal, so identifier-bearing references +> resolve meaningfully higher. This fixture measures the hard, fuzzy-only floor. + +## Results + +| Regime | P@1 | MRR | R@3 | R@5 | p50/p95 | +|---|---|---|---|---|---| +| **Within-workspace** (you already know the repo) | 97.3% | 0.986 | 100% | 100% | 406 / 1248 ms | +| **Cross-workspace** — naive per-ws merge (BASELINE) | **21.6%** | 0.440 | 62.2% | 73.0% | 2440 / 4966 ms | +| **Cross-workspace** — GLOBAL fusion (P1 slice 1) | **35.1%** | 0.578 | 78.4% | 86.5% | 10 / 28 ms | +| **Cross-workspace** — GLOBAL + rerank (P1 slice 2) | **73.0%** | 0.820 | 89.2% | 91.9% | ~43 / 88 ms | + +## The finding — a fusion/ranking problem, not a recall problem + +- **Within a workspace the resolver is already strong** (97% P@1): the hybrid recall + primitives are sound. +- **Cross-workspace the naive merge collapses to 21.6% P@1.** Failure mode: the bulk + of the misses return the same wrong session at #1 (two *small* workspaces, by + opaque id). `searchSessions` normalizes BM25 **batch-relative per workspace**, so a + small workspace yields inflated scores that dominate a naive cross-workspace merge, + regardless of semantic relevance. Scores are not comparable across workspaces. +- **R@5 stays ~86–92% throughout** — so the right session is almost always in the top + few; the *ranking* is what's broken, not recall. + +## P1 slice 1 — global fusion + native cross-workspace (DONE) + +`engine.recallGlobal()` unions FTS + vector candidates across all workspaces and +ranks them in ONE batch (BM25 normalization is global); `searchSessions()` goes +global when no `workspaceId` is passed. Effect vs naive merge: + +- **Latency 4966 → 28 ms p95** (~200×): one query embed + one global search instead + of 11 per-workspace searches. Comfortably under the 2 s budget. +- **R@5 73% → 86.5%**, R@3 62% → 78% (better, globally-comparable candidate pool). +- **MRR 0.44 → 0.58**; **P@1 21.6% → 35.1%** (modest — big verbose sessions now fill + the #1 slot; the misses are mostly rank 2–3). + +## P1 slice 2 — cross-encoder rerank (DONE) — gate cleared + +A `Reranker` interface + a transformers.js cross-encoder (`ms-marco-MiniLM-L-6-v2`, +swappable for bge-reranker-v2-m3). The engine reranks the top-8 candidate sessions +by (query, evidence) when a reranker is present; `searchSessions({ rerank })` gates +it. Effect: + +- **P@1 35.1% → 73.0%**, MRR 0.58 → 0.82, R@3 78% → 89%. Latency +~30 ms (88 ms p95). +- Converts the ~92% R@5 into precision@1 by reading (query, session) jointly — the + near-ties the first stage can't resolve. + +**P1 go/no-go gate: CLEARED** — cross-workspace P@1 **21.6% → 73.0%** on pure-fuzzy +references (identifier-bearing references land higher still), p95 < 100 ms vs the 2 s +budget. Remaining P1 slices (BGE-M3 embedder, identifier-aware lexical, session +cards) are optional polish now — revisit if the number regresses as the corpus grows. + +> Corpus note: 16 sessions is a modest snapshot; re-run as usage grows. `memory.db` +> is **not** committed (real session content); only the derived (genericized) +> fuzzy-reference fixture is. diff --git a/src/daemon/eval/baseline.ts b/src/daemon/eval/baseline.ts new file mode 100644 index 0000000..c97a5a9 --- /dev/null +++ b/src/daemon/eval/baseline.ts @@ -0,0 +1,154 @@ +/** + * P0 baseline runner — measures the CURRENT session resolver against a labeled + * fixture, establishing the precision@1 number that P1 must beat. + * + * The current resolver (`MemoryEngine.searchSessions`) is **workspace-scoped**, + * so we measure two regimes: + * - **within-workspace** — query the gold session's workspace only. Upper bound + * of today's capability (assumes you already know which repo). + * - **cross-workspace** — the conductor's real need. Today's code can't do this + * natively, so the baseline is the naive "run per-workspace, merge by + * aggregateScore" approach P1 replaces with global fusion + rerank. + * + * Run (points at a real memory.db + the local BGE-small model cache): + * MEMORY_DB=/path/to/memory.db bun run src/daemon/eval/baseline.ts + * optional: FIXTURE=... MODEL_CACHE=~/.codeoid/models + */ + +import { Database } from "bun:sqlite"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { SqliteEpisodeStore } from "../memory/store.js"; +import { createEmbedder } from "../memory/embedder.js"; +import { createReranker } from "../memory/reranker.js"; +import { MemoryEngine, type SessionSearchHit } from "../memory/engine.js"; +import { precisionAt1, mrr, recallAtK, percentile, type EvalCase } from "./metrics.js"; + +interface FixtureCase { + reference: string; + expectedSessionId: string; + expectedWorkspaceId: string; +} + +const HOME = process.env.HOME ?? ""; +const MEMORY_DB = process.env.MEMORY_DB ?? `${HOME}/.codeoid/memory.db`; +const MODEL_CACHE = process.env.MODEL_CACHE ?? `${HOME}/.codeoid/models`; +const FIXTURE = + process.env.FIXTURE ?? + fileURLToPath(new URL("./fixtures/session-resolution.json", import.meta.url)); + +const cases: FixtureCase[] = JSON.parse(readFileSync(FIXTURE, "utf8")); +const evalCases: EvalCase[] = cases.map((c) => ({ + reference: c.reference, + expectedSessionId: c.expectedSessionId, +})); + +const store = new SqliteEpisodeStore(MEMORY_DB); +const embedder = await createEmbedder({ cacheDir: MODEL_CACHE }); +const reranker = await createReranker({ cacheDir: MODEL_CACHE }); +const engine = new MemoryEngine({ store, embedder, reranker }); +await engine.init(); + +// Enumerate workspaces + count sessions straight from the DB (read-only). +const raw = new Database(MEMORY_DB, { readonly: true }); +const workspaces = ( + raw.query("SELECT DISTINCT workspace_id AS w FROM episodes").all() as { w: string }[] +).map((r) => r.w); +const sessionCount = ( + raw.query("SELECT count(DISTINCT session_id) AS c FROM episodes").get() as { c: number } +).c; + +/** Cross-workspace: run per-workspace searchSessions, merge, rank by aggregateScore. */ +async function crossRank(query: string): Promise { + const all: SessionSearchHit[] = []; + for (const ws of workspaces) { + all.push(...(await engine.searchSessions({ query, workspaceId: ws, limit: 10 }))); + } + const best = new Map(); + for (const h of all) { + const cur = best.get(h.sessionId); + if (cur === undefined || h.aggregateScore > cur) best.set(h.sessionId, h.aggregateScore); + } + return [...best.entries()].sort((a, b) => b[1] - a[1]).map((e) => e[0]); +} + +/** Within-workspace: query only the gold session's workspace. */ +async function withinRank(query: string, ws: string): Promise { + const hits = await engine.searchSessions({ query, workspaceId: ws, limit: 10 }); + return hits.map((h) => h.sessionId); +} + +/** Cross-workspace GLOBAL fusion, no rerank (P1 slice 1). */ +async function globalRank(query: string): Promise { + const hits = await engine.searchSessions({ query, limit: 15, rerank: false }); + return hits.map((h) => h.sessionId); +} + +/** Cross-workspace GLOBAL fusion + cross-encoder rerank (P1 slice 2). */ +async function globalRerankRank(query: string): Promise { + const hits = await engine.searchSessions({ query, limit: 15, rerank: true }); + return hits.map((h) => h.sessionId); +} + +const crossRanked: string[][] = []; +const crossLat: number[] = []; +const withinRanked: string[][] = []; +const withinLat: number[] = []; +const globalRanked: string[][] = []; +const globalLat: number[] = []; +const rerankRanked: string[][] = []; +const rerankLat: number[] = []; + +for (const c of cases) { + let t = performance.now(); + crossRanked.push(await crossRank(c.reference)); + crossLat.push(performance.now() - t); + + t = performance.now(); + withinRanked.push(await withinRank(c.reference, c.expectedWorkspaceId)); + withinLat.push(performance.now() - t); + + t = performance.now(); + globalRanked.push(await globalRank(c.reference)); + globalLat.push(performance.now() - t); + + t = performance.now(); + rerankRanked.push(await globalRerankRank(c.reference)); + rerankLat.push(performance.now() - t); +} + +function report(label: string, ranked: string[][], lat: number[]): void { + const pct = (x: number) => `${(x * 100).toFixed(1)}%`; + console.log(`\n${label}`); + console.log(` P@1 = ${pct(precisionAt1(evalCases, ranked))} MRR = ${mrr(evalCases, ranked).toFixed(3)}`); + console.log(` R@3 = ${pct(recallAtK(evalCases, ranked, 3))} R@5 = ${pct(recallAtK(evalCases, ranked, 5))}`); + console.log(` latency p50/p95 = ${percentile(lat, 50).toFixed(0)}/${percentile(lat, 95).toFixed(0)} ms`); +} + +console.log("═".repeat(72)); +console.log( + `Baseline: ${sessionCount} sessions across ${workspaces.length} workspaces; ${cases.length} labeled references`, +); +console.log(`Embedder: ${embedder.modelName} (${embedder.dimensions}d)`); +report("WITHIN-workspace (upper bound — you already know the repo):", withinRanked, withinLat); +report("CROSS-workspace (BASELINE — naive per-ws merge):", crossRanked, crossLat); +report("CROSS-workspace GLOBAL fusion (P1 slice 1 — no rerank):", globalRanked, globalLat); +report("CROSS-workspace GLOBAL + rerank (P1 slice 2):", rerankRanked, rerankLat); + +console.log("\nCross-workspace GLOBAL+rerank (P1) P@1 misses:"); +let misses = 0; +cases.forEach((c, i) => { + const top = rerankRanked[i]?.[0]; + if (top !== c.expectedSessionId) { + misses++; + const r = rerankRanked[i]?.indexOf(c.expectedSessionId) ?? -1; + console.log( + ` ✗ "${c.reference.slice(0, 58)}" → got ${top?.slice(0, 8) ?? "∅"}, want ${c.expectedSessionId.slice(0, 8)} (rank ${r < 0 ? "NF" : r + 1})`, + ); + } +}); +if (misses === 0) console.log(" (none)"); +console.log("═".repeat(72)); + +await engine.close(); +raw.close(); diff --git a/src/daemon/eval/fixtures/session-resolution.json b/src/daemon/eval/fixtures/session-resolution.json new file mode 100644 index 0000000..2fa7486 --- /dev/null +++ b/src/daemon/eval/fixtures/session-resolution.json @@ -0,0 +1,53 @@ +[ + {"reference": "the work to run multiple model providers behind one interface", "expectedSessionId": "2ad22a46-2bcc-4dcb-9d48-0441558f8c12", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "making the harness support codex and gemini as pluggable backends", "expectedSessionId": "2ad22a46-2bcc-4dcb-9d48-0441558f8c12", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "the common provider-interface feature for multi-provider support", "expectedSessionId": "2ad22a46-2bcc-4dcb-9d48-0441558f8c12", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + + {"reference": "the telegram bug where switching sessions did not change the view", "expectedSessionId": "65f645ce-6095-4b6b-a22b-f4fd9cb794b0", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "the session-switch fix with unit and integration tests", "expectedSessionId": "65f645ce-6095-4b6b-a22b-f4fd9cb794b0", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "fixing the telegram UI not updating when I switch sessions", "expectedSessionId": "65f645ce-6095-4b6b-a22b-f4fd9cb794b0", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + + {"reference": "the bug where sending a message mid-turn cancelled the whole conversation", "expectedSessionId": "96e93b8c-82a0-4375-b5ea-18cb7f9e0da4", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "the harness ending the conversation when I send another message during a reply", "expectedSessionId": "96e93b8c-82a0-4375-b5ea-18cb7f9e0da4", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "the mid-turn interruption and cancelled tool call issue", "expectedSessionId": "96e93b8c-82a0-4375-b5ea-18cb7f9e0da4", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + + {"reference": "the deep comparison against the swarmclaw project", "expectedSessionId": "c94d2cdb-1e16-47f9-8d5c-c457d4c70cdf", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "comparing our session recall to how txtai does it", "expectedSessionId": "c94d2cdb-1e16-47f9-8d5c-c457d4c70cdf", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + {"reference": "the swarmclaw analysis where we filed issues for high-value features", "expectedSessionId": "c94d2cdb-1e16-47f9-8d5c-c457d4c70cdf", "expectedWorkspaceId": "ws_1a7cf3ca10b7c8ad"}, + + {"reference": "the research on high-leverage AI agents to adopt", "expectedSessionId": "1f32d975-4533-4615-97fc-62853a6b9a43", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "the epic ticket planning agent work in build order", "expectedSessionId": "1f32d975-4533-4615-97fc-62853a6b9a43", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "what changed yesterday and the agents-strategy planning", "expectedSessionId": "1f32d975-4533-4615-97fc-62853a6b9a43", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + + {"reference": "implementing the unified policy configuration screen", "expectedSessionId": "6ebcf4d5-6bbb-41c7-babb-f62464a168aa", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "the policy-unification work in the architecture docs", "expectedSessionId": "6ebcf4d5-6bbb-41c7-babb-f62464a168aa", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "the unified policy screen implementation from the design doc", "expectedSessionId": "6ebcf4d5-6bbb-41c7-babb-f62464a168aa", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + + {"reference": "prioritizing the simplest bugs on my sprint board", "expectedSessionId": "8ddeafbd-fb9c-42ee-8670-d6fcc1f71a6d", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "the sprint-board bug triage for the week", "expectedSessionId": "8ddeafbd-fb9c-42ee-8670-d6fcc1f71a6d", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + {"reference": "triaging the bugs assigned to me this sprint", "expectedSessionId": "8ddeafbd-fb9c-42ee-8670-d6fcc1f71a6d", "expectedWorkspaceId": "ws_7863bab6bc25eb7a"}, + + {"reference": "changing a service's model provider to a GLM model", "expectedSessionId": "c7557f9f-9600-4063-be9b-382038d7627a", "expectedWorkspaceId": "ws_6c0a43399afe4436"}, + {"reference": "the provider switch that reads its API key from an env var", "expectedSessionId": "c7557f9f-9600-4063-be9b-382038d7627a", "expectedWorkspaceId": "ws_6c0a43399afe4436"}, + + {"reference": "comparing the Hermes agent's sandboxing against Claude Code", "expectedSessionId": "591fc609-8904-4773-a763-6c32fe60093c", "expectedWorkspaceId": "ws_4e1e0ef9653e8b21"}, + {"reference": "the agent sandboxing-feature comparison notes", "expectedSessionId": "591fc609-8904-4773-a763-6c32fe60093c", "expectedWorkspaceId": "ws_4e1e0ef9653e8b21"}, + + {"reference": "the SDK audit of what's broken after the new agent-registration APIs", "expectedSessionId": "a5ecf55a-5b4d-4b21-bdc4-958f392edd87", "expectedWorkspaceId": "ws_5fc1444e46534d62"}, + {"reference": "auditing the SDK for required changes", "expectedSessionId": "a5ecf55a-5b4d-4b21-bdc4-958f392edd87", "expectedWorkspaceId": "ws_5fc1444e46534d62"}, + + {"reference": "learning about a WIMSE agent-identity project and its mission-intent linking", "expectedSessionId": "b6b56014-ad0d-4a85-b621-b56fbc007bd0", "expectedWorkspaceId": "ws_17b4e88098eca64c"}, + {"reference": "the agent-identity mission-intent work", "expectedSessionId": "b6b56014-ad0d-4a85-b621-b56fbc007bd0", "expectedWorkspaceId": "ws_17b4e88098eca64c"}, + + {"reference": "the vulnerability agentic harness we cloned to evaluate", "expectedSessionId": "8f3bfa46-b21d-4842-84e8-e2a2575c9805", "expectedWorkspaceId": "ws_e7e44be431f8b89a"}, + {"reference": "evaluating an agentic vulnerability harness for the regression suite", "expectedSessionId": "8f3bfa46-b21d-4842-84e8-e2a2575c9805", "expectedWorkspaceId": "ws_e7e44be431f8b89a"}, + + {"reference": "the viral post announcing open-sourcing a project", "expectedSessionId": "1f31f357-7fe3-4380-ae2b-251c4d09ccb2", "expectedWorkspaceId": "ws_edaab3bb0114ef1c"}, + {"reference": "the linkedin post about open sourcing, fixing the hook line", "expectedSessionId": "1f31f357-7fe3-4380-ae2b-251c4d09ccb2", "expectedWorkspaceId": "ws_edaab3bb0114ef1c"}, + + {"reference": "the remote part-time software dev job search", "expectedSessionId": "9dce1005-6cff-4ba9-82bf-817926cbbeae", "expectedWorkspaceId": "ws_a3cd85f9be85b73d"}, + {"reference": "finding a part-time developer role to supplement income", "expectedSessionId": "9dce1005-6cff-4ba9-82bf-817926cbbeae", "expectedWorkspaceId": "ws_a3cd85f9be85b73d"}, + + {"reference": "alternatives to the IDE-hooks approach for capturing agent events", "expectedSessionId": "4e2614f7-b251-4d65-9d71-feef414102b1", "expectedWorkspaceId": "ws_d478dd6869244b16"}, + {"reference": "the code-agents feature as an alternative to installing IDE hooks", "expectedSessionId": "4e2614f7-b251-4d65-9d71-feef414102b1", "expectedWorkspaceId": "ws_d478dd6869244b16"} +] diff --git a/src/daemon/eval/metrics.test.ts b/src/daemon/eval/metrics.test.ts new file mode 100644 index 0000000..6efde13 --- /dev/null +++ b/src/daemon/eval/metrics.test.ts @@ -0,0 +1,57 @@ +import { describe, test, expect } from "bun:test"; +import { + precisionAt1, + mrr, + recallAtK, + percentile, + runEval, + type EvalCase, +} from "./metrics"; + +const cases: EvalCase[] = [ + { reference: "the authz latest_only fix", expectedSessionId: "A" }, + { reference: "the durga extraction eval", expectedSessionId: "B" }, + { reference: "studio receipt badge", expectedSessionId: "C" }, +]; + +// A: correct top-1. B: correct at rank 3. C: not found. +const ranked: string[][] = [ + ["A", "X", "Y"], + ["X", "Y", "B"], + ["X", "Y", "Z"], +]; + +describe("eval metrics", () => { + test("precision@1 counts only top-1 hits", () => { + expect(precisionAt1(cases, ranked)).toBeCloseTo(1 / 3, 5); + }); + + test("MRR averages reciprocal ranks (1 + 1/3 + 0)/3", () => { + expect(mrr(cases, ranked)).toBeCloseTo((1 + 1 / 3 + 0) / 3, 5); + }); + + test("recall@k = hit@k for known-item", () => { + expect(recallAtK(cases, ranked, 1)).toBeCloseTo(1 / 3, 5); // only A in top-1 + expect(recallAtK(cases, ranked, 5)).toBeCloseTo(2 / 3, 5); // A and B, not C + }); + + test("percentile is nearest-rank", () => { + const xs = [10, 20, 30, 40, 100]; + expect(percentile(xs, 50)).toBe(30); + expect(percentile(xs, 100)).toBe(100); + expect(percentile([], 50)).toBe(0); + }); + + test("runEval wires a resolver into a full report", async () => { + const byRef = new Map([ + [cases[0]!.reference, ranked[0]!], + [cases[1]!.reference, ranked[1]!], + [cases[2]!.reference, ranked[2]!], + ]); + const report = await runEval((ref) => byRef.get(ref) ?? [], cases); + expect(report.n).toBe(3); + expect(report.precisionAt1).toBeCloseTo(1 / 3, 5); + expect(report.recallAt5).toBeCloseTo(2 / 3, 5); + expect(report.p95Ms).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/src/daemon/eval/metrics.ts b/src/daemon/eval/metrics.ts new file mode 100644 index 0000000..bc512e7 --- /dev/null +++ b/src/daemon/eval/metrics.ts @@ -0,0 +1,115 @@ +/** + * Session-resolution eval harness — metrics. + * + * The conductor's linchpin capability (docs/conductor-session-resolution.md) is + * only trustworthy against a labeled set. This module measures a resolver: + * given a fuzzy natural-language reference, does it rank the correct session + * first (precision@1), and how fast? + * + * Known-item retrieval: each case has exactly ONE correct session, so recall@k + * equals hit@k. Precision@1 is the metric that matters — a wrong top-1 routes a + * command to the wrong repo. + */ + +/** One labeled example: a reference and the session it should resolve to. */ +export interface EvalCase { + reference: string; + expectedSessionId: string; +} + +/** A resolver returns session ids in rank order (best first). */ +export type Resolver = (reference: string) => Promise | string[]; + +export interface EvalReport { + n: number; + precisionAt1: number; + mrr: number; + recallAt5: number; + recallAt10: number; + p50Ms: number; + p95Ms: number; +} + +/** Rank of the expected id within a ranked list (0-based), or -1 if absent. */ +function rankOf(expected: string, ranked: string[]): number { + return ranked.indexOf(expected); +} + +/** Fraction of cases whose top-1 result is the expected session. */ +export function precisionAt1(cases: EvalCase[], ranked: string[][]): number { + if (cases.length === 0) return 0; + let hits = 0; + for (let i = 0; i < cases.length; i++) { + if (ranked[i]?.[0] === cases[i]!.expectedSessionId) hits++; + } + return hits / cases.length; +} + +/** Mean reciprocal rank — 1/(rank+1) averaged over cases (0 if not found). */ +export function mrr(cases: EvalCase[], ranked: string[][]): number { + if (cases.length === 0) return 0; + let sum = 0; + for (let i = 0; i < cases.length; i++) { + const r = rankOf(cases[i]!.expectedSessionId, ranked[i] ?? []); + if (r >= 0) sum += 1 / (r + 1); + } + return sum / cases.length; +} + +/** Fraction of cases whose expected session appears in the top-k (= hit@k). */ +export function recallAtK(cases: EvalCase[], ranked: string[][], k: number): number { + if (cases.length === 0) return 0; + let hits = 0; + for (let i = 0; i < cases.length; i++) { + const r = rankOf(cases[i]!.expectedSessionId, ranked[i] ?? []); + if (r >= 0 && r < k) hits++; + } + return hits / cases.length; +} + +/** Nearest-rank percentile (p in [0,100]) of a numeric sample. */ +export function percentile(values: number[], p: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const idx = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil((p / 100) * sorted.length) - 1), + ); + return sorted[idx]!; +} + +/** + * Run a resolver over every case, timing each call, and compute the report. + * Resolvers may be sync or async. + */ +export async function runEval(resolver: Resolver, cases: EvalCase[]): Promise { + const ranked: string[][] = []; + const latencies: number[] = []; + + for (const c of cases) { + const t0 = performance.now(); + const result = await resolver(c.reference); + latencies.push(performance.now() - t0); + ranked.push(result); + } + + return { + n: cases.length, + precisionAt1: precisionAt1(cases, ranked), + mrr: mrr(cases, ranked), + recallAt5: recallAtK(cases, ranked, 5), + recallAt10: recallAtK(cases, ranked, 10), + p50Ms: percentile(latencies, 50), + p95Ms: percentile(latencies, 95), + }; +} + +/** One-line human-readable summary for logging a run. */ +export function formatReport(r: EvalReport): string { + const pct = (x: number) => `${(x * 100).toFixed(1)}%`; + return ( + `n=${r.n} P@1=${pct(r.precisionAt1)} MRR=${r.mrr.toFixed(3)} ` + + `R@5=${pct(r.recallAt5)} R@10=${pct(r.recallAt10)} ` + + `p50=${r.p50Ms.toFixed(0)}ms p95=${r.p95Ms.toFixed(0)}ms` + ); +} diff --git a/src/daemon/memory/cards.test.ts b/src/daemon/memory/cards.test.ts new file mode 100644 index 0000000..5373dd4 --- /dev/null +++ b/src/daemon/memory/cards.test.ts @@ -0,0 +1,156 @@ +import { describe, test, expect } from "bun:test"; +import { SessionCardStore } from "./cards"; + +function freshStore(): SessionCardStore { + return new SessionCardStore(":memory:"); +} + +describe("SessionCardStore — cards", () => { + test("upsert + get round-trips all fields", () => { + const s = freshStore(); + s.upsertCard({ + sessionId: "sess-1", + workspaceId: "ws-a", + repo: "highflame-authz", + branch: "fix/latest-only", + task: "fix the latest_only tenant-scope bug", + state: "WIP", + lastAction: "wrote shared latestVersionIDs helper", + openThreads: ["add regression test"], + entities: ["studio#870", "latest_only"], + createdAt: 1000, + updatedAt: 1000, + }); + + const c = s.getCard("sess-1")!; + expect(c.repo).toBe("highflame-authz"); + expect(c.branch).toBe("fix/latest-only"); + expect(c.state).toBe("WIP"); + expect(c.openThreads).toEqual(["add regression test"]); + expect(c.entities).toEqual(["studio#870", "latest_only"]); + expect(c.createdAt).toBe(1000); + s.close(); + }); + + test("upsert preserves createdAt but advances updatedAt", () => { + const s = freshStore(); + s.upsertCard({ sessionId: "s", workspaceId: "w", openThreads: [], entities: [], createdAt: 100, updatedAt: 100 }); + s.upsertCard({ sessionId: "s", workspaceId: "w", state: "merged", openThreads: [], entities: [], updatedAt: 200 }); + const c = s.getCard("s")!; + expect(c.createdAt).toBe(100); + expect(c.updatedAt).toBe(200); + expect(c.state).toBe("merged"); + s.close(); + }); + + test("FTS finds a card by an exact identifier and by a fuzzy term", () => { + const s = freshStore(); + s.upsertCard({ + sessionId: "sess-870", + workspaceId: "ws-a", + repo: "highflame-studio", + task: "latest_only tenant scope regression", + openThreads: [], + entities: ["studio#870"], + updatedAt: 1, + }); + s.upsertCard({ + sessionId: "sess-durga", + workspaceId: "ws-b", + repo: "durga", + task: "extraction eval field accuracy", + openThreads: [], + entities: [], + updatedAt: 2, + }); + + const byId = s.ftsSearchCards("studio#870"); + expect(byId[0]?.sessionId).toBe("sess-870"); + + const byFuzzy = s.ftsSearchCards("extraction eval"); + expect(byFuzzy[0]?.sessionId).toBe("sess-durga"); + s.close(); + }); + + test("listCards orders by updatedAt desc and scopes by workspace", () => { + const s = freshStore(); + s.upsertCard({ sessionId: "a", workspaceId: "w1", openThreads: [], entities: [], updatedAt: 10 }); + s.upsertCard({ sessionId: "b", workspaceId: "w1", openThreads: [], entities: [], updatedAt: 30 }); + s.upsertCard({ sessionId: "c", workspaceId: "w2", openThreads: [], entities: [], updatedAt: 20 }); + + expect(s.listCards().map((c) => c.sessionId)).toEqual(["b", "c", "a"]); + expect(s.listCards(50, "w1").map((c) => c.sessionId)).toEqual(["b", "a"]); + s.close(); + }); +}); + +describe("SessionCardStore — bi-temporal facts", () => { + test("assertFact supersedes: invalidate-don't-delete", () => { + const s = freshStore(); + const subject = "session:sess-1"; + + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", validAt: 100, now: 100 }); + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "merged", validAt: 200, now: 210 }); + + // Current belief = merged, single open fact. + const cur = s.currentFacts(subject); + expect(cur).toHaveLength(1); + expect(cur[0]!.object).toBe("merged"); + + // Nothing deleted — both versions still on disk. + const all = s.factsForSession("sess-1"); + expect(all).toHaveLength(2); + const wip = all.find((f) => f.object === "WIP")!; + expect(wip.invalidAt).toBe(200); // closed at the new fact's validAt (event time) + expect(wip.expiredAt).toBe(210); // superseded at now (system time) + s.close(); + }); + + test("factsAsOf time-travels to the right version", () => { + const s = freshStore(); + const subject = "session:sess-1"; + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", validAt: 100, now: 100 }); + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "merged", validAt: 200, now: 200 }); + + expect(s.factsAsOf(subject, 150).map((f) => f.object)).toEqual(["WIP"]); + expect(s.factsAsOf(subject, 250).map((f) => f.object)).toEqual(["merged"]); + expect(s.factsAsOf(subject, 50)).toHaveLength(0); // before anything was true + s.close(); + }); + + test("re-asserting the same object is a no-op (no churn)", () => { + const s = freshStore(); + const subject = "session:sess-1"; + const a = s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", validAt: 100, now: 100 }); + const b = s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", validAt: 150, now: 150 }); + expect(b.id).toBe(a.id); + expect(s.factsForSession("sess-1")).toHaveLength(1); + s.close(); + }); + + test("distinct predicates coexist as open facts", () => { + const s = freshStore(); + const subject = "session:sess-1"; + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", now: 1 }); + s.assertFact({ sessionId: "sess-1", subject, predicate: "branch", object: "fix/x", now: 1 }); + expect(s.currentFacts(subject)).toHaveLength(2); + s.close(); + }); + + test("out-of-order validAt is rejected — never corrupts the open fact", () => { + const s = freshStore(); + const subject = "session:sess-1"; + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "merged", validAt: 200, now: 200 }); + + // Closing the open fact at validAt=100 would set invalid_at < valid_at, + // making it unsatisfiable for every factsAsOf() — reject instead. + expect(() => + s.assertFact({ sessionId: "sess-1", subject, predicate: "status", object: "WIP", validAt: 100, now: 300 }), + ).toThrow(/out-of-order validAt/); + + // The open fact survived untouched, both live and in time-travel reads. + expect(s.currentFacts(subject).map((f) => f.object)).toEqual(["merged"]); + expect(s.factsAsOf(subject, 250).map((f) => f.object)).toEqual(["merged"]); + s.close(); + }); +}); diff --git a/src/daemon/memory/cards.ts b/src/daemon/memory/cards.ts new file mode 100644 index 0000000..ff3e6f4 --- /dev/null +++ b/src/daemon/memory/cards.ts @@ -0,0 +1,470 @@ +/** + * SessionCardStore — durable, addressable per-session digests ("cards") plus a + * bi-temporal fact log for session state. + * + * This is the storage foundation for the conductor's session-resolution + * capability (see docs/conductor-session-resolution.md): + * + * - `session_cards` — one compact digest per session (repo, branch, task, + * state, open threads, entities) + an embedding of its fuzzy fields. A + * standalone FTS5 mirror gives keyword/identifier recall. + * - `facts` — bi-temporal state log (Zep/Graphiti pattern). State changes are + * recorded, never mutated in place: superseding a fact sets the prior fact's + * `invalid_at` (event time) + `expired_at` (system time) and inserts the new + * one. This yields free time-travel ("what was this session's state last + * Tuesday?") and a lossless audit trail. + * + * P0 scope: schema + CRUD + the bi-temporal primitive + FTS. Embedding + * generation, cross-workspace hybrid recall, and rerank land in P1. + */ + +import { Database } from "bun:sqlite"; +import { randomUUID } from "node:crypto"; + +export interface SessionCard { + sessionId: string; + workspaceId: string; + repo?: string; + branch?: string; + /** NL description of what this session is doing — the primary fuzzy field. */ + task?: string; + /** Current status descriptor (e.g. "WIP", "merged", "blocked"). */ + state?: string; + lastAction?: string; + /** Open threads of work, free-form strings. */ + openThreads: string[]; + /** Salient entities: files, symbols, ticket ids, branch names. */ + entities: string[]; + /** Embedding of the fuzzy fields (task + open threads). Filled in P1. */ + embedding?: Float32Array; + embeddingModel?: string; + createdAt: number; + updatedAt: number; +} + +/** A bi-temporal fact. `invalidAt`/`expiredAt` null means true-now / current-belief. */ +export interface Fact { + id: string; + sessionId: string; + subject: string; + predicate: string; + object: string; + /** Event time: when the fact became true in the world. */ + validAt: number; + /** Event time: when it stopped being true (null = still true). */ + invalidAt: number | null; + /** System time: when we recorded it. */ + createdAt: number; + /** System time: when we superseded our belief (null = current belief). */ + expiredAt: number | null; +} + +interface CardRow { + session_id: string; + workspace_id: string; + repo: string | null; + branch: string | null; + task: string | null; + state: string | null; + last_action: string | null; + open_threads: string; + entities: string; + embedding: Uint8Array | null; + embedding_model: string | null; + created_at: number; + updated_at: number; +} + +interface FactRow { + id: string; + session_id: string; + subject: string; + predicate: string; + object: string; + valid_at: number; + invalid_at: number | null; + created_at: number; + expired_at: number | null; +} + +export class SessionCardStore { + #db: Database; + + constructor(dbPath: string) { + this.#db = new Database(dbPath, { create: true }); + this.#db.exec("PRAGMA journal_mode = WAL"); + this.#db.exec("PRAGMA synchronous = NORMAL"); + this.#migrate(); + } + + #migrate(): void { + this.#db.exec(` + CREATE TABLE IF NOT EXISTS session_cards ( + session_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + repo TEXT, + branch TEXT, + task TEXT, + state TEXT, + last_action TEXT, + open_threads TEXT NOT NULL DEFAULT '[]', + entities TEXT NOT NULL DEFAULT '[]', + embedding BLOB, + embedding_model TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_cards_workspace + ON session_cards(workspace_id, updated_at DESC); + + -- Standalone FTS mirror (populated manually in upsertCard) so identifier + -- and keyword recall over card text works without external-content triggers. + CREATE VIRTUAL TABLE IF NOT EXISTS session_cards_fts USING fts5( + session_id UNINDEXED, + text + ); + + CREATE TABLE IF NOT EXISTS facts ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + subject TEXT NOT NULL, + predicate TEXT NOT NULL, + object TEXT NOT NULL, + valid_at INTEGER NOT NULL, + invalid_at INTEGER, + created_at INTEGER NOT NULL, + expired_at INTEGER + ); + + CREATE INDEX IF NOT EXISTS idx_facts_subject_pred + ON facts(subject, predicate, valid_at DESC); + CREATE INDEX IF NOT EXISTS idx_facts_session + ON facts(session_id, valid_at DESC); + `); + } + + // ── Cards ─────────────────────────────────────────────────────────────── + + /** Insert or replace a card. Refreshes the FTS mirror row. */ + upsertCard( + card: Omit & { + createdAt?: number; + updatedAt?: number; + }, + ): SessionCard { + const now = card.updatedAt ?? Date.now(); + const existing = this.getCard(card.sessionId); + const createdAt = existing?.createdAt ?? card.createdAt ?? now; + + const embeddingBuf = card.embedding + ? new Uint8Array( + card.embedding.buffer, + card.embedding.byteOffset, + card.embedding.byteLength, + ) + : null; + + // One transaction for the card row + its FTS mirror: a crash between the + // two statements must not leave session_cards_fts drifted from + // session_cards. + this.#db.transaction(() => { + this.#db + .prepare( + `INSERT INTO session_cards ( + session_id, workspace_id, repo, branch, task, state, last_action, + open_threads, entities, embedding, embedding_model, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + workspace_id = excluded.workspace_id, + repo = excluded.repo, + branch = excluded.branch, + task = excluded.task, + state = excluded.state, + last_action = excluded.last_action, + open_threads = excluded.open_threads, + entities = excluded.entities, + embedding = excluded.embedding, + embedding_model = excluded.embedding_model, + updated_at = excluded.updated_at`, + ) + .run( + card.sessionId, + card.workspaceId, + card.repo ?? null, + card.branch ?? null, + card.task ?? null, + card.state ?? null, + card.lastAction ?? null, + JSON.stringify(card.openThreads ?? []), + JSON.stringify(card.entities ?? []), + embeddingBuf, + card.embeddingModel ?? null, + createdAt, + now, + ); + + // Refresh the FTS mirror: delete any prior row for this session, re-insert. + this.#db + .prepare("DELETE FROM session_cards_fts WHERE session_id = ?") + .run(card.sessionId); + this.#db + .prepare("INSERT INTO session_cards_fts (session_id, text) VALUES (?, ?)") + .run(card.sessionId, cardFtsText(card)); + })(); + + return this.getCard(card.sessionId)!; + } + + getCard(sessionId: string): SessionCard | null { + const row = this.#db + .prepare("SELECT * FROM session_cards WHERE session_id = ?") + .get(sessionId) as CardRow | null; + return row ? rowToCard(row) : null; + } + + /** Most-recently-updated cards, optionally scoped to a workspace. */ + listCards(limit = 50, workspaceId?: string): SessionCard[] { + const rows = workspaceId + ? (this.#db + .prepare( + "SELECT * FROM session_cards WHERE workspace_id = ? ORDER BY updated_at DESC LIMIT ?", + ) + .all(workspaceId, limit) as CardRow[]) + : (this.#db + .prepare("SELECT * FROM session_cards ORDER BY updated_at DESC LIMIT ?") + .all(limit) as CardRow[]); + return rows.map(rowToCard); + } + + /** FTS keyword search over card text. Returns session ids + bm25 (lower = better). */ + ftsSearchCards( + query: string, + limit = 24, + ): Array<{ sessionId: string; bm25: number }> { + const sanitized = sanitizeFtsQuery(query); + if (!sanitized) return []; + return this.#db + .prepare( + `SELECT session_id AS sessionId, bm25(session_cards_fts) AS bm25 + FROM session_cards_fts + WHERE session_cards_fts MATCH ? + ORDER BY bm25 + LIMIT ?`, + ) + .all(sanitized, limit) as Array<{ sessionId: string; bm25: number }>; + } + + // ── Bi-temporal facts ───────────────────────────────────────────────────── + + /** + * Assert a fact, superseding any currently-open fact for the same + * (subject, predicate) whose object differs. Invalidate-don't-delete: the + * prior fact stays, with `invalid_at` (event time) set to the new fact's + * `validAt` and `expired_at` (system time) set to now. + * + * Returns the newly-inserted fact (or the existing open fact when the object + * is unchanged — no churn). + */ + assertFact(input: { + sessionId: string; + subject: string; + predicate: string; + object: string; + validAt?: number; + now?: number; + }): Fact { + const now = input.now ?? Date.now(); + const validAt = input.validAt ?? now; + + // Lookup + supersede + insert commit atomically: a crash between the + // close-out UPDATE and the INSERT must never leave (subject, predicate) + // with no open fact. + return this.#db.transaction((): Fact => { + const open = this.#db + .prepare( + `SELECT * FROM facts + WHERE subject = ? AND predicate = ? AND invalid_at IS NULL AND expired_at IS NULL + ORDER BY valid_at DESC LIMIT 1`, + ) + .get(input.subject, input.predicate) as FactRow | null; + + if (open && open.object === input.object) { + return rowToFact(open); // unchanged — no new version + } + + if (open) { + // Closing the open fact with invalid_at < valid_at would make that + // row unsatisfiable for every factsAsOf() query — the fact would + // silently vanish from all time-travel reads. Reject out-of-order + // assertions instead of corrupting the bi-temporal history. + if (validAt < open.valid_at) { + throw new Error( + `assertFact: out-of-order validAt (${validAt}) precedes open fact's validAt (${open.valid_at}) for ${input.subject}/${input.predicate}`, + ); + } + // Supersede: close the prior belief in both time axes. + this.#db + .prepare("UPDATE facts SET invalid_at = ?, expired_at = ? WHERE id = ?") + .run(validAt, now, open.id); + } + + const id = randomUUID(); + this.#db + .prepare( + `INSERT INTO facts + (id, session_id, subject, predicate, object, valid_at, invalid_at, created_at, expired_at) + VALUES (?, ?, ?, ?, ?, ?, NULL, ?, NULL)`, + ) + .run( + id, + input.sessionId, + input.subject, + input.predicate, + input.object, + validAt, + now, + ); + + return { + id, + sessionId: input.sessionId, + subject: input.subject, + predicate: input.predicate, + object: input.object, + validAt, + invalidAt: null, + createdAt: now, + expiredAt: null, + }; + })(); + } + + /** Facts currently believed true for a subject (invalidAt + expiredAt null). */ + currentFacts(subject: string): Fact[] { + const rows = this.#db + .prepare( + `SELECT * FROM facts + WHERE subject = ? AND invalid_at IS NULL AND expired_at IS NULL + ORDER BY predicate ASC`, + ) + .all(subject) as FactRow[]; + return rows.map(rowToFact); + } + + /** + * Facts that were true (in event time) at `asOf`, regardless of when we + * learned them. Time-travel query. + */ + factsAsOf(subject: string, asOf: number): Fact[] { + const rows = this.#db + .prepare( + `SELECT * FROM facts + WHERE subject = ? + AND valid_at <= ? + AND (invalid_at IS NULL OR invalid_at > ?) + ORDER BY predicate ASC`, + ) + .all(subject, asOf, asOf) as FactRow[]; + return rows.map(rowToFact); + } + + /** All facts for a session (including superseded), newest event-time first. */ + factsForSession(sessionId: string): Fact[] { + const rows = this.#db + .prepare("SELECT * FROM facts WHERE session_id = ? ORDER BY valid_at DESC") + .all(sessionId) as FactRow[]; + return rows.map(rowToFact); + } + + close(): void { + this.#db.close(); + } +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +/** The text blob an FTS card row indexes — all human-fuzzy + identifier fields. */ +function cardFtsText(card: { + repo?: string; + branch?: string; + task?: string; + state?: string; + lastAction?: string; + openThreads?: string[]; + entities?: string[]; +}): string { + return [ + card.repo, + card.branch, + card.task, + card.state, + card.lastAction, + ...(card.openThreads ?? []), + ...(card.entities ?? []), + ] + .filter((x): x is string => typeof x === "string" && x.length > 0) + .join(" "); +} + +function rowToCard(row: CardRow): SessionCard { + return { + sessionId: row.session_id, + workspaceId: row.workspace_id, + repo: row.repo ?? undefined, + branch: row.branch ?? undefined, + task: row.task ?? undefined, + state: row.state ?? undefined, + lastAction: row.last_action ?? undefined, + openThreads: safeJsonArray(row.open_threads), + entities: safeJsonArray(row.entities), + embedding: row.embedding ? uint8ToFloat32(row.embedding) : undefined, + embeddingModel: row.embedding_model ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +function rowToFact(row: FactRow): Fact { + return { + id: row.id, + sessionId: row.session_id, + subject: row.subject, + predicate: row.predicate, + object: row.object, + validAt: row.valid_at, + invalidAt: row.invalid_at, + createdAt: row.created_at, + expiredAt: row.expired_at, + }; +} + +function uint8ToFloat32(buf: Uint8Array): Float32Array { + const copy = new Uint8Array(buf.byteLength); + copy.set(buf); + return new Float32Array(copy.buffer, 0, copy.byteLength / 4); +} + +function safeJsonArray(s: string): string[] { + try { + const parsed = JSON.parse(s); + return Array.isArray(parsed) + ? parsed.filter((x): x is string => typeof x === "string") + : []; + } catch { + return []; + } +} + +/** + * Escape FTS5 operators by quoting each whitespace-split term (implicit AND). + * `\s+` handles all whitespace incl. control chars, so no separate strip needed. + */ +function sanitizeFtsQuery(q: string): string { + const terms = q + .split(/\s+/) + .map((t) => t.replace(/["']/g, "").trim()) + .filter((t) => t.length > 0); + if (terms.length === 0) return ""; + return terms.map((t) => `"${t}"`).join(" "); +} diff --git a/src/daemon/memory/engine.test.ts b/src/daemon/memory/engine.test.ts new file mode 100644 index 0000000..2d02cd3 --- /dev/null +++ b/src/daemon/memory/engine.test.ts @@ -0,0 +1,120 @@ +import { describe, test, expect } from "bun:test"; +import { SqliteEpisodeStore } from "./store"; +import { MemoryEngine } from "./engine"; +import type { Embedder } from "./embedder"; +import type { Episode } from "./types"; +import type { Reranker } from "./reranker"; + +/** + * Deterministic fake embedder — an 8-dim char-histogram vector. Related text + * shares buckets → similar vectors, with zero model/network dependency. Good + * enough to exercise the vector code path; assertions lean on the FTS/keyword + * signal so they don't hinge on fake-vector quality. + */ +class FakeEmbedder implements Embedder { + readonly modelName = "fake-test"; + readonly dimensions = 8; + async init(): Promise {} + async embed(texts: string[]): Promise { + return texts.map((t) => { + const v = new Float32Array(this.dimensions); + for (const ch of t.toLowerCase()) { + const c = ch.charCodeAt(0); + if (c >= 97 && c <= 122) v[(c - 97) % this.dimensions]! += 1; + } + return v; + }); + } + async close(): Promise {} +} + +/** Deterministic fake reranker — boosts any doc mentioning "telescope" so the + * rerank stage visibly reorders regardless of the fusion order. */ +class FakeReranker implements Reranker { + readonly modelName = "fake-rerank"; + async init(): Promise {} + async rerank(_query: string, docs: string[]): Promise { + return docs.map((d) => (d.toLowerCase().includes("telescope") ? 10 : 0)); + } + async close(): Promise {} +} + +function ep( + workspaceId: string, + sessionId: string, + summary: string, + content: string, +): Omit { + return { + workspaceId, + sessionId, + kind: "user_turn", + summary, + content, + filePaths: [], + tokenEstimate: Math.ceil(content.length / 4), + createdAt: 1_000_000, + createdBy: "test", + }; +} + +async function seededEngine(reranker?: Reranker): Promise { + const store = new SqliteEpisodeStore(":memory:"); + const engine = new MemoryEngine({ store, embedder: new FakeEmbedder(), reranker }); + await engine.init(); + engine.ingest(ep("wsA", "sA1", "unicorn deploy", "alpha unicorn deployment pipeline")); + engine.ingest(ep("wsA", "sA2", "widget refactor", "beta widget refactor cleanup")); + engine.ingest(ep("wsB", "sB1", "telescope migration", "gamma telescope schema migration")); + engine.ingest(ep("wsB", "sB2", "penguin caching", "delta penguin cache eviction")); + await engine.drain(); + return engine; +} + +describe("cross-workspace (global) resolution", () => { + test("searchSessions with no workspaceId resolves across workspaces", async () => { + const engine = await seededEngine(); + const a = await engine.searchSessions({ query: "unicorn deployment" }); + expect(a[0]?.sessionId).toBe("sA1"); + const b = await engine.searchSessions({ query: "telescope migration" }); + expect(b[0]?.sessionId).toBe("sB1"); + await engine.close(); + }); + + test("passing a workspaceId still scopes to that workspace", async () => { + const engine = await seededEngine(); + // "unicorn" lives in wsA; scoping to wsB must not surface sA1. + const scoped = await engine.searchSessions({ query: "unicorn", workspaceId: "wsB" }); + expect(scoped.every((h) => h.sessionId !== "sA1")).toBe(true); + await engine.close(); + }); + + test("recallGlobal unions candidates across workspaces", async () => { + const engine = await seededEngine(); + const hits = await engine.recallGlobal({ query: "widget telescope", limit: 10 }); + const sessions = new Set(hits.map((h) => h.episode.sessionId)); + // A cross-workspace query should pull episodes from both wsA and wsB. + expect(sessions.has("sA2")).toBe(true); + expect(sessions.has("sB1")).toBe(true); + await engine.close(); + }); + + test("store cross-workspace primitives", async () => { + const store = new SqliteEpisodeStore(":memory:"); + store.insert({ ...ep("w1", "s1", "one", "keyword-one body"), id: "e1" }); + store.insert({ ...ep("w2", "s2", "two", "keyword-two body"), id: "e2" }); + expect(new Set(store.listWorkspaceIds())).toEqual(new Set(["w1", "w2"])); + expect(store.ftsSearchGlobal("keyword-one", 10).map((r) => r.id)).toEqual(["e1"]); + expect(store.episodesByIds(["e2", "e1"]).map((e) => e.id)).toEqual(["e2", "e1"]); + store.close(); + }); + + test("rerank reorders the top-k by cross-encoder score", async () => { + const engine = await seededEngine(new FakeReranker()); + const noRerank = await engine.searchSessions({ query: "widget", rerank: false }); + const withRerank = await engine.searchSessions({ query: "widget", rerank: true }); + // The fake reranker boosts the "telescope" session (sB1) to #1. + expect(withRerank[0]?.sessionId).toBe("sB1"); + expect(withRerank[0]?.sessionId).not.toBe(noRerank[0]?.sessionId); + await engine.close(); + }); +}); diff --git a/src/daemon/memory/engine.ts b/src/daemon/memory/engine.ts index 3e31ebd..d8eeffe 100644 --- a/src/daemon/memory/engine.ts +++ b/src/daemon/memory/engine.ts @@ -15,6 +15,7 @@ import type { Embedder } from "./embedder.js"; import { normalize } from "./embedder.js"; import type { SqliteEpisodeStore } from "./store.js"; import { DEFAULT_WEIGHTS, rank, type RankerWeights } from "./ranker.js"; +import type { Reranker } from "./reranker.js"; import type { Episode, EpisodeKind, @@ -47,9 +48,15 @@ export interface SessionSearchHit { }>; } +/** Ceiling on the reranker's first-run model download + load. Past this the + * engine degrades to fusion-only rather than holding up daemon startup. */ +const RERANKER_INIT_TIMEOUT_MS = 120_000; + export interface MemoryEngineOptions { store: SqliteEpisodeStore; embedder: Embedder; + /** Optional cross-encoder for the final precision@1 rerank stage. */ + reranker?: Reranker; weights?: RankerWeights; /** Top-K FTS hits to consider during recall. Default 24. */ ftsCandidateK?: number; @@ -60,6 +67,7 @@ export interface MemoryEngineOptions { export class MemoryEngine { #store: SqliteEpisodeStore; #embedder: Embedder; + #reranker: Reranker | undefined; #weights: RankerWeights; #ftsK: number; #vectorK: number; @@ -72,10 +80,14 @@ export class MemoryEngine { * keyword recall, episode persistence and usage tracking all keep working; * only the vector signal is disabled. */ #embedderReady = false; + /** False until the reranker loads (or none provided). When not ready the + * rerank stage is skipped and resolution degrades to fusion-only. */ + #rerankerReady = false; constructor(opts: MemoryEngineOptions) { this.#store = opts.store; this.#embedder = opts.embedder; + this.#reranker = opts.reranker; this.#weights = opts.weights ?? DEFAULT_WEIGHTS; this.#ftsK = opts.ftsCandidateK ?? 24; this.#vectorK = opts.vectorCandidateK ?? 24; @@ -97,6 +109,39 @@ export class MemoryEngine { ); } + if (this.#reranker) { + // Bounded: init downloads model weights on first run, and a stalled + // download must degrade to fusion-only instead of wedging daemon + // startup (memory boots before session resume and the listen socket). + let timer: ReturnType | undefined; + try { + await Promise.race([ + this.#reranker.init(), + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + `init timed out after ${RERANKER_INIT_TIMEOUT_MS}ms`, + ), + ), + RERANKER_INIT_TIMEOUT_MS, + ); + }), + ]); + this.#rerankerReady = true; + } catch (err) { + this.#rerankerReady = false; + console.error( + `[codeoid/memory] reranker init failed — resolution runs fusion-only: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } finally { + clearTimeout(timer); + } + } + // Bound the file-read dedup cache (pure cache — safe to prune, no // semantic loss). episodes/audit_log retention is a policy decision, // tracked in #14. @@ -186,6 +231,68 @@ export class MemoryEngine { return hits.slice(0, limit); } + /** + * Cross-workspace hybrid retrieval — the conductor's path. Unlike recall() + * (scoped to one workspace), this unions FTS + vector candidates across ALL + * workspaces and ranks them in a SINGLE batch, so the ranker's BM25 + * min-max normalization is GLOBAL and scores are comparable across + * workspaces. That fixes the small-workspace-domination failure a naive + * per-workspace merge has (a small workspace's batch-relative scores no + * longer inflate past a semantically better hit elsewhere). + */ + async recallGlobal(q: { + query: string; + limit?: number; + filePaths?: string[]; + /** Candidate pool per channel before ranking. Larger than the per-workspace + * default so many sessions are represented across the whole corpus. */ + candidateK?: number; + }): Promise { + const limit = q.limit ?? 8; + const candidateK = q.candidateK ?? Math.max(this.#ftsK, 100); + const now = Date.now(); + + const queryVector = + this.#embedderReady && q.query.trim() + ? normalize((await this.#embedder.embed([q.query]))[0]!) + : null; + + // Global FTS candidates (all workspaces). + const ftsRows = this.#store.ftsSearchGlobal(q.query, candidateK); + const ftsHits = new Map(ftsRows.map((r) => [r.id, r.bm25])); + + // Global vector candidates: cosine over every workspace's (in-sync) matrix. + const vectorIds: string[] = []; + if (queryVector) { + const scored: Array<{ id: string; score: number }> = []; + for (const ws of this.#store.listWorkspaceIds()) { + const { ids, vectors } = this.#store.loadVectorMatrix(ws); + for (let i = 0; i < vectors.length; i++) { + const v = vectors[i]!; + if (v.length !== queryVector.length) continue; + let sum = 0; + for (let j = 0; j < v.length; j++) sum += v[j]! * queryVector[j]!; + scored.push({ id: ids[i]!, score: sum }); + } + } + scored.sort((a, b) => b.score - a.score); + for (const s of scored.slice(0, candidateK)) vectorIds.push(s.id); + } + + const candidateIds = [...new Set([...ftsHits.keys(), ...vectorIds])]; + if (candidateIds.length === 0) return []; + + const episodes = this.#store.episodesByIds(candidateIds); + const hits = rank(episodes, { + queryVector, + ftsHits, + queryFilePaths: q.filePaths ?? [], + now, + weights: this.#weights, + }); + return hits.slice(0, limit); + } + /** Fetch a single episode by id (for recall_turn-style lookup). */ getEpisode(id: string): Episode | null { return this.#store.getEpisode(id); @@ -208,7 +315,8 @@ export class MemoryEngine { */ async searchSessions(opts: { query: string; - workspaceId: string; + /** Absent = cross-workspace (global) resolution — the conductor's path. */ + workspaceId?: string; limit?: number; /** Episode-hit candidates to consider before grouping. */ candidatePoolSize?: number; @@ -220,16 +328,25 @@ export class MemoryEngine { * boost. Purely additive; absence just means no name boost. */ sessionNames?: Map; + /** Cross-encoder rerank of the top-k. Defaults on when a reranker is ready. */ + rerank?: boolean; }): Promise { const limit = opts.limit ?? 10; - const pool = opts.candidatePoolSize ?? Math.max(40, limit * 5); + const global = opts.workspaceId === undefined; + // Cross-workspace needs a bigger candidate pool so many sessions across the + // corpus are represented (a single busy session would otherwise fill it). + const pool = + opts.candidatePoolSize ?? + (global ? Math.max(150, limit * 15) : Math.max(40, limit * 5)); const snippetsPerSession = opts.snippetsPerSession ?? 3; - const hits = await this.recall({ - query: opts.query, - workspaceId: opts.workspaceId, - limit: pool, - }); + const hits = global + ? await this.recallGlobal({ query: opts.query, limit: pool, candidateK: pool }) + : await this.recall({ + query: opts.query, + workspaceId: opts.workspaceId!, + limit: pool, + }); if (hits.length === 0) return []; // Group by session id. @@ -305,11 +422,41 @@ export class MemoryEngine { } scored.sort((a, b) => b.aggregateScore - a.aggregateScore); + + // Cross-encoder rerank of the top-k — the precision@1 stage. After fusion + // the right session is usually already top-k; a cross-encoder reading + // (query, evidence) jointly pulls it to #1. Bounded to RERANK_K pairs. + const doRerank = (opts.rerank ?? this.#rerankerReady) && this.#rerankerReady; + if (doRerank && scored.length > 1) { + const RERANK_K = 8; + const head = scored.slice(0, RERANK_K); + const docs = head.map((h) => + h.snippets + .map((s) => s.summary) + .join(". ") + .slice(0, 500), + ); + try { + const rerankScores = await this.#reranker!.rerank(opts.query, docs); + const reordered = head + .map((h, i) => ({ h, s: rerankScores[i] ?? Number.NEGATIVE_INFINITY })) + .sort((a, b) => b.s - a.s) + .map((x) => x.h); + return [...reordered, ...scored.slice(RERANK_K)].slice(0, limit); + } catch (err) { + console.error( + `[codeoid/memory] rerank failed — returning fusion order: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + } return scored.slice(0, limit); } async close(): Promise { await this.#embedder.close(); + if (this.#reranker) await this.#reranker.close(); this.#store.close(); } diff --git a/src/daemon/memory/reranker-transformersjs.ts b/src/daemon/memory/reranker-transformersjs.ts new file mode 100644 index 0000000..609708a --- /dev/null +++ b/src/daemon/memory/reranker-transformersjs.ts @@ -0,0 +1,93 @@ +/** + * TransformersJsReranker — pure-WASM cross-encoder reranking via + * @xenova/transformers. Runs a (query, doc) sequence-classification model and + * returns the relevance logit per doc. First init downloads the model to the + * cache dir; subsequent runs load from cache. + * + * Only called on the top-k (~8) candidates per query, so cost is bounded and + * off the recall hot path's critical width. + */ + +import { homedir } from "node:os"; +import { join } from "node:path"; +import { mkdirSync, existsSync } from "node:fs"; +import type { Reranker } from "./reranker.js"; + +// Type-only — real types imported dynamically to keep startup fast. +// biome-ignore lint/suspicious/noExplicitAny: dynamic import of optional dep +type Any = any; + +export class TransformersJsReranker implements Reranker { + readonly modelName: string; + #cacheDir: string; + #tokenizer: Any = null; + #model: Any = null; + #initPromise: Promise | null = null; + + constructor(modelName: string, cacheDir?: string) { + this.modelName = modelName; + this.#cacheDir = cacheDir ?? join(homedir(), ".codeoid", "models"); + if (!existsSync(this.#cacheDir)) { + mkdirSync(this.#cacheDir, { recursive: true }); + } + } + + async init(): Promise { + if (this.#model) return; + if (this.#initPromise) return this.#initPromise; + this.#initPromise = this.#doInit(); + return this.#initPromise; + } + + async #doInit(): Promise { + const mod = await import("@xenova/transformers").catch((err) => { + throw new Error( + `Failed to load @xenova/transformers — is it installed? (${err instanceof Error ? err.message : String(err)})`, + ); + }); + mod.env.cacheDir = this.#cacheDir; + mod.env.allowLocalModels = true; + + this.#tokenizer = await mod.AutoTokenizer.from_pretrained(this.modelName); + this.#model = await mod.AutoModelForSequenceClassification.from_pretrained(this.modelName, { + quantized: true, + }); + } + + async rerank(query: string, docs: string[]): Promise { + if (docs.length === 0) return []; + if (!this.#model) await this.init(); + + // Tokenize (query, doc) pairs as a batch: text = query repeated, text_pair = docs. + const inputs = await this.#tokenizer(new Array(docs.length).fill(query), { + text_pair: docs, + padding: true, + truncation: true, + }); + const { logits } = await this.#model(inputs); + const data = logits.data as Float32Array; + const n = docs.length; + const dim = data.length / n; // 1 for ms-marco (single relevance logit), 2 for some models + + const scores: number[] = []; + for (let i = 0; i < n; i++) { + // dim===1 → the logit; dim===2 → positive-class logit. + scores.push(dim === 1 ? data[i]! : data[i * dim + 1]!); + } + return scores; + } + + async close(): Promise { + // Release the ONNX sessions' WASM memory — dropping the JS reference + // alone doesn't free it. Best-effort: a "cannot release session" from + // the runtime must not fail close(). (AutoTokenizer has no dispose().) + try { + await this.#model?.dispose?.(); + } catch { + // Best-effort. + } + this.#tokenizer = null; + this.#model = null; + this.#initPromise = null; + } +} diff --git a/src/daemon/memory/reranker.ts b/src/daemon/memory/reranker.ts new file mode 100644 index 0000000..02a971f --- /dev/null +++ b/src/daemon/memory/reranker.ts @@ -0,0 +1,37 @@ +/** + * Reranker — pluggable cross-encoder for the final precision@1 stage of session + * resolution. A bi-encoder (the embedder) compresses each doc into one + * query-agnostic vector; a cross-encoder reads the (query, doc) pair jointly and + * scores relevance for *this* query — far better at picking THE one right result + * from a good top-k, which is exactly the conductor's need (right session is + * usually top-3 after global fusion; the rerank pulls it to #1). + * + * Default is a small English cross-encoder (fast, CPU-friendly). Swap to + * bge-reranker-v2-m3 (multilingual, heavier) behind this same interface later. + */ + +/** Small, fast, English cross-encoder. ~22M params; ms-marco trained. */ +export const DEFAULT_RERANKER_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2"; + +export interface Reranker { + readonly modelName: string; + /** Load the model. Safe to call multiple times; only the first does work. */ + init(): Promise; + /** Relevance score per doc vs the query (higher = more relevant). */ + rerank(query: string, docs: string[]): Promise; + /** Free model resources. */ + close(): Promise; +} + +export interface RerankerConfig { + /** HuggingFace model id (default: ms-marco-MiniLM-L-6-v2). */ + model?: string; + /** Cache dir for model weights (default: ~/.codeoid/models). */ + cacheDir?: string; +} + +/** Factory — returns a ready-to-init Reranker. */ +export async function createReranker(config: RerankerConfig = {}): Promise { + const { TransformersJsReranker } = await import("./reranker-transformersjs.js"); + return new TransformersJsReranker(config.model ?? DEFAULT_RERANKER_MODEL, config.cacheDir); +} diff --git a/src/daemon/memory/store.ts b/src/daemon/memory/store.ts index 6961e67..1e7f9f0 100644 --- a/src/daemon/memory/store.ts +++ b/src/daemon/memory/store.ts @@ -1058,6 +1058,59 @@ export class SqliteEpisodeStore { return ordered; } + // ── Cross-workspace (global) retrieval primitives ───────────────────────── + // The conductor resolves a fuzzy reference to the right session across ALL + // workspaces on the machine, not within one. These are the unscoped twins of + // ftsSearch / loadVectorMatrix / filter. Ranking a single GLOBAL candidate set + // (rather than merging per-workspace results) is what makes BM25 scores + // comparable across workspaces — see engine.recallGlobal(). + // + // NOTE: "global" here means the whole store, which is single-tenant per user + // (workspace ids already fold in account+project). When the conductor grows a + // multi-tenant surface, scope these to the caller's tenant workspace set. + + /** Distinct workspace ids present in the store. */ + listWorkspaceIds(): string[] { + return ( + this.#db + .prepare("SELECT DISTINCT workspace_id AS w FROM episodes") + .all() as Array<{ w: string }> + ).map((r) => r.w); + } + + /** FTS5 keyword search across ALL workspaces. */ + ftsSearchGlobal(query: string, limit: number): Array<{ id: string; bm25: number }> { + if (!query.trim()) return []; + const sanitized = sanitizeFtsQuery(query); + if (!sanitized) return []; + return this.#db + .prepare( + `SELECT e.id AS id, bm25(episodes_fts) AS bm25 + FROM episodes_fts + JOIN episodes e ON e.rowid = episodes_fts.rowid + WHERE episodes_fts MATCH ? + ORDER BY bm25 + LIMIT ?`, + ) + .all(sanitized, limit) as Array<{ id: string; bm25: number }>; + } + + /** Fetch episodes by id (no workspace scoping), preserving input order. */ + episodesByIds(ids: string[]): Episode[] { + if (ids.length === 0) return []; + const placeholders = ids.map(() => "?").join(","); + const rows = this.#db + .prepare(`SELECT * FROM episodes WHERE id IN (${placeholders})`) + .all(...ids) as EpisodeRow[]; + const byId = new Map(rows.map((r) => [r.id, r])); + const ordered: Episode[] = []; + for (const id of ids) { + const row = byId.get(id); + if (row) ordered.push(this.#rowToEpisode(row)); + } + return ordered; + } + close(): void { this.#db.close(); } diff --git a/src/daemon/session-manager.ts b/src/daemon/session-manager.ts index da0d741..411e7cb 100644 --- a/src/daemon/session-manager.ts +++ b/src/daemon/session-manager.ts @@ -123,6 +123,18 @@ export class SessionManager { * Rebuilds in-memory session objects and scrollback buffers. */ async resumeSessions(): Promise { + // Reload the durable conductor identity first (design R2): the persisted + // {identityId, wimseUri, apiKey} row is reused instead of re-registering, + // so the conductor keeps ONE stable WIMSE URI across daemon restarts. + // Best-effort and null-safe — a missing or stale row just means the next + // registerConductor() starts fresh. + const conductor = await this.#identityManager?.resumeConductor(); + if (conductor) { + console.log( + `[codeoid] resumed conductor identity ${conductor.wimseUri}`, + ); + } + const allMetas = await this.#transcriptStore.loadAllMeta(); // Newest-first by last activity so the cap keeps the most relevant // sessions when there are more than RESUME_MAX_SESSIONS on disk. diff --git a/src/daemon/store.ts b/src/daemon/store.ts index 05075e0..c1121ed 100644 --- a/src/daemon/store.ts +++ b/src/daemon/store.ts @@ -84,6 +84,22 @@ export class Store { models_json TEXT NOT NULL, cached_at TEXT NOT NULL DEFAULT (datetime('now')) ); + + -- Durable conductor identity (design R2): one row per tenant, reloaded + -- on daemon restart so the conductor keeps a stable WIMSE URI across + -- process lifetimes. api_key is the ONE credential at rest — the + -- conductor's working token is re-minted per boot by owner delegation, + -- and its actor keypair never touches disk. + CREATE TABLE IF NOT EXISTS conductor_identity ( + account_id TEXT NOT NULL, + project_id TEXT NOT NULL, + identity_id TEXT NOT NULL, + wimse_uri TEXT NOT NULL, + api_key TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (account_id, project_id) + ); `); // Pre-release single-row predecessor of provider_model_catalogs — never // shipped in a tagged version; drop from dev databases that ran the branch. @@ -312,6 +328,51 @@ export class Store { this.#db.prepare("DELETE FROM sessions WHERE id = ?").run(id); } + // ── Conductor identity ──────────────────────────────────────────────── + + saveConductorIdentity(row: { + accountId: string; + projectId: string; + identityId: string; + wimseUri: string; + apiKey: string; + }): void { + this.#db + .prepare( + `INSERT INTO conductor_identity (account_id, project_id, identity_id, wimse_uri, api_key) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(account_id, project_id) DO UPDATE SET + identity_id = excluded.identity_id, + wimse_uri = excluded.wimse_uri, + api_key = excluded.api_key, + updated_at = datetime('now')`, + ) + .run(row.accountId, row.projectId, row.identityId, row.wimseUri, row.apiKey); + } + + getConductorIdentity( + accountId: string, + projectId: string, + ): { identityId: string; wimseUri: string; apiKey: string } | null { + const row = this.#db + .prepare( + `SELECT identity_id AS identityId, wimse_uri AS wimseUri, api_key AS apiKey + FROM conductor_identity WHERE account_id = ? AND project_id = ?`, + ) + .get(accountId, projectId) as + | { identityId: string; wimseUri: string; apiKey: string } + | null; + return row ?? null; + } + + deleteConductorIdentity(accountId: string, projectId: string): void { + this.#db + .prepare( + "DELETE FROM conductor_identity WHERE account_id = ? AND project_id = ?", + ) + .run(accountId, projectId); + } + // ── Model catalog cache ─────────────────────────────────────────────── /** diff --git a/src/integration/conductor-zeroid.test.ts b/src/integration/conductor-zeroid.test.ts new file mode 100644 index 0000000..c409e10 --- /dev/null +++ b/src/integration/conductor-zeroid.test.ts @@ -0,0 +1,360 @@ +/** + * Conductor identity integration test — P2 exit criteria, run against a LIVE + * ZeroID (default http://localhost:8899, override with ZEROID_URL): + * + * 1. Durable identity: registerConductor persists {identityId, wimseUri, + * apiKey} to the Store; a fresh manager (simulated daemon restart) + * resumes the SAME identity — one stable WIMSE URI. + * 2. Owner delegation: the owner's token delegates to the conductor via + * RFC 8693; the chain owner → conductor → child → sub-agent mints at + * delegation_depth 3 with a verifiable act chain at every hop. + * 3. Attenuation: no hop can mint tools:write / tools:execute — the + * conductor's scope ceiling caps its whole subtree. + * 4. Cascading revocation: deactivating the conductor kills the subtree + * (conductor/child/sub-agent tokens all introspect inactive) while the + * owner's own token stays live. + * + * NOT in the default `bun test` globs (CI has no ZeroID) — run with + * `bun run test:integration`. Skips itself when ZeroID is unreachable or no + * real (account_id, project_id) can be resolved. Every identity registered + * here uses external_id `codeoid-conductor-test-*` and is deactivated in + * afterAll. + */ + +import { Database } from "bun:sqlite"; +import { afterAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { type RegisterAgentRequest, ZeroIDClient } from "@highflame/sdk"; +import { + generateAgentKeypair, + signActorAssertion, +} from "../daemon/agent-assertion.js"; +import { + AgentIdentityManager, + CONDUCTOR_SCOPES, +} from "../daemon/agent-identity.js"; +import { Store } from "../daemon/store.js"; +import { ALL_SCOPES_STRING } from "../protocol/scopes.js"; + +const BASE_URL = process.env.ZEROID_URL ?? "http://localhost:8899"; +const TIMEOUT = 30_000; + +async function zeroidUp(): Promise { + try { + const res = await fetch(`${BASE_URL}/health`, { + signal: AbortSignal.timeout(2_000), + }); + return res.ok; + } catch { + return false; + } +} + +/** + * A real (account_id, project_id) — env override first, else the most-used + * tenant in the local ~/.codeoid/codeoid.db sessions table. + */ +function resolveTenant(): { accountId: string; projectId: string } | null { + const envAccount = process.env.ZEROID_TEST_ACCOUNT; + const envProject = process.env.ZEROID_TEST_PROJECT; + if (envAccount && envProject) { + return { accountId: envAccount, projectId: envProject }; + } + const dbPath = join(homedir(), ".codeoid", "codeoid.db"); + if (!existsSync(dbPath)) return null; + try { + const db = new Database(dbPath, { readonly: true }); + const row = db + .prepare( + `SELECT account_id AS accountId, project_id AS projectId + FROM sessions GROUP BY account_id, project_id + ORDER BY COUNT(*) DESC LIMIT 1`, + ) + .get() as { accountId: string; projectId: string } | null; + db.close(); + return row ?? null; + } catch { + return null; + } +} + +const up = await zeroidUp(); +const tenant = up ? resolveTenant() : null; +const ready = up && tenant !== null; +if (!up) { + console.warn( + `[conductor-integration] skipping — ZeroID not reachable at ${BASE_URL}`, + ); +} else if (!tenant) { + console.warn( + "[conductor-integration] skipping — no tenant in ~/.codeoid/codeoid.db and no ZEROID_TEST_ACCOUNT/ZEROID_TEST_PROJECT", + ); +} + +const d = ready ? describe : describe.skip; + +d("conductor identity against live ZeroID (P2)", () => { + const { accountId, projectId } = tenant!; + const run = crypto.randomUUID().slice(0, 8); + const client = new ZeroIDClient({ baseUrl: BASE_URL, accountId, projectId }); + + const tmpDir = mkdtempSync(join(tmpdir(), "codeoid-conductor-it-")); + const store = new Store(join(tmpDir, "store.db")); + const managerConfig = { + auth: { baseUrl: BASE_URL }, + accountId, + projectId, + conductorExternalIdPrefix: "codeoid-conductor-test", + }; + + /** Identity ids to deactivate in afterAll, whatever state the run died in. */ + const cleanupIds: string[] = []; + + let owner: { identityId: string; wimseUri: string; apiKey: string }; + let manager2: AgentIdentityManager; + let conductor: { identityId: string; wimseUri: string }; + let ownerToken = ""; + let conductorToken = ""; + let childToken = ""; + let subagentToken = ""; + + afterAll(async () => { + for (const id of cleanupIds) { + try { + await client.agents.deactivate(id); + } catch { + // Best-effort — already deactivated by the test itself is the norm. + } + } + store.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + /** + * Register a throwaway `codeoid-conductor-test-*` actor identity with a + * fresh keypair, so it can appear as an RFC 8693 actor in the chain. + */ + async function registerActor( + role: string, + allowedScopes: string[], + createdBy: string, + ) { + const keypair = await generateAgentKeypair(); + const registerReq = { + name: `codeoid-test/${role}`, + external_id: `codeoid-conductor-test-${role}-${run}`, + sub_type: "tool_agent" as const, + trust_level: "first_party" as const, + framework: "claude-agent-sdk", + publisher: "codeoid", + created_by: createdBy, + allowed_scopes: allowedScopes, + public_key_pem: keypair.publicKeyPem, + }; + const resp = await client.agents.register( + registerReq as RegisterAgentRequest, + ); + cleanupIds.push(resp.identity.id); + return { + identityId: resp.identity.id, + wimseUri: resp.identity.wimse_uri, + apiKey: resp.api_key, + keypair, + }; + } + + test( + "registerConductor persists a durable identity to the Store", + async () => { + // The delegation root: a stand-in for the human owner, holding the + // full codeoid scope set the way a real owner token does. + const ownerReq = { + name: "codeoid-test/owner", + external_id: `codeoid-conductor-test-owner-${run}`, + sub_type: "human_proxy" as const, + trust_level: "first_party" as const, + framework: "claude-agent-sdk", + publisher: "codeoid", + // created_by becomes the identity's required owner_user_id. + created_by: `codeoid-conductor-test-user-${run}`, + allowed_scopes: ALL_SCOPES_STRING.split(" "), + }; + const ownerResp = await client.agents.register( + ownerReq as RegisterAgentRequest, + ); + cleanupIds.push(ownerResp.identity.id); + owner = { + identityId: ownerResp.identity.id, + wimseUri: ownerResp.identity.wimse_uri, + apiKey: ownerResp.api_key, + }; + + const manager1 = new AgentIdentityManager(managerConfig, store); + const registered = await manager1.registerConductor(owner.wimseUri); + expect(registered).not.toBeNull(); + conductor = registered!; + cleanupIds.push(conductor.identityId); + + const row = store.getConductorIdentity(accountId, projectId); + expect(row).not.toBeNull(); + expect(row!.identityId).toBe(conductor.identityId); + expect(row!.wimseUri).toBe(conductor.wimseUri); + expect(row!.apiKey).toStartWith("zid_sk_"); + }, + TIMEOUT, + ); + + test( + "a fresh manager resumes the SAME identity — stable WIMSE URI across restarts", + async () => { + // New manager over the same store = daemon restart. resumeConductor is + // what SessionManager.resumeSessions calls on boot. + manager2 = new AgentIdentityManager(managerConfig, store); + const resumed = await manager2.resumeConductor(); + expect(resumed).not.toBeNull(); + expect(resumed!.identityId).toBe(conductor.identityId); + expect(resumed!.wimseUri).toBe(conductor.wimseUri); + + // registerConductor must also reload, never mint a second identity. + const again = await manager2.registerConductor(owner.wimseUri); + expect(again!.wimseUri).toBe(conductor.wimseUri); + }, + TIMEOUT, + ); + + test( + "owner → conductor delegation mints depth 1 with the owner in the act chain", + async () => { + ownerToken = ( + await client.tokens.issueApiKey(owner.apiKey, { + scope: ALL_SCOPES_STRING, + }) + ).access_token; + + const minted = await manager2.mintConductorToken(ownerToken); + expect(minted).not.toBeNull(); + conductorToken = minted!; + + const intro = await client.tokens.introspect(conductorToken); + expect(intro.active).toBe(true); + expect(intro.delegation_depth).toBe(1); + expect(intro.sub).toBe(conductor.wimseUri); + expect(intro.act?.sub).toBe(owner.wimseUri); + + // Scope ceiling: exactly the conductor profile, nothing tool-shaped. + const scopes = (intro.scope ?? "").split(" ").sort(); + expect(scopes).toEqual([...CONDUCTOR_SCOPES].sort()); + expect(scopes).not.toContain("tools:write"); + expect(scopes).not.toContain("tools:execute"); + }, + TIMEOUT, + ); + + test( + "conductor → child → sub-agent extends the chain to delegation_depth 3", + async () => { + const child = await registerActor( + "child", + ["session:read", "session:dispatch"], + conductor.wimseUri, + ); + const childAssertion = await signActorAssertion( + child.keypair.privateKey, + child.wimseUri, + BASE_URL, + ); + childToken = ( + await client.tokens.issueTokenExchange(conductorToken, childAssertion, { + scope: "session:read session:dispatch", + }) + ).access_token; + + const childIntro = await client.tokens.introspect(childToken); + expect(childIntro.active).toBe(true); + expect(childIntro.delegation_depth).toBe(2); + expect(childIntro.sub).toBe(child.wimseUri); + expect(childIntro.act?.sub).toBe(conductor.wimseUri); + + const subagent = await registerActor( + "subagent", + ["session:read"], + child.wimseUri, + ); + const subAssertion = await signActorAssertion( + subagent.keypair.privateKey, + subagent.wimseUri, + BASE_URL, + ); + subagentToken = ( + await client.tokens.issueTokenExchange(childToken, subAssertion, { + scope: "session:read", + }) + ).access_token; + + const subIntro = await client.tokens.introspect(subagentToken); + expect(subIntro.active).toBe(true); + expect(subIntro.delegation_depth).toBe(3); + expect(subIntro.sub).toBe(subagent.wimseUri); + expect(subIntro.act?.sub).toBe(child.wimseUri); + expect((subIntro.scope ?? "").split(" ")).toEqual(["session:read"]); + }, + TIMEOUT, + ); + + test( + "no hop below the conductor can mint tools:write — attenuation holds", + async () => { + // Ask for tools:write mid-chain: the subject (conductor) never held it, + // so the three-way intersection must strip it (or reject outright). + const child = await registerActor( + "grabby-child", + ["session:read", "tools:write"], + conductor.wimseUri, + ); + const assertion = await signActorAssertion( + child.keypair.privateKey, + child.wimseUri, + BASE_URL, + ); + let granted: string[] = []; + try { + const resp = await client.tokens.issueTokenExchange( + conductorToken, + assertion, + { scope: "session:read tools:write" }, + ); + granted = (resp.scope ?? "").split(" "); + } catch { + // invalid_scope rejection is an equally acceptable outcome. + } + expect(granted).not.toContain("tools:write"); + expect(granted).not.toContain("tools:execute"); + }, + TIMEOUT, + ); + + test( + "deactivating the conductor cascade-revokes the whole subtree", + async () => { + await manager2.deactivateConductor(); + + // Every credential under the conductor dies with it, walked via the + // parent_jti chain: its own owner-delegated token, the child's, the + // sub-agent's. Introspection is the revocation-aware path. + for (const token of [conductorToken, childToken, subagentToken]) { + const intro = await client.tokens.introspect(token); + expect(intro.active).toBe(false); + } + + // Revocation must not climb UP the chain — the owner keeps working. + const ownerIntro = await client.tokens.introspect(ownerToken); + expect(ownerIntro.active).toBe(true); + + // The persisted row is gone; the next registerConductor starts fresh. + expect(store.getConductorIdentity(accountId, projectId)).toBeNull(); + }, + TIMEOUT, + ); +}); diff --git a/src/tests/agent-identity-conductor.test.ts b/src/tests/agent-identity-conductor.test.ts new file mode 100644 index 0000000..2ea179d --- /dev/null +++ b/src/tests/agent-identity-conductor.test.ts @@ -0,0 +1,429 @@ +/** + * Conductor identity unit tests — the durable, owner-delegated conductor + * lifecycle (P2) against a fetch-stubbed ZeroID, so the register / resume / + * mint / deactivate flows and their Store persistence are exercised without a + * live server. The live-server counterpart (depth-3 chain, real cascade + * revocation) is src/integration/conductor-zeroid.test.ts. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + AgentIdentityManager, + CONDUCTOR_SCOPES, +} from "../daemon/agent-identity.js"; +import { Store } from "../daemon/store.js"; + +const BASE_URL = "http://zeroid.test"; +const ACCOUNT = "acct_t"; +const PROJECT = "proj_t"; + +/** Decode a JWS payload (the actor assertion) without verifying. */ +function decodeJwtPayload(jwt: string): Record { + const b64 = jwt.split(".")[1]!.replace(/-/g, "+").replace(/_/g, "/"); + return JSON.parse(Buffer.from(b64, "base64").toString("utf8")); +} + +/** + * In-memory fake of the ZeroID endpoints the conductor lifecycle touches. + * Tracks calls so tests can assert on the wire contract, and simulates + * api-key revocation on deactivation (the liveness probe's failure mode). + */ +class FakeZeroID { + registerCalls: Array> = []; + tokenCalls: Array> = []; + keyRotations: Array<{ identityId: string; publicKeyPem: unknown }> = []; + deactivateCalls: string[] = []; + /** When true, POST /api/v1/agents/register returns 422. */ + failRegister = false; + /** When true, the deactivate endpoint returns 422. */ + failDeactivate = false; + + #nextId = 0; + #identities = new Map(); + #apiKeys = new Map(); // api_key -> identity id + + /** Directly mark an identity dead (out-of-band deactivation). */ + killIdentity(identityId: string): void { + const identity = this.#identities.get(identityId); + if (identity) identity.active = false; + } + + install(): void { + globalThis.fetch = (async ( + input: string | URL | Request, + init?: RequestInit, + ): Promise => { + const url = String(input); + const path = url.replace(BASE_URL, ""); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + + if (path === "/api/v1/agents/register") { + this.registerCalls.push(body); + if (this.failRegister) { + return Response.json({ detail: "validation failed" }, { status: 422 }); + } + const id = `ident-${++this.#nextId}`; + const wimseUri = `wimse://zeroid.test/agent/${id}`; + const apiKey = `zid_sk_${id}`; + this.#identities.set(id, { wimseUri, active: true }); + this.#apiKeys.set(apiKey, id); + return Response.json({ + identity: { id, wimse_uri: wimseUri, external_id: body.external_id }, + api_key: apiKey, + }); + } + + if (path === "/oauth2/token") { + this.tokenCalls.push(body); + if (body.grant_type === "api_key") { + const identityId = this.#apiKeys.get(body.api_key); + const identity = identityId + ? this.#identities.get(identityId) + : undefined; + if (!identity?.active) { + return Response.json( + { detail: "invalid or revoked api key" }, + { status: 401 }, + ); + } + return Response.json({ + access_token: `tok-${identityId}`, + token_type: "Bearer", + expires_in: 3600, + jti: crypto.randomUUID(), + iat: 0, + scope: body.scope ?? "", + }); + } + // RFC 8693 token exchange (owner -> conductor) + return Response.json({ + access_token: "delegated-conductor-token", + token_type: "Bearer", + expires_in: 3600, + jti: crypto.randomUUID(), + iat: 0, + scope: body.scope ?? "", + }); + } + + const patchMatch = path.match(/^\/api\/v1\/identities\/([^/]+)$/); + if (patchMatch && init?.method === "PATCH") { + this.keyRotations.push({ + identityId: patchMatch[1]!, + publicKeyPem: body.public_key_pem, + }); + return Response.json({ id: patchMatch[1] }); + } + + const deactivateMatch = path.match( + /^\/api\/v1\/agents\/registry\/([^/]+)\/deactivate$/, + ); + if (deactivateMatch) { + const id = deactivateMatch[1]!; + this.deactivateCalls.push(id); + if (this.failDeactivate) { + return Response.json({ detail: "deactivate failed" }, { status: 422 }); + } + this.killIdentity(id); + return Response.json({ id, status: "deactivated" }); + } + + return Response.json( + { detail: `unexpected route: ${init?.method ?? "GET"} ${path}` }, + { status: 404 }, + ); + }) as typeof fetch; + } +} + +const realFetch = globalThis.fetch; + +describe("AgentIdentityManager conductor lifecycle", () => { + let tmpDir: string; + let store: Store; + let zeroid: FakeZeroID; + + const config = { + auth: { baseUrl: BASE_URL }, + accountId: ACCOUNT, + projectId: PROJECT, + }; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codeoid-conductor-unit-")); + store = new Store(join(tmpDir, "store.db")); + zeroid = new FakeZeroID(); + zeroid.install(); + }); + + afterEach(() => { + globalThis.fetch = realFetch; + store.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("registerConductor registers an orchestrator with the conductor scope ceiling and persists it", async () => { + const manager = new AgentIdentityManager(config, store); + const conductor = await manager.registerConductor("user:owner@test"); + + expect(conductor).not.toBeNull(); + expect(manager.conductorUri).toBe(conductor!.wimseUri); + + expect(zeroid.registerCalls).toHaveLength(1); + const req = zeroid.registerCalls[0]!; + expect(req.sub_type).toBe("orchestrator"); + expect(req.created_by).toBe("user:owner@test"); + expect(req.allowed_scopes).toEqual([...CONDUCTOR_SCOPES]); + expect(req.allowed_scopes).not.toContain("tools:write"); + expect(req.allowed_scopes).not.toContain("tools:execute"); + expect(String(req.external_id)).toStartWith("codeoid-conductor-"); + expect(String(req.public_key_pem)).toContain("BEGIN PUBLIC KEY"); + + const row = store.getConductorIdentity(ACCOUNT, PROJECT); + expect(row).toEqual({ + identityId: conductor!.identityId, + wimseUri: conductor!.wimseUri, + apiKey: expect.stringMatching(/^zid_sk_/) as unknown as string, + }); + }); + + test("registerConductor honors conductorExternalIdPrefix", async () => { + const manager = new AgentIdentityManager( + { ...config, conductorExternalIdPrefix: "codeoid-conductor-test" }, + store, + ); + await manager.registerConductor("user:owner@test"); + expect(String(zeroid.registerCalls[0]!.external_id)).toStartWith( + "codeoid-conductor-test-", + ); + }); + + test("second registerConductor reuses the in-memory identity — no re-registration", async () => { + const manager = new AgentIdentityManager(config, store); + const first = await manager.registerConductor("user:owner@test"); + const second = await manager.registerConductor("user:owner@test"); + + expect(second).toEqual(first!); + expect(zeroid.registerCalls).toHaveLength(1); + }); + + test("a fresh manager resumes the persisted identity — same WIMSE URI, key rotated, no re-registration", async () => { + const manager1 = new AgentIdentityManager(config, store); + const registered = await manager1.registerConductor("user:owner@test"); + + // Simulated daemon restart: new manager over the same store. + const manager2 = new AgentIdentityManager(config, store); + const resumed = await manager2.resumeConductor(); + + expect(resumed).toEqual(registered!); + expect(zeroid.registerCalls).toHaveLength(1); + + // Liveness probe minted from the persisted api key… + const probe = zeroid.tokenCalls.find((c) => c.grant_type === "api_key"); + expect(probe?.api_key).toBe( + store.getConductorIdentity(ACCOUNT, PROJECT)!.apiKey, + ); + // …and the process-local actor keypair was re-registered. + expect(zeroid.keyRotations).toHaveLength(1); + expect(zeroid.keyRotations[0]!.identityId).toBe(registered!.identityId); + expect(String(zeroid.keyRotations[0]!.publicKeyPem)).toContain( + "BEGIN PUBLIC KEY", + ); + }); + + test("resumeConductor without a persisted row is a no-op", async () => { + const manager = new AgentIdentityManager(config, store); + expect(await manager.resumeConductor()).toBeNull(); + expect(manager.conductorUri).toBeUndefined(); + expect(zeroid.tokenCalls).toHaveLength(0); + }); + + test("a stale persisted identity is dropped and the next register starts fresh", async () => { + const manager1 = new AgentIdentityManager(config, store); + const first = await manager1.registerConductor("user:owner@test"); + + // The identity dies out-of-band (revoked on the server); the persisted + // api key stops minting. + zeroid.killIdentity(first!.identityId); + + const manager2 = new AgentIdentityManager(config, store); + expect(await manager2.resumeConductor()).toBeNull(); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)).toBeNull(); + + // registerConductor now mints a NEW identity and persists it. + const second = await manager2.registerConductor("user:owner@test"); + expect(second).not.toBeNull(); + expect(second!.identityId).not.toBe(first!.identityId); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)!.identityId).toBe( + second!.identityId, + ); + }); + + test("registerConductor returns null when ZeroID rejects registration", async () => { + zeroid.failRegister = true; + const manager = new AgentIdentityManager(config, store); + expect(await manager.registerConductor("user:owner@test")).toBeNull(); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)).toBeNull(); + }); + + test("mintConductorToken exchanges the owner's token with a self-signed actor assertion", async () => { + const manager = new AgentIdentityManager(config, store); + const conductor = await manager.registerConductor("user:owner@test"); + + const token = await manager.mintConductorToken("owner-subject-token"); + expect(token).toBe("delegated-conductor-token"); + + const exchange = zeroid.tokenCalls.find( + (c) => c.grant_type !== "api_key", + )!; + expect(exchange.subject_token).toBe("owner-subject-token"); + expect(exchange.scope).toBe(CONDUCTOR_SCOPES.join(" ")); + + // The actor assertion is self-signed by the conductor: iss = sub = its + // WIMSE URI, aud = the ZeroID base URL. + const assertion = decodeJwtPayload(String(exchange.actor_token)); + expect(assertion.iss).toBe(conductor!.wimseUri); + expect(assertion.sub).toBe(conductor!.wimseUri); + expect(assertion.aud).toBe(BASE_URL); + }); + + test("mintConductorToken without a conductor returns null", async () => { + const manager = new AgentIdentityManager(config, store); + expect(await manager.mintConductorToken("owner-subject-token")).toBeNull(); + expect(zeroid.tokenCalls).toHaveLength(0); + }); + + test("deactivateConductor deactivates in ZeroID and clears the persisted row", async () => { + const manager = new AgentIdentityManager(config, store); + const conductor = await manager.registerConductor("user:owner@test"); + + await manager.deactivateConductor(); + + expect(zeroid.deactivateCalls).toEqual([conductor!.identityId]); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)).toBeNull(); + expect(manager.conductorUri).toBeUndefined(); + expect(await manager.resumeConductor()).toBeNull(); + }); + + test("deactivateConductor works from the persisted row alone (no in-memory conductor)", async () => { + const manager1 = new AgentIdentityManager(config, store); + const conductor = await manager1.registerConductor("user:owner@test"); + + // Fresh manager that never registered nor resumed. + const manager2 = new AgentIdentityManager(config, store); + await manager2.deactivateConductor(); + + expect(zeroid.deactivateCalls).toEqual([conductor!.identityId]); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)).toBeNull(); + }); + + test("deactivateConductor with nothing registered is a no-op", async () => { + const manager = new AgentIdentityManager(config, store); + await manager.deactivateConductor(); + expect(zeroid.deactivateCalls).toHaveLength(0); + }); + + test("a failed remote deactivation keeps the persisted row for retry", async () => { + const manager = new AgentIdentityManager(config, store); + const conductor = await manager.registerConductor("user:owner@test"); + + zeroid.failDeactivate = true; + await manager.deactivateConductor(); + + // Locally stopped, but the durable record of the still-live identity + // survives so a later call can retry against it. + expect(manager.conductorUri).toBeUndefined(); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)!.identityId).toBe( + conductor!.identityId, + ); + + zeroid.failDeactivate = false; + await manager.deactivateConductor(); + expect(zeroid.deactivateCalls).toEqual([ + conductor!.identityId, + conductor!.identityId, + ]); + expect(store.getConductorIdentity(ACCOUNT, PROJECT)).toBeNull(); + }); +}); + +describe("Store conductor_identity persistence", () => { + let tmpDir: string; + let store: Store; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codeoid-conductor-store-")); + store = new Store(join(tmpDir, "store.db")); + }); + + afterEach(() => { + store.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + const row = { + accountId: "acct_a", + projectId: "proj_a", + identityId: "ident-1", + wimseUri: "wimse://zeroid.test/agent/ident-1", + apiKey: "zid_sk_1", + }; + + test("save + get round-trips", () => { + store.saveConductorIdentity(row); + expect(store.getConductorIdentity("acct_a", "proj_a")).toEqual({ + identityId: row.identityId, + wimseUri: row.wimseUri, + apiKey: row.apiKey, + }); + }); + + test("get returns null for an unknown tenant", () => { + store.saveConductorIdentity(row); + expect(store.getConductorIdentity("acct_b", "proj_a")).toBeNull(); + expect(store.getConductorIdentity("acct_a", "proj_b")).toBeNull(); + }); + + test("save upserts — one row per tenant, latest wins", () => { + store.saveConductorIdentity(row); + store.saveConductorIdentity({ + ...row, + identityId: "ident-2", + wimseUri: "wimse://zeroid.test/agent/ident-2", + apiKey: "zid_sk_2", + }); + expect(store.getConductorIdentity("acct_a", "proj_a")!.identityId).toBe( + "ident-2", + ); + }); + + test("tenants are isolated", () => { + store.saveConductorIdentity(row); + store.saveConductorIdentity({ + ...row, + accountId: "acct_b", + identityId: "ident-b", + }); + expect(store.getConductorIdentity("acct_a", "proj_a")!.identityId).toBe( + "ident-1", + ); + expect(store.getConductorIdentity("acct_b", "proj_a")!.identityId).toBe( + "ident-b", + ); + }); + + test("delete removes only the addressed tenant's row", () => { + store.saveConductorIdentity(row); + store.saveConductorIdentity({ + ...row, + accountId: "acct_b", + identityId: "ident-b", + }); + store.deleteConductorIdentity("acct_a", "proj_a"); + expect(store.getConductorIdentity("acct_a", "proj_a")).toBeNull(); + expect(store.getConductorIdentity("acct_b", "proj_a")).not.toBeNull(); + }); +}); diff --git a/src/tests/reranker.test.ts b/src/tests/reranker.test.ts new file mode 100644 index 0000000..2c62211 --- /dev/null +++ b/src/tests/reranker.test.ts @@ -0,0 +1,149 @@ +/** + * Reranker tests — the createReranker factory and the TransformersJsReranker + * batching/score-extraction logic, with @xenova/transformers mocked so no + * model is downloaded. The logit layout handling (dim 1 = single relevance + * logit vs dim 2 = [neg, pos] classes) is the part worth pinning down: a + * wrong stride silently reranks by garbage. + */ + +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** Logits the fake model returns on its next call (flat, row-major). */ +let nextLogits: number[] = []; +let tokenizerCalls: Array<{ texts: string[]; opts: Record }> = + []; +let fromPretrainedCalls: Array<{ model: string; opts?: unknown }> = []; + +const fakeEnv = { cacheDir: "", allowLocalModels: false }; + +mock.module("@xenova/transformers", () => ({ + env: fakeEnv, + AutoTokenizer: { + from_pretrained: async (model: string) => { + fromPretrainedCalls.push({ model }); + return async (texts: string[], opts: Record) => { + tokenizerCalls.push({ texts, opts }); + return { input_ids: texts }; + }; + }, + }, + AutoModelForSequenceClassification: { + from_pretrained: async (model: string, opts?: unknown) => { + fromPretrainedCalls.push({ model, opts }); + return async (_inputs: unknown) => ({ + logits: { data: new Float32Array(nextLogits) }, + }); + }, + }, +})); + +// Import AFTER mock.module so the dynamic import inside init() resolves to +// the fake. +const { createReranker, DEFAULT_RERANKER_MODEL } = await import( + "../daemon/memory/reranker.js" +); +const { TransformersJsReranker } = await import( + "../daemon/memory/reranker-transformersjs.js" +); + +describe("TransformersJsReranker", () => { + let cacheDir: string; + + beforeEach(() => { + cacheDir = mkdtempSync(join(tmpdir(), "codeoid-reranker-")); + nextLogits = []; + tokenizerCalls = []; + fromPretrainedCalls = []; + }); + + afterEach(() => { + rmSync(cacheDir, { recursive: true, force: true }); + }); + + test("init loads tokenizer + model once and points the cache at cacheDir", async () => { + const reranker = new TransformersJsReranker("test/model", cacheDir); + await reranker.init(); + await reranker.init(); + + expect(fakeEnv.cacheDir).toBe(cacheDir); + expect(fakeEnv.allowLocalModels).toBe(true); + // One tokenizer + one model load despite the double init. + expect(fromPretrainedCalls).toHaveLength(2); + expect(fromPretrainedCalls.map((c) => c.model)).toEqual([ + "test/model", + "test/model", + ]); + }); + + test("rerank batches (query, doc) pairs and returns single-logit scores (dim 1)", async () => { + const reranker = new TransformersJsReranker("test/model", cacheDir); + nextLogits = [0.9, -1.2, 3.4]; + + const scores = await reranker.rerank("which session?", ["a", "b", "c"]); + expect(scores).toEqual([ + expect.closeTo(0.9), + expect.closeTo(-1.2), + expect.closeTo(3.4), + ]); + + // Query repeated per doc, docs as text_pair — the cross-encoder contract. + expect(tokenizerCalls).toHaveLength(1); + expect(tokenizerCalls[0]!.texts).toEqual([ + "which session?", + "which session?", + "which session?", + ]); + expect(tokenizerCalls[0]!.opts.text_pair).toEqual(["a", "b", "c"]); + }); + + test("rerank extracts the positive-class logit for two-class models (dim 2)", async () => { + const reranker = new TransformersJsReranker("test/model", cacheDir); + // Rows of [negative, positive]: scores must be 0.7, -0.3. + nextLogits = [0.1, 0.7, 0.5, -0.3]; + + const scores = await reranker.rerank("q", ["a", "b"]); + expect(scores).toEqual([expect.closeTo(0.7), expect.closeTo(-0.3)]); + }); + + test("rerank on an empty doc list returns [] without loading the model", async () => { + const reranker = new TransformersJsReranker("test/model", cacheDir); + expect(await reranker.rerank("q", [])).toEqual([]); + expect(fromPretrainedCalls).toHaveLength(0); + }); + + test("rerank auto-inits, and close() releases so the next call re-inits", async () => { + const reranker = new TransformersJsReranker("test/model", cacheDir); + nextLogits = [1]; + await reranker.rerank("q", ["a"]); + expect(fromPretrainedCalls).toHaveLength(2); + + await reranker.close(); + nextLogits = [2]; + expect(await reranker.rerank("q", ["a"])).toEqual([2]); + expect(fromPretrainedCalls).toHaveLength(4); + }); +}); + +describe("createReranker factory", () => { + test("defaults to the ms-marco cross-encoder", async () => { + const reranker = await createReranker(); + expect(reranker.modelName).toBe(DEFAULT_RERANKER_MODEL); + expect(reranker).toBeInstanceOf(TransformersJsReranker); + }); + + test("honors a custom model + cache dir", async () => { + const dir = mkdtempSync(join(tmpdir(), "codeoid-reranker-factory-")); + try { + const reranker = await createReranker({ + model: "custom/model", + cacheDir: dir, + }); + expect(reranker.modelName).toBe("custom/model"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/tests/scopes.test.ts b/src/tests/scopes.test.ts index 14acbb2..7203982 100644 --- a/src/tests/scopes.test.ts +++ b/src/tests/scopes.test.ts @@ -15,8 +15,8 @@ import { } from "../protocol/scopes.js"; describe("SCOPES constants", () => { - test("all 9 scopes are defined", () => { - expect(Object.keys(SCOPES)).toHaveLength(9); + test("all 11 scopes are defined", () => { + expect(Object.keys(SCOPES)).toHaveLength(11); expect(SCOPES.SESSION_CREATE).toBe("session:create"); expect(SCOPES.SESSION_ATTACH).toBe("session:attach"); expect(SCOPES.SESSION_WATCH).toBe("session:watch"); @@ -25,11 +25,13 @@ describe("SCOPES constants", () => { expect(SCOPES.SESSION_APPROVE).toBe("session:approve"); expect(SCOPES.SESSION_DESTROY).toBe("session:destroy"); expect(SCOPES.SESSION_LIST).toBe("session:list"); + expect(SCOPES.SESSION_READ).toBe("session:read"); + expect(SCOPES.SESSION_DISPATCH).toBe("session:dispatch"); expect(SCOPES.FS_READ).toBe("fs:read"); }); - test("ALL_SCOPES contains all 9", () => { - expect(ALL_SCOPES).toHaveLength(9); + test("ALL_SCOPES contains all 11", () => { + expect(ALL_SCOPES).toHaveLength(11); for (const scope of Object.values(SCOPES)) { expect(ALL_SCOPES).toContain(scope); } @@ -37,7 +39,7 @@ describe("SCOPES constants", () => { test("ALL_SCOPES_STRING is space-delimited", () => { const parts = ALL_SCOPES_STRING.split(" "); - expect(parts).toHaveLength(9); + expect(parts).toHaveLength(11); for (const scope of ALL_SCOPES) { expect(parts).toContain(scope); } @@ -83,6 +85,15 @@ describe("OPERATOR_SCOPES", () => { }); }); +describe("conductor scopes stay conductor-only", () => { + test("watcher and operator profiles do not gain fleet scopes", () => { + for (const profile of [WATCHER_SCOPES, OPERATOR_SCOPES]) { + expect(profile).not.toContain(SCOPES.SESSION_READ); + expect(profile).not.toContain(SCOPES.SESSION_DISPATCH); + } + }); +}); + describe("hasScope", () => { test("returns true when scope is present", () => { expect(hasScope(["session:create", "session:list"], SCOPES.SESSION_CREATE)).toBe(true);