From 6712af1e374cdc72569648ec1a99ade6561df64f Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:28:42 +0300 Subject: [PATCH 1/6] feat(context): add index_freshness trust metadata (slice 1) Surface commit drift, watcher pending sync, and disk-ahead signals on codemap context / MCP context so agents know when structural queries may lag the checkout. Watcher exposes debouncer state; slice 2 will extend MCP/HTTP tool responses per plan. --- docs/agents.md | 2 + docs/plans/index-freshness-trust-bundle.md | 86 ++++++++++++ src/application/context-engine.ts | 4 + src/application/index-freshness.test.ts | 152 +++++++++++++++++++++ src/application/index-freshness.ts | 142 +++++++++++++++++++ src/application/mcp-server.test.ts | 4 + src/application/watcher.ts | 28 +++- 7 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 docs/plans/index-freshness-trust-bundle.md create mode 100644 src/application/index-freshness.test.ts create mode 100644 src/application/index-freshness.ts diff --git a/docs/agents.md b/docs/agents.md index 4e137894..c2966b3c 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -103,6 +103,8 @@ Recipe ids cited in the playbook are machine-validated in tests against the live ## MCP tool allowlist +**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). + **`CODEMAP_MCP_TOOLS`** — comma-separated snake_case MCP tool names. When set, only listed tools register (stderr lists the active set). Unknown names are ignored with a warning. Unset = all tools (default). **`query_batch`** registers only when listed or when unset (eval ablation). Example: `CODEMAP_MCP_TOOLS=query,context,show codemap mcp --no-watch` diff --git a/docs/plans/index-freshness-trust-bundle.md b/docs/plans/index-freshness-trust-bundle.md new file mode 100644 index 00000000..20dc4d28 --- /dev/null +++ b/docs/plans/index-freshness-trust-bundle.md @@ -0,0 +1,86 @@ +# Index freshness trust bundle — plan + +> **Status:** in progress · **Priority:** agent session · **Effort:** S (~3–5 days) · **Roadmap:** [§ Index staleness surfacing](../roadmap.md#agent-session--warm-path-economics), [§ HEAD / index freshness warning](../roadmap.md#agent-session--warm-path-economics) +> +> **Motivator:** Agents treat MCP / HTTP / `context` output as ground truth. Today they can query during watcher debounce (disk ahead of index), after a branch switch (`last_indexed_commit` ≠ `HEAD`), or with a dirty working tree when watch is off — with no signal except running `validate` manually. Wrong structural verdicts follow. + +--- + +## Pre-locked decisions + +| # | Decision | Source | +| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| L.1 | **`index_freshness` is metadata, not a verdict** — structured fields + optional `warning` string; never `pass`/`fail`. | [Moat A](../roadmap.md#moats-load-bearing) | +| L.2 | **Canonical shape in one module** — `src/application/index-freshness.ts`; `context`, MCP, HTTP, and CLI all call `computeIndexFreshness()`. | Same seam as `validate-engine` / `context-engine` | +| L.3 | **`pending_sync` = watcher queue OR in-flight reindex** — true when debouncer has paths **or** a targeted `--files` reindex is running. Not “SQLite mid-transaction” (writes stay transactional). | [roadmap § No split-brain](../roadmap.md#floors-v1-product-shape) | +| L.4 | **Cheap vs full freshness** — every transport gets cheap signals (HEAD drift, pending sync, watch active). **Disk drift** (`getChangedFiles`) runs on `context` and opt-in full mode only (git cost). | Avoid git subprocess on every `query` row | +| L.5 | **JSON tool payloads stay backward-compatible in v1 slice** — slice 1 enriches `context` only; slice 2 adds HTTP headers + MCP `_meta` wrapper without breaking array-shaped `query` results. | Agent eval / golden harness consume raw arrays today | + +--- + +## Freshness envelope + +```typescript +interface IndexFreshness { + head_commit: string | null; + last_indexed_commit: string | null; + commit_drift: boolean; + watch_active: boolean; + pending_sync: boolean; + pending_paths: number; + reindex_in_flight: boolean; + /** Present when `include_disk_drift: true` */ + disk_ahead_of_index?: boolean; + unindexed_change_count?: number; + history_incompatible?: boolean; + /** Single agent-readable line when any concern is active; null when fresh */ + warning: string | null; +} +``` + +--- + +## Shipping cadence (tracer bullets) + +### Slice 1 — `context` + watcher state (this PR) + +1. **`getWatchSyncState()`** on `watcher.ts` — expose debouncer pending count + reindex-in-flight flag (module-scoped; test reset hook). +2. **`index-freshness.ts`** — `computeIndexFreshness(db, { include_disk_drift })`. +3. **`ContextEnvelope.index_freshness`** — `buildContextEnvelope` calls full mode (`include_disk_drift: true`). +4. **CLI `codemap context`** — inherits via `buildContextEnvelope` (no separate code path). +5. **Unit tests** — freshness permutations (drift, pending, disk-ahead, history rewrite). +6. **Docs** — one paragraph in [`agents.md`](../agents.md) + roadmap checkbox note when shipped. + +**Acceptance** + +- [ ] `codemap context --json` includes `index_freshness` with `warning` when HEAD ≠ `last_indexed_commit` +- [ ] Fake watcher with pending debounce → `pending_sync: true` +- [ ] Non-git fixture → `head_commit: null`, no throw + +### Slice 2 — all MCP / HTTP tool responses + +1. **HTTP response headers** on `POST /tool/*`: `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, `X-Codemap-Warning` (when set). +2. **MCP `wrapToolResult`** — for JSON tools, wrap as `{ result, index_freshness }` **or** append a second `content` block with type `text` prefixed `@codemap/index_freshness` (pick in plan-PR review; default: wrapper object for object payloads, header-equivalent second block for arrays — document in MCP instructions). +3. **`/health`** — include cheap freshness when DB exists (optional). + +### Slice 3 — stderr + MCP initialize (optional) + +1. **`codemap mcp` / `serve` boot** — one-line stderr when commit drift detected at prime time. +2. **MCP `instructions` / `codemap://mcp-instructions`** — document freshness fields + agent guidance (“if `pending_sync`, retry after debounce or call `validate`”). + +--- + +## Dependencies + +- [`watcher.ts`](../../src/application/watcher.ts) — debouncer + `isWatchActive()` +- [`index-engine.ts`](../../src/application/index-engine.ts) — `getChangedFiles`, `getCurrentCommit` +- [`context-engine.ts`](../../src/application/context-engine.ts) — envelope builder +- [`validate-engine.ts`](../../src/application/validate-engine.ts) — conceptual sibling (per-file staleness vs index-level) + +--- + +## Out of scope + +- Blocking queries when stale (agents decide). +- Changing incremental indexing semantics ([No split-brain floor](../roadmap.md#floors-v1-product-shape)). +- Shared daemon / multi-session lifecycle (separate roadmap item). diff --git a/src/application/context-engine.ts b/src/application/context-engine.ts index e4f471ed..f3d41ffa 100644 --- a/src/application/context-engine.ts +++ b/src/application/context-engine.ts @@ -1,6 +1,8 @@ import { getMeta, SCHEMA_VERSION } from "../db"; import type { CodemapDatabase } from "../db"; import { CODEMAP_VERSION } from "../version"; +import { computeIndexFreshness } from "./index-freshness"; +import type { IndexFreshness } from "./index-freshness"; import { QUERY_RECIPES } from "./query-recipes"; /** @@ -32,6 +34,7 @@ export interface ContextEnvelope { content: string; }[]; recipes: { id: string; description: string }[]; + index_freshness: IndexFreshness; intent?: { input: string; classified_as: string; @@ -136,6 +139,7 @@ export function buildContextEnvelope( id, description: meta.description, })), + index_freshness: computeIndexFreshness(db, { include_disk_drift: true }), }; if (!opts.compact) { diff --git a/src/application/index-freshness.test.ts b/src/application/index-freshness.test.ts new file mode 100644 index 00000000..d0acadf5 --- /dev/null +++ b/src/application/index-freshness.test.ts @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { resolveCodemapConfig } from "../config"; +import { closeDb, createTables, openDb, setMeta } from "../db"; +import { initCodemap } from "../runtime"; +import { buildContextEnvelope } from "./context-engine"; +import * as indexEngine from "./index-engine"; +import { computeIndexFreshness } from "./index-freshness"; +import { _resetWatchStateForTests, runWatchLoop } from "./watcher"; +import type { WatchBackend } from "./watcher"; + +let benchDir: string; + +beforeEach(() => { + benchDir = mkdtempSync(join(tmpdir(), "index-freshness-")); + mkdirSync(join(benchDir, ".codemap"), { recursive: true }); + initCodemap(resolveCodemapConfig(benchDir, undefined)); + _resetWatchStateForTests(); +}); + +afterEach(() => { + rmSync(benchDir, { recursive: true, force: true }); + _resetWatchStateForTests(); +}); + +function withEmptyDb(fn: (db: ReturnType) => T): T { + const db = openDb(); + try { + createTables(db); + return fn(db); + } finally { + closeDb(db); + } +} + +function fakeBackend(): WatchBackend & { + fire: (kind: "add" | "change" | "unlink", abs: string) => void; +} { + let onEvent: + | ((k: "add" | "change" | "unlink", p: string) => void) + | undefined; + return { + start(opts) { + onEvent = opts.onEvent; + }, + async stop() {}, + fire(kind, abs) { + onEvent?.(kind, abs); + }, + }; +} + +describe("computeIndexFreshness", () => { + it("reports no warning when HEAD matches last_indexed_commit", () => { + const head = "abc123def456789012345678901234567890abcd"; + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + head, + ); + + try { + withEmptyDb((db) => { + setMeta(db, "last_indexed_commit", head); + const f = computeIndexFreshness(db); + expect(f.commit_drift).toBe(false); + expect(f.warning).toBeNull(); + expect(f.head_commit).toBe(head); + }); + } finally { + revParse.mockRestore(); + } + }); + + it("warns on commit drift", () => { + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ); + + try { + withEmptyDb((db) => { + setMeta( + db, + "last_indexed_commit", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ); + const f = computeIndexFreshness(db); + expect(f.commit_drift).toBe(true); + expect(f.warning).toContain("HEAD is bbbbbbb"); + }); + } finally { + revParse.mockRestore(); + } + }); + + it("reports pending_sync when the watcher debouncer has queued paths", async () => { + spyOn(indexEngine, "getCurrentCommit").mockReturnValue(""); + mkdirSync(join(benchDir, "src"), { recursive: true }); + const backend = fakeBackend(); + const handle = runWatchLoop({ + root: benchDir, + excludeDirNames: new Set(["node_modules", ".git", "dist"]), + onChange: () => {}, + debounceMs: 60_000, + backend, + }); + + backend.fire("change", join(benchDir, "src/a.ts")); + + withEmptyDb((db) => { + const f = computeIndexFreshness(db); + expect(f.pending_sync).toBe(true); + expect(f.pending_paths).toBe(1); + expect(f.warning).toContain("pending"); + }); + + await handle.stop(); + }); +}); + +describe("buildContextEnvelope", () => { + it("includes index_freshness with disk drift opt-in", () => { + const head = "cccccccccccccccccccccccccccccccccccccccc"; + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + head, + ); + const changedFiles = spyOn(indexEngine, "getChangedFiles").mockReturnValue({ + changed: [], + deleted: [], + existingPaths: new Set(), + sourceCache: new Map(), + existingHashes: new Map(), + }); + + try { + withEmptyDb((db) => { + setMeta(db, "last_indexed_commit", head); + const envelope = buildContextEnvelope(db, benchDir, { + compact: true, + intent: null, + }); + expect(envelope.index_freshness.head_commit).toBe(head); + expect(envelope.index_freshness.disk_ahead_of_index).toBe(false); + expect(envelope.index_freshness.warning).toBeNull(); + }); + } finally { + revParse.mockRestore(); + changedFiles.mockRestore(); + } + }); +}); diff --git a/src/application/index-freshness.ts b/src/application/index-freshness.ts new file mode 100644 index 00000000..3761cc9b --- /dev/null +++ b/src/application/index-freshness.ts @@ -0,0 +1,142 @@ +import type { CodemapDatabase } from "../db"; +import { getMeta } from "../db"; +import { getChangedFiles, getCurrentCommit } from "./index-engine"; +import { getWatchSyncState, isWatchActive } from "./watcher"; + +/** + * Index-level freshness metadata for agent transports. Complements per-file + * `validate` / snippet `stale` — answers "is the whole index behind the + * checkout?" not "did this one file drift?". + */ +export interface IndexFreshness { + head_commit: string | null; + last_indexed_commit: string | null; + commit_drift: boolean; + watch_active: boolean; + pending_sync: boolean; + pending_paths: number; + reindex_in_flight: boolean; + disk_ahead_of_index?: boolean; + unindexed_change_count?: number; + history_incompatible?: boolean; + warning: string | null; +} + +export interface ComputeIndexFreshnessOpts { + /** Runs `getChangedFiles` (git subprocess). Default false — use on `context`. */ + include_disk_drift?: boolean; +} + +/** + * Compute cheap + optional disk-drift freshness signals from an open DB. + * Pure over DB + module-level watcher state + git subprocesses when opted in. + */ +export function computeIndexFreshness( + db: CodemapDatabase, + opts: ComputeIndexFreshnessOpts = {}, +): IndexFreshness { + const lastIndexed = getMeta(db, "last_indexed_commit") ?? null; + const headRaw = readHeadCommit(); + const headCommit = headRaw === "" ? null : headRaw; + + const watchActive = isWatchActive(); + const sync = getWatchSyncState(); + const pendingSync = sync.pending_paths > 0 || sync.reindex_in_flight === true; + + const commitDrift = + headCommit !== null && lastIndexed !== null && headCommit !== lastIndexed; + + const freshness: IndexFreshness = { + head_commit: headCommit, + last_indexed_commit: lastIndexed, + commit_drift: commitDrift, + watch_active: watchActive, + pending_sync: pendingSync, + pending_paths: sync.pending_paths, + reindex_in_flight: sync.reindex_in_flight, + warning: null, + }; + + if (opts.include_disk_drift === true) { + if (lastIndexed === null) { + freshness.disk_ahead_of_index = true; + freshness.unindexed_change_count = undefined; + freshness.history_incompatible = false; + } else { + const drift = readDiskDrift(db); + if (drift !== undefined) { + freshness.disk_ahead_of_index = drift.disk_ahead_of_index; + freshness.unindexed_change_count = drift.unindexed_change_count; + freshness.history_incompatible = drift.history_incompatible; + } + } + } + + freshness.warning = buildFreshnessWarning(freshness, lastIndexed); + return freshness; +} + +function readHeadCommit(): string { + try { + return getCurrentCommit(); + } catch { + return ""; + } +} + +function readDiskDrift(db: CodemapDatabase): + | { + disk_ahead_of_index: boolean; + unindexed_change_count: number; + history_incompatible: boolean; + } + | undefined { + try { + const changed = getChangedFiles(db); + if (changed === null) { + return { + disk_ahead_of_index: true, + unindexed_change_count: 0, + history_incompatible: true, + }; + } + const count = changed.changed.length + changed.deleted.length; + return { + disk_ahead_of_index: count > 0, + unindexed_change_count: count, + history_incompatible: false, + }; + } catch { + return undefined; + } +} + +function buildFreshnessWarning( + f: IndexFreshness, + lastIndexed: string | null, +): string | null { + if (f.pending_sync) { + const n = f.pending_paths; + if (n > 0) { + return `Index sync pending — ${n} file(s) queued; query results may not reflect the latest edits yet.`; + } + return "Index reindex in progress; query results may not reflect the latest edits yet."; + } + if (f.history_incompatible === true) { + return "Git history is incompatible with the indexed commit; run `codemap --full` to rebuild."; + } + if (lastIndexed === null) { + return "No indexed commit recorded; run `codemap` to build the index."; + } + if (f.commit_drift) { + return `Index was built at ${f.last_indexed_commit?.slice(0, 7) ?? "?"} but HEAD is ${f.head_commit?.slice(0, 7) ?? "?"}; run \`codemap\` to catch up.`; + } + if (f.disk_ahead_of_index === true) { + const n = f.unindexed_change_count ?? 0; + if (n > 0) { + return `Working tree has ${n} unindexed change(s); run \`codemap\` or enable watch before trusting structural queries.`; + } + return "Working tree may be ahead of the index; run `codemap` before trusting structural queries."; + } + return null; +} diff --git a/src/application/mcp-server.test.ts b/src/application/mcp-server.test.ts index dc6ef9ec..e719f4fd 100644 --- a/src/application/mcp-server.test.ts +++ b/src/application/mcp-server.test.ts @@ -604,6 +604,10 @@ describe("MCP server — audit / context / validate tools", () => { expect(json).toMatchObject({ codemap: { schema_version: expect.any(Number) }, project: { root: expect.any(String), file_count: expect.any(Number) }, + index_freshness: expect.objectContaining({ + pending_sync: expect.any(Boolean), + commit_drift: expect.any(Boolean), + }), }); } finally { await server.close(); diff --git a/src/application/watcher.ts b/src/application/watcher.ts index 4dc417d9..5ed5c133 100644 --- a/src/application/watcher.ts +++ b/src/application/watcher.ts @@ -179,6 +179,20 @@ export const DEFAULT_DEBOUNCE_MS = 250; */ let watchActive = false; +/** Debouncer + in-flight reindex state for index freshness metadata. */ +let watchDebouncer: Debouncer | undefined; +let watchReindexInFlight = false; + +export function getWatchSyncState(): Readonly<{ + pending_paths: number; + reindex_in_flight: boolean; +}> { + return { + pending_paths: watchDebouncer?.pendingSize() ?? 0, + reindex_in_flight: watchReindexInFlight, + }; +} + export function isWatchActive(): boolean { return watchActive; } @@ -186,6 +200,8 @@ export function isWatchActive(): boolean { /** Test-only escape hatch — drops the flag so a test that booted a fake watcher leaves a clean slate for siblings. */ export function _resetWatchStateForTests(): void { watchActive = false; + watchDebouncer = undefined; + watchReindexInFlight = false; } /** Test-only escape hatch — flips the flag without booting a real watcher (for handleAudit prelude-skip tests). */ @@ -331,13 +347,21 @@ export function runWatchLoop(opts: WatchLoopOpts): { let inFlight: Promise = Promise.resolve(); const debouncer = createDebouncer((paths) => { inFlight = inFlight - .then(() => Promise.resolve(opts.onChange(paths))) + .then(async () => { + watchReindexInFlight = true; + try { + await Promise.resolve(opts.onChange(paths)); + } finally { + watchReindexInFlight = false; + } + }) .catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); // eslint-disable-next-line no-console -- intentional onChange-error log console.error(`codemap watch: onChange failed — ${msg}`); }); }, debounceMs); + watchDebouncer = debouncer; const backend: WatchBackend = opts.backend ?? createChokidarBackend({ recipesPrefix, stateDirRel }); @@ -404,6 +428,8 @@ export function runWatchLoop(opts: WatchLoopOpts): { // ones still re-indexing). await inFlight; await backend.stop(); + watchDebouncer = undefined; + watchReindexInFlight = false; watchActive = false; }, }; From 068812a6150a0eae48309b3df951816d88c5b2ef Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:33:15 +0300 Subject: [PATCH 2/6] feat(mcp,serve): surface index freshness on all tool responses (slice 2) HTTP tools emit X-Codemap-* freshness headers without changing JSON bodies; MCP merges index_freshness into object payloads and appends a second content block for array-shaped query results. /health includes cheap freshness when DB exists. --- docs/agents.md | 2 +- docs/plans/index-freshness-trust-bundle.md | 18 +++-- src/application/http-server.test.ts | 2 + src/application/http-server.ts | 15 +++- src/application/index-freshness.test.ts | 36 +++++++++- src/application/index-freshness.ts | 79 ++++++++++++++++++++++ src/application/mcp-server.test.ts | 34 ++++++++-- src/application/mcp-server.ts | 21 +++++- 8 files changed, 191 insertions(+), 16 deletions(-) diff --git a/docs/agents.md b/docs/agents.md index c2966b3c..cd2f2d0f 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -103,7 +103,7 @@ Recipe ids cited in the playbook are machine-validated in tests against the live ## MCP tool allowlist -**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). +**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. **MCP:** array-shaped JSON tools (`query`, …) keep row payloads verbatim and append a second `content` block prefixed `@codemap/index_freshness`; object-shaped tools merge `index_freshness` inline. **HTTP:** `POST /tool/*` adds `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, and `X-Codemap-Warning` headers without changing JSON bodies. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). **`CODEMAP_MCP_TOOLS`** — comma-separated snake_case MCP tool names. When set, only listed tools register (stderr lists the active set). Unknown names are ignored with a warning. Unset = all tools (default). **`query_batch`** registers only when listed or when unset (eval ablation). diff --git a/docs/plans/index-freshness-trust-bundle.md b/docs/plans/index-freshness-trust-bundle.md index 20dc4d28..70aa09ce 100644 --- a/docs/plans/index-freshness-trust-bundle.md +++ b/docs/plans/index-freshness-trust-bundle.md @@ -1,6 +1,6 @@ # Index freshness trust bundle — plan -> **Status:** in progress · **Priority:** agent session · **Effort:** S (~3–5 days) · **Roadmap:** [§ Index staleness surfacing](../roadmap.md#agent-session--warm-path-economics), [§ HEAD / index freshness warning](../roadmap.md#agent-session--warm-path-economics) +> **Status:** in progress (slice 2 shipped on branch) · **Priority:** agent session · **Effort:** S (~3–5 days) · **Roadmap:** [§ Index staleness surfacing](../roadmap.md#agent-session--warm-path-economics), [§ HEAD / index freshness warning](../roadmap.md#agent-session--warm-path-economics) > > **Motivator:** Agents treat MCP / HTTP / `context` output as ground truth. Today they can query during watcher debounce (disk ahead of index), after a branch switch (`last_indexed_commit` ≠ `HEAD`), or with a dirty working tree when watch is off — with no signal except running `validate` manually. Wrong structural verdicts follow. @@ -53,15 +53,21 @@ interface IndexFreshness { **Acceptance** -- [ ] `codemap context --json` includes `index_freshness` with `warning` when HEAD ≠ `last_indexed_commit` -- [ ] Fake watcher with pending debounce → `pending_sync: true` -- [ ] Non-git fixture → `head_commit: null`, no throw +- [x] `codemap context --json` includes `index_freshness` with `warning` when HEAD ≠ `last_indexed_commit` +- [x] Fake watcher with pending debounce → `pending_sync: true` +- [x] Non-git fixture → `head_commit: null`, no throw ### Slice 2 — all MCP / HTTP tool responses 1. **HTTP response headers** on `POST /tool/*`: `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, `X-Codemap-Warning` (when set). -2. **MCP `wrapToolResult`** — for JSON tools, wrap as `{ result, index_freshness }` **or** append a second `content` block with type `text` prefixed `@codemap/index_freshness` (pick in plan-PR review; default: wrapper object for object payloads, header-equivalent second block for arrays — document in MCP instructions). -3. **`/health`** — include cheap freshness when DB exists (optional). +2. **MCP `wrapToolResult`** — object JSON payloads merge `index_freshness` inline; array payloads append a second `content` block prefixed `@codemap/index_freshness`. +3. **`/health`** — include cheap `index_freshness` when DB exists. + +**Acceptance** + +- [x] HTTP `query` row array body unchanged; freshness headers present +- [x] MCP `query` array → second content block with `@codemap/index_freshness` +- [x] MCP object payloads (`query` summary, `show`, …) merge `index_freshness` inline ### Slice 3 — stderr + MCP initialize (optional) diff --git a/src/application/http-server.test.ts b/src/application/http-server.test.ts index 220acd0d..08347db3 100644 --- a/src/application/http-server.test.ts +++ b/src/application/http-server.test.ts @@ -142,6 +142,8 @@ describe("http-server — POST /tool/query", () => { }); expect(r.status).toBe(200); expect(r.headers.get("content-type")).toContain("application/json"); + expect(r.headers.get("X-Codemap-Pending-Sync")).toBe("false"); + expect(r.headers.get("X-Codemap-Commit-Drift")).toBe("false"); expect(r.json).toEqual([ { name: "bar", kind: "const" }, { name: "foo", kind: "function" }, diff --git a/src/application/http-server.ts b/src/application/http-server.ts index 48ff3b5c..b254ba68 100644 --- a/src/application/http-server.ts +++ b/src/application/http-server.ts @@ -12,6 +12,10 @@ import { getTsconfigPath, initCodemap, } from "../runtime"; +import { + applyIndexFreshnessHeaders, + readCheapIndexFreshness, +} from "./index-freshness"; import { listResources, readResource } from "./resource-handlers"; import { affectedArgsSchema, @@ -248,10 +252,16 @@ export async function handleRequest( // Liveness probe — auth-exempt so monitoring works without the token. if (method === "GET" && path === "/health") { + const freshness = readCheapIndexFreshness(); + applyIndexFreshnessHeaders(res, freshness); return writeJson( res, 200, - { ok: true, version: opts.version }, + { + ok: true, + version: opts.version, + ...(freshness !== null ? { index_freshness: freshness } : {}), + }, opts.version, ); } @@ -390,6 +400,9 @@ function writeToolResult( result: ToolResult, version: string, ): void { + const freshness = result.ok ? readCheapIndexFreshness() : null; + applyIndexFreshnessHeaders(res, freshness); + if (!result.ok) { return writeJson( res, diff --git a/src/application/index-freshness.test.ts b/src/application/index-freshness.test.ts index d0acadf5..18c65f9c 100644 --- a/src/application/index-freshness.test.ts +++ b/src/application/index-freshness.test.ts @@ -8,7 +8,10 @@ import { closeDb, createTables, openDb, setMeta } from "../db"; import { initCodemap } from "../runtime"; import { buildContextEnvelope } from "./context-engine"; import * as indexEngine from "./index-engine"; -import { computeIndexFreshness } from "./index-freshness"; +import { + computeIndexFreshness, + mergeIndexFreshnessIntoJsonPayload, +} from "./index-freshness"; import { _resetWatchStateForTests, runWatchLoop } from "./watcher"; import type { WatchBackend } from "./watcher"; @@ -119,6 +122,37 @@ describe("computeIndexFreshness", () => { }); }); +describe("mergeIndexFreshnessIntoJsonPayload", () => { + const freshness = { + head_commit: "a".repeat(40), + last_indexed_commit: "b".repeat(40), + commit_drift: true, + watch_active: false, + pending_sync: false, + pending_paths: 0, + reindex_in_flight: false, + warning: "drift", + }; + + it("leaves array payloads unchanged", () => { + const rows = [{ path: "src/a.ts" }]; + expect(mergeIndexFreshnessIntoJsonPayload(rows, freshness)).toBe(rows); + }); + + it("merges into object payloads", () => { + expect(mergeIndexFreshnessIntoJsonPayload({ count: 3 }, freshness)).toEqual( + { count: 3, index_freshness: freshness }, + ); + }); + + it("skips when index_freshness is already present", () => { + const payload = { index_freshness: freshness, file_count: 1 }; + expect(mergeIndexFreshnessIntoJsonPayload(payload, freshness)).toBe( + payload, + ); + }); +}); + describe("buildContextEnvelope", () => { it("includes index_freshness with disk drift opt-in", () => { const head = "cccccccccccccccccccccccccccccccccccccccc"; diff --git a/src/application/index-freshness.ts b/src/application/index-freshness.ts index 3761cc9b..f3a707f2 100644 --- a/src/application/index-freshness.ts +++ b/src/application/index-freshness.ts @@ -1,3 +1,6 @@ +import type { ServerResponse } from "node:http"; + +import { closeDb, openDb } from "../db"; import type { CodemapDatabase } from "../db"; import { getMeta } from "../db"; import { getChangedFiles, getCurrentCommit } from "./index-engine"; @@ -140,3 +143,79 @@ function buildFreshnessWarning( } return null; } + +/** Prefix for the MCP secondary content block carrying cheap freshness metadata. */ +export const INDEX_FRESHNESS_MCP_PREFIX = "@codemap/index_freshness\n"; + +/** + * Read cheap freshness (no disk-drift git walk) from the live index DB. + * Returns null when the DB is unavailable — transports omit headers/blocks. + */ +export function readCheapIndexFreshness(): IndexFreshness | null { + try { + const db = openDb(); + try { + return computeIndexFreshness(db); + } finally { + closeDb(db, { readonly: true }); + } + } catch { + return null; + } +} + +/** HTTP response headers mirroring cheap freshness signals (slice 2). */ +export function applyIndexFreshnessHeaders( + res: ServerResponse, + freshness: IndexFreshness | null, +): void { + if (freshness === null) return; + res.setHeader( + "X-Codemap-Pending-Sync", + freshness.pending_sync ? "true" : "false", + ); + res.setHeader( + "X-Codemap-Commit-Drift", + freshness.commit_drift ? "true" : "false", + ); + if (freshness.warning !== null) { + res.setHeader("X-Codemap-Warning", freshness.warning); + } +} + +export function formatIndexFreshnessMcpBlock( + freshness: IndexFreshness, +): string { + return INDEX_FRESHNESS_MCP_PREFIX + JSON.stringify(freshness); +} + +/** + * Merge `index_freshness` into JSON object tool payloads. Array payloads stay + * verbatim — MCP attaches {@link formatIndexFreshnessMcpBlock} as a second block. + * Skips when the payload already carries freshness (`context` tool). + */ +export function mergeIndexFreshnessIntoJsonPayload( + payload: unknown, + freshness: IndexFreshness | null, +): unknown { + if (freshness === null) return payload; + if (Array.isArray(payload)) return payload; + if ( + payload !== null && + typeof payload === "object" && + "index_freshness" in payload + ) { + return payload; + } + if (payload !== null && typeof payload === "object") { + return { + ...(payload as Record), + index_freshness: freshness, + }; + } + return { result: payload, index_freshness: freshness }; +} + +export function jsonPayloadNeedsMcpFreshnessBlock(payload: unknown): boolean { + return Array.isArray(payload); +} diff --git a/src/application/mcp-server.test.ts b/src/application/mcp-server.test.ts index e719f4fd..d5d4c889 100644 --- a/src/application/mcp-server.test.ts +++ b/src/application/mcp-server.test.ts @@ -151,7 +151,33 @@ describe("MCP server — query tool", () => { name: "query", arguments: { sql: "SELECT path FROM files", summary: true }, }); - expect(readJson(r)).toEqual({ count: 3 }); + expect(readJson(r)).toMatchObject({ + count: 3, + index_freshness: expect.objectContaining({ + pending_sync: expect.any(Boolean), + }), + }); + } finally { + await server.close(); + } + }); + + it("query array payloads keep rows verbatim and attach freshness block", async () => { + const { client, server } = await makeClient(); + try { + const r = await client.callTool({ + name: "query", + arguments: { sql: "SELECT path FROM files ORDER BY path" }, + }); + const blocks = + (r as { content?: Array<{ text?: string }> }).content ?? []; + expect(blocks).toHaveLength(2); + expect(JSON.parse(blocks[0]!.text!)).toEqual([ + { path: "docs/c.md" }, + { path: "src/a.ts" }, + { path: "src/b.ts" }, + ]); + expect(blocks[1]!.text).toStartWith("@codemap/index_freshness\n"); } finally { await server.close(); } @@ -364,7 +390,7 @@ describe("MCP server — query_recipe tool", () => { name: "query_recipe", arguments: { recipe: "deprecated-symbols", summary: true }, }); - expect(readJson(r)).toEqual({ count: 0 }); + expect(readJson(r)).toMatchObject({ count: 0 }); } finally { await server.close(); } @@ -746,7 +772,7 @@ describe("MCP server — baseline tools", () => { name: "drop_baseline", arguments: { name: "to-drop" }, }); - expect(readJson(first)).toEqual({ dropped: "to-drop" }); + expect(readJson(first)).toMatchObject({ dropped: "to-drop" }); const second = await client.callTool({ name: "drop_baseline", @@ -1105,7 +1131,7 @@ describe("MCP server — show + snippet tools", () => { }); expect(r.isError).not.toBe(true); const json = readJson(r); - expect(json).toEqual({ matches: [] }); + expect(json).toMatchObject({ matches: [] }); } finally { await server.close(); } diff --git a/src/application/mcp-server.ts b/src/application/mcp-server.ts index 14b15f12..d85c8e11 100644 --- a/src/application/mcp-server.ts +++ b/src/application/mcp-server.ts @@ -13,6 +13,12 @@ import { initCodemap, } from "../runtime"; import { assembleMcpInstructions } from "./agent-content"; +import { + formatIndexFreshnessMcpBlock, + jsonPayloadNeedsMcpFreshnessBlock, + mergeIndexFreshnessIntoJsonPayload, + readCheapIndexFreshness, +} from "./index-freshness"; import { isMcpToolEnabled, logMcpToolAllowlist, @@ -110,9 +116,18 @@ function wrapToolResult(r: ToolResult) { }; } if (r.format === "json") { - return { - content: [{ type: "text" as const, text: JSON.stringify(r.payload) }], - }; + const freshness = readCheapIndexFreshness(); + const payload = mergeIndexFreshnessIntoJsonPayload(r.payload, freshness); + const content: Array<{ type: "text"; text: string }> = [ + { type: "text", text: JSON.stringify(payload) }, + ]; + if (freshness !== null && jsonPayloadNeedsMcpFreshnessBlock(r.payload)) { + content.push({ + type: "text", + text: formatIndexFreshnessMcpBlock(freshness), + }); + } + return { content }; } return { content: [{ type: "text" as const, text: r.payload }] }; } From a6764b6207d713eac4b8a4bc4776495bb859d321 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:34:27 +0300 Subject: [PATCH 3/6] feat(mcp,serve): boot stderr freshness warnings and MCP playbook (slice 3) Warn on stderr when index_freshness concerns remain after bootstrap or watch prime; document index_freshness fields and agent retry guidance in MCP instructions. Marks trust-bundle plan and roadmap items shipped. --- docs/plans/index-freshness-trust-bundle.md | 17 +++++-- docs/roadmap.md | 4 +- src/application/http-server.ts | 10 +++- src/application/index-freshness.test.ts | 54 +++++++++++++++++++++ src/application/index-freshness.ts | 30 ++++++++++-- src/application/mcp-server.test.ts | 2 + src/application/mcp-server.ts | 10 +++- templates/agent-content/mcp-instructions.md | 24 ++++++++- 8 files changed, 137 insertions(+), 14 deletions(-) diff --git a/docs/plans/index-freshness-trust-bundle.md b/docs/plans/index-freshness-trust-bundle.md index 70aa09ce..b2e446e3 100644 --- a/docs/plans/index-freshness-trust-bundle.md +++ b/docs/plans/index-freshness-trust-bundle.md @@ -1,6 +1,6 @@ # Index freshness trust bundle — plan -> **Status:** in progress (slice 2 shipped on branch) · **Priority:** agent session · **Effort:** S (~3–5 days) · **Roadmap:** [§ Index staleness surfacing](../roadmap.md#agent-session--warm-path-economics), [§ HEAD / index freshness warning](../roadmap.md#agent-session--warm-path-economics) +> **Status:** shipped · **Priority:** agent session · **Effort:** S (~3–5 days) · **PR:** [#149](https://github.com/stainless-code/codemap/pull/149) > > **Motivator:** Agents treat MCP / HTTP / `context` output as ground truth. Today they can query during watcher debounce (disk ahead of index), after a branch switch (`last_indexed_commit` ≠ `HEAD`), or with a dirty working tree when watch is off — with no signal except running `validate` manually. Wrong structural verdicts follow. @@ -69,13 +69,22 @@ interface IndexFreshness { - [x] MCP `query` array → second content block with `@codemap/index_freshness` - [x] MCP object payloads (`query` summary, `show`, …) merge `index_freshness` inline -### Slice 3 — stderr + MCP initialize (optional) +### Slice 3 — stderr + MCP initialize -1. **`codemap mcp` / `serve` boot** — one-line stderr when commit drift detected at prime time. -2. **MCP `instructions` / `codemap://mcp-instructions`** — document freshness fields + agent guidance (“if `pending_sync`, retry after debounce or call `validate`”). +1. **`codemap mcp` / `serve` boot** — one-line stderr when freshness concerns remain after bootstrap / watch prime. +2. **MCP `instructions` / `codemap://mcp-instructions`** — document freshness fields + agent guidance. + +**Acceptance** + +- [x] `codemap mcp` / `serve` stderr warns when `index_freshness.warning` is set after prime +- [x] MCP initialize instructions document `index_freshness`, `pending_sync`, and agent retry guidance --- +## Shipped + +Trust bundle complete in [#149](https://github.com/stainless-code/codemap/pull/149) (slices 1–3). Roadmap items **Index staleness surfacing** and **HEAD / index freshness warning** satisfied via `index_freshness` metadata + transport headers + boot stderr. + ## Dependencies - [`watcher.ts`](../../src/application/watcher.ts) — debouncer + `isWatchActive()` diff --git a/docs/roadmap.md b/docs/roadmap.md index 614b786a..a0f20cce 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -78,11 +78,11 @@ Long-running MCP / HTTP sessions dominate agent workflows; one-shot CLI keeps th - [ ] **MCP shared daemon per project** — one watcher + one SQLite writer per indexed root; Unix socket / named pipe so concurrent agent sessions share a live index instead of each spawning watchers and contending on WAL. Complements perf item **6.1** (read pool) but is a separate write-side + lifecycle concern. Effort: L. - [ ] **Rich `context` composer** — expand session bootstrap beyond counts + recipe catalog: ranked hub files, sample markers, optional inlined signatures, fan-in leaders, and links to high-value recipes for the detected intent. Goal: one bootstrap call replaces a common show → explore chain on session start. Effort: M. - [ ] **Codebase map in bootstrap responses** — hash-stable structural summary (top hubs, CLI entry hints, schema version, index freshness) auto-included in `context` / MCP initialize payload. Opt-out via flag. Effort: S–M. -- [ ] **Index staleness surfacing** — when the watcher has pending files (debounce window), MCP/HTTP responses and `context` carry an explicit pending-sync indicator; align with `validate` staleness rows. Effort: S. +- [x] **Index staleness surfacing** — `index_freshness.pending_sync` on `context`, MCP tool metadata, and HTTP headers when the watcher debounce queue or in-flight reindex is active. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). Shipped [#149](https://github.com/stainless-code/codemap/pull/149). - [ ] **Adaptive output budgets** — scale explore/trace/node snippet char caps and row limits from indexed file/symbol counts so large trees do not blow token budgets. Effort: S. - [ ] **MCP session lifecycle hygiene** — idle timeout, client disconnect detection, graceful watcher shutdown on last client; avoid orphaned watchers after agent host crashes. Effort: S–M. - [ ] **`agents init` uninstall (teardown)** — symmetric inverse of init for failed pilots, template mistakes, or leaving a repo: remove codemap-managed MCP entries, pointer sections, and IDE symlinks only (same scoped paths as init; never delete user-authored `.agents/` siblings). `--target` filter, `--yes` non-interactive. Not the happy-path docs story — adoption stays `init --mcp --git-hooks` + committed `.agents/`. Effort: S. -- [ ] **HEAD / index freshness warning** — stderr or `context` field when `meta.last_indexed_commit` ≠ current `HEAD`, or when the checkout (including linked git worktrees) may not match the tree the index was built for. Effort: S. +- [x] **HEAD / index freshness warning** — `index_freshness.commit_drift` + `warning` on `context` / tool metadata; boot stderr on `codemap mcp` / `serve` when concerns remain after prime. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). Shipped [#149](https://github.com/stainless-code/codemap/pull/149). ### Recipe & audit enrichment diff --git a/src/application/http-server.ts b/src/application/http-server.ts index b254ba68..55b245c5 100644 --- a/src/application/http-server.ts +++ b/src/application/http-server.ts @@ -15,6 +15,7 @@ import { import { applyIndexFreshnessHeaders, readCheapIndexFreshness, + warnIndexFreshnessToStderr, } from "./index-freshness"; import { listResources, readResource } from "./resource-handlers"; import { @@ -121,6 +122,9 @@ const TOOL_NAMES = [ */ export async function runHttpServer(opts: HttpServerOpts): Promise { await bootstrapForServe(opts); + if (opts.watch !== true) { + warnIndexFreshnessToStderr("codemap serve"); + } const server = createServer((req, res) => { handleRequest(req, res, opts).catch((err: unknown) => { @@ -145,12 +149,16 @@ export async function runHttpServer(opts: HttpServerOpts): Promise { let stopWatch: (() => Promise) | undefined; if (opts.watch === true) { try { + const prime = createPrimeIndex({ quiet: false, label: "codemap serve" }); const handle = runWatchLoop({ root: getProjectRoot(), excludeDirNames: getExcludeDirNames(), recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, - onPrime: createPrimeIndex({ quiet: false, label: "codemap serve" }), + onPrime: async () => { + await prime(); + warnIndexFreshnessToStderr("codemap serve"); + }, onChange: createReindexOnChange({ quiet: false, label: "codemap serve", diff --git a/src/application/index-freshness.test.ts b/src/application/index-freshness.test.ts index 18c65f9c..42eec120 100644 --- a/src/application/index-freshness.test.ts +++ b/src/application/index-freshness.test.ts @@ -11,6 +11,7 @@ import * as indexEngine from "./index-engine"; import { computeIndexFreshness, mergeIndexFreshnessIntoJsonPayload, + warnIndexFreshnessToStderr, } from "./index-freshness"; import { _resetWatchStateForTests, runWatchLoop } from "./watcher"; import type { WatchBackend } from "./watcher"; @@ -153,6 +154,59 @@ describe("mergeIndexFreshnessIntoJsonPayload", () => { }); }); +describe("warnIndexFreshnessToStderr", () => { + it("logs a one-line warning when freshness concerns remain", () => { + const errLog = spyOn(console, "error").mockImplementation(() => {}); + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ); + + try { + withEmptyDb((db) => { + setMeta( + db, + "last_indexed_commit", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ); + warnIndexFreshnessToStderr("codemap mcp"); + }); + expect(errLog).toHaveBeenCalledWith( + expect.stringContaining("codemap mcp:"), + ); + } finally { + errLog.mockRestore(); + revParse.mockRestore(); + } + }); + + it("stays silent when warning is null", () => { + const head = "cccccccccccccccccccccccccccccccccccccccc"; + const errLog = spyOn(console, "error").mockImplementation(() => {}); + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + head, + ); + const changedFiles = spyOn(indexEngine, "getChangedFiles").mockReturnValue({ + changed: [], + deleted: [], + existingPaths: new Set(), + sourceCache: new Map(), + existingHashes: new Map(), + }); + + try { + withEmptyDb((db) => { + setMeta(db, "last_indexed_commit", head); + warnIndexFreshnessToStderr("codemap serve"); + }); + expect(errLog).not.toHaveBeenCalled(); + } finally { + errLog.mockRestore(); + revParse.mockRestore(); + changedFiles.mockRestore(); + } + }); +}); + describe("buildContextEnvelope", () => { it("includes index_freshness with disk drift opt-in", () => { const head = "cccccccccccccccccccccccccccccccccccccccc"; diff --git a/src/application/index-freshness.ts b/src/application/index-freshness.ts index f3a707f2..b320e27c 100644 --- a/src/application/index-freshness.ts +++ b/src/application/index-freshness.ts @@ -148,14 +148,17 @@ function buildFreshnessWarning( export const INDEX_FRESHNESS_MCP_PREFIX = "@codemap/index_freshness\n"; /** - * Read cheap freshness (no disk-drift git walk) from the live index DB. - * Returns null when the DB is unavailable — transports omit headers/blocks. + * Read freshness from the live index DB. Returns null when unavailable. + * Default is cheap mode (no disk-drift git walk) — pass `{ include_disk_drift: true }` + * for boot-time / `context` diagnostics. */ -export function readCheapIndexFreshness(): IndexFreshness | null { +export function readIndexFreshness( + opts: ComputeIndexFreshnessOpts = {}, +): IndexFreshness | null { try { const db = openDb(); try { - return computeIndexFreshness(db); + return computeIndexFreshness(db, opts); } finally { closeDb(db, { readonly: true }); } @@ -164,6 +167,25 @@ export function readCheapIndexFreshness(): IndexFreshness | null { } } +/** Cheap per-tool freshness — no disk-drift git walk. */ +export function readCheapIndexFreshness(): IndexFreshness | null { + return readIndexFreshness(); +} + +/** + * One-line stderr warning when freshness concerns remain after boot / prime. + * Used by `codemap mcp` and `codemap serve` only — stdout stays JSON-RPC clean. + */ +export function warnIndexFreshnessToStderr( + label: string, + opts: ComputeIndexFreshnessOpts = { include_disk_drift: true }, +): void { + const freshness = readIndexFreshness(opts); + if (freshness?.warning === null || freshness?.warning === undefined) return; + // eslint-disable-next-line no-console -- intentional bootstrap warning on stderr + console.error(`${label}: ${freshness.warning}`); +} + /** HTTP response headers mirroring cheap freshness signals (slice 2). */ export function applyIndexFreshnessHeaders( res: ServerResponse, diff --git a/src/application/mcp-server.test.ts b/src/application/mcp-server.test.ts index d5d4c889..0865a60d 100644 --- a/src/application/mcp-server.test.ts +++ b/src/application/mcp-server.test.ts @@ -67,6 +67,8 @@ describe("MCP server — initialize instructions", () => { expect(instructions!.length).toBeGreaterThan(500); expect(instructions).toContain("Session start"); expect(instructions).toContain("codemap://rule"); + expect(instructions).toContain("index_freshness"); + expect(instructions).toContain("pending_sync"); } finally { await server.close(); } diff --git a/src/application/mcp-server.ts b/src/application/mcp-server.ts index d85c8e11..760350d7 100644 --- a/src/application/mcp-server.ts +++ b/src/application/mcp-server.ts @@ -18,6 +18,7 @@ import { jsonPayloadNeedsMcpFreshnessBlock, mergeIndexFreshnessIntoJsonPayload, readCheapIndexFreshness, + warnIndexFreshnessToStderr, } from "./index-freshness"; import { isMcpToolEnabled, @@ -579,6 +580,9 @@ async function bootstrapForMcp(opts: ServerOpts): Promise { */ export async function runMcpServer(opts: ServerOpts): Promise { await bootstrapForMcp(opts); + if (opts.watch !== true) { + warnIndexFreshnessToStderr("codemap mcp"); + } const server = createMcpServer(opts); const transport = new StdioServerTransport(); await server.connect(transport); @@ -588,12 +592,16 @@ export async function runMcpServer(opts: ServerOpts): Promise { // eslint-disable-next-line no-console -- intentional bootstrap log on stderr console.error("codemap mcp: --watch enabled, booting file watcher..."); try { + const prime = createPrimeIndex({ quiet: false, label: "codemap mcp" }); const handle = runWatchLoop({ root: getProjectRoot(), excludeDirNames: getExcludeDirNames(), recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, - onPrime: createPrimeIndex({ quiet: false, label: "codemap mcp" }), + onPrime: async () => { + await prime(); + warnIndexFreshnessToStderr("codemap mcp"); + }, onChange: createReindexOnChange({ quiet: false, label: "codemap mcp", diff --git a/templates/agent-content/mcp-instructions.md b/templates/agent-content/mcp-instructions.md index 24328e9b..c89a9ea7 100644 --- a/templates/agent-content/mcp-instructions.md +++ b/templates/agent-content/mcp-instructions.md @@ -4,10 +4,29 @@ Operational playbook injected into the MCP initialize handshake. Full schema, re ## Session start -1. **`context`** — project root, schema version, file count, language breakdown, recipe summary (one call replaces 4–5 queries). +1. **`context`** — project root, schema version, file count, language breakdown, recipe summary, **`index_freshness`** (one call replaces 4–5 queries). 2. **`codemap://rule`** — always-on priming: query the index for structure, don't grep. 3. When you need the catalog or DDL: **`codemap://recipes`**, **`codemap://schema`**. +## Index freshness + +Every JSON tool response carries index-level freshness metadata (not a pass/fail verdict): + +| Surface | Where to read it | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | +| **`context`** | `index_freshness` object (includes disk-drift counts) | +| **Object payloads** (`show`, `query` summary, …) | `index_freshness` merged inline | +| **Array payloads** (`query` rows) | second `content` block prefixed `@codemap/index_freshness` | +| **HTTP** | `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, `X-Codemap-Warning` headers (JSON body unchanged) | + +Key fields: `pending_sync` (watcher debounce queue or in-flight reindex), `commit_drift` (`HEAD` ≠ `last_indexed_commit`), `warning` (single agent-readable line when anything is off). + +**Agent guidance** + +- If **`pending_sync: true`** — wait ~250ms (debounce) and retry, or call **`validate`** for per-file drift. +- If **`commit_drift: true`** or **`warning`** is set — run **`codemap`** (or rely on watch prime) before treating structural queries as authoritative. +- Prefer **`context`** at session start for the full disk-drift picture; use **`validate`** / snippet `stale` for individual files. + ## Common tasks | Goal | MCP tool | Recipe twin (`query_recipe`) | @@ -25,7 +44,8 @@ Operational playbook injected into the MCP initialize handshake. Full schema, re | CI / SARIF | **`query_recipe`** + `format: "sarif"` | `deprecated-symbols`, `boundary-violations`, … | | Ad-hoc SQL | **`query`** | — | | N statements / one round-trip | **`query_batch`** (no CLI verb; MCP + HTTP) | N × `query` | -| Index freshness | **`validate`** | — | +| Index freshness (index-level) | **`context`** (`index_freshness`) + tool metadata above | — | +| Per-file staleness | **`validate`** | — | | Drift vs baseline | **`audit`** (`baseline_prefix` and/or per-delta `baselines`) | save via **`save_baseline`**; CLI-only diff via `codemap query --baseline` | | Apply recipe diff rows | **`apply`** | recipe must emit `{file_path, line_start, before_pattern, after_pattern}` rows | From 4a037fcc0d5a94fdcb65dbd54c43ea442ba0c807 Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:42:36 +0300 Subject: [PATCH 4/6] fix(freshness): gate transport boot on watch prime and track prime in-flight Address PR review: expose watch prime as reindex_in_flight/pending_sync, await watch ready before MCP connect / HTTP listen, skip redundant DB reads for context payloads, move HTTP headers to http-server, and extend tests/docs. --- docs/agents.md | 2 +- docs/architecture.md | 2 +- docs/plans/index-freshness-trust-bundle.md | 14 ++-- src/application/http-server.ts | 80 ++++++++++++------- src/application/index-freshness.test.ts | 64 +++++++++++++++ src/application/index-freshness.ts | 42 +++++----- src/application/mcp-server.ts | 60 +++++++------- src/application/watcher.test.ts | 5 +- src/application/watcher.ts | 11 ++- templates/agent-content/mcp-instructions.md | 2 +- .../agent-content/skill/50-maintenance.md | 7 +- 11 files changed, 191 insertions(+), 98 deletions(-) diff --git a/docs/agents.md b/docs/agents.md index cd2f2d0f..72923ad2 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -103,7 +103,7 @@ Recipe ids cited in the playbook are machine-validated in tests against the live ## MCP tool allowlist -**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. **MCP:** array-shaped JSON tools (`query`, …) keep row payloads verbatim and append a second `content` block prefixed `@codemap/index_freshness`; object-shaped tools merge `index_freshness` inline. **HTTP:** `POST /tool/*` adds `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, and `X-Codemap-Warning` headers without changing JSON bodies. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). +**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. **MCP:** array-shaped JSON tools (`query`, …) keep row payloads verbatim and append a second `content` block prefixed `@codemap/index_freshness`; object-shaped tools merge `index_freshness` inline. **HTTP:** `POST /tool/*` adds `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, and `X-Codemap-Warning` headers without changing JSON bodies; **`GET /health`** includes full cheap `index_freshness` when the DB is readable. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). **`CODEMAP_MCP_TOOLS`** — comma-separated snake_case MCP tool names. When set, only listed tools register (stderr lists the active set). Unknown names are ignored with a warning. Unset = all tools (default). **`query_batch`** registers only when listed or when unset (eval ablation). diff --git a/docs/architecture.md b/docs/architecture.md index f4eca66e..d5ff83fc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -129,7 +129,7 @@ A local SQLite database (`.codemap/index.db`) indexes the project tree and store **PR-comment wiring:** **`src/cli/cmd-pr-comment.ts`** (argv — `` (or `-` for stdin) + `--shape audit|sarif` + `--json`) + **`src/application/pr-comment-engine.ts`** (engine — `renderAuditComment` / `renderSarifComment` / `detectCommentInputShape`). Renders an audit-JSON envelope or SARIF doc as a markdown PR-summary comment; designed for surfaces SARIF→Code-Scanning doesn't cover (private repos without GHAS, aggregate audit deltas without `file:line` anchors, bot-context seeding). Output: bare markdown by default; `--json` envelope `{markdown, findings_count, kind}` for action.yml steps. Audit-mode groups by delta with `
` sections (added + removed); SARIF-mode groups by `ruleId`. Lists >50 entries collapse to `… and N more`. v1.0 ships the (b) summary-comment shape; (c) inline-review comments deferred per Q4 of [`plans/github-marketplace-action.md`](./plans/github-marketplace-action.md). -**Context wiring:** **`src/cli/cmd-context.ts`** (argv + render) + **`src/application/context-engine.ts`** (engine — **`buildContextEnvelope`**, **`classifyIntent`**, `ContextEnvelope` type). `buildContextEnvelope` composes the JSON envelope from existing recipes (`fan-in` for `hubs`, `markers` SELECT for `sample_markers`, `QUERY_RECIPES` map for the catalog). **`classifyIntent`** maps `--for ""` to one of `refactor | debug | test | feature | explore | other` via regex against the trimmed input; whitespace-only intents are rejected. `--compact` drops `hubs` + `sample_markers` and emits one-line JSON; otherwise pretty-prints with 2-space indent. +**Context wiring:** **`src/cli/cmd-context.ts`** (argv + render) + **`src/application/context-engine.ts`** (engine — **`buildContextEnvelope`**, **`classifyIntent`**, `ContextEnvelope` type). `buildContextEnvelope` composes the JSON envelope from existing recipes (`fan-in` for `hubs`, `markers` SELECT for `sample_markers`, `QUERY_RECIPES` map for the catalog) and **`index_freshness`** via **`src/application/index-freshness.ts`** (`computeIndexFreshness` with disk-drift). **`classifyIntent`** maps `--for ""` to one of `refactor | debug | test | feature | explore | other` via regex against the trimmed input; whitespace-only intents are rejected. `--compact` drops `hubs` + `sample_markers` and emits one-line JSON; otherwise pretty-prints with 2-space indent. Product-shape constraint: [No split-brain incremental index](./roadmap.md#floors-v1-product-shape). **Impact wiring:** **`src/cli/cmd-impact.ts`** (argv — `` + `--direction up|down|both` + `--depth N` + `--via dependencies|calls|imports|all` + `--limit N` + `--summary` + `--json`; bootstrap absorbs `--root`/`--config`) + **`src/application/impact-engine.ts`** (engine — `findImpact({db, target, direction?, via?, depth?, limit?})`). Pure transport-agnostic walker over the calls + dependencies + imports graphs; CLI / MCP / HTTP all dispatch the same engine function via `tool-handlers.ts`'s `handleImpact`. Target auto-resolves: contains `/` or matches `files.path` → file target; otherwise symbol (case-sensitive). Walks compatible backends per resolved kind: **symbol** → `calls` (callers / callees by `caller_name` / `callee_name`); **file** → `dependencies` (`from_path` / `to_path`) + `imports` (`file_path` / `resolved_path`, `IS NOT NULL` filter). `--via ` overrides; mismatched explicit choices land in `skipped_backends` (no error — agents see why their backend selection yielded fewer rows than expected). One `WITH RECURSIVE` per (direction, backend) combo with cycle detection via path-string `instr` check (SQLite has no native cycle predicate); JS-side merge + dedup by `(direction, kind, name?, file_path)` keeping the shallowest depth. `--depth 0` uses an unbounded sentinel (`UNBOUNDED_DEPTH_SENTINEL = 1_000_000`); cycle detection + `LIMIT` keep cyclic graphs cheap regardless. Termination reason classification: `limit` (truncated) > `depth` (any node sat at the cap) > `exhausted`. Result envelope: `{target, direction, via, depth_limit, matches: [{depth, direction, edge, kind, name?, file_path}], summary: {nodes, max_depth_reached, by_kind, terminated_by}, skipped_backends?}`. `--summary` blanks `matches` (transport bandwidth saver) but preserves `summary.nodes` so CI gates (`jq '.summary.nodes'`) still see the count. SARIF / annotations not supported (graph traversal, not findings — the parser accepts the flag combos but the engine only emits JSON). diff --git a/docs/plans/index-freshness-trust-bundle.md b/docs/plans/index-freshness-trust-bundle.md index b2e446e3..6bcc2fc4 100644 --- a/docs/plans/index-freshness-trust-bundle.md +++ b/docs/plans/index-freshness-trust-bundle.md @@ -8,13 +8,13 @@ ## Pre-locked decisions -| # | Decision | Source | -| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| L.1 | **`index_freshness` is metadata, not a verdict** — structured fields + optional `warning` string; never `pass`/`fail`. | [Moat A](../roadmap.md#moats-load-bearing) | -| L.2 | **Canonical shape in one module** — `src/application/index-freshness.ts`; `context`, MCP, HTTP, and CLI all call `computeIndexFreshness()`. | Same seam as `validate-engine` / `context-engine` | -| L.3 | **`pending_sync` = watcher queue OR in-flight reindex** — true when debouncer has paths **or** a targeted `--files` reindex is running. Not “SQLite mid-transaction” (writes stay transactional). | [roadmap § No split-brain](../roadmap.md#floors-v1-product-shape) | -| L.4 | **Cheap vs full freshness** — every transport gets cheap signals (HEAD drift, pending sync, watch active). **Disk drift** (`getChangedFiles`) runs on `context` and opt-in full mode only (git cost). | Avoid git subprocess on every `query` row | -| L.5 | **JSON tool payloads stay backward-compatible in v1 slice** — slice 1 enriches `context` only; slice 2 adds HTTP headers + MCP `_meta` wrapper without breaking array-shaped `query` results. | Agent eval / golden harness consume raw arrays today | +| # | Decision | Source | +| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| L.1 | **`index_freshness` is metadata, not a verdict** — structured fields + optional `warning` string; never `pass`/`fail`. | [Moat A](../roadmap.md#moats-load-bearing) | +| L.2 | **Canonical shape in one module** — `src/application/index-freshness.ts`; `context`, MCP, HTTP, and CLI all call `computeIndexFreshness()`. | Same seam as `validate-engine` / `context-engine` | +| L.3 | **`pending_sync` = watcher queue OR in-flight reindex** — true when debouncer has paths **or** a targeted `--files` reindex is running. Not “SQLite mid-transaction” (writes stay transactional). | [roadmap § No split-brain](../roadmap.md#floors-v1-product-shape) | +| L.4 | **Cheap vs full freshness** — every transport gets cheap signals (HEAD drift, pending sync, watch active). **Disk drift** (`getChangedFiles`) runs on `context` and opt-in full mode only (git cost). | Avoid git subprocess on every `query` row | +| L.5 | **JSON tool payloads stay backward-compatible** — HTTP headers only (array bodies unchanged); MCP object payloads merge `index_freshness` inline; MCP array payloads append `@codemap/index_freshness` second block. | Agent eval / golden harness consume raw arrays today | --- diff --git a/src/application/http-server.ts b/src/application/http-server.ts index 55b245c5..3130d1c6 100644 --- a/src/application/http-server.ts +++ b/src/application/http-server.ts @@ -13,10 +13,11 @@ import { initCodemap, } from "../runtime"; import { - applyIndexFreshnessHeaders, readCheapIndexFreshness, + resolveTransportIndexFreshness, warnIndexFreshnessToStderr, } from "./index-freshness"; +import type { IndexFreshness } from "./index-freshness"; import { listResources, readResource } from "./resource-handlers"; import { affectedArgsSchema, @@ -122,10 +123,33 @@ const TOOL_NAMES = [ */ export async function runHttpServer(opts: HttpServerOpts): Promise { await bootstrapForServe(opts); - if (opts.watch !== true) { + + let stopWatch: (() => Promise) | undefined; + let watchReady: Promise = Promise.resolve(); + if (opts.watch === true) { + const prime = createPrimeIndex({ quiet: false, label: "codemap serve" }); + const handle = runWatchLoop({ + root: getProjectRoot(), + excludeDirNames: getExcludeDirNames(), + recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), + debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, + onPrime: async () => { + await prime(); + warnIndexFreshnessToStderr("codemap serve"); + }, + onChange: createReindexOnChange({ + quiet: false, + label: "codemap serve", + }), + }); + stopWatch = handle.stop; + watchReady = handle.ready; + } else { warnIndexFreshnessToStderr("codemap serve"); } + await watchReady; + const server = createServer((req, res) => { handleRequest(req, res, opts).catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); @@ -146,34 +170,6 @@ export async function runHttpServer(opts: HttpServerOpts): Promise { }); }); - let stopWatch: (() => Promise) | undefined; - if (opts.watch === true) { - try { - const prime = createPrimeIndex({ quiet: false, label: "codemap serve" }); - const handle = runWatchLoop({ - root: getProjectRoot(), - excludeDirNames: getExcludeDirNames(), - recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), - debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, - onPrime: async () => { - await prime(); - warnIndexFreshnessToStderr("codemap serve"); - }, - onChange: createReindexOnChange({ - quiet: false, - label: "codemap serve", - }), - }); - stopWatch = handle.stop; - } catch (err) { - // Watcher boot threw AFTER `server.listen()` resolved — close - // the listener so we don't leak an orphaned HTTP socket on a - // failed boot. Caught by CodeRabbit on PR #47. - await new Promise((res) => server.close(() => res())); - throw err; - } - } - await new Promise((resolve) => { const shutdown = (signal: string) => { // eslint-disable-next-line no-console -- intentional shutdown log on stderr @@ -403,12 +399,34 @@ async function readJsonBody( * 4xx / 5xx with `{"error": "..."}` (same shape `codemap query --json` * prints on failure — agents and CLI consumers unwrap identically). */ +function applyIndexFreshnessHeaders( + res: ServerResponse, + freshness: IndexFreshness | null, +): void { + if (freshness === null) return; + res.setHeader( + "X-Codemap-Pending-Sync", + freshness.pending_sync ? "true" : "false", + ); + res.setHeader( + "X-Codemap-Commit-Drift", + freshness.commit_drift ? "true" : "false", + ); + if (freshness.warning !== null) { + res.setHeader("X-Codemap-Warning", freshness.warning); + } +} + function writeToolResult( res: ServerResponse, result: ToolResult, version: string, ): void { - const freshness = result.ok ? readCheapIndexFreshness() : null; + const freshness = result.ok + ? result.format === "json" + ? resolveTransportIndexFreshness(result.payload) + : readCheapIndexFreshness() + : null; applyIndexFreshnessHeaders(res, freshness); if (!result.ok) { diff --git a/src/application/index-freshness.test.ts b/src/application/index-freshness.test.ts index 42eec120..c4ffda8e 100644 --- a/src/application/index-freshness.test.ts +++ b/src/application/index-freshness.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveCodemapConfig } from "../config"; +import * as dbModule from "../db"; import { closeDb, createTables, openDb, setMeta } from "../db"; import { initCodemap } from "../runtime"; import { buildContextEnvelope } from "./context-engine"; @@ -11,6 +12,7 @@ import * as indexEngine from "./index-engine"; import { computeIndexFreshness, mergeIndexFreshnessIntoJsonPayload, + resolveTransportIndexFreshness, warnIndexFreshnessToStderr, } from "./index-freshness"; import { _resetWatchStateForTests, runWatchLoop } from "./watcher"; @@ -121,6 +123,68 @@ describe("computeIndexFreshness", () => { await handle.stop(); }); + + it("reports history_incompatible when getChangedFiles returns null", () => { + spyOn(indexEngine, "getCurrentCommit").mockReturnValue("d".repeat(40)); + const changedFiles = spyOn(indexEngine, "getChangedFiles").mockReturnValue( + null, + ); + + try { + withEmptyDb((db) => { + setMeta(db, "last_indexed_commit", "e".repeat(40)); + const f = computeIndexFreshness(db, { include_disk_drift: true }); + expect(f.history_incompatible).toBe(true); + expect(f.warning).toContain("Git history is incompatible"); + }); + } finally { + changedFiles.mockRestore(); + } + }); + + it("reports disk-ahead with unindexed change count", () => { + const head = "f".repeat(40); + spyOn(indexEngine, "getCurrentCommit").mockReturnValue(head); + spyOn(indexEngine, "getChangedFiles").mockReturnValue({ + changed: ["src/a.ts", "src/b.ts"], + deleted: ["src/c.ts"], + existingPaths: new Set(), + sourceCache: new Map(), + existingHashes: new Map(), + }); + + withEmptyDb((db) => { + setMeta(db, "last_indexed_commit", head); + const f = computeIndexFreshness(db, { include_disk_drift: true }); + expect(f.disk_ahead_of_index).toBe(true); + expect(f.unindexed_change_count).toBe(3); + expect(f.warning).toContain("3 unindexed change"); + }); + }); +}); + +describe("resolveTransportIndexFreshness", () => { + it("reuses embedded index_freshness without opening the DB", () => { + const embedded = { + head_commit: null, + last_indexed_commit: null, + commit_drift: false, + watch_active: false, + pending_sync: false, + pending_paths: 0, + reindex_in_flight: false, + warning: null, + }; + const openDbSpy = spyOn(dbModule, "openDb"); + try { + expect( + resolveTransportIndexFreshness({ index_freshness: embedded }), + ).toBe(embedded); + expect(openDbSpy).not.toHaveBeenCalled(); + } finally { + openDbSpy.mockRestore(); + } + }); }); describe("mergeIndexFreshnessIntoJsonPayload", () => { diff --git a/src/application/index-freshness.ts b/src/application/index-freshness.ts index b320e27c..d212bebe 100644 --- a/src/application/index-freshness.ts +++ b/src/application/index-freshness.ts @@ -1,5 +1,3 @@ -import type { ServerResponse } from "node:http"; - import { closeDb, openDb } from "../db"; import type { CodemapDatabase } from "../db"; import { getMeta } from "../db"; @@ -172,6 +170,27 @@ export function readCheapIndexFreshness(): IndexFreshness | null { return readIndexFreshness(); } +/** + * Freshness for MCP/HTTP JSON tool wrappers. Reuses embedded + * `index_freshness` from `context` (avoids a second DB open). + */ +export function resolveTransportIndexFreshness( + payload: unknown, +): IndexFreshness | null { + if ( + payload !== null && + typeof payload === "object" && + !Array.isArray(payload) && + "index_freshness" in payload + ) { + const embedded = (payload as { index_freshness?: unknown }).index_freshness; + if (embedded !== null && typeof embedded === "object") { + return embedded as IndexFreshness; + } + } + return readCheapIndexFreshness(); +} + /** * One-line stderr warning when freshness concerns remain after boot / prime. * Used by `codemap mcp` and `codemap serve` only — stdout stays JSON-RPC clean. @@ -186,25 +205,6 @@ export function warnIndexFreshnessToStderr( console.error(`${label}: ${freshness.warning}`); } -/** HTTP response headers mirroring cheap freshness signals (slice 2). */ -export function applyIndexFreshnessHeaders( - res: ServerResponse, - freshness: IndexFreshness | null, -): void { - if (freshness === null) return; - res.setHeader( - "X-Codemap-Pending-Sync", - freshness.pending_sync ? "true" : "false", - ); - res.setHeader( - "X-Codemap-Commit-Drift", - freshness.commit_drift ? "true" : "false", - ); - if (freshness.warning !== null) { - res.setHeader("X-Codemap-Warning", freshness.warning); - } -} - export function formatIndexFreshnessMcpBlock( freshness: IndexFreshness, ): string { diff --git a/src/application/mcp-server.ts b/src/application/mcp-server.ts index 760350d7..b5c3c34b 100644 --- a/src/application/mcp-server.ts +++ b/src/application/mcp-server.ts @@ -17,7 +17,7 @@ import { formatIndexFreshnessMcpBlock, jsonPayloadNeedsMcpFreshnessBlock, mergeIndexFreshnessIntoJsonPayload, - readCheapIndexFreshness, + resolveTransportIndexFreshness, warnIndexFreshnessToStderr, } from "./index-freshness"; import { @@ -117,7 +117,7 @@ function wrapToolResult(r: ToolResult) { }; } if (r.format === "json") { - const freshness = readCheapIndexFreshness(); + const freshness = resolveTransportIndexFreshness(r.payload); const payload = mergeIndexFreshnessIntoJsonPayload(r.payload, freshness); const content: Array<{ type: "text"; text: string }> = [ { type: "text", text: JSON.stringify(payload) }, @@ -580,43 +580,39 @@ async function bootstrapForMcp(opts: ServerOpts): Promise { */ export async function runMcpServer(opts: ServerOpts): Promise { await bootstrapForMcp(opts); - if (opts.watch !== true) { - warnIndexFreshnessToStderr("codemap mcp"); - } - const server = createMcpServer(opts); - const transport = new StdioServerTransport(); - await server.connect(transport); let stopWatch: (() => Promise) | undefined; + let watchReady: Promise = Promise.resolve(); if (opts.watch === true) { // eslint-disable-next-line no-console -- intentional bootstrap log on stderr console.error("codemap mcp: --watch enabled, booting file watcher..."); - try { - const prime = createPrimeIndex({ quiet: false, label: "codemap mcp" }); - const handle = runWatchLoop({ - root: getProjectRoot(), - excludeDirNames: getExcludeDirNames(), - recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), - debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, - onPrime: async () => { - await prime(); - warnIndexFreshnessToStderr("codemap mcp"); - }, - onChange: createReindexOnChange({ - quiet: false, - label: "codemap mcp", - }), - }); - stopWatch = handle.stop; - } catch (err) { - // Watcher boot threw — close the MCP transport so the agent host - // sees the disconnect cleanly instead of a half-alive server. - // Caught by CodeRabbit on PR #47. - await server.close(); - throw err; - } + const prime = createPrimeIndex({ quiet: false, label: "codemap mcp" }); + const handle = runWatchLoop({ + root: getProjectRoot(), + excludeDirNames: getExcludeDirNames(), + recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), + debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, + onPrime: async () => { + await prime(); + warnIndexFreshnessToStderr("codemap mcp"); + }, + onChange: createReindexOnChange({ + quiet: false, + label: "codemap mcp", + }), + }); + stopWatch = handle.stop; + watchReady = handle.ready; + } else { + warnIndexFreshnessToStderr("codemap mcp"); } + await watchReady; + + const server = createMcpServer(opts); + const transport = new StdioServerTransport(); + await server.connect(transport); + await new Promise((resolve) => { transport.onclose = () => resolve(); }); diff --git a/src/application/watcher.test.ts b/src/application/watcher.test.ts index 35119675..487b5f47 100644 --- a/src/application/watcher.test.ts +++ b/src/application/watcher.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "bun:test"; import { _resetWatchStateForTests, createDebouncer, + getWatchSyncState, isWatchActive, runWatchLoop, shouldIndexPath, @@ -345,10 +346,12 @@ describe("runWatchLoop — backend dispatch + path filter", () => { // Backend started, but flag still false because prime hasn't run. expect(backend.started).toBe(true); expect(isWatchActive()).toBe(false); + expect(getWatchSyncState().reindex_in_flight).toBe(true); // Release the prime → flag flips. releasePrime!(); - await new Promise((r) => setTimeout(r, 10)); + await handle.ready; expect(isWatchActive()).toBe(true); + expect(getWatchSyncState().reindex_in_flight).toBe(false); await handle.stop(); expect(isWatchActive()).toBe(false); }); diff --git a/src/application/watcher.ts b/src/application/watcher.ts index 5ed5c133..fdd5e6d9 100644 --- a/src/application/watcher.ts +++ b/src/application/watcher.ts @@ -182,6 +182,7 @@ let watchActive = false; /** Debouncer + in-flight reindex state for index freshness metadata. */ let watchDebouncer: Debouncer | undefined; let watchReindexInFlight = false; +let watchPrimeInFlight = false; export function getWatchSyncState(): Readonly<{ pending_paths: number; @@ -189,7 +190,7 @@ export function getWatchSyncState(): Readonly<{ }> { return { pending_paths: watchDebouncer?.pendingSize() ?? 0, - reindex_in_flight: watchReindexInFlight, + reindex_in_flight: watchReindexInFlight || watchPrimeInFlight, }; } @@ -202,6 +203,7 @@ export function _resetWatchStateForTests(): void { watchActive = false; watchDebouncer = undefined; watchReindexInFlight = false; + watchPrimeInFlight = false; } /** Test-only escape hatch — flips the flag without booting a real watcher (for handleAudit prelude-skip tests). */ @@ -334,6 +336,8 @@ export interface WatchBackend { */ export function runWatchLoop(opts: WatchLoopOpts): { stop: () => Promise; + /** Resolves when optional `onPrime` finishes (immediate when absent). */ + ready: Promise; } { const debounceMs = opts.debounceMs ?? DEFAULT_DEBOUNCE_MS; const recipesPrefix = opts.recipesWatchPrefix ?? DEFAULT_RECIPES_WATCH_PREFIX; @@ -396,6 +400,7 @@ export function runWatchLoop(opts: WatchLoopOpts): { primingDone = Promise.resolve(); } else { primingDone = (async () => { + watchPrimeInFlight = true; try { await opts.onPrime!(); if (!stopped) watchActive = true; @@ -408,11 +413,14 @@ export function runWatchLoop(opts: WatchLoopOpts): { // Leave watchActive = false; embedder may decide to stop or // tolerate. We don't tear down here — the watcher is still // catching new edits, just can't promise historical freshness. + } finally { + watchPrimeInFlight = false; } })(); } return { + ready: primingDone, async stop() { // Stop early so handleAudit doesn't keep skipping prelude while // we're shutting down (any in-flight audit reads a "stale" @@ -430,6 +438,7 @@ export function runWatchLoop(opts: WatchLoopOpts): { await backend.stop(); watchDebouncer = undefined; watchReindexInFlight = false; + watchPrimeInFlight = false; watchActive = false; }, }; diff --git a/templates/agent-content/mcp-instructions.md b/templates/agent-content/mcp-instructions.md index c89a9ea7..c764ba45 100644 --- a/templates/agent-content/mcp-instructions.md +++ b/templates/agent-content/mcp-instructions.md @@ -10,7 +10,7 @@ Operational playbook injected into the MCP initialize handshake. Full schema, re ## Index freshness -Every JSON tool response carries index-level freshness metadata (not a pass/fail verdict): +Every successful JSON tool response carries index-level freshness metadata (not a pass/fail verdict): | Surface | Where to read it | | ------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | diff --git a/templates/agent-content/skill/50-maintenance.md b/templates/agent-content/skill/50-maintenance.md index 6019cc0f..8c49c01b 100644 --- a/templates/agent-content/skill/50-maintenance.md +++ b/templates/agent-content/skill/50-maintenance.md @@ -12,8 +12,11 @@ codemap # Full rebuild — after rebase, branch switch, or stale index codemap --full -# Check index freshness -codemap query --json "SELECT key, value FROM meta" +# Check index freshness (index-level — HEAD drift, pending sync, disk-ahead) +codemap context --compact --json | jq '.index_freshness' + +# Per-file staleness (content_hash drift) +codemap validate --json ``` **Prefer `--files`** when you know which files you changed — it skips git diff and filesystem scanning for the rest of the tree. Deleted files passed to `--files` are auto-removed from the index. From 76f161da79d81995fdf3a235ab41baf2391e565b Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:44:47 +0300 Subject: [PATCH 5/6] chore(release): add changeset and retire shipped freshness plan Patch changeset for user-facing index_freshness surfacing; delete the shipped plan per docs-governance and point docs at architecture instead. --- .changeset/index-freshness-trust-bundle.md | 5 + docs/agents.md | 2 +- docs/plans/index-freshness-trust-bundle.md | 101 --------------------- docs/roadmap.md | 4 +- 4 files changed, 8 insertions(+), 104 deletions(-) create mode 100644 .changeset/index-freshness-trust-bundle.md delete mode 100644 docs/plans/index-freshness-trust-bundle.md diff --git a/.changeset/index-freshness-trust-bundle.md b/.changeset/index-freshness-trust-bundle.md new file mode 100644 index 00000000..44359029 --- /dev/null +++ b/.changeset/index-freshness-trust-bundle.md @@ -0,0 +1,5 @@ +--- +"@stainless-code/codemap": patch +--- + +Add `index_freshness` metadata on `context`, MCP tool responses, HTTP headers, and boot stderr warnings so agents can detect commit drift, pending watcher sync, and disk-ahead-of-index states before trusting structural queries. diff --git a/docs/agents.md b/docs/agents.md index 72923ad2..063908b3 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -103,7 +103,7 @@ Recipe ids cited in the playbook are machine-validated in tests against the live ## MCP tool allowlist -**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. **MCP:** array-shaped JSON tools (`query`, …) keep row payloads verbatim and append a second `content` block prefixed `@codemap/index_freshness`; object-shaped tools merge `index_freshness` inline. **HTTP:** `POST /tool/*` adds `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, and `X-Codemap-Warning` headers without changing JSON bodies; **`GET /health`** includes full cheap `index_freshness` when the DB is readable. Complements per-file `validate` / snippet `stale`. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). +**`context.index_freshness`** — session bootstrap includes index-level freshness metadata: `commit_drift` (HEAD ≠ `last_indexed_commit`), `pending_sync` (watcher debounce queue or in-flight reindex), optional disk-drift counts when watch is off, and a single `warning` string when agents should pause or re-index. **MCP:** array-shaped JSON tools (`query`, …) keep row payloads verbatim and append a second `content` block prefixed `@codemap/index_freshness`; object-shaped tools merge `index_freshness` inline. **HTTP:** `POST /tool/*` adds `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, and `X-Codemap-Warning` headers without changing JSON bodies; **`GET /health`** includes full cheap `index_freshness` when the DB is readable. Complements per-file `validate` / snippet `stale`. See [`architecture.md` § Context wiring](./architecture.md). **`CODEMAP_MCP_TOOLS`** — comma-separated snake_case MCP tool names. When set, only listed tools register (stderr lists the active set). Unknown names are ignored with a warning. Unset = all tools (default). **`query_batch`** registers only when listed or when unset (eval ablation). diff --git a/docs/plans/index-freshness-trust-bundle.md b/docs/plans/index-freshness-trust-bundle.md deleted file mode 100644 index 6bcc2fc4..00000000 --- a/docs/plans/index-freshness-trust-bundle.md +++ /dev/null @@ -1,101 +0,0 @@ -# Index freshness trust bundle — plan - -> **Status:** shipped · **Priority:** agent session · **Effort:** S (~3–5 days) · **PR:** [#149](https://github.com/stainless-code/codemap/pull/149) -> -> **Motivator:** Agents treat MCP / HTTP / `context` output as ground truth. Today they can query during watcher debounce (disk ahead of index), after a branch switch (`last_indexed_commit` ≠ `HEAD`), or with a dirty working tree when watch is off — with no signal except running `validate` manually. Wrong structural verdicts follow. - ---- - -## Pre-locked decisions - -| # | Decision | Source | -| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| L.1 | **`index_freshness` is metadata, not a verdict** — structured fields + optional `warning` string; never `pass`/`fail`. | [Moat A](../roadmap.md#moats-load-bearing) | -| L.2 | **Canonical shape in one module** — `src/application/index-freshness.ts`; `context`, MCP, HTTP, and CLI all call `computeIndexFreshness()`. | Same seam as `validate-engine` / `context-engine` | -| L.3 | **`pending_sync` = watcher queue OR in-flight reindex** — true when debouncer has paths **or** a targeted `--files` reindex is running. Not “SQLite mid-transaction” (writes stay transactional). | [roadmap § No split-brain](../roadmap.md#floors-v1-product-shape) | -| L.4 | **Cheap vs full freshness** — every transport gets cheap signals (HEAD drift, pending sync, watch active). **Disk drift** (`getChangedFiles`) runs on `context` and opt-in full mode only (git cost). | Avoid git subprocess on every `query` row | -| L.5 | **JSON tool payloads stay backward-compatible** — HTTP headers only (array bodies unchanged); MCP object payloads merge `index_freshness` inline; MCP array payloads append `@codemap/index_freshness` second block. | Agent eval / golden harness consume raw arrays today | - ---- - -## Freshness envelope - -```typescript -interface IndexFreshness { - head_commit: string | null; - last_indexed_commit: string | null; - commit_drift: boolean; - watch_active: boolean; - pending_sync: boolean; - pending_paths: number; - reindex_in_flight: boolean; - /** Present when `include_disk_drift: true` */ - disk_ahead_of_index?: boolean; - unindexed_change_count?: number; - history_incompatible?: boolean; - /** Single agent-readable line when any concern is active; null when fresh */ - warning: string | null; -} -``` - ---- - -## Shipping cadence (tracer bullets) - -### Slice 1 — `context` + watcher state (this PR) - -1. **`getWatchSyncState()`** on `watcher.ts` — expose debouncer pending count + reindex-in-flight flag (module-scoped; test reset hook). -2. **`index-freshness.ts`** — `computeIndexFreshness(db, { include_disk_drift })`. -3. **`ContextEnvelope.index_freshness`** — `buildContextEnvelope` calls full mode (`include_disk_drift: true`). -4. **CLI `codemap context`** — inherits via `buildContextEnvelope` (no separate code path). -5. **Unit tests** — freshness permutations (drift, pending, disk-ahead, history rewrite). -6. **Docs** — one paragraph in [`agents.md`](../agents.md) + roadmap checkbox note when shipped. - -**Acceptance** - -- [x] `codemap context --json` includes `index_freshness` with `warning` when HEAD ≠ `last_indexed_commit` -- [x] Fake watcher with pending debounce → `pending_sync: true` -- [x] Non-git fixture → `head_commit: null`, no throw - -### Slice 2 — all MCP / HTTP tool responses - -1. **HTTP response headers** on `POST /tool/*`: `X-Codemap-Pending-Sync`, `X-Codemap-Commit-Drift`, `X-Codemap-Warning` (when set). -2. **MCP `wrapToolResult`** — object JSON payloads merge `index_freshness` inline; array payloads append a second `content` block prefixed `@codemap/index_freshness`. -3. **`/health`** — include cheap `index_freshness` when DB exists. - -**Acceptance** - -- [x] HTTP `query` row array body unchanged; freshness headers present -- [x] MCP `query` array → second content block with `@codemap/index_freshness` -- [x] MCP object payloads (`query` summary, `show`, …) merge `index_freshness` inline - -### Slice 3 — stderr + MCP initialize - -1. **`codemap mcp` / `serve` boot** — one-line stderr when freshness concerns remain after bootstrap / watch prime. -2. **MCP `instructions` / `codemap://mcp-instructions`** — document freshness fields + agent guidance. - -**Acceptance** - -- [x] `codemap mcp` / `serve` stderr warns when `index_freshness.warning` is set after prime -- [x] MCP initialize instructions document `index_freshness`, `pending_sync`, and agent retry guidance - ---- - -## Shipped - -Trust bundle complete in [#149](https://github.com/stainless-code/codemap/pull/149) (slices 1–3). Roadmap items **Index staleness surfacing** and **HEAD / index freshness warning** satisfied via `index_freshness` metadata + transport headers + boot stderr. - -## Dependencies - -- [`watcher.ts`](../../src/application/watcher.ts) — debouncer + `isWatchActive()` -- [`index-engine.ts`](../../src/application/index-engine.ts) — `getChangedFiles`, `getCurrentCommit` -- [`context-engine.ts`](../../src/application/context-engine.ts) — envelope builder -- [`validate-engine.ts`](../../src/application/validate-engine.ts) — conceptual sibling (per-file staleness vs index-level) - ---- - -## Out of scope - -- Blocking queries when stale (agents decide). -- Changing incremental indexing semantics ([No split-brain floor](../roadmap.md#floors-v1-product-shape)). -- Shared daemon / multi-session lifecycle (separate roadmap item). diff --git a/docs/roadmap.md b/docs/roadmap.md index a0f20cce..d7b3a975 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -78,11 +78,11 @@ Long-running MCP / HTTP sessions dominate agent workflows; one-shot CLI keeps th - [ ] **MCP shared daemon per project** — one watcher + one SQLite writer per indexed root; Unix socket / named pipe so concurrent agent sessions share a live index instead of each spawning watchers and contending on WAL. Complements perf item **6.1** (read pool) but is a separate write-side + lifecycle concern. Effort: L. - [ ] **Rich `context` composer** — expand session bootstrap beyond counts + recipe catalog: ranked hub files, sample markers, optional inlined signatures, fan-in leaders, and links to high-value recipes for the detected intent. Goal: one bootstrap call replaces a common show → explore chain on session start. Effort: M. - [ ] **Codebase map in bootstrap responses** — hash-stable structural summary (top hubs, CLI entry hints, schema version, index freshness) auto-included in `context` / MCP initialize payload. Opt-out via flag. Effort: S–M. -- [x] **Index staleness surfacing** — `index_freshness.pending_sync` on `context`, MCP tool metadata, and HTTP headers when the watcher debounce queue or in-flight reindex is active. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). Shipped [#149](https://github.com/stainless-code/codemap/pull/149). +- [x] **Index staleness surfacing** — `index_freshness.pending_sync` on `context`, MCP tool metadata, and HTTP headers when the watcher debounce queue or in-flight reindex is active. Shipped [#149](https://github.com/stainless-code/codemap/pull/149). - [ ] **Adaptive output budgets** — scale explore/trace/node snippet char caps and row limits from indexed file/symbol counts so large trees do not blow token budgets. Effort: S. - [ ] **MCP session lifecycle hygiene** — idle timeout, client disconnect detection, graceful watcher shutdown on last client; avoid orphaned watchers after agent host crashes. Effort: S–M. - [ ] **`agents init` uninstall (teardown)** — symmetric inverse of init for failed pilots, template mistakes, or leaving a repo: remove codemap-managed MCP entries, pointer sections, and IDE symlinks only (same scoped paths as init; never delete user-authored `.agents/` siblings). `--target` filter, `--yes` non-interactive. Not the happy-path docs story — adoption stays `init --mcp --git-hooks` + committed `.agents/`. Effort: S. -- [x] **HEAD / index freshness warning** — `index_freshness.commit_drift` + `warning` on `context` / tool metadata; boot stderr on `codemap mcp` / `serve` when concerns remain after prime. Plan: [`plans/index-freshness-trust-bundle.md`](./plans/index-freshness-trust-bundle.md). Shipped [#149](https://github.com/stainless-code/codemap/pull/149). +- [x] **HEAD / index freshness warning** — `index_freshness.commit_drift` + `warning` on `context` / tool metadata; boot stderr on `codemap mcp` / `serve` when concerns remain after prime. Shipped [#149](https://github.com/stainless-code/codemap/pull/149). ### Recipe & audit enrichment From cbf71a8c0d68e86fae3a7b1c42fbd751ffaae7de Mon Sep 17 00:00:00 2001 From: Sutu Sebastian Date: Wed, 27 May 2026 13:47:11 +0300 Subject: [PATCH 6/6] fix(freshness): keep pending_sync true across chained watcher batches Track queued inFlight batches so pending_sync does not false-negative between serialized reindexes; always emit boot freshness warning after watch prime settles, including on prime failure. --- src/application/http-server.ts | 7 ++-- src/application/index-freshness.test.ts | 44 +++++++++++++++++++++++++ src/application/mcp-server.ts | 7 ++-- src/application/watcher.test.ts | 38 +++++++++++++++++++++ src/application/watcher.ts | 10 +++++- 5 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/application/http-server.ts b/src/application/http-server.ts index 3130d1c6..1d79b20c 100644 --- a/src/application/http-server.ts +++ b/src/application/http-server.ts @@ -134,8 +134,11 @@ export async function runHttpServer(opts: HttpServerOpts): Promise { recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, onPrime: async () => { - await prime(); - warnIndexFreshnessToStderr("codemap serve"); + try { + await prime(); + } finally { + warnIndexFreshnessToStderr("codemap serve"); + } }, onChange: createReindexOnChange({ quiet: false, diff --git a/src/application/index-freshness.test.ts b/src/application/index-freshness.test.ts index c4ffda8e..97653a95 100644 --- a/src/application/index-freshness.test.ts +++ b/src/application/index-freshness.test.ts @@ -243,6 +243,50 @@ describe("warnIndexFreshnessToStderr", () => { } }); + it("logs boot freshness warning when watch prime fails", async () => { + _resetWatchStateForTests(); + const errLog = spyOn(console, "error").mockImplementation(() => {}); + const revParse = spyOn(indexEngine, "getCurrentCommit").mockReturnValue( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ); + const backend = fakeBackend(); + + try { + withEmptyDb((db) => { + setMeta( + db, + "last_indexed_commit", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ); + }); + const handle = runWatchLoop({ + root: benchDir, + excludeDirNames: new Set(["node_modules", ".git", "dist"]), + onChange: () => {}, + debounceMs: 60_000, + backend, + onPrime: async () => { + try { + throw new Error("prime failed"); + } finally { + warnIndexFreshnessToStderr("codemap mcp"); + } + }, + }); + await handle.ready; + await handle.stop(); + expect(errLog).toHaveBeenCalledWith( + expect.stringContaining("codemap mcp:"), + ); + expect(errLog).toHaveBeenCalledWith( + expect.stringContaining("prime failed"), + ); + } finally { + errLog.mockRestore(); + revParse.mockRestore(); + } + }); + it("stays silent when warning is null", () => { const head = "cccccccccccccccccccccccccccccccccccccccc"; const errLog = spyOn(console, "error").mockImplementation(() => {}); diff --git a/src/application/mcp-server.ts b/src/application/mcp-server.ts index b5c3c34b..a7db13dc 100644 --- a/src/application/mcp-server.ts +++ b/src/application/mcp-server.ts @@ -593,8 +593,11 @@ export async function runMcpServer(opts: ServerOpts): Promise { recipesWatchPrefix: resolveRecipesWatchPrefix(getProjectRoot()), debounceMs: opts.debounceMs ?? DEFAULT_DEBOUNCE_MS, onPrime: async () => { - await prime(); - warnIndexFreshnessToStderr("codemap mcp"); + try { + await prime(); + } finally { + warnIndexFreshnessToStderr("codemap mcp"); + } }, onChange: createReindexOnChange({ quiet: false, diff --git a/src/application/watcher.test.ts b/src/application/watcher.test.ts index 487b5f47..e2645a40 100644 --- a/src/application/watcher.test.ts +++ b/src/application/watcher.test.ts @@ -415,6 +415,44 @@ describe("runWatchLoop — backend dispatch + path filter", () => { expect(onChangeFinished).toBe(true); }); + it("reindex_in_flight stays true while chained batches wait on inFlight", async () => { + _resetWatchStateForTests(); + const backend = fakeBackend(); + let releaseBatch1: (() => void) | undefined; + const batch1Gate = new Promise((resolve) => { + releaseBatch1 = resolve; + }); + let batchCount = 0; + + const handle = runWatchLoop({ + root: "/tmp/proj", + excludeDirNames: exclude, + onChange: async () => { + batchCount++; + if (batchCount === 1) { + await batch1Gate; + } + }, + debounceMs: 10, + backend, + }); + + backend.fire("change", "/tmp/proj/src/a.ts"); + await new Promise((r) => setTimeout(r, 25)); + expect(getWatchSyncState().reindex_in_flight).toBe(true); + + backend.fire("change", "/tmp/proj/src/b.ts"); + await new Promise((r) => setTimeout(r, 25)); + expect(getWatchSyncState().pending_paths).toBe(0); + expect(getWatchSyncState().reindex_in_flight).toBe(true); + + releaseBatch1!(); + await new Promise((r) => setTimeout(r, 25)); + expect(getWatchSyncState().reindex_in_flight).toBe(false); + + await handle.stop(); + }); + it("treats unlink as a path requiring reindex (caller handles deletes)", async () => { const backend = fakeBackend(); const calls: ReadonlySet[] = []; diff --git a/src/application/watcher.ts b/src/application/watcher.ts index fdd5e6d9..439f7e35 100644 --- a/src/application/watcher.ts +++ b/src/application/watcher.ts @@ -183,6 +183,8 @@ let watchActive = false; let watchDebouncer: Debouncer | undefined; let watchReindexInFlight = false; let watchPrimeInFlight = false; +/** Batches chained on `inFlight` but not yet running or still running. */ +let watchReindexPendingBatches = 0; export function getWatchSyncState(): Readonly<{ pending_paths: number; @@ -190,7 +192,10 @@ export function getWatchSyncState(): Readonly<{ }> { return { pending_paths: watchDebouncer?.pendingSize() ?? 0, - reindex_in_flight: watchReindexInFlight || watchPrimeInFlight, + reindex_in_flight: + watchReindexInFlight || + watchPrimeInFlight || + watchReindexPendingBatches > 0, }; } @@ -204,6 +209,7 @@ export function _resetWatchStateForTests(): void { watchDebouncer = undefined; watchReindexInFlight = false; watchPrimeInFlight = false; + watchReindexPendingBatches = 0; } /** Test-only escape hatch — flips the flag without booting a real watcher (for handleAudit prelude-skip tests). */ @@ -350,6 +356,7 @@ export function runWatchLoop(opts: WatchLoopOpts): { // re-indexing left the DB in a half-state). let inFlight: Promise = Promise.resolve(); const debouncer = createDebouncer((paths) => { + watchReindexPendingBatches++; inFlight = inFlight .then(async () => { watchReindexInFlight = true; @@ -357,6 +364,7 @@ export function runWatchLoop(opts: WatchLoopOpts): { await Promise.resolve(opts.onChange(paths)); } finally { watchReindexInFlight = false; + watchReindexPendingBatches--; } }) .catch((err: unknown) => {