diff --git a/packages/loopover-mcp/bin/loopover-mcp.js b/packages/loopover-mcp/bin/loopover-mcp.js index ca8b16550..25d8d19bb 100644 --- a/packages/loopover-mcp/bin/loopover-mcp.js +++ b/packages/loopover-mcp/bin/loopover-mcp.js @@ -99,7 +99,7 @@ const CLI_COMMAND_SPEC = { profile: ["list", "create", "switch", "remove"], cache: ["status", "clear", "list"], agent: ["plan", "status", "explain", "packet"], - maintain: ["status", "queue", "approve", "reject", "pause", "resume", "set-level", "precision", "onboarding-pack", "audit-feed"], + maintain: ["status", "queue", "approve", "reject", "pause", "resume", "set-level", "precision", "outcome-calibration", "onboarding-pack", "audit-feed"], }; const COMPLETION_SHELLS = ["bash", "zsh", "fish", "powershell"]; const AGENT_PROFILE_IDS = ["miner-planner", "miner-auto-dev", "maintainer-triage", "repo-owner-intake"]; @@ -3044,6 +3044,8 @@ function printMaintainHelp() { ` actions: ${MAINTAIN_ACTION_CLASSES.join(", ")}`, ` levels: ${MAINTAIN_AUTONOMY_LEVELS.join(", ")}`, " precision [--window-days N] Show gate false-positive telemetry (blocked-then-merged per gate type).", + " outcome-calibration Show slop-band merge rates and recommendation-outcome calibration.", + " [--window-days N] Bound the recommendation window (default: full history).", " onboarding-pack [--refresh] Preview the repo's contributor onboarding pack.", " audit-feed [--since ISO] Show the agent audit feed (who did what, when).", " [--limit N] Cap the events returned (1-200).", @@ -3154,6 +3156,26 @@ async function maintainCli(args) { emit(payload, lines.join("\n")); return; } + if (subcommand === "outcome-calibration") { + // #6735 outcome calibration: read-only measurement of whether higher-slop bands merge less often and how + // agent recommendations panned out. Same --window-days handling the sibling precision command uses (a + // non-positive value omits ?windowDays, so the server reports full history). + const windowDays = Number(options.windowDays); + const query = windowDays > 0 ? `?windowDays=${encodeURIComponent(windowDays)}` : ""; + const payload = await apiGet(`${repoBase}/outcome-calibration${query}`); + const window = payload.windowDays ? `last ${payload.windowDays}d` : "all history"; + const recommendations = payload.recommendations ?? {}; + const rate = (value) => (value === null || value === undefined ? "n/a (below sample)" : `${Math.round(value * 100)}%`); + const lines = [ + `Outcome calibration for ${repoFullName} (${window}): recommendations ${recommendations.positive ?? 0} positive, ${recommendations.negative ?? 0} negative, ${recommendations.pending ?? 0} pending (positive rate ${rate(recommendations.positiveRate)}).`, + ...(payload.slop ?? []).map( + (band) => `- ${band.band}: ${rate(band.mergeRate)} merge rate over ${band.sampleSize ?? 0} PR(s) (${band.merged ?? 0} merged, ${band.closed ?? 0} closed)`, + ), + ...(payload.signals ?? []), + ]; + emit(payload, lines.join("\n")); + return; + } if (subcommand === "onboarding-pack") { // #6738: session-authenticated mirror of GET /onboarding-pack/preview (and the remote // loopover_get_repo_onboarding_pack tool). Bare `--refresh` becomes options.refresh === true via diff --git a/test/unit/mcp-cli-basics.test.ts b/test/unit/mcp-cli-basics.test.ts index 05d9cd5cd..b7ad8e5a3 100644 --- a/test/unit/mcp-cli-basics.test.ts +++ b/test/unit/mcp-cli-basics.test.ts @@ -221,7 +221,7 @@ describe("loopover-mcp CLI — basics", () => { expect(ps).toContain("[System.Management.Automation.CompletionResult]::new"); expect(ps).toContain("$commands = @('login', 'logout'"); expect(ps).toContain( - "'maintain' = @('status', 'queue', 'approve', 'reject', 'pause', 'resume', 'set-level', 'precision', 'onboarding-pack', 'audit-feed')", + "'maintain' = @('status', 'queue', 'approve', 'reject', 'pause', 'resume', 'set-level', 'precision', 'outcome-calibration', 'onboarding-pack', 'audit-feed')", ); }); diff --git a/test/unit/mcp-cli-maintain.test.ts b/test/unit/mcp-cli-maintain.test.ts index 02b99152b..40940ce82 100644 --- a/test/unit/mcp-cli-maintain.test.ts +++ b/test/unit/mcp-cli-maintain.test.ts @@ -95,6 +95,24 @@ describe("loopover-mcp CLI — maintain (#784)", () => { expect(scoped).toMatch(/Gate precision for owner\/repo \(last 30d\)/); }); + it("outcome-calibration reports slop-band merge rates + recommendation outcomes (plain + json), passing the window through (#6735)", async () => { + const e = await env(); + const out = await runAsync(["maintain", "outcome-calibration", "--repo", "owner/repo"], e); + expect(out).toMatch(/Outcome calibration for owner\/repo \(all history\): recommendations 14 positive, 3 negative, 3 pending \(positive rate 82%\)/); + expect(out).toMatch(/clean: 75% merge rate over 12 PR\(s\) \(9 merged, 3 closed\)/); + expect(out).toMatch(/high: 25% merge rate over 4 PR\(s\)/); + expect(out).toMatch(/Higher-slop bands merge less often/); + const json = JSON.parse(await runAsync(["maintain", "outcome-calibration", "--repo", "owner/repo", "--json"], e)) as { + recommendations: { positive: number; positiveRate: number }; + slop: Array<{ band: string }>; + }; + expect(json.recommendations).toMatchObject({ positive: 14, positiveRate: 0.82 }); + expect(json.slop.map((band) => band.band)).toEqual(["clean", "high"]); + // --window-days bounds the recommendation window; the CLI forwards it as ?windowDays and reflects it. + const scoped = await runAsync(["maintain", "outcome-calibration", "--repo", "owner/repo", "--window-days", "30"], e); + expect(scoped).toMatch(/Outcome calibration for owner\/repo \(last 30d\)/); + }); + it("onboarding-pack mirrors the session-gated API payload and forwards refresh", async () => { const requests: string[] = []; const e = await env((request) => requests.push(request.url ?? "")); diff --git a/test/unit/support/mcp-cli-harness.ts b/test/unit/support/mcp-cli-harness.ts index 682ec3290..515e3da7e 100644 --- a/test/unit/support/mcp-cli-harness.ts +++ b/test/unit/support/mcp-cli-harness.ts @@ -516,6 +516,23 @@ export async function startFixtureServer( ); return; } + if (request.url?.startsWith("/v1/repos/owner/repo/outcome-calibration") && request.method === "GET") { + const windowDays = new URL(request.url, "http://localhost").searchParams.get("windowDays"); + response.end( + JSON.stringify({ + repoFullName: "owner/repo", + generatedAt: "2026-05-30T00:00:00.000Z", + windowDays: windowDays ? Number(windowDays) : null, + slop: [ + { band: "clean", sampleSize: 12, merged: 9, closed: 3, mergeRate: 0.75 }, + { band: "high", sampleSize: 4, merged: 1, closed: 3, mergeRate: 0.25 }, + ], + recommendations: { total: 20, positive: 14, negative: 3, pending: 3, positiveRate: 0.82 }, + signals: ["Higher-slop bands merge less often — the slop signal is tracking real outcomes."], + }), + ); + return; + } const onboardingPackUrl = new URL(request.url ?? "/", "http://localhost"); if (onboardingPackUrl.pathname === "/v1/repos/owner/repo/onboarding-pack/preview" && request.method === "GET") { const refresh = onboardingPackUrl.searchParams.get("refresh");