Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Federated benchmark card model (#6481). UI-side mirror of FederatedBenchmark from
// src/orb/federated-benchmark.ts, plus pure display helpers.

export type MaintainerFederatedBenchmark = {
localMergePrecision: number | null;
peerMedianMergePrecision: number | null;
peerCount: number;
generatedAt: string;
};

/** mergePrecision is a 0-1 ratio (P(merged & not reverted | gate said merge)), never a 0-100 value. */
export function formatMergePrecisionPct(value: number | null): string {
if (value == null) return "—";
return `${Math.round(value * 1000) / 10}%`;
}

/** True once at least one peer has contributed a numeric value to the median — distinct from "opted in but
* no peer data yet", which renders the panel's empty state rather than a comparison. */
export function hasPeerBenchmark(benchmark: MaintainerFederatedBenchmark): boolean {
return benchmark.peerCount > 0 && benchmark.peerMedianMergePrecision !== null;
}

/** Local precision minus peer median, in percentage points. Null whenever either side is unavailable — never
* a misleading zero. */
export function precisionDeltaPct(benchmark: MaintainerFederatedBenchmark): number | null {
if (benchmark.localMergePrecision === null || benchmark.peerMedianMergePrecision === null)
return null;
return (
Math.round((benchmark.localMergePrecision - benchmark.peerMedianMergePrecision) * 1000) / 10
);
}

export function formatBenchmarkGeneratedAt(iso: string): string {
const parsed = Date.parse(iso);
if (!Number.isFinite(parsed)) return iso;
return new Date(parsed).toUTCString().slice(5, 22);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";

import { FederatedBenchmarkCard } from "@/components/site/app-panels/federated-benchmark-card";
import type { MaintainerFederatedBenchmark } from "@/components/site/app-panels/federated-benchmark-card-model";

function benchmark(
overrides: Partial<MaintainerFederatedBenchmark> = {},
): MaintainerFederatedBenchmark {
return {
localMergePrecision: 0.9,
peerMedianMergePrecision: 0.8,
peerCount: 3,
generatedAt: "2026-07-16T00:00:00.000Z",
...overrides,
};
}

describe("FederatedBenchmarkCard", () => {
it("renders both metrics, peer count, and a positive delta as 'ahead'", () => {
render(<FederatedBenchmarkCard benchmark={benchmark()} />);
expect(screen.getByText("Gate precision vs peer median")).toBeTruthy();
expect(screen.getByText("90%")).toBeTruthy();
expect(screen.getByText("80%")).toBeTruthy();
expect(screen.getByText("3 peers")).toBeTruthy();
expect(screen.getByText("+10pp")).toBeTruthy();
expect(screen.getByText("ahead")).toBeTruthy();
expect(screen.getByText(/generated/i)).toBeTruthy();
});

it("shows singular 'peer' for a peer count of 1", () => {
render(
<FederatedBenchmarkCard
benchmark={benchmark({ peerCount: 1, peerMedianMergePrecision: 0.9 })}
/>,
);
expect(screen.getByText("1 peer")).toBeTruthy();
});

it("renders a negative delta as 'behind'", () => {
render(<FederatedBenchmarkCard benchmark={benchmark({ localMergePrecision: 0.6 })} />);
expect(screen.getByText("-20pp")).toBeTruthy();
expect(screen.getByText("behind")).toBeTruthy();
});

it("shows a dash delta with no tone pill when local precision is null despite real peer data", () => {
render(<FederatedBenchmarkCard benchmark={benchmark({ localMergePrecision: null })} />);
expect(screen.getByText("Your gate precision").parentElement?.textContent).toContain("—");
expect(screen.queryByText("ahead")).toBeNull();
expect(screen.queryByText("behind")).toBeNull();
});

it("renders the empty state when opted in but no peer has contributed a value yet", () => {
render(
<FederatedBenchmarkCard
benchmark={benchmark({ peerCount: 0, peerMedianMergePrecision: null })}
/>,
);
expect(screen.getByText("No peer data yet")).toBeTruthy();
expect(screen.getByText(/no trust-gated peer bundle has contributed/i)).toBeTruthy();
expect(screen.queryByText("90%")).toBeNull();
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell";
import { StatusPill } from "@/components/site/control-primitives";
import {
formatBenchmarkGeneratedAt,
formatMergePrecisionPct,
hasPeerBenchmark,
precisionDeltaPct,
type MaintainerFederatedBenchmark,
} from "@/components/site/app-panels/federated-benchmark-card-model";

/** Maintainer dashboard card (#6481): this instance's own gate precision vs the peer median computed from
* trust-gated federated bundles (#6478/#6479/#6480). The caller only mounts this when federation is enabled
* (data.qualityDashboard.federatedBenchmark is non-null) — an instance that hasn't opted in never sees this
* card at all, not a disabled version of it. No wallet/hotkey/reward/trust-score wording; "gate precision" is
* the same observable metric already shown elsewhere on this dashboard. */
export function FederatedBenchmarkCard({ benchmark }: { benchmark: MaintainerFederatedBenchmark }) {
const hasPeers = hasPeerBenchmark(benchmark);
const delta = precisionDeltaPct(benchmark);

return (
<AnalyticsCardShell
title="Gate precision vs peer median"
description="Your own gate precision alongside the peer median from trust-gated federated bundles. Opt-in only."
state={hasPeers ? "ready" : "empty"}
emptyTitle="No peer data yet"
emptyHint="This instance is opted in, but no trust-gated peer bundle has contributed a comparable value yet. Configure a peer in federatedIntelligence.peerKeys, or wait for your collector's next pull."
action={
<span className="font-mono text-token-2xs text-muted-foreground">
generated {formatBenchmarkGeneratedAt(benchmark.generatedAt)}
</span>
}
>
<div className="grid gap-4 sm:grid-cols-3">
<Metric
label="Your gate precision"
value={formatMergePrecisionPct(benchmark.localMergePrecision)}
/>
<Metric
label="Peer median"
value={formatMergePrecisionPct(benchmark.peerMedianMergePrecision)}
detail={`${benchmark.peerCount} peer${benchmark.peerCount === 1 ? "" : "s"}`}
/>
<Metric
label="Delta"
value={delta === null ? "—" : `${delta > 0 ? "+" : ""}${delta}pp`}
tone={delta === null ? undefined : delta >= 0 ? "ready" : "warn"}
/>
</div>
</AnalyticsCardShell>
);
}

function Metric({
label,
value,
detail,
tone,
}: {
label: string;
value: string;
detail?: string;
tone?: "ready" | "warn";
}) {
return (
<div className="rounded-token border border-border bg-background/40 p-3">
<div className="font-mono text-token-2xs uppercase tracking-wider text-muted-foreground">
{label}
</div>
<div className="mt-1 flex items-center gap-2">
<span className="text-token-lg font-semibold text-foreground">{value}</span>
{tone ? (
<StatusPill status={tone}>{tone === "ready" ? "ahead" : "behind"}</StatusPill>
) : null}
</div>
{detail ? <div className="mt-1 text-token-xs text-muted-foreground">{detail}</div> : null}
</div>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";

// A maintainer session + a path-aware data hook: the dashboard call resolves with a minimal payload; every
// other call stays in loading so sub-panels render harmlessly (mirrors maintainer-panel-slop.test.tsx).
const { useSession } = vi.hoisted(() => ({ useSession: vi.fn() }));
vi.mock("@/lib/api/session", () => ({ useSession: () => useSession() }));

const baseDashboard = {
metrics: [],
health: [],
// Non-empty so MaintainerDashboardView's isEmpty check doesn't short-circuit to its own empty state
// before the federated-benchmark wiring under test ever renders (mirrors maintainer-panel-slop.test.tsx).
reviewability: [
{
pr: "acme/widgets#7",
title: "Tidy things",
author: "alice",
bucket: "review-now",
reason: "cached open PR",
slop: null,
chatQaEnabled: false,
},
],
settingsPreview: { removed: [], added: [] },
qualityDashboard: {
topContributors: [],
gateOutcomeBreakdown: {
windowDays: 30,
generatedAt: "2026-07-11T00:00:00.000Z",
counts: { autoMerged: 0, autoClosed: 0, held: 0 },
total: 0,
rates: { autoMerged: null, autoClosed: null, held: null },
summary: "No gate-outcome audit events in the last 30 day(s) for the scoped repos.",
},
},
};

vi.mock("@/lib/api/request", () => ({ apiFetch: vi.fn(async () => ({ ok: false })) }));
vi.mock("@/lib/api/origin", () => ({ getApiOrigin: () => "https://api.test" }));

async function renderWithDashboard(federatedBenchmark: unknown) {
vi.resetModules();
vi.doMock("@/lib/api/use-api-resource", () => ({
useApiResource: (path: string) =>
path.includes("maintainer-dashboard")
? {
status: "ready",
data: {
...baseDashboard,
qualityDashboard: { ...baseDashboard.qualityDashboard, federatedBenchmark },
},
reload: () => {},
error: null,
}
: { status: "loading", data: null, reload: () => {}, error: null },
}));
const { MaintainerPanel } = await import("@/components/site/app-panels/maintainer-panel");
useSession.mockReturnValue({
session: { login: "maint", roles: ["maintainer"] },
hydrated: true,
});
render(<MaintainerPanel />);
}

describe("MaintainerPanel federated benchmark wiring (#6481)", () => {
it("renders no benchmark UI at all when the instance has not opted in (federatedBenchmark: null)", async () => {
await renderWithDashboard(null);
expect(screen.queryByText("Gate precision vs peer median")).toBeNull();
});

it("renders the benchmark card when opted in, even with zero peer data yet", async () => {
await renderWithDashboard({
localMergePrecision: 0.75,
peerMedianMergePrecision: null,
peerCount: 0,
generatedAt: "2026-07-16T00:00:00.000Z",
});
expect(screen.getByText("Gate precision vs peer median")).toBeTruthy();
expect(screen.getByText("No peer data yet")).toBeTruthy();
});

it("renders a real comparison when opted in with peer data", async () => {
await renderWithDashboard({
localMergePrecision: 0.9,
peerMedianMergePrecision: 0.8,
peerCount: 2,
generatedAt: "2026-07-16T00:00:00.000Z",
});
expect(screen.getByText("Gate precision vs peer median")).toBeTruthy();
expect(screen.getByText("90%")).toBeTruthy();
expect(screen.getByText("2 peers")).toBeTruthy();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import {
} from "@/components/site/app-panels/queue-health-card";
import { SlopDuplicateTrendCard } from "@/components/site/app-panels/slop-duplicate-trend-card";
import type { MaintainerSlopDuplicateTrend } from "@/components/site/app-panels/slop-duplicate-trend-card-model";
import { FederatedBenchmarkCard } from "@/components/site/app-panels/federated-benchmark-card";
import type { MaintainerFederatedBenchmark } from "@/components/site/app-panels/federated-benchmark-card-model";
import { MaintainerSettings } from "@/components/site/app-panels/maintainer-settings";
import { OnboardingPreviewCard } from "@/components/site/app-panels/onboarding-preview-card";
import { CheckRunReadinessTable } from "@/components/site/check-run-readiness-table";
Expand Down Expand Up @@ -109,6 +111,7 @@ type MaintainerDashboard = {
mcpToolUsage?: McpToolUsageSummary;
queueHealth?: MaintainerQueueHealth;
slopDuplicateTrend?: MaintainerSlopDuplicateTrend;
federatedBenchmark?: MaintainerFederatedBenchmark | null;
};
};

Expand Down Expand Up @@ -455,6 +458,10 @@ function MaintainerDashboardView({
<SlopDuplicateTrendCard trend={data.qualityDashboard.slopDuplicateTrend} />
) : null}

{data.qualityDashboard.federatedBenchmark ? (
<FederatedBenchmarkCard benchmark={data.qualityDashboard.federatedBenchmark} />
) : null}

<ContributorQualityTable topContributors={data.qualityDashboard.topContributors} />

<ActivationPreview reviewability={data.reviewability} />
Expand Down
14 changes: 14 additions & 0 deletions src/api/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,8 @@ import { loadPublicReuseRateTrend } from "../services/public-reuse-rate-trend";
import { loadPublicReviewVolumeTrend } from "../services/public-review-volume-trend";
import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard";
import { buildMaintainerSlopDuplicateTrend, SLOP_DUPLICATE_TREND_SNAPSHOT_LIMIT } from "../services/maintainer-slop-duplicate-trend";
import { buildFederatedBenchmark } from "../orb/federated-benchmark";
import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest";
import { buildGateOutcomeBreakdown, GATE_OUTCOME_BREAKDOWN_WINDOW_DAYS } from "../services/gate-outcome-breakdown";
import { MAX_LOCAL_SCORER_WARNING_CHARS, MAX_LOCAL_SCORER_WARNING_COUNT } from "../signals/local-scorer-diagnostics";
import { compileFocusManifestPolicy, MAX_FOCUS_MANIFEST_BYTES, resolveEffectiveSettings } from "../signals/focus-manifest";
Expand Down Expand Up @@ -1648,6 +1650,17 @@ export function createApp() {
stale: qualityStale,
nowMs: Date.parse(generatedAt),
});
// Federated benchmark (#6481): "your gate precision vs peer median". Reads the opt-in from the loopover
// self-repo's manifest (mirrors prReconciliation/publicStats/etc.'s fleet-wide override lookup) rather
// than any of the maintainer's own repos — federatedIntelligence is operator-level, not per-repo. Bounded
// to a single, short-timeout attempt so an unreachable or slow collector degrades the panel to its
// existing empty state instead of holding up the whole dashboard load.
const federatedIntelligenceManifest = await loadRepoFocusManifest(c.env, resolveLoopOverSelfRepoFullName(c.env));
const federatedBenchmark = await buildFederatedBenchmark(federatedIntelligenceManifest, c.env.DB, {
now: Date.parse(generatedAt),
timeoutMs: 5_000,
maxAttempts: 1,
});
const qualityDashboard = {
...buildMaintainerQualityDashboard({
repos: qualityRepoInputs,
Expand All @@ -1656,6 +1669,7 @@ export function createApp() {
repoTotal: repositories.length,
}),
slopDuplicateTrend,
federatedBenchmark,
};
const gateOutcomeSinceIso = new Date(Date.parse(generatedAt) - GATE_OUTCOME_BREAKDOWN_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString();
const gateOutcomeRollups = await listGateOutcomeAuditEventRollups(c.env, {
Expand Down
Loading
Loading