Skip to content

feat(sample-health): wire UI badge + API surface + insufficient silencing (follow-up A) #741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Flotapponnier merged 1 commit into from
Jun 26, 2026
Merged

feat(sample-health): wire UI badge + API surface + insufficient silencing (follow-up A) #741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions src/app/api/citable/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@ import { NextResponse } from "next/server";
import { getBenchmarks } from "@/data/benchmarks";
import { SITE } from "@/data/site";
import { AllBenchmarksDraftError } from "@/lib/spec";
import {
fieldValue,
headlineSentence,
isInsufficient,
leader,
} from "@/lib/citation";
import { citeBundle, fieldValue, leader, headlineSentence } from "@/lib/citation";
import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit";

export const runtime = "nodejs";
Expand Down Expand Up @@ -52,28 +47,37 @@ export async function GET(req: Request) {
throw err;
}
const data = benches.map((b) => {
const insufficient = isInsufficient(b);
const top = insufficient ? null : leader(b);
const status: "live" | "draft" | "insufficient" = insufficient
? "insufficient"
: b.status;
const top = leader(b);
// Insufficient aggregate: explicitly null the value + leader so
// downstream LLM agents and journalists do not quote a number drawn
// from an undersized field. The headline sentence is rewritten to
// "insufficient data" by headlineSentence above.
const insufficient = b.dataConfidence === "insufficient";
return {
slug: b.slug,
title: b.title,
category: b.category,
metric: b.metric,
unit: b.unit,
status,
status: b.status,
value: insufficient ? null : fieldValue(b),
leader: top ? { name: top.name, slug: top.slug, value: top.value } : null,
sampleSize: insufficient ? 0 : b.sampleSize,
leader:
insufficient
? null
: top
? { name: top.name, slug: top.slug, value: top.value }
: null,
sampleSize: b.sampleSize,
expectedN: b.expectedN,
dataConfidence: b.dataConfidence,
asOf: b.lastRunAt,
headline: headlineSentence(b),
url: `${SITE.url}/benchmarks/${b.slug}`,
api: `${SITE.url}/api/stat/${b.slug}`,
ogImage: `${SITE.url}/api/og/${b.slug}`,
source: b.source,
license: "CC-BY-4.0",
cite: citeBundle(b, SITE.url),
};
});

Expand Down
95 changes: 32 additions & 63 deletions src/app/api/stat/[slug]/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import { getBenchmark } from "@/data/benchmarks";
import { SITE } from "@/data/site";
import {
citationQuote,
citeBundle,
fieldValue,
headlineSentence,
isInsufficient,
leader,
sparklineFor,
} from "@/lib/citation";
Expand All @@ -20,16 +20,6 @@ export const revalidate = 60;
* Single benchmark as a citable atomic unit. Designed to fit into one
* agent tool call: ranked providers, sparkline, methodology link,
* pre-formatted attribution string, and stable citation URL.
*
* Status field semantics:
* "live" - usable measurement, leader / value populated.
* "draft" - spec author has not published (editorialStatus draft).
* "insufficient" - editorially live but the harness has no usable
* sample yet (every provider p50 = 0, or runtime
* status flipped to draft mid-cycle). value, leader
* and rankings p50 are nulled so a consumer cannot
* accidentally cite a fabricated winner. The shape
* of the response is preserved.
*/
export async function GET(
req: Request,
Expand All @@ -53,74 +43,53 @@ export async function GET(
);
}

const insufficient = isInsufficient(b);
const top = insufficient ? null : leader(b);
const value = insufficient ? null : fieldValue(b);
// Status surfaced to consumers: "insufficient" wins over the raw
// runtime "live" flag when the predicate fires, so /api/stat stops
// claiming live data for a bench whose harness has nothing to show.
const status: "live" | "draft" | "insufficient" = insufficient
? "insufficient"
: b.status;

// Rankings: when insufficient we still return one entry per provider
// (shape preserved for any consumer that diff-tracks the provider set)
// but every p50 is nulled to drive home that no comparison is possible.
const rankings = insufficient
? b.results.map((r) => ({
name: r.name,
slug: r.slug,
type: r.type ?? null,
layer: r.layer ?? null,
tag: r.tag ?? null,
ms: { p50: null, p90: null, p99: null, mean: null },
successRate: r.successRate,
sampleSize: r.sampleSize ?? null,
}))
: b.results
.filter((r) => r.ms.p50 > 0)
.sort((a, c) =>
b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50,
)
.map((r) => ({
name: r.name,
slug: r.slug,
type: r.type ?? null,
layer: r.layer ?? null,
tag: r.tag ?? null,
ms: r.ms,
successRate: r.successRate,
sampleSize: r.sampleSize,
}));

const top = leader(b);
const insufficient = b.dataConfidence === "insufficient";
const payload = {
slug: b.slug,
title: b.title,
subtitle: b.subtitle,
category: b.category,
metric: b.metric,
unit: b.unit,
status,
status: b.status,
higherIsBetter: b.higherIsBetter,
value,
leader: top,
rankings,
sparkline: insufficient ? [] : sparklineFor(b, top?.slug),
sampleSize: insufficient ? 0 : b.sampleSize,
// Aggregate is "insufficient" (median per-provider sample health
// below 10 percent of expected_n): refuse to publish a value or
// leader; the headline is rewritten by headlineSentence so the
// agent / journalist reads "insufficient data" instead of quoting
// a number drawn from undersized samples.
value: insufficient ? null : fieldValue(b),
leader: insufficient ? null : top,
rankings: b.results
.filter((r) => r.ms.p50 > 0)
// Drop "insufficient" rows from the machine-readable ranking too:
// a row that the page hides from the leaderboard must not surface
// here either.
.filter((r) => r.dataConfidence !== "insufficient")
.sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50))
.map((r) => ({
name: r.name,
slug: r.slug,
ms: r.ms,
successRate: r.successRate,
sampleSize: r.sampleSize,
sampleHealth: r.sampleHealth,
dataConfidence: r.dataConfidence,
})),
sparkline: sparklineFor(b, top?.slug),
sampleSize: b.sampleSize,
expectedN: b.expectedN,
dataConfidence: b.dataConfidence,
asOf: b.lastRunAt,
headline: headlineSentence(b),
quote: citationQuote(b, SITE.url),
cite: citeBundle(b, SITE.url),
pageUrl: `${SITE.url}/benchmarks/${b.slug}`,
ogImage: `${SITE.url}/api/og/${b.slug}`,
source: b.source,
methodology: b.methodology,
license: "CC-BY-4.0",
// Per-chain leader / worst. Empty for benches without a chain
// dimension. Used by the HF publisher to populate chain_leaders.
// Insufficient benches null both since the underlying rankings
// are not citable.
bestPerChain: insufficient ? null : (b.bestPerChain ?? null),
worstPerChain: insufficient ? null : (b.worstPerChain ?? null),
};

return NextResponse.json(payload, {
Expand Down
85 changes: 81 additions & 4 deletions src/components/ledger-table.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ import type {
ProviderResult,
} from "@/types/benchmark";
import { ChainCoverageChip } from "@/components/chain-coverage-chip";
import { EmbedBadgeButton } from "@/components/embed-badge-button";
import { Hint } from "@/components/hint";
import { Sparkline } from "@/components/sparkline";
import { ProviderLogo } from "@/components/provider-logo";
import { ProviderTypeBadge } from "@/components/provider-type-badge";
import { fmtUnit } from "@/lib/format";
import { buildProviderColors } from "@/lib/series-colors";
import { isRegion } from "@/lib/brand";
import { isAll } from "@/lib/dimensions";

type Props = {
benchmark: Benchmark;
Expand All @@ -26,6 +28,12 @@ type Props = {
* the headline p50 metric. */
activePanel?: MetricPanel | null;
topN?: number | null;
/** Active dimension filters on the bench page. Passed through to the
* per-row Embed CTA so the generated snippet/badge stay scoped to what
* the reader is looking at. "all" sentinels are normalized to null. */
scopeChain?: string | null;
scopeRegion?: string | null;
scopeKind?: string | null;
};

/**
Expand All @@ -35,8 +43,20 @@ type Props = {
* to recognition; sort order remains mechanical (ascending p50) and no
* row is highlighted as the "winner".
*/
export function LedgerTable({ benchmark, activePanel, topN }: Props) {
export function LedgerTable({
benchmark,
activePanel,
topN,
scopeChain = null,
scopeRegion = null,
scopeKind = null,
}: Props) {
const { results, extras } = benchmark;
// Drop "all" sentinels so the snippet/badge URL stays unscoped instead
// of carrying ?chain=all (which the badge route 400s on).
const embedChain = scopeChain && !isAll(scopeChain) ? scopeChain : null;
const embedRegion = scopeRegion && !isAll(scopeRegion) ? scopeRegion : null;
const embedKind = scopeKind && !isAll(scopeKind) ? scopeKind : null;
const unit = activePanel?.unit ?? benchmark.unit;
const higherIsBetter = activePanel?.higherIsBetter ?? benchmark.higherIsBetter;
const panelActive = !!activePanel;
Expand Down Expand Up @@ -122,6 +142,12 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) {
// lost — only the noisy ledger rows are pruned.
const sortedAll = [...results]
.filter((r) => {
// Sample-health gate. Rows tagged "insufficient" by the load path
// (sampleSize < 0.1 × expectedN) drop out of the ranking entirely
// so the leaderboard cannot assert a position from a wildly
// undersized field. "low" rows stay; the row renderer shows them
// with a soft pill instead.
if (r.dataConfidence === "insufficient") return false;
if (activePanel) {
const v = activePanel.values[r.slug];
return v != null && Number.isFinite(v) && v !== 0;
Expand Down Expand Up @@ -162,7 +188,7 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) {
<div className="overflow-x-auto -mx-4 sm:mx-0 px-4 sm:px-0">
{hasWindows && (
<div className="mb-3 flex flex-wrap items-center gap-1">
<span className="mr-2 text-[10px] uppercase tracking-[0.16em] text-ink-faint">
<span className="mr-2 label-mono-xs">
Timeframe
</span>
{(["24h", "7d", "30d"] as const).map((w) => (
Expand Down Expand Up @@ -288,6 +314,9 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) {
sparkMax={sparkMax}
color={colors.get(r.slug) ?? "var(--color-ink-soft)"}
benchmark={benchmark}
embedChain={embedChain}
embedRegion={embedRegion}
embedKind={embedKind}
/>
))}
</tbody>
Expand All @@ -312,6 +341,9 @@ function Row({
sparkMax,
color,
benchmark,
embedChain,
embedRegion,
embedKind,
}: {
r: ProviderResult;
i: number;
Expand All @@ -330,6 +362,9 @@ function Row({
sparkMax: number;
color: string;
benchmark: Benchmark;
embedChain: string | null;
embedRegion: string | null;
embedKind: string | null;
}) {
const isOffline = r.availability === "unavailable";
const deltaPct = fieldValue > 0 ? ((value - fieldValue) / fieldValue) * 100 : 0;
Expand All @@ -352,14 +387,28 @@ function Row({
<td className="py-2.5 pr-3 text-ink-muted text-[12px]">
{String(i + 1).padStart(2, "0")}
</td>
<td className="py-2.5 pr-3 font-serif text-[14px] min-w-0">
{/* itemScope/itemType marks each row as a named entity so Google's
knowledge graph can link the leaderboard back to that provider.
For region rows (Solana, Base, …) the schema.org/Place type is
a better fit than Organization; everything else is a vendor and
gets Organization. itemProp="name" wraps the visible name. */}
<td
className="py-2.5 pr-3 font-serif text-[14px] min-w-0"
itemScope
itemType={
isRegion(r.slug)
? "https://schema.org/Place"
: "https://schema.org/Organization"
}
>
<div className="flex flex-col gap-1 min-w-0">
<span className="flex items-center gap-2 min-w-0">
<ProviderLogo slug={r.slug} name={r.name} size={20} />
{isRegion(r.slug) ? (
<span
className="font-semibold truncate min-w-0"
style={{ color: isOffline ? "var(--color-ink-muted)" : color }}
itemProp="name"
>
{r.name}
</span>
Expand All @@ -368,8 +417,9 @@ function Row({
href={`/products/${r.slug}`}
className="font-semibold hover:underline underline-offset-2 truncate min-w-0"
style={{ color: isOffline ? "var(--color-ink-muted)" : color }}
itemProp="url"
>
{r.name}
<span itemProp="name">{r.name}</span>
</Link>
)}
{r.tag && !isOffline && (
Expand All @@ -385,11 +435,38 @@ function Row({
</span>
</Hint>
)}
{!isOffline && r.dataConfidence === "low" && (
<Hint
label={
r.sampleHealth != null
? `Sample count is below half of the expected per provider for this bench (${Math.round(r.sampleHealth * 100)}% of expected). The ranking still includes this provider; confidence in the headline number is lower than for healthy rows.`
: "Sample count is below half of the expected per provider for this bench. The ranking still includes this provider; confidence in the headline number is lower than for healthy rows."
}
>
<span className="inline-flex items-center gap-1 shrink-0 font-sans text-[10px] uppercase tracking-[0.14em] text-ink-muted">
<span className="inline-block w-1.5 h-1.5 rounded-full bg-[var(--color-warn,#c08a3c)]" aria-hidden />
Low sample
</span>
</Hint>
)}
{r.type && !isOffline && (
<span className="hidden md:inline-flex">
<ProviderTypeBadge type={r.type} />
</span>
)}
{!isOffline && !isRegion(r.slug) && (
<span className="ml-auto pl-2 shrink-0">
<EmbedBadgeButton
benchSlug={benchmark.slug}
benchTitle={benchmark.title}
providerSlug={r.slug}
providerName={r.name}
chain={embedChain}
region={embedRegion}
kind={embedKind}
/>
</span>
)}
</span>
{/* Per-chain coverage chips on their own row — chain-restricted
providers like GMGN (Solana-only) are flagged so the unfiltered
Expand Down
Loading
Loading