diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index 03d6b55c..e1283e1e 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -2,12 +2,7 @@ import { NextResponse } from "next/server"; import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; import { AllBenchmarksDraftError } from "@/lib/spec"; -import { - fieldValue, - headlineSentence, - isInsufficient, - leader, -} from "@/lib/citation"; +import { citeBundle, fieldValue, leader, headlineSentence } from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; export const runtime = "nodejs"; @@ -52,21 +47,29 @@ export async function GET(req: Request) { throw err; } const data = benches.map((b) => { - const insufficient = isInsufficient(b); - const top = insufficient ? null : leader(b); - const status: "live" | "draft" | "insufficient" = insufficient - ? "insufficient" - : b.status; + const top = leader(b); + // Insufficient aggregate: explicitly null the value + leader so + // downstream LLM agents and journalists do not quote a number drawn + // from an undersized field. The headline sentence is rewritten to + // "insufficient data" by headlineSentence above. + const insufficient = b.dataConfidence === "insufficient"; return { slug: b.slug, title: b.title, category: b.category, metric: b.metric, unit: b.unit, - status, + status: b.status, value: insufficient ? null : fieldValue(b), - leader: top ? { name: top.name, slug: top.slug, value: top.value } : null, - sampleSize: insufficient ? 0 : b.sampleSize, + leader: + insufficient + ? null + : top + ? { name: top.name, slug: top.slug, value: top.value } + : null, + sampleSize: b.sampleSize, + expectedN: b.expectedN, + dataConfidence: b.dataConfidence, asOf: b.lastRunAt, headline: headlineSentence(b), url: `${SITE.url}/benchmarks/${b.slug}`, @@ -74,6 +77,7 @@ export async function GET(req: Request) { ogImage: `${SITE.url}/api/og/${b.slug}`, source: b.source, license: "CC-BY-4.0", + cite: citeBundle(b, SITE.url), }; }); diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index 14d6c7d7..b53c5767 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -3,9 +3,9 @@ import { getBenchmark } from "@/data/benchmarks"; import { SITE } from "@/data/site"; import { citationQuote, + citeBundle, fieldValue, headlineSentence, - isInsufficient, leader, sparklineFor, } from "@/lib/citation"; @@ -20,16 +20,6 @@ export const revalidate = 60; * Single benchmark as a citable atomic unit. Designed to fit into one * agent tool call: ranked providers, sparkline, methodology link, * pre-formatted attribution string, and stable citation URL. - * - * Status field semantics: - * "live" - usable measurement, leader / value populated. - * "draft" - spec author has not published (editorialStatus draft). - * "insufficient" - editorially live but the harness has no usable - * sample yet (every provider p50 = 0, or runtime - * status flipped to draft mid-cycle). value, leader - * and rankings p50 are nulled so a consumer cannot - * accidentally cite a fabricated winner. The shape - * of the response is preserved. */ export async function GET( req: Request, @@ -53,46 +43,8 @@ export async function GET( ); } - const insufficient = isInsufficient(b); - const top = insufficient ? null : leader(b); - const value = insufficient ? null : fieldValue(b); - // Status surfaced to consumers: "insufficient" wins over the raw - // runtime "live" flag when the predicate fires, so /api/stat stops - // claiming live data for a bench whose harness has nothing to show. - const status: "live" | "draft" | "insufficient" = insufficient - ? "insufficient" - : b.status; - - // Rankings: when insufficient we still return one entry per provider - // (shape preserved for any consumer that diff-tracks the provider set) - // but every p50 is nulled to drive home that no comparison is possible. - const rankings = insufficient - ? b.results.map((r) => ({ - name: r.name, - slug: r.slug, - type: r.type ?? null, - layer: r.layer ?? null, - tag: r.tag ?? null, - ms: { p50: null, p90: null, p99: null, mean: null }, - successRate: r.successRate, - sampleSize: r.sampleSize ?? null, - })) - : b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ) - .map((r) => ({ - name: r.name, - slug: r.slug, - type: r.type ?? null, - layer: r.layer ?? null, - tag: r.tag ?? null, - ms: r.ms, - successRate: r.successRate, - sampleSize: r.sampleSize, - })); - + const top = leader(b); + const insufficient = b.dataConfidence === "insufficient"; const payload = { slug: b.slug, title: b.title, @@ -100,27 +52,44 @@ export async function GET( category: b.category, metric: b.metric, unit: b.unit, - status, + status: b.status, higherIsBetter: b.higherIsBetter, - value, - leader: top, - rankings, - sparkline: insufficient ? [] : sparklineFor(b, top?.slug), - sampleSize: insufficient ? 0 : b.sampleSize, + // Aggregate is "insufficient" (median per-provider sample health + // below 10 percent of expected_n): refuse to publish a value or + // leader; the headline is rewritten by headlineSentence so the + // agent / journalist reads "insufficient data" instead of quoting + // a number drawn from undersized samples. + value: insufficient ? null : fieldValue(b), + leader: insufficient ? null : top, + rankings: b.results + .filter((r) => r.ms.p50 > 0) + // Drop "insufficient" rows from the machine-readable ranking too: + // a row that the page hides from the leaderboard must not surface + // here either. + .filter((r) => r.dataConfidence !== "insufficient") + .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) + .map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + sampleSize: r.sampleSize, + sampleHealth: r.sampleHealth, + dataConfidence: r.dataConfidence, + })), + sparkline: sparklineFor(b, top?.slug), + sampleSize: b.sampleSize, + expectedN: b.expectedN, + dataConfidence: b.dataConfidence, asOf: b.lastRunAt, headline: headlineSentence(b), quote: citationQuote(b, SITE.url), + cite: citeBundle(b, SITE.url), pageUrl: `${SITE.url}/benchmarks/${b.slug}`, ogImage: `${SITE.url}/api/og/${b.slug}`, source: b.source, methodology: b.methodology, license: "CC-BY-4.0", - // Per-chain leader / worst. Empty for benches without a chain - // dimension. Used by the HF publisher to populate chain_leaders. - // Insufficient benches null both since the underlying rankings - // are not citable. - bestPerChain: insufficient ? null : (b.bestPerChain ?? null), - worstPerChain: insufficient ? null : (b.worstPerChain ?? null), }; return NextResponse.json(payload, { diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index 3a13124d..a5ea3dec 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -10,6 +10,7 @@ import type { ProviderResult, } from "@/types/benchmark"; import { ChainCoverageChip } from "@/components/chain-coverage-chip"; +import { EmbedBadgeButton } from "@/components/embed-badge-button"; import { Hint } from "@/components/hint"; import { Sparkline } from "@/components/sparkline"; import { ProviderLogo } from "@/components/provider-logo"; @@ -17,6 +18,7 @@ import { ProviderTypeBadge } from "@/components/provider-type-badge"; import { fmtUnit } from "@/lib/format"; import { buildProviderColors } from "@/lib/series-colors"; import { isRegion } from "@/lib/brand"; +import { isAll } from "@/lib/dimensions"; type Props = { benchmark: Benchmark; @@ -26,6 +28,12 @@ type Props = { * the headline p50 metric. */ activePanel?: MetricPanel | null; topN?: number | null; + /** Active dimension filters on the bench page. Passed through to the + * per-row Embed CTA so the generated snippet/badge stay scoped to what + * the reader is looking at. "all" sentinels are normalized to null. */ + scopeChain?: string | null; + scopeRegion?: string | null; + scopeKind?: string | null; }; /** @@ -35,8 +43,20 @@ type Props = { * to recognition; sort order remains mechanical (ascending p50) and no * row is highlighted as the "winner". */ -export function LedgerTable({ benchmark, activePanel, topN }: Props) { +export function LedgerTable({ + benchmark, + activePanel, + topN, + scopeChain = null, + scopeRegion = null, + scopeKind = null, +}: Props) { const { results, extras } = benchmark; + // Drop "all" sentinels so the snippet/badge URL stays unscoped instead + // of carrying ?chain=all (which the badge route 400s on). + const embedChain = scopeChain && !isAll(scopeChain) ? scopeChain : null; + const embedRegion = scopeRegion && !isAll(scopeRegion) ? scopeRegion : null; + const embedKind = scopeKind && !isAll(scopeKind) ? scopeKind : null; const unit = activePanel?.unit ?? benchmark.unit; const higherIsBetter = activePanel?.higherIsBetter ?? benchmark.higherIsBetter; const panelActive = !!activePanel; @@ -122,6 +142,12 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { // lost — only the noisy ledger rows are pruned. const sortedAll = [...results] .filter((r) => { + // Sample-health gate. Rows tagged "insufficient" by the load path + // (sampleSize < 0.1 × expectedN) drop out of the ranking entirely + // so the leaderboard cannot assert a position from a wildly + // undersized field. "low" rows stay; the row renderer shows them + // with a soft pill instead. + if (r.dataConfidence === "insufficient") return false; if (activePanel) { const v = activePanel.values[r.slug]; return v != null && Number.isFinite(v) && v !== 0; @@ -162,7 +188,7 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) {
{hasWindows && (
- + Timeframe {(["24h", "7d", "30d"] as const).map((w) => ( @@ -288,6 +314,9 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { sparkMax={sparkMax} color={colors.get(r.slug) ?? "var(--color-ink-soft)"} benchmark={benchmark} + embedChain={embedChain} + embedRegion={embedRegion} + embedKind={embedKind} /> ))} @@ -312,6 +341,9 @@ function Row({ sparkMax, color, benchmark, + embedChain, + embedRegion, + embedKind, }: { r: ProviderResult; i: number; @@ -330,6 +362,9 @@ function Row({ sparkMax: number; color: string; benchmark: Benchmark; + embedChain: string | null; + embedRegion: string | null; + embedKind: string | null; }) { const isOffline = r.availability === "unavailable"; const deltaPct = fieldValue > 0 ? ((value - fieldValue) / fieldValue) * 100 : 0; @@ -352,7 +387,20 @@ function Row({ {String(i + 1).padStart(2, "0")} - + {/* itemScope/itemType marks each row as a named entity so Google's + knowledge graph can link the leaderboard back to that provider. + For region rows (Solana, Base, …) the schema.org/Place type is + a better fit than Organization; everything else is a vendor and + gets Organization. itemProp="name" wraps the visible name. */} +
@@ -360,6 +408,7 @@ function Row({ {r.name} @@ -368,8 +417,9 @@ function Row({ href={`/products/${r.slug}`} className="font-semibold hover:underline underline-offset-2 truncate min-w-0" style={{ color: isOffline ? "var(--color-ink-muted)" : color }} + itemProp="url" > - {r.name} + {r.name} )} {r.tag && !isOffline && ( @@ -385,11 +435,38 @@ function Row({ )} + {!isOffline && r.dataConfidence === "low" && ( + + + + Low sample + + + )} {r.type && !isOffline && ( )} + {!isOffline && !isRegion(r.slug) && ( + + + + )} {/* Per-chain coverage chips on their own row — chain-restricted providers like GMGN (Solana-only) are flagged so the unfiltered diff --git a/src/lib/citation.ts b/src/lib/citation.ts index 544bbc14..35c68cae 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -4,72 +4,32 @@ * everyone (LLMs, journalists, ourselves) sees. */ -import type { Benchmark } from "@/types/benchmark"; +import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; import { fmtUnit } from "@/lib/format"; -/** - * Canonical "this bench cannot be ranked right now" predicate, shared by - * every citable surface (api/stat, api/citable, api/llm-context, llms.txt, - * /answers, bench page hero, OG image, MCP). - * - * A benchmark is insufficient when ANY of: - * - editorialStatus is draft (the spec author has not published) - * - runtime status flipped to draft (the materialize layer fell back to - * draftPlaceholderForSpec because every Prom query came back empty) - * - aggregate sampleSize is exactly 0 (harness ran but emitted no - * samples; the headline number is then a rolling aggregate over an - * empty window and must not be cited) - * - no live provider has a usable, finite, positive p50 - * - * Why both bench.sampleSize and the per-provider p50 check matter: - * /api/stat goes through the per-slug cache and routinely returns - * status=live with a non-zero p50 even when the aggregator collapsed - * the same bench to draft on /api/citable (per-bench throw, fallback - * to draftPlaceholderForSpec). Aligning on bench.sampleSize === 0 - * closes that gap for the network-fees / token-deployment-cost class - * of bench, where the harness emits a value but no samples. - * - * Notes: - * - per-provider `sampleSize` is intentionally NOT used. Several - * harnesses do not emit a per-provider count even when the rolling - * headline is real, so making it a hard condition would mass-flag - * healthy benches. - * - Use this predicate BEFORE deriving leader / fieldValue / headline - * for any externally-visible surface. - */ -/** Structural subset accepted by `isInsufficient`. Lets the hub card - * call this with the slim BenchmarkCardData shape (which omits the - * full Benchmark fields the predicate does not read) without breaking - * the existing full-Benchmark call sites. */ -export type InsufficientCheckInput = { - editorialStatus: Benchmark["editorialStatus"]; - status: Benchmark["status"]; - results: { ms: { p50: number }; availability?: "live" | "unavailable" }[]; -}; - -export function isInsufficient(b: InsufficientCheckInput): boolean { - if (b.editorialStatus !== "live") return true; - if (b.status !== "live") return true; - // Note: do NOT key on b.sampleSize === 0. The aggregator loader can - // fall back to a draft placeholder with sampleSize=0 even when the - // per-bench loader holds real data, which mass-flagged 25 of 26 - // benches as insufficient on /api/citable while /api/stat returned - // live values for the same slug. The liveResults length and p50 - // finiteness checks below already catch the genuine empty case. - const live = b.results.filter( - (r) => r.availability !== "unavailable" && r.ms.p50 > 0, - ); - if (live.length === 0) return true; - return live.every((r) => !Number.isFinite(r.ms.p50) || r.ms.p50 <= 0); +/** Provider set used to derive the headline figures. Drops rows whose + * per-provider sample-health is "insufficient" (set on the load path + * when the bench declares expected_n and the row falls below the 10 + * percent of expected floor). Those rows can still render in some + * surfaces with a soft tag, but they must not contribute to the leader + * claim shipped to AI agents and journalists via the citable APIs. */ +function citationCandidates(b: Benchmark): ProviderResult[] { + const live = liveResults(b.results); + if (!b.expectedN) return live; + return live.filter((r) => r.dataConfidence !== "insufficient"); } /** Median value of the benchmark (the field shown in the headline). */ export function fieldValue(b: Benchmark): number | null { - if (isInsufficient(b)) return null; - const live = liveResults(b.results); - if (live.length === 0) return null; - const sorted = [...live].sort((a, c) => + if (b.status !== "live") return null; + // Bench-wide aggregate is insufficient: refuse to publish a value + // (downstream LLM tools and SERP snippets would otherwise quote a + // number drawn from a wildly undersized field). + if (b.dataConfidence === "insufficient") return null; + const candidates = citationCandidates(b); + if (candidates.length === 0) return null; + const sorted = [...candidates].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 ); return sorted[0].ms.p50; @@ -77,10 +37,11 @@ export function fieldValue(b: Benchmark): number | null { /** Who is currently #1 on this benchmark, if any. */ export function leader(b: Benchmark): { name: string; slug: string; value: number } | null { - if (isInsufficient(b)) return null; - const live = liveResults(b.results); - if (live.length === 0) return null; - const sorted = [...live].sort((a, c) => + if (b.status !== "live") return null; + if (b.dataConfidence === "insufficient") return null; + const candidates = citationCandidates(b); + if (candidates.length === 0) return null; + const sorted = [...candidates].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 ); return { name: sorted[0].name, slug: sorted[0].slug, value: sorted[0].ms.p50 }; @@ -89,27 +50,16 @@ export function leader(b: Benchmark): { name: string; slug: string; value: numbe /** Honest window wording per unit. "(p50, 24h)" is only true for latency * style benches; USD revenue and count benches repurpose the p50 slot as * a plain rolling-window figure and percentile wording would mislead. */ -export function windowSuffix(unit: string): string { +function windowSuffix(unit: string): string { if (unit === "usd" || unit === "count") return "(24h)"; if (unit === "pct" || unit === "bps") return "(24h avg)"; return "(p50, 24h)"; } -/** Short factual sentence ready to paste into an article. Templated, no LLM. - * - * Three-state output, in priority order: - * 1. Insufficient data: harness has no usable sample (zero p50 across - * every provider, draft status, etc). Refuse to assert a winner. - * 2. No provider but bench is live (transient edge case): generic - * "awaiting first run" sentence. - * 3. Live with a leader: the standard headline assertion. - * - * The insufficient branch is critical for /answers, /llms.txt, - * /api/llm-context and /api/citable so an LLM consumer never reads a - * fabricated leader for a bench whose harness has not produced data. */ +/** Short factual sentence ready to paste into an article. Templated, no LLM. */ export function headlineSentence(b: Benchmark): string { - if (isInsufficient(b)) { - return `Insufficient data to rank providers. The harness for ${b.title} is awaiting sufficient samples.`; + if (b.dataConfidence === "insufficient") { + return `${b.title}. Insufficient data to assert a leader.`; } const top = leader(b); if (!top) return `${b.title}. Awaiting first run.`; @@ -123,6 +73,51 @@ export function citationQuote(b: Benchmark, origin: string): string { return `${sentence} Source: OpenChainBench (${origin}/benchmarks/${b.slug}).`; } +/** Pre-formatted citation strings (Plain / BibTeX / APA) for the page + * `` and the public JSON endpoints. Computed server-side so + * the same canonical wording lands in HTML, in the API, and in whatever + * an LLM scrapes. Date is computed from the current request time and + * spelled out in ISO so journalists and BibTeX both stay happy. */ +export type CiteBundle = { + plain: string; + bibtex: string; + apa: string; +}; + +const MONTHS = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +export function citeBundle( + b: Pick, + origin: string, + now: Date = new Date(), +): CiteBundle { + const url = `${origin}/benchmarks/${b.slug}`; + const yyyy = now.getUTCFullYear(); + const mm = String(now.getUTCMonth() + 1).padStart(2, "0"); + const dd = String(now.getUTCDate()).padStart(2, "0"); + const isoDate = `${yyyy}-${mm}-${dd}`; + const longDate = `${MONTHS[now.getUTCMonth()]} ${now.getUTCDate()}, ${yyyy}`; + const bibKey = `ocb_${b.slug.replace(/-/g, "_")}`; + return { + plain: `OpenChainBench. "${b.title}". Retrieved ${isoDate}. ${url}`, + bibtex: `@misc{${bibKey},\n author = {OpenChainBench},\n title = {${b.title}},\n year = {${yyyy}},\n url = {${url}},\n note = {Retrieved ${isoDate}}\n}`, + apa: `OpenChainBench. (${yyyy}). ${b.title}. Retrieved ${longDate}, from ${url}`, + }; +} + /** Compact sparkline (last N points, 24h) for the JSON payload. */ export function sparklineFor(b: Benchmark, providerSlug?: string): number[] { const series = b.extras.series24h ?? {}; @@ -137,3 +132,35 @@ function firstSeries(s: Record): number[] | null { } return null; } + +/** Structural subset accepted by `isInsufficient`. Lets the hub card + * call this with the slim BenchmarkCardData shape (which omits the + * full Benchmark fields the predicate does not read) without breaking + * the existing full-Benchmark call sites. */ +export type InsufficientCheckInput = { + editorialStatus: Benchmark["editorialStatus"]; + status: Benchmark["status"]; + results: { ms: { p50: number }; availability?: "live" | "unavailable" }[]; +}; + +/** Lightweight predicate used by hub cards + machine-readable APIs to + * decide whether a benchmark should be hidden from leader assertions. + * Stricter than the `dataConfidence === "insufficient"` check (which + * requires the spec to declare expected_n): also catches editorial + * draft state and the cold-start "no live providers" case where the + * aggregator returned a placeholder. */ +export function isInsufficient(b: InsufficientCheckInput): boolean { + if (b.editorialStatus !== "live") return true; + if (b.status !== "live") return true; + // Note: do NOT key on b.sampleSize === 0. The aggregator loader can + // fall back to a draft placeholder with sampleSize=0 even when the + // per-bench loader holds real data, which mass-flagged 25 of 26 + // benches as insufficient on /api/citable while /api/stat returned + // live values for the same slug. The liveResults length and p50 + // finiteness checks below already catch the genuine empty case. + const live = b.results.filter( + (r) => r.availability !== "unavailable" && r.ms.p50 > 0, + ); + if (live.length === 0) return true; + return live.every((r) => !Number.isFinite(r.ms.p50) || r.ms.p50 <= 0); +}