Skip to content
37 changes: 37 additions & 0 deletions app/src/app/model/[id]/page.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import type { Metadata } from "next";
import Link from "next/link";
import { notFound } from "next/navigation";
import {
formatDelta,
servingSensitivityFor,
} from "../../../lib/servingSensitivity";
import { wouldRank } from "../../../lib/wouldRank";

import rawData from "../../../data-summary.json";
import versionRegistryJson from "../../../data.versions.json";
Expand Down Expand Up @@ -183,6 +188,7 @@ export default async function ModelPage({
const rows = programRows(bench, id);
const cases = hardestCases(bench, id);
const coverage = coverageAccuracy(bench, id);
const sensitivity = servingSensitivityFor(id);
const currencySymbol = summary.country === "uk" ? "£" : "$";
return (
<section
Expand Down Expand Up @@ -224,6 +230,37 @@ export default async function ModelPage({
}
/>
</div>
{summary.country === "us" && sensitivity ? (
<p className="mt-4 max-w-3xl text-sm leading-relaxed text-text-secondary">
<span className="text-[10px] uppercase tracking-[0.14em] text-text-muted font-medium">
Serving sensitivity
</span>{" "}
This row {sensitivity.boardTreatment}. The same model,{" "}
{sensitivity.autoTreatment}, scores{" "}
<span className="font-[family-name:var(--font-mono)] text-text">
{sensitivity.autoExact.toFixed(1)}%
</span>{" "}
exact (
{formatDelta(
sensitivity.autoExact,
summary.stat.exact ?? summary.stat.score,
)}
) and would rank #
{wouldRank(
sensitivity.autoExact,
bench.modelStats.filter((row) => row.condition === "no_tools"),
)}{" "}
on the live board; the board keeps the request shape its
model card records.{" "}
<Link
href={sensitivity.noteHref}
className="text-primary-strong underline-offset-2 hover:underline"
>
Read the note
</Link>
.
</p>
) : null}

<div className="mt-8 grid gap-8 lg:grid-cols-2">
<div>
Expand Down
92 changes: 52 additions & 40 deletions app/src/components/ModelLeaderboard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,12 @@ import { canonicalScoreByModel } from "../lib/canonicalScore";
import { wouldRank } from "../lib/wouldRank";
import { isCurrentBoard, modelPageHref } from "../lib/boardScope";

// Exact-match scores of the tool_choice: auto sensitivity runs
// (sensitivity/claude-thinking-2026-08.md), scored on the same 1,973 outputs
// the board scores (eleven outputs are excluded for every model). Their
// "would rank" positions are derived from the live board rows at render time,
// never typed by hand.
const SENSITIVITY_EXACT = {
"claude-fable-5": 87.5,
"claude-opus-5": 86.2,
"claude-sonnet-5": 80.8,
"claude-fable-5.1": 88.2,
} as const;
import ServingSensitivityChip from "./ServingSensitivityChip";
import {
NEXT_BOARD_HREF,
SENSITIVITY_DOC_HREF,
servingSensitivityFor,
} from "../lib/servingSensitivity";
import {
rankWithFallbackScore,
rankWithRecomputedScores,
Expand Down Expand Up @@ -238,6 +233,16 @@ export default function ModelLeaderboard({
canRecomputeScores && baseNoTools.length > 0 && noTools.length === 0;

const activeView = SENSITIVITY_VIEWS.find((v) => v.id === sensitivityView)!;
// The serving-sensitivity markers describe the live US board's rows. Their
// comparison is against the row's canonical exact-match score on the
// unfiltered board (the universe the re-runs were scored on), whatever
// metric, weighting or program filter the reader has selected.
const showSensitivity =
selectedView === "us" && isCurrentBoard(versionId, liveVersionId);
const unfilteredExact = (model: string): number => {
const row = baseNoTools.find((entry) => entry.model === model);
return row?.exact ?? row?.score ?? 0;
};

// "Exact" means "within one currency unit," and that unit is country-
// specific. Surface the right word in tooltips, captions, and the Options
Expand Down Expand Up @@ -291,7 +296,7 @@ export default function ModelLeaderboard({
>
Model rankings
</h2>
{selectedView === "us" && isCurrentBoard(versionId, liveVersionId) && (
{showSensitivity && (
<div
className="card mt-5 px-5 py-4 animate-fade-up"
style={{ animationDelay: "120ms" }}
Expand All @@ -300,35 +305,20 @@ export default function ModelLeaderboard({
Serving sensitivity · August 2026
</div>
<p className="mt-2 text-sm leading-relaxed text-text-secondary">
Claude models skip extended thinking when the answer tool call is
forced, as it is in the request this board sends every model whose
selected answer contract is the forced tool (rows on the JSON
contract, whether their provider rejects a forced tool or their
model card selects JSON, answer as a JSON object); other
reasoning-by-default providers reason regardless. Re-run with <code>tool_choice: auto</code>,
Claude Fable 5 scores {SENSITIVITY_EXACT["claude-fable-5"]} (would
rank #{wouldRank(SENSITIVITY_EXACT["claude-fable-5"], baseNoTools)}
), Claude Opus 5 {SENSITIVITY_EXACT["claude-opus-5"]} (#
{wouldRank(SENSITIVITY_EXACT["claude-opus-5"], baseNoTools)}), and
Claude Sonnet 5 {SENSITIVITY_EXACT["claude-sonnet-5"]} (#
{wouldRank(SENSITIVITY_EXACT["claude-sonnet-5"], baseNoTools)}).
Claude Fable 5.1 rejects forced tool calls outright, so its board
row answers as a JSON object and reasons at the provider default;
with the tool declared under <code>auto</code> it scores{" "}
{SENSITIVITY_EXACT["claude-fable-5.1"]} (#
{wouldRank(SENSITIVITY_EXACT["claude-fable-5.1"], baseNoTools)}).
The board below is unchanged — those runs sit beside it as a{" "}
<a
href="https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md"
className="text-primary hover:underline"
>
labeled sensitivity
Three Claude rows ran without extended thinking: this board forces
the answer tool call for every row whose model card selects the
tool contract, which switches Claude&apos;s thinking off. Claude
Fable 5.1 rejects forced calls, so its row selects JSON, answers as
a JSON object and reasons either way; its marker compares
transports. The{" "}
<code>tool_choice: auto</code> re-runs are marked on the four rows
(open a marker for the score and why); the board itself is
unchanged. The{" "}
<a href={SENSITIVITY_DOC_HREF} className="text-primary hover:underline">
sensitivity note
</a>{" "}
— and the{" "}
<a
href="https://github.com/PolicyEngine/policybench/issues/139"
className="text-primary hover:underline"
>
has all four runs, and the{" "}
<a href={NEXT_BOARD_HREF} className="text-primary hover:underline">
next board version
</a>{" "}
moves every model to auto.
Expand Down Expand Up @@ -432,6 +422,17 @@ export default function ModelLeaderboard({
>
{MODEL_LABELS[m.model] || m.model}
</Link>
{showSensitivity && servingSensitivityFor(m.model) ? (
<ServingSensitivityChip
modelLabel={MODEL_LABELS[m.model] || m.model}
boardExact={unfilteredExact(m.model)}
sensitivity={servingSensitivityFor(m.model)!}
wouldRank={wouldRank(
servingSensitivityFor(m.model)!.autoExact,
baseNoTools,
)}
/>
) : null}
</div>
<div className="mt-1.5 pl-[26px] font-[family-name:var(--font-mono)] text-[11px] text-text-muted">
{fmtCost(m.costPerHousehold, currencySymbol)} per
Expand Down Expand Up @@ -470,6 +471,17 @@ export default function ModelLeaderboard({
>
{MODEL_LABELS[m.model] || m.model}
</Link>
{showSensitivity && servingSensitivityFor(m.model) ? (
<ServingSensitivityChip
modelLabel={MODEL_LABELS[m.model] || m.model}
boardExact={unfilteredExact(m.model)}
sensitivity={servingSensitivityFor(m.model)!}
wouldRank={wouldRank(
servingSensitivityFor(m.model)!.autoExact,
baseNoTools,
)}
/>
) : null}
</div>

<div className="col-span-2 text-right font-[family-name:var(--font-mono)] text-sm text-text-muted">
Expand Down
Loading