Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions cmd/routing-report/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ func main() {
corpusPath = flag.String("corpus", "internal/router/cluster/testdata/register_probes.jsonl", "labeled probe corpus")
embPath = flag.String("emb", "internal/router/cluster/testdata/register_probes.emb", "precomputed probe embeddings")
topP = flag.Int("top-p", 2, "top-P clusters blended per decision (prod runs 2)")
qualityBias = flag.Float64("quality-bias", -1, "QualityBias dial position in [0,1] to route at; <0 uses the bundle's default knobs (no dial)")
outPath = flag.String("out", "", "write markdown here instead of stdout")
)
flag.Parse()
Expand Down Expand Up @@ -174,13 +175,13 @@ func main() {
}
embedder := buildEmbedder(hdr, probes, vecs)

tgt, err := routeCorpus(*artifactsDir, *target, probes, embedder, *topP)
tgt, err := routeCorpus(*artifactsDir, *target, probes, embedder, *topP, *qualityBias)
if err != nil {
fatal("route target %s: %v", *target, err)
}
var base *routeResult
if *baseline != *target {
base, err = routeCorpus(*artifactsDir, *baseline, probes, embedder, *topP)
base, err = routeCorpus(*artifactsDir, *baseline, probes, embedder, *topP, *qualityBias)
if err != nil {
fatal("route baseline %s: %v", *baseline, err)
}
Expand All @@ -206,7 +207,7 @@ type routeResult struct {
nearest []int // per probe single closest cluster (for auto-labels)
}

func routeCorpus(artifactsDir, version string, probes []probe, embedder *staticEmbedder, topP int) (*routeResult, error) {
func routeCorpus(artifactsDir, version string, probes []probe, embedder *staticEmbedder, topP int, qualityBias float64) (*routeResult, error) {
dir := filepath.Join(artifactsDir, version)
bundle, err := cluster.LoadBundleFromDir(dir, version)
if err != nil {
Expand All @@ -230,8 +231,13 @@ func routeCorpus(artifactsDir, version string, probes []probe, embedder *staticE
nearest: make([]int, len(probes)),
}
ctx := context.Background()
var knobs *router.Overrides
if qualityBias >= 0 {
qb := qualityBias
knobs = &router.Overrides{QualityBias: &qb}
}
for i, p := range probes {
dec, err := scorer.Route(ctx, router.Request{PromptText: p.Text})
dec, err := scorer.Route(ctx, router.Request{PromptText: p.Text, RoutingKnobs: knobs})
if err != nil {
return nil, fmt.Errorf("route probe %s: %w", p.ID, err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/routing-report/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func TestReportRunsOnLatest(t *testing.T) {
}
emb := buildEmbedder(hdr, probes, vecs)

res, err := routeCorpus(testArtifacts, latest, probes, emb, 2)
res, err := routeCorpus(testArtifacts, latest, probes, emb, 2, -1)
if err != nil {
t.Fatalf("route corpus on %s: %v", latest, err)
}
Expand Down
10 changes: 9 additions & 1 deletion internal/router/cluster/artifacts.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,15 @@ type ArtifactEmbedder struct {
}

type DefaultRoutingKnobs struct {
Alpha []float64 `yaml:"alpha"`
Alpha []float64 `yaml:"alpha"`
// AlphaFloor is an optional per-cluster minimum the QualityBias dial may not
// pull a cluster's alpha below. It lets a bundle declare, per cluster, the
// LOWEST quality weight it will tolerate at maximum price-sensitivity — so a
// price-leaning dial still routes each cluster to the best model available
// for that budget instead of collapsing the whole vector to the cheapest
// model. Length must equal K when present; nil/empty disables flooring (the
// dial maps to a uniform alpha, the legacy behavior). See applyDialAlpha.
AlphaFloor []float64 `yaml:"alpha_floor,omitempty"`
SpeedWeight float64 `yaml:"speed_weight"`
OutputCostRatio float64 `yaml:"output_cost_ratio"`
ExpectedOutputTokens int `yaml:"expected_output_tokens"`
Expand Down
28 changes: 28 additions & 0 deletions internal/router/cluster/artifacts/v0.70/metadata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,34 @@ training:
- 0.96 # c13 hard-code
- 0.96 # c14
- 0.80 # c15 trivial-NL
# Per-cluster floor the QualityBias dial may not pull alpha below — the
# LOWEST quality weight tolerated at maximum price-sensitivity. The dial
# collapses the per-cluster alpha vector to one uniform value, so without a
# floor a price-leaning dial cheaps out EVERY cluster, stranding agentic
# main-loop turns on models that can't drive the harness (tool/skill
# protocol — observed: minimax-m3 grepped for a skill instead of running
# it). Coding/agentic clusters floor at 0.88 so they stay on premium models
# at any dial; conversational/NL clusters floor at 0.55 — still much
# cheaper than their 0.80 default, but not collapsed to the cheapest model,
# so price-sensitive turns get the best quality their budget allows.
# Verify shifts with `cmd/routing-report --quality-bias`.
alpha_floor:
- 0.88 # c0 agentic/coding catch-all
- 0.88 # c1
- 0.55 # c2 conversational
- 0.88 # c3
- 0.88 # c4
- 0.88 # c5 coding/agentic
- 0.88 # c6
- 0.88 # c7
- 0.88 # c8
- 0.55 # c9 conversational/NL
- 0.55 # c10 conversational/NL (primary)
- 0.55 # c11 trivial-NL
- 0.88 # c12
- 0.88 # c13 hard-code
- 0.88 # c14
- 0.55 # c15 trivial-NL
speed_weight: 0.0
output_cost_ratio: 0.0
expected_output_tokens: 2000
Expand Down
6 changes: 2 additions & 4 deletions internal/router/cluster/distribution.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func (s *Scorer) defaultActiveKnobs() DefaultRoutingKnobs {
if s.metadata != nil && s.metadata.Training.DefaultRoutingKnobs != nil {
knobs := *s.metadata.Training.DefaultRoutingKnobs
knobs.Alpha = append([]float64(nil), knobs.Alpha...)
knobs.AlphaFloor = append([]float64(nil), knobs.AlphaFloor...)
return knobs
}
knobs := DefaultRoutingKnobs{
Expand Down Expand Up @@ -86,10 +87,7 @@ func (s *Scorer) RoutingDistribution(gridN int) ([]DistributionPoint, error) {
t := float64(g) / float64(gridN-1)

knobs := s.defaultActiveKnobs()
a := s.dialToAlpha(t)
for i := range knobs.Alpha {
knobs.Alpha[i] = a
}
s.applyDialAlpha(t, knobs.Alpha, knobs.AlphaFloor)

counts := make(map[string]int, len(s.models))
for c := 0; c < k; c++ {
Expand Down
95 changes: 95 additions & 0 deletions internal/router/cluster/distribution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,101 @@ func TestRoutingDistribution_MidDialIsPricierThanLowDial(t *testing.T) {
"the 50%% dial must route a meaningfully pricier (higher-quality) mix than the 20%% dial")
}

// loadV0_70 loads the committed v0.70 bundle (the first shaped bundle whose
// default alpha protects agentic/code clusters at 0.96 and lets conversational
// clusters cheapen at 0.8). Like loadV0_67 it uses a fake embedder; the dial
// tests below never embed a prompt — they score cluster centroids directly.
func loadV0_70(t *testing.T) *Scorer {
t.Helper()
bundle, err := LoadBundle("v0.70")
require.NoError(t, err)
require.True(t, bundle.IsV2)
s, err := NewScorer(bundle, DefaultConfig(), &fakeEmbedder{dim: bundle.Centroids.Dim}, allProviders())
require.NoError(t, err)
return s
}

// clusterWinnerAt scores the centroid of cluster c at the given dial position
// through the exact production path (defaultActiveKnobs -> applyDialAlpha ->
// blendScoresV2 -> argmax over the top-P union) and returns the winning model.
// withFloor=false reproduces the OLD uniform-dial behavior (the bug) so a test
// can assert the floor changed the realized routing.
func clusterWinnerAt(s *Scorer, c int, t float64, withFloor bool) string {
knobs := s.defaultActiveKnobs()
floor := knobs.AlphaFloor
if !withFloor {
floor = nil // reproduce the old uniform dial (no alpha_floor)
}
s.applyDialAlpha(t, knobs.Alpha, floor)
top := topPNearest(s.centroids.Row(c), s.centroids, s.cfg.TopP)
scores := s.blendScoresV2(top, knobs, s.models, nil)
winner, _ := argmax(scores, s.models)
return winner
}

func TestApplyDialAlpha_HoldsEachClusterAtItsDeclaredFloor(t *testing.T) {
s := loadV0_70(t)
knobs := s.defaultActiveKnobs()
floor := knobs.AlphaFloor
require.Len(t, floor, s.centroids.K, "v0.70 must ship a full per-cluster alpha_floor")

// At the price extreme (t=0, dialToAlpha=0) every cluster is held at exactly
// its declared floor — no cluster collapses to alpha 0 (the cheapest model).
s.applyDialAlpha(0.0, knobs.Alpha, floor)
for i := range knobs.Alpha {
assert.InDelta(t, floor[i], knobs.Alpha[i], 1e-9,
"cluster %d must be held at its declared floor at the price extreme", i)
}

// Above a cluster's floor the dial governs (max(dialAlpha, floor)); at the
// quality extreme (t=1, dialToAlpha=1) every cluster reaches 1.0.
knobs2 := s.defaultActiveKnobs()
s.applyDialAlpha(1.0, knobs2.Alpha, knobs2.AlphaFloor)
for i := range knobs2.Alpha {
assert.InDelta(t, 1.0, knobs2.Alpha[i], 1e-9, "cluster %d must reach 1.0 at the quality extreme", i)
}
}

func TestApplyDialAlpha_NilFloorIsUniformDial(t *testing.T) {
// A bundle that ships no alpha_floor keeps the legacy uniform-dial behavior:
// every cluster gets dialToAlpha(t) verbatim.
s := loadV0_70(t)
knobs := s.defaultActiveKnobs()
s.applyDialAlpha(0.3, knobs.Alpha, nil)
want := s.dialToAlpha(0.3)
for i := range knobs.Alpha {
assert.Equal(t, want, knobs.Alpha[i], "cluster %d must equal the uniform dial alpha with no floor", i)
}
}

func TestApplyDialAlpha_AgenticStaysOffCheapModelAtLowDial(t *testing.T) {
// The reported bug, end to end: under a price-leaning dial the agentic
// cluster routed to minimax-m3 (a model that can't drive the Claude Code
// skill/tool protocol). Cluster 0 is the agentic catch-all. Without the
// floor (old uniform dial) the centroid routes to a cheap OSS model; with
// the floor it stays on a frontier model.
s := loadV0_70(t)
const lowDial = 0.2

without := clusterWinnerAt(s, 0, lowDial, false)
with := clusterWinnerAt(s, 0, lowDial, true)

assert.NotEqual(t, with, without,
"the floor must change the agentic winner at a low dial (old=%s)", without)
// The fixed winner must be a genuine agentic-capable frontier model, not the
// cheap pack the uniform dial fell through to.
frontier := map[string]struct{}{
"claude-opus-4-8": {},
"claude-opus-4-7": {},
"claude-sonnet-4-6": {},
"gemini-3.1-pro-preview": {},
"gpt-5.5": {},
"deepseek/deepseek-v4-pro": {},
}
_, isFrontier := frontier[with]
assert.True(t, isFrontier, "with the floor the agentic cluster must route to a frontier model, got %s", with)
}

// mixSignatureOf renders a DistributionPoint's model shares as a stable key for
// comparing whether two dial positions route the same mix.
func mixSignatureOf(models []ModelShare) string {
Expand Down
55 changes: 46 additions & 9 deletions internal/router/cluster/scorer.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,20 @@ func NewScorer(bundle *Bundle, cfg Config, embed Embedder, availableProviders ma
if len(dk.Alpha) != bundle.Centroids.K {
return nil, fmt.Errorf("cluster %s: default_routing_knobs.alpha length %d must equal K=%d", bundle.Version, len(dk.Alpha), bundle.Centroids.K)
}
// alpha_floor is optional, but when present must be a full per-cluster
// vector with each entry a valid quality weight: applyDialAlpha writes
// floor[i] straight into the alpha slot, so a bad length or out-of-range
// value would silently produce an invalid blend.
if dk.AlphaFloor != nil {
if len(dk.AlphaFloor) != bundle.Centroids.K {
return nil, fmt.Errorf("cluster %s: default_routing_knobs.alpha_floor length %d must equal K=%d", bundle.Version, len(dk.AlphaFloor), bundle.Centroids.K)
}
for i, f := range dk.AlphaFloor {
if f < 0 || f > 1 {
return nil, fmt.Errorf("cluster %s: default_routing_knobs.alpha_floor[%d] (%f) must be in [0, 1]", bundle.Version, i, f)
}
}
}
}
} else {
for k := 0; k < bundle.Centroids.K; k++ {
Expand Down Expand Up @@ -327,6 +341,30 @@ func (s *Scorer) dialToAlpha(t float64) float64 {
return bp[i] + frac*(bp[i+1]-bp[i])
}

// applyDialAlpha resolves the QualityBias dial position t into the effective
// per-cluster alpha, writing in place into alpha. It maps t through the
// bundle's mix-change calibration (dialToAlpha) to a uniform alpha, then floors
// each cluster at the bundle-declared floor[i]: alpha[i] = max(dialAlpha,
// floor[i]). The floor is the LOWEST quality weight the bundle tolerates per
// cluster at maximum price-sensitivity, so a price-leaning dial still routes
// each cluster to the best model for that budget instead of collapsing the
// whole vector to the cheapest model (which stranded agentic main-loop turns
// on models that can't drive the harness). floor==nil disables flooring (the
// plain uniform dial, legacy behavior for bundles that ship no alpha_floor).
// Single source of truth for the dial->alpha resolution shared by Route and
// RoutingDistribution. Caller guarantees len(floor) == len(alpha) when non-nil
// (validated against K at load time).
func (s *Scorer) applyDialAlpha(t float64, alpha, floor []float64) {
a := s.dialToAlpha(t)
for i := range alpha {
if floor != nil && floor[i] > a {
alpha[i] = floor[i]
} else {
alpha[i] = a
}
}
}

// resolveProviderFor walks the catalog's ordered ProviderBinding list for
// modelID and returns the first binding whose Provider name is in
// available. Falls back to the registry's recorded provider for models
Expand Down Expand Up @@ -553,19 +591,18 @@ func (s *Scorer) Route(ctx context.Context, req router.Request) (router.Decision
switch {
case req.RoutingKnobs.QualityBias != nil:
// Dial path: map the single dial position through this bundle's
// mix-change calibration to a uniform alpha, so the slider spends
// equal travel on each distinct routed mix (no dead zone) while
// the endpoints still pin to all-cheapest / best-per-cluster.
// Takes precedence over a co-set Alpha (the higher-level intent
// wins).
// mix-change calibration (so the slider spends equal travel on
// each distinct routed mix, no dead zone), then floor each
// cluster at a fraction of its shipped default so the dial
// preserves the bundle's per-cluster alpha shape instead of
// flattening it — a price-leaning dial cheapens conversational
// turns but never strands agentic on a cheap model. Takes
// precedence over a co-set Alpha (the higher-level intent wins).
t := *req.RoutingKnobs.QualityBias
if math.IsNaN(t) || math.IsInf(t, 0) || t < 0 || t > 1 {
return router.Decision{}, fmt.Errorf("%w: quality_bias (%f) must be a finite value in [0, 1]", ErrInvalidRoutingKnobs, t)
}
a := s.dialToAlpha(t)
for i := range activeKnobs.Alpha {
activeKnobs.Alpha[i] = a
}
s.applyDialAlpha(t, activeKnobs.Alpha, activeKnobs.AlphaFloor)
case req.RoutingKnobs.Alpha != nil:
// Sledgehammer behavior: uniformly replace every alpha with the
// scalar (eval/debug lever, ignores per-cluster dispersion).
Expand Down
Loading