Let the gateway pin a model to one upstream deployment

.gitignore

@@ -15,3 +15,6 @@ wheels/
 secrets.env
 *.secrets.env
 !*.example
+
+# Codex CLI local state, including the worktrees it checks out for its own runs.
+.codex/

harness-engineering-bench/terminal-bench/baseline/build.azure.yaml

@@ -0,0 +1,191 @@
+name: vero/optimize-terminal-bench-baseline
+description: >-
+  Improve a terminal agent on Terminal-Bench 2.1 while preserving the Harbor
+  agent interface. Each task gives the agent a container and a goal; the task's
+  own tests decide pass or fail, so the reward is a pass rate.
+
+# Target model and baseline_reward are both measured and pinned. See
+# ../README.md for the model probe and runs/BASELINES.md for the pinning.
+
+agent_repo: target
+task_source: terminal-bench/terminal-bench-2-1@sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a
+task_manifest: ../partitions/manifest.json
+agent_import_path: terminal_bench_agent.agent:TerminalBenchAgent
+harbor_requirement: harbor[modal]==0.20.0
+
+partition_files:
+  development: ../partitions/development.json
+  validation: ../partitions/validation.json
+  test: ../partitions/test.json
+
+# total_cases = 4 full passes per partition (dev 17*4, validation 36*4).
+agent_access:
+  - partition: development
+    disclosure: full
+    expose_case_resources: true
+    total_runs: 100
+    total_cases: 68
+  - partition: validation
+    disclosure: aggregate
+    expose_case_resources: false
+    min_aggregate_cases: 5
+    total_runs: 100
+    total_cases: 144
+
+selection_partition: validation
+targets:
+  - partition: test
+    reward_key: reward
+    # Seed-killed trials score 0 (4 UnicodeDecodeError from the seed's
+    # undecodable-output bug, 4 AgentTimeoutError the verifier could not score;
+    # the other 14 timeouts scored on final container state and were always in
+    # the pin); platform-killed trials are dropped. See runs/recompute.py.
+    baseline_reward: 0.2407  # K=3: 0.250 / 0.222 / 0.250 (sd 0.0131, n=108); was 0.2600 excluding seed-killed trials. See runs/BASELINES.md
+    # 36 held-out cases was expected to be noisy -- it is the smallest test set in
+    # the suite -- but the measured sd is the second tightest of the six, behind
+    # only tau3. Deterministic per-task tests instead of an LLM judge is the
+    # likely reason. So n_attempts stays at 3; it does not need raising to 5.
+    failure_value: 0.0
+    max_attempts: 1
+    # Score the selected candidate 3x per case and average, so the reported
+    # reward carries the same standard error as a 3-round pinned baseline.
+    # Per-target override - search/validation keep the global n_attempts (1).
+    n_attempts: 3
+    aggregate_attempts: mean
+
+evaluation_set_name: terminal-bench
+objective:
+  selector:
+    metric: score
+  direction: maximize
+reward_mode: submit  # agent picks; falls back to auto_best, then current version
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
+score_baseline: false
+rescore_top_k: 3
+rescore_attempts: 1
+
+# Chosen from a measured probe: seed harness, all 17 development tasks, one round
+# each, direct to the proxy.
+#
+#   model                          solved   reward   $/Mtok*  notes
+#   xai/grok-build-0.1              6/17    0.3529    0.380
+#   azure_ai/gpt-5-nano             2/17    0.1176    0.044
+#   xai/grok-4-1-fast-reasoning     2/17    0.1176    0.095   dominated by nano
+#   gemini/gemini-2.5-flash-lite    0/12    0.0000    0.049   5 trials crashed
+#   * 90% cache-read + 10% output, our measured token mix
+#
+# grok-build wins despite 8.6x the cheapest rate, because what matters for a
+# benchmark is a baseline with headroom, not reward per dollar. 0.35 sits where
+# officeqa's baseline does (0.3412), and officeqa's best cell moved +0.41 from
+# there; 0.12 is swe-atlas-qna territory (0.0676), the least informative cell in
+# the suite. Estimated ~$130/cell, ~$1,170 for a nine-cell pass.
+#
+# Not on Fireworks, deliberately: the shared per-minute generated-token quota is
+# what limits how many cells can run at once, so a target off that provider does
+# not contend with the officeqa or browsecomp-plus grids.
+model: xai/grok-build-0.1
+environment_name: ${inner_env:-modal}
+extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
+harbor_python_version: "3.12"
+n_attempts: 1
+max_retries: 4
+retry_max_wait_seconds: 120
+infrastructure_max_attempts: 3
+infrastructure_retry_delay_seconds: 5
+aggregate_attempts: best
+feedback_transcripts: true
+feedback_max_bytes: 16000
+expose_attempt_detail: false
+
+# Unlike every other benchmark here, Terminal-Bench declares a DIFFERENT agent
+# timeout per task: 600s to 12,000s, 48 of 89 at 900s, 13 at 3600s. That is fine
+# and needs no special handling, because vero passes Harbor a single ratio
+# (`case_timeout_seconds / task_agent_timeout_seconds`) which Harbor applies to
+# each task's own declared budget. Keeping the pair equal therefore gives every
+# task exactly the clock its author intended. The absolute value is irrelevant as
+# long as the two match -- do not "fix" this by setting them to different numbers.
+case_timeout_seconds: 900
+task_agent_timeout_seconds: 900
+
+# Worst-case finalize wall: 36 test x 3 attempts = 108 case-runs, ceil(108/24) =
+# 5 waves, slowest test task declares 7200s -> 36,000s. Set above that so a
+# pathological run cannot lose a scored result to the job clock.
+timeout_seconds: 43200
+# Worst-case finalize (36,000) plus a rescore_top_k=3 validation pass, whose
+# widest single eval is ceil(36/24) = 2 waves x the slowest validation task
+# (12,000s) = 24,000s. A verifier timeout loses the score outright.
+verifier_timeout_seconds: 64800
+max_concurrency: 24
+error_rate_threshold: 0.1
+
+secrets:
+  - MODAL_TOKEN_ID
+  - MODAL_TOKEN_SECRET
+  - WANDB_API_KEY
+  - WANDB_BASE_URL
+
+harness_user: harness
+agent_env:
+  # Above this benchmark's widest single search eval: a full validation pass is
+  # ceil(36/24) = 2 waves x 12,000s = 24,000s worst case. The optimizer must be
+  # able to block on one evaluation in a single foreground call -- a headless run
+  # that backgrounds a long call is never re-woken and the search dies there.
+  BASH_MAX_TIMEOUT_MS: "28800000"
+  BASH_DEFAULT_TIMEOUT_MS: "28800000"
+  ENABLE_BACKGROUND_TASKS: "0"
+  FORCE_AUTO_BACKGROUND_TASKS: "0"
+  UV_TOOL_BIN_DIR: "/home/agent/.local/bin"
+
+wandb:
+  project: harness-engineering-bench  # one project for the whole suite
+  group: terminal-bench
+  name: ${wandb_run:-terminal-bench}
+  tags: [terminal-bench]
+  log_traces: true
+
+inference_gateway:
+  upstream_api_key_env: OPENAI_API_KEY
+  upstream_base_url_env: OPENAI_BASE_URL
+  request_log_attribution: true
+  producer:
+    allowed_models: ["${optimizer_model:-openai/gpt-5.4}"]
+    # THE ONLY BLOCK THAT MAY DIFFER FROM build.yaml, and a test asserts exactly
+    # that. A 183-line duplicate is a drift hazard: if the two files diverge in a
+    # budget, timeout or partition, this cell silently stops being comparable to
+    # the nine that use build.yaml, and nothing would fail to tell us.
+    #
+    # Pin gpt-5.6-sol to one Azure deployment. codex reduces a model id to its
+    # last path component, so a codex cell cannot ask for `azure_ai/gpt-5.6-sol`
+    # itself -- the gateway has to supply it. Without this the request lands on
+    # the unqualified load-balanced group, and the Responses API's encrypted
+    # reasoning content is decryptable only by the deployment that produced it:
+    # every turn after the first fails with invalid_encrypted_content, which
+    # killed this cell on r2, r3 and r4. Measured 2026-08-01 against the live
+    # proxy: bare `gpt-5.6-sol` failed 5 of 5 encrypted-content replays,
+    # `azure_ai/gpt-5.6-sol` passed 5 of 5.
+    model_aliases:
+      gpt-5.6-sol: azure_ai/gpt-5.6-sol
+    max_concurrency: 8
+  # The case budget is the spend control; these caps only stop a runaway. Sized
+  # per CONFIGURATION.md at the same per-case-run rate as the other benchmarks:
+  # 212 agent case-runs here (68 dev + 144 validation).
+  evaluation:
+    allowed_models: [xai/grok-build-0.1]
+    max_requests: 200000
+    max_tokens: 1200000000
+    max_concurrency: 64
+  # Reserved so a search-phase overspend can never starve held-out scoring.
+  finalization:
+    allowed_models: [xai/grok-build-0.1]
+    max_requests: 200000
+    # 216 case-runs (36 test x3 attempts, plus rescore_top_k=3 over 36 validation)
+    # at the suite's per-case-run rate of ~5.05M, which officeqa and
+    # browsecomp-plus both use. Was 700M, i.e. 3.24M per case-run -- 36% tighter
+    # than the rest of the suite for no stated reason. Erring low here is the
+    # expensive direction: an exhausted finalization budget surfaces as an
+    # upstream 429 and silently starves held-out scoring, which is exactly how
+    # officeqa lost a re-score. The case budget is the real spend control.
+    max_tokens: 1100000000
+    max_concurrency: 64
+instruct_multifidelity: true
+instruct_exhaust_budget: true

harness-engineering-bench/terminal-bench/baseline/build.yaml

@@ -162,7 +162,14 @@ inference_gateway:
   finalization:
     allowed_models: [xai/grok-build-0.1]
     max_requests: 200000
-    max_tokens: 700000000  # 36 test cases x3 attempts + rescore headroom
+    # 216 case-runs (36 test x3 attempts, plus rescore_top_k=3 over 36 validation)
+    # at the suite's per-case-run rate of ~5.05M, which officeqa and
+    # browsecomp-plus both use. Was 700M, i.e. 3.24M per case-run -- 36% tighter
+    # than the rest of the suite for no stated reason. Erring low here is the
+    # expensive direction: an exhausted finalization budget surfaces as an
+    # upstream 429 and silently starves held-out scoring, which is exactly how
+    # officeqa lost a re-score. The case budget is the real spend control.
+    max_tokens: 1100000000
     max_concurrency: 64
 instruct_multifidelity: true
 instruct_exhaust_budget: true

vero/src/vero/gateway/inference.py

@@ -44,6 +44,13 @@ class InferenceScopeConfig(StrictModel):
 
     token_sha256: str
     allowed_models: list[str]
+    # Applied AFTER the allow-list check, on the way upstream. Keeps two
+    # concerns apart: allowed_models says what the caller may ask for, and
+    # therefore what a cell's label means; model_aliases says which upstream
+    # deployment serves it. Rewriting before the check would force the aliased
+    # name into allowed_models, and the allow-list would stop describing the
+    # contestant.
+    model_aliases: dict[str, str] = Field(default_factory=dict)
     max_requests: int | None = Field(default=None, ge=1)
     max_tokens: int | None = Field(default=None, ge=1)
     max_concurrency: int = Field(default=8, ge=1)
@@ -858,6 +865,11 @@ async def proxy(
         # Parameters the upstream refused and we retried without. Recorded so a
         # degraded request is auditable rather than a silent behaviour change.
         dropped_params: list[str] = []
+        # Set when scope.model_aliases rewrote the requested model. Same reason:
+        # a substitution the caller cannot see must be visible in the log. Bound
+        # here rather than at the rewrite, because log_request reads it and fires
+        # on the model_denied path before the rewrite is reached.
+        aliased_from: str | None = None
 
         async def log_request(
             *,
@@ -876,6 +888,7 @@ async def log_request(
                 attribution=attribution,
                 endpoint=endpoint,
                 model=value.get("model") if isinstance(value, dict) else None,
+                aliased_from=aliased_from,
                 dropped_params=dropped_params or None,
                 stream=stream,
                 status=status,
@@ -908,6 +921,29 @@ async def log_request(
             return _provider_error(
                 403, "model is not allowed for this scope", "model_denied"
             )
+
+        # Declared model rewrite, applied only now that the allow-list has passed.
+        # Some harnesses cannot put a provider-qualified id on the wire at all --
+        # codex reduces a model id to its last path component -- so a build that
+        # must reach one specific upstream deployment has no client-side way to
+        # ask for it. Aliasing here is the only place that can.
+        #
+        # Why that matters concretely: an unqualified model group is load-balanced
+        # across deployments, and the Responses API's encrypted reasoning content
+        # is decryptable only by the deployment that produced it. Replaying it
+        # against a sibling fails with invalid_encrypted_content on every turn
+        # after the first. Measured 2026-08-01: bare `gpt-5.6-sol` failed 5 of 5
+        # replays, `azure_ai/gpt-5.6-sol` passed 5 of 5.
+        #
+        # Both names are recorded, so the log shows the substitution rather than
+        # quietly reporting only what went upstream.
+        upstream_model = scope.model_aliases.get(model, model)
+        if upstream_model != model:
+            aliased_from = model
+            value["model"] = upstream_model
+            body = json.dumps(value).encode()
+        else:
+            aliased_from = None
         if not attribution or any(
             not (character.isalnum() or character in "_.-") for character in attribution
         ):
@@ -953,7 +989,10 @@ async def send(payload: bytes) -> httpx.Response:
         # the parameter on nearly every call -- 52 of 66 in a conformance run --
         # so retrying each one would double that traffic against the RPM limit for
         # no new information.
-        refused = app.state.refused_params.get(model, frozenset())
+        # Keyed on the model that will actually serve the request: refusing a
+        # parameter is a property of the upstream deployment, not of the name the
+        # caller used for it.
+        refused = app.state.refused_params.get(upstream_model, frozenset())
         if refused and isinstance(value, dict):
             known = [name for name in refused if name in value]
             if known:
@@ -985,10 +1024,15 @@ async def send(payload: bytes) -> httpx.Response:
                     # Re-serialize so the request log records what we really sent.
                     body = json.dumps(value).encode()
                     dropped_params.extend(blamed)
-                    # Bounded by allowed_models across scopes, so this cannot grow
-                    # with traffic: an unlisted model is rejected before reaching
-                    # here.
-                    app.state.refused_params[model] = refused.union(blamed)
+                    # Keyed on upstream_model to match the read above: keying the
+                    # write on the requested name would never hit for an aliased
+                    # model, and the cache would re-discover the same refusal on
+                    # every request.
+                    #
+                    # Bounded by allowed_models (and the finite alias map) across
+                    # scopes, so this cannot grow with traffic: an unlisted model
+                    # is rejected before reaching here.
+                    app.state.refused_params[upstream_model] = refused.union(blamed)
                     upstream = await send(body)
         except httpx.HTTPError:
             await asyncio.shield(

vero/src/vero/harbor/build/specs.py

@@ -126,13 +126,21 @@ class InferenceBudgetSpec(StrictModel):
     Attributes:
         allowed_models: Models this scope may request. A request naming anything
             else is refused with 403 model_denied.
+        model_aliases: Optional rewrite of the requested model, applied *after*
+            the allow-list check on the way upstream. Use it when the model that
+            must serve a request cannot be named by the caller -- codex, for
+            instance, reduces a model id to its last path component, so it can
+            never ask for a provider-qualified deployment. Declared here rather
+            than inferred, so the substitution is auditable in the build config
+            and in the request log.
         max_requests: Cap on proxied requests; unlimited when omitted.
         max_tokens: Cap on cumulative tokens; unlimited when omitted. Checked
             before a request starts, so a single request can overshoot it.
         max_concurrency: Requests this scope may have in flight at once.
     """
 
     allowed_models: list[str]
+    model_aliases: dict[str, str] = Field(default_factory=dict)
     max_requests: int | None = Field(default=None, ge=1)
     max_tokens: int | None = Field(default=None, ge=1)
     max_concurrency: int = Field(default=8, ge=1)
@@ -146,6 +154,30 @@ def validate_models(cls, value: list[str]) -> list[str]:
             raise ValueError("allowed_models must be unique")
         return value
 
+    @field_validator("model_aliases")
+    @classmethod
+    def validate_aliases(cls, value: dict[str, str]) -> dict[str, str]:
+        # Self-aliases are dropped rather than rejected, and keys outside
+        # allowed_models are permitted. Both concessions exist because one build
+        # config is shared by every cell of a grid, with allowed_models templated
+        # per launch: a map that pins the right deployment for one optimizer
+        # necessarily carries keys the other launches never request. Rejecting
+        # those would make the field unusable exactly where it is needed.
+        #
+        # Nothing is lost by allowing them. An alias can only fire for a model
+        # the allow-list already admitted, so an unreachable key is inert, and a
+        # self-alias is a no-op. The failure this field could cause -- a request
+        # served by a model other than the one named -- is bounded by the
+        # allow-list and made visible by the `aliased_from` field in the request
+        # log, not by validation here.
+        cleaned = {}
+        for requested, upstream in value.items():
+            if not requested.strip() or not upstream.strip():
+                raise ValueError("model_aliases names must not be empty")
+            if requested != upstream:
+                cleaned[requested] = upstream
+        return cleaned
+
 
 class InferenceGatewaySpec(StrictModel):
     """Credential source and independent producer/evaluator policies.