diff --git a/.gitignore b/.gitignore index e1ec0d66..90e55be3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ wheels/ secrets.env *.secrets.env !*.example + +# Codex CLI local state, including the worktrees it checks out for its own runs. +.codex/ diff --git a/harness-engineering-bench/terminal-bench/baseline/build.azure.yaml b/harness-engineering-bench/terminal-bench/baseline/build.azure.yaml new file mode 100644 index 00000000..39d70cee --- /dev/null +++ b/harness-engineering-bench/terminal-bench/baseline/build.azure.yaml @@ -0,0 +1,191 @@ +name: vero/optimize-terminal-bench-baseline +description: >- + Improve a terminal agent on Terminal-Bench 2.1 while preserving the Harbor + agent interface. Each task gives the agent a container and a goal; the task's + own tests decide pass or fail, so the reward is a pass rate. + +# Target model and baseline_reward are both measured and pinned. See +# ../README.md for the model probe and runs/BASELINES.md for the pinning. + +agent_repo: target +task_source: terminal-bench/terminal-bench-2-1@sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a +task_manifest: ../partitions/manifest.json +agent_import_path: terminal_bench_agent.agent:TerminalBenchAgent +harbor_requirement: harbor[modal]==0.20.0 + +partition_files: + development: ../partitions/development.json + validation: ../partitions/validation.json + test: ../partitions/test.json + +# total_cases = 4 full passes per partition (dev 17*4, validation 36*4). +agent_access: + - partition: development + disclosure: full + expose_case_resources: true + total_runs: 100 + total_cases: 68 + - partition: validation + disclosure: aggregate + expose_case_resources: false + min_aggregate_cases: 5 + total_runs: 100 + total_cases: 144 + +selection_partition: validation +targets: + - partition: test + reward_key: reward + # Seed-killed trials score 0 (4 UnicodeDecodeError from the seed's + # undecodable-output bug, 4 AgentTimeoutError the verifier could not score; + # the other 14 timeouts scored on final container state and were always in + # the pin); platform-killed trials are dropped. See runs/recompute.py. + baseline_reward: 0.2407 # K=3: 0.250 / 0.222 / 0.250 (sd 0.0131, n=108); was 0.2600 excluding seed-killed trials. See runs/BASELINES.md + # 36 held-out cases was expected to be noisy -- it is the smallest test set in + # the suite -- but the measured sd is the second tightest of the six, behind + # only tau3. Deterministic per-task tests instead of an LLM judge is the + # likely reason. So n_attempts stays at 3; it does not need raising to 5. + failure_value: 0.0 + max_attempts: 1 + # Score the selected candidate 3x per case and average, so the reported + # reward carries the same standard error as a 3-round pinned baseline. + # Per-target override - search/validation keep the global n_attempts (1). + n_attempts: 3 + aggregate_attempts: mean + +evaluation_set_name: terminal-bench +objective: + selector: + metric: score + direction: maximize +reward_mode: submit # agent picks; falls back to auto_best, then current version +baseline_floor: false # gates on validation while reward is on test; opt-in only +score_baseline: false +rescore_top_k: 3 +rescore_attempts: 1 + +# Chosen from a measured probe: seed harness, all 17 development tasks, one round +# each, direct to the proxy. +# +# model solved reward $/Mtok* notes +# xai/grok-build-0.1 6/17 0.3529 0.380 +# azure_ai/gpt-5-nano 2/17 0.1176 0.044 +# xai/grok-4-1-fast-reasoning 2/17 0.1176 0.095 dominated by nano +# gemini/gemini-2.5-flash-lite 0/12 0.0000 0.049 5 trials crashed +# * 90% cache-read + 10% output, our measured token mix +# +# grok-build wins despite 8.6x the cheapest rate, because what matters for a +# benchmark is a baseline with headroom, not reward per dollar. 0.35 sits where +# officeqa's baseline does (0.3412), and officeqa's best cell moved +0.41 from +# there; 0.12 is swe-atlas-qna territory (0.0676), the least informative cell in +# the suite. Estimated ~$130/cell, ~$1,170 for a nine-cell pass. +# +# Not on Fireworks, deliberately: the shared per-minute generated-token quota is +# what limits how many cells can run at once, so a target off that provider does +# not contend with the officeqa or browsecomp-plus grids. +model: xai/grok-build-0.1 +environment_name: ${inner_env:-modal} +extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +harbor_python_version: "3.12" +n_attempts: 1 +max_retries: 4 +retry_max_wait_seconds: 120 +infrastructure_max_attempts: 3 +infrastructure_retry_delay_seconds: 5 +aggregate_attempts: best +feedback_transcripts: true +feedback_max_bytes: 16000 +expose_attempt_detail: false + +# Unlike every other benchmark here, Terminal-Bench declares a DIFFERENT agent +# timeout per task: 600s to 12,000s, 48 of 89 at 900s, 13 at 3600s. That is fine +# and needs no special handling, because vero passes Harbor a single ratio +# (`case_timeout_seconds / task_agent_timeout_seconds`) which Harbor applies to +# each task's own declared budget. Keeping the pair equal therefore gives every +# task exactly the clock its author intended. The absolute value is irrelevant as +# long as the two match -- do not "fix" this by setting them to different numbers. +case_timeout_seconds: 900 +task_agent_timeout_seconds: 900 + +# Worst-case finalize wall: 36 test x 3 attempts = 108 case-runs, ceil(108/24) = +# 5 waves, slowest test task declares 7200s -> 36,000s. Set above that so a +# pathological run cannot lose a scored result to the job clock. +timeout_seconds: 43200 +# Worst-case finalize (36,000) plus a rescore_top_k=3 validation pass, whose +# widest single eval is ceil(36/24) = 2 waves x the slowest validation task +# (12,000s) = 24,000s. A verifier timeout loses the score outright. +verifier_timeout_seconds: 64800 +max_concurrency: 24 +error_rate_threshold: 0.1 + +secrets: + - MODAL_TOKEN_ID + - MODAL_TOKEN_SECRET + - WANDB_API_KEY + - WANDB_BASE_URL + +harness_user: harness +agent_env: + # Above this benchmark's widest single search eval: a full validation pass is + # ceil(36/24) = 2 waves x 12,000s = 24,000s worst case. The optimizer must be + # able to block on one evaluation in a single foreground call -- a headless run + # that backgrounds a long call is never re-woken and the search dies there. + BASH_MAX_TIMEOUT_MS: "28800000" + BASH_DEFAULT_TIMEOUT_MS: "28800000" + ENABLE_BACKGROUND_TASKS: "0" + FORCE_AUTO_BACKGROUND_TASKS: "0" + UV_TOOL_BIN_DIR: "/home/agent/.local/bin" + +wandb: + project: harness-engineering-bench # one project for the whole suite + group: terminal-bench + name: ${wandb_run:-terminal-bench} + tags: [terminal-bench] + log_traces: true + +inference_gateway: + upstream_api_key_env: OPENAI_API_KEY + upstream_base_url_env: OPENAI_BASE_URL + request_log_attribution: true + producer: + allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + # THE ONLY BLOCK THAT MAY DIFFER FROM build.yaml, and a test asserts exactly + # that. A 183-line duplicate is a drift hazard: if the two files diverge in a + # budget, timeout or partition, this cell silently stops being comparable to + # the nine that use build.yaml, and nothing would fail to tell us. + # + # Pin gpt-5.6-sol to one Azure deployment. codex reduces a model id to its + # last path component, so a codex cell cannot ask for `azure_ai/gpt-5.6-sol` + # itself -- the gateway has to supply it. Without this the request lands on + # the unqualified load-balanced group, and the Responses API's encrypted + # reasoning content is decryptable only by the deployment that produced it: + # every turn after the first fails with invalid_encrypted_content, which + # killed this cell on r2, r3 and r4. Measured 2026-08-01 against the live + # proxy: bare `gpt-5.6-sol` failed 5 of 5 encrypted-content replays, + # `azure_ai/gpt-5.6-sol` passed 5 of 5. + model_aliases: + gpt-5.6-sol: azure_ai/gpt-5.6-sol + max_concurrency: 8 + # The case budget is the spend control; these caps only stop a runaway. Sized + # per CONFIGURATION.md at the same per-case-run rate as the other benchmarks: + # 212 agent case-runs here (68 dev + 144 validation). + evaluation: + allowed_models: [xai/grok-build-0.1] + max_requests: 200000 + max_tokens: 1200000000 + max_concurrency: 64 + # Reserved so a search-phase overspend can never starve held-out scoring. + finalization: + allowed_models: [xai/grok-build-0.1] + max_requests: 200000 + # 216 case-runs (36 test x3 attempts, plus rescore_top_k=3 over 36 validation) + # at the suite's per-case-run rate of ~5.05M, which officeqa and + # browsecomp-plus both use. Was 700M, i.e. 3.24M per case-run -- 36% tighter + # than the rest of the suite for no stated reason. Erring low here is the + # expensive direction: an exhausted finalization budget surfaces as an + # upstream 429 and silently starves held-out scoring, which is exactly how + # officeqa lost a re-score. The case budget is the real spend control. + max_tokens: 1100000000 + max_concurrency: 64 +instruct_multifidelity: true +instruct_exhaust_budget: true diff --git a/harness-engineering-bench/terminal-bench/baseline/build.yaml b/harness-engineering-bench/terminal-bench/baseline/build.yaml index 4cc5fa79..9d8b68e0 100644 --- a/harness-engineering-bench/terminal-bench/baseline/build.yaml +++ b/harness-engineering-bench/terminal-bench/baseline/build.yaml @@ -162,7 +162,14 @@ inference_gateway: finalization: allowed_models: [xai/grok-build-0.1] max_requests: 200000 - max_tokens: 700000000 # 36 test cases x3 attempts + rescore headroom + # 216 case-runs (36 test x3 attempts, plus rescore_top_k=3 over 36 validation) + # at the suite's per-case-run rate of ~5.05M, which officeqa and + # browsecomp-plus both use. Was 700M, i.e. 3.24M per case-run -- 36% tighter + # than the rest of the suite for no stated reason. Erring low here is the + # expensive direction: an exhausted finalization budget surfaces as an + # upstream 429 and silently starves held-out scoring, which is exactly how + # officeqa lost a re-score. The case budget is the real spend control. + max_tokens: 1100000000 max_concurrency: 64 instruct_multifidelity: true instruct_exhaust_budget: true diff --git a/vero/src/vero/gateway/inference.py b/vero/src/vero/gateway/inference.py index c024ed05..0965f7fc 100644 --- a/vero/src/vero/gateway/inference.py +++ b/vero/src/vero/gateway/inference.py @@ -44,6 +44,13 @@ class InferenceScopeConfig(StrictModel): token_sha256: str allowed_models: list[str] + # Applied AFTER the allow-list check, on the way upstream. Keeps two + # concerns apart: allowed_models says what the caller may ask for, and + # therefore what a cell's label means; model_aliases says which upstream + # deployment serves it. Rewriting before the check would force the aliased + # name into allowed_models, and the allow-list would stop describing the + # contestant. + model_aliases: dict[str, str] = Field(default_factory=dict) max_requests: int | None = Field(default=None, ge=1) max_tokens: int | None = Field(default=None, ge=1) max_concurrency: int = Field(default=8, ge=1) @@ -858,6 +865,11 @@ async def proxy( # Parameters the upstream refused and we retried without. Recorded so a # degraded request is auditable rather than a silent behaviour change. dropped_params: list[str] = [] + # Set when scope.model_aliases rewrote the requested model. Same reason: + # a substitution the caller cannot see must be visible in the log. Bound + # here rather than at the rewrite, because log_request reads it and fires + # on the model_denied path before the rewrite is reached. + aliased_from: str | None = None async def log_request( *, @@ -876,6 +888,7 @@ async def log_request( attribution=attribution, endpoint=endpoint, model=value.get("model") if isinstance(value, dict) else None, + aliased_from=aliased_from, dropped_params=dropped_params or None, stream=stream, status=status, @@ -908,6 +921,29 @@ async def log_request( return _provider_error( 403, "model is not allowed for this scope", "model_denied" ) + + # Declared model rewrite, applied only now that the allow-list has passed. + # Some harnesses cannot put a provider-qualified id on the wire at all -- + # codex reduces a model id to its last path component -- so a build that + # must reach one specific upstream deployment has no client-side way to + # ask for it. Aliasing here is the only place that can. + # + # Why that matters concretely: an unqualified model group is load-balanced + # across deployments, and the Responses API's encrypted reasoning content + # is decryptable only by the deployment that produced it. Replaying it + # against a sibling fails with invalid_encrypted_content on every turn + # after the first. Measured 2026-08-01: bare `gpt-5.6-sol` failed 5 of 5 + # replays, `azure_ai/gpt-5.6-sol` passed 5 of 5. + # + # Both names are recorded, so the log shows the substitution rather than + # quietly reporting only what went upstream. + upstream_model = scope.model_aliases.get(model, model) + if upstream_model != model: + aliased_from = model + value["model"] = upstream_model + body = json.dumps(value).encode() + else: + aliased_from = None if not attribution or any( not (character.isalnum() or character in "_.-") for character in attribution ): @@ -953,7 +989,10 @@ async def send(payload: bytes) -> httpx.Response: # the parameter on nearly every call -- 52 of 66 in a conformance run -- # so retrying each one would double that traffic against the RPM limit for # no new information. - refused = app.state.refused_params.get(model, frozenset()) + # Keyed on the model that will actually serve the request: refusing a + # parameter is a property of the upstream deployment, not of the name the + # caller used for it. + refused = app.state.refused_params.get(upstream_model, frozenset()) if refused and isinstance(value, dict): known = [name for name in refused if name in value] if known: @@ -985,10 +1024,15 @@ async def send(payload: bytes) -> httpx.Response: # Re-serialize so the request log records what we really sent. body = json.dumps(value).encode() dropped_params.extend(blamed) - # Bounded by allowed_models across scopes, so this cannot grow - # with traffic: an unlisted model is rejected before reaching - # here. - app.state.refused_params[model] = refused.union(blamed) + # Keyed on upstream_model to match the read above: keying the + # write on the requested name would never hit for an aliased + # model, and the cache would re-discover the same refusal on + # every request. + # + # Bounded by allowed_models (and the finite alias map) across + # scopes, so this cannot grow with traffic: an unlisted model + # is rejected before reaching here. + app.state.refused_params[upstream_model] = refused.union(blamed) upstream = await send(body) except httpx.HTTPError: await asyncio.shield( diff --git a/vero/src/vero/harbor/build/specs.py b/vero/src/vero/harbor/build/specs.py index 1d32e420..06c565b8 100644 --- a/vero/src/vero/harbor/build/specs.py +++ b/vero/src/vero/harbor/build/specs.py @@ -126,6 +126,13 @@ class InferenceBudgetSpec(StrictModel): Attributes: allowed_models: Models this scope may request. A request naming anything else is refused with 403 model_denied. + model_aliases: Optional rewrite of the requested model, applied *after* + the allow-list check on the way upstream. Use it when the model that + must serve a request cannot be named by the caller -- codex, for + instance, reduces a model id to its last path component, so it can + never ask for a provider-qualified deployment. Declared here rather + than inferred, so the substitution is auditable in the build config + and in the request log. max_requests: Cap on proxied requests; unlimited when omitted. max_tokens: Cap on cumulative tokens; unlimited when omitted. Checked before a request starts, so a single request can overshoot it. @@ -133,6 +140,7 @@ class InferenceBudgetSpec(StrictModel): """ allowed_models: list[str] + model_aliases: dict[str, str] = Field(default_factory=dict) max_requests: int | None = Field(default=None, ge=1) max_tokens: int | None = Field(default=None, ge=1) max_concurrency: int = Field(default=8, ge=1) @@ -146,6 +154,30 @@ def validate_models(cls, value: list[str]) -> list[str]: raise ValueError("allowed_models must be unique") return value + @field_validator("model_aliases") + @classmethod + def validate_aliases(cls, value: dict[str, str]) -> dict[str, str]: + # Self-aliases are dropped rather than rejected, and keys outside + # allowed_models are permitted. Both concessions exist because one build + # config is shared by every cell of a grid, with allowed_models templated + # per launch: a map that pins the right deployment for one optimizer + # necessarily carries keys the other launches never request. Rejecting + # those would make the field unusable exactly where it is needed. + # + # Nothing is lost by allowing them. An alias can only fire for a model + # the allow-list already admitted, so an unreachable key is inert, and a + # self-alias is a no-op. The failure this field could cause -- a request + # served by a model other than the one named -- is bounded by the + # allow-list and made visible by the `aliased_from` field in the request + # log, not by validation here. + cleaned = {} + for requested, upstream in value.items(): + if not requested.strip() or not upstream.strip(): + raise ValueError("model_aliases names must not be empty") + if requested != upstream: + cleaned[requested] = upstream + return cleaned + class InferenceGatewaySpec(StrictModel): """Credential source and independent producer/evaluator policies. diff --git a/vero/tests/test_v05_benchmark_configs.py b/vero/tests/test_v05_benchmark_configs.py index 55e6dbaa..8e328e84 100644 --- a/vero/tests/test_v05_benchmark_configs.py +++ b/vero/tests/test_v05_benchmark_configs.py @@ -168,3 +168,48 @@ def test_build_params_override_run_time_knobs_without_rebuild(): # The rest of the measurement substrate is untemplated and stays fixed. assert overridden.model == default.model assert overridden.task_source == default.task_source + + +def test_terminal_bench_azure_variant_differs_only_by_the_model_alias(): + """build.azure.yaml must be build.yaml plus one alias, and nothing else. + + The variant exists only because codex cannot put a provider-qualified model + id on the wire, so the gateway has to pin `gpt-5.6-sol` to a single Azure + deployment on its behalf. Everything else -- budgets, timeouts, partitions, + the pinned baseline -- has to stay identical, or the cell that uses this file + stops being comparable to the nine that use build.yaml. + + There is no include/extends mechanism for these YAMLs, so the variant is a + 183-line copy. That is a drift hazard with no natural alarm: divergence in a + timeout or a budget would change results and fail nothing. This test is the + alarm. If you deliberately change build.yaml, mirror it here and the test + passes again; if you forget, it does not. + """ + baseline = BENCHMARK_ROOT / "terminal-bench" / "baseline" + shared = yaml.safe_load((baseline / "build.yaml").read_text(encoding="utf-8")) + variant = yaml.safe_load((baseline / "build.azure.yaml").read_text(encoding="utf-8")) + + alias = variant["inference_gateway"]["producer"].pop("model_aliases") + assert alias == {"gpt-5.6-sol": "azure_ai/gpt-5.6-sol"} + # Compared after popping the alias: the two documents must now be equal. + assert variant == shared, ( + "build.azure.yaml has drifted from build.yaml beyond the model alias; " + "mirror the change or the azure cell is no longer comparable" + ) + + # And the alias must actually reach the gateway for the cell that needs it, + # while staying inert for a cell whose optimizer is something else. + sol = load_harbor_build_config( + baseline / "build.azure.yaml", params={"optimizer_model": "gpt-5.6-sol"} + ) + other = load_harbor_build_config( + baseline / "build.azure.yaml", params={"optimizer_model": "claude-opus-5"} + ) + assert sol.inference_gateway.producer.allowed_models == ["gpt-5.6-sol"] + assert sol.inference_gateway.producer.model_aliases == { + "gpt-5.6-sol": "azure_ai/gpt-5.6-sol" + } + # Present but unreachable: the allow-list admits only claude-opus-5, and an + # alias can fire only for a model the allow-list already passed. + assert other.inference_gateway.producer.allowed_models == ["claude-opus-5"] + assert "claude-opus-5" not in other.inference_gateway.producer.model_aliases diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 77e074bc..448462ef 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1452,6 +1452,46 @@ def case(name: str, **updates): case("pinned", task_source="gaia/gaia@sha256:abc123") +def test_model_alias_tolerates_keys_a_given_launch_never_requests(): + """One build config serves every cell of a grid, so the alias map must + tolerate keys the current launch cannot request. + + `allowed_models` is templated per launch (``${optimizer_model}``), so a map + that pins the right deployment for one optimizer necessarily carries keys the + other launches never name. Those entries are inert -- an alias can only fire + for a model the allow-list already admitted -- so rejecting them would make + the field unusable in exactly the shared-config case it exists for. + """ + spec = InferenceBudgetSpec( + allowed_models=["claude-opus-5"], + model_aliases={"gpt-5.6-sol": "azure_ai/gpt-5.6-sol"}, + ) + assert spec.model_aliases == {"gpt-5.6-sol": "azure_ai/gpt-5.6-sol"} + + # A self-alias is a no-op and is dropped rather than rejected, so a template + # that resolves to `{X: X}` when no override is passed still validates. + assert ( + InferenceBudgetSpec( + allowed_models=["gpt-producer"], + model_aliases={"gpt-producer": "gpt-producer"}, + ).model_aliases + == {} + ) + + with pytest.raises(ValidationError, match="must not be empty"): + InferenceBudgetSpec( + allowed_models=["gpt-producer"], model_aliases={"gpt-producer": " "} + ) + + # Reaches the gateway via model_dump, which is how the compiler lowers it. + assert InferenceBudgetSpec( + allowed_models=["gpt-producer"], + model_aliases={"gpt-producer": "vendor_x/gpt-producer"}, + ).model_dump(mode="json")["model_aliases"] == { + "gpt-producer": "vendor_x/gpt-producer" + } + + def test_harbor_itself_routes_the_rescue_hook_into_the_sidecar_collection_pass( tmp_path, ): diff --git a/vero/tests/test_v05_harbor_inference.py b/vero/tests/test_v05_harbor_inference.py index 981c1e39..b8db35e6 100644 --- a/vero/tests/test_v05_harbor_inference.py +++ b/vero/tests/test_v05_harbor_inference.py @@ -1111,3 +1111,78 @@ def upstream(request: httpx.Request): # straight through. Retrying per request would have cost ten. assert attempts == [True, False, False, False, False, False], attempts assert attempts.count(True) == 1, "the refusal must be discovered only once" + + +def test_model_alias_rewrites_upstream_after_the_allow_list(tmp_path): + """A declared alias changes which deployment serves the request, not what the + caller may ask for. + + The distinction is the point. `allowed_models` is what makes "this cell ran + gpt-test" a true statement, so the alias is applied only once that check has + passed: the caller still cannot reach an unlisted model by naming its alias + key, and the log records both names. + """ + observed: list[dict] = [] + + def upstream(request: httpx.Request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json={"id": "response", "usage": {}}) + + config = InferenceGatewayConfig( + state_path=str(tmp_path / "usage.json"), + request_log=InferenceRequestLogConfig(directory=str(tmp_path / "log")), + scopes={ + "producer": InferenceScopeConfig( + token_sha256=token_digest("scoped-token"), + allowed_models=["gpt-test", "plain"], + model_aliases={"gpt-test": "vendor_x/gpt-test"}, + max_concurrency=1, + ) + }, + ) + app = create_inference_gateway_app( + config=config, + upstream_api_key="upstream-secret", + upstream_base_url="https://provider.example/v1", + transport=httpx.MockTransport(upstream), + ) + with TestClient(app) as client: + aliased = client.post( + "/scopes/producer/optimizer/v1/responses", + headers={"Authorization": "Bearer scoped-token"}, + json={"model": "gpt-test", "input": "hello"}, + ) + untouched = client.post( + "/scopes/producer/optimizer/v1/responses", + headers={"Authorization": "Bearer scoped-token"}, + json={"model": "plain", "input": "hello"}, + ) + # The alias TARGET is not itself allow-listed, so naming it directly is + # still refused. An alias must not become a second way in. + target_direct = client.post( + "/scopes/producer/optimizer/v1/responses", + headers={"Authorization": "Bearer scoped-token"}, + json={"model": "vendor_x/gpt-test", "input": "hello"}, + ) + + assert aliased.status_code == 200 + assert untouched.status_code == 200 + assert target_direct.status_code == 403 + assert target_direct.json()["error"]["code"] == "model_denied" + + # Rewritten on the way upstream; the unaliased model passes through as-is. + assert [payload["model"] for payload in observed] == [ + "vendor_x/gpt-test", + "plain", + ] + + records = [ + json.loads(line) + for path in sorted((tmp_path / "log").glob("requests-*.jsonl")) + for line in path.read_text(encoding="utf-8").splitlines() + ] + proxied = [record for record in records if record["status"] == 200] + assert [(r["model"], r.get("aliased_from")) for r in proxied] == [ + ("vendor_x/gpt-test", "gpt-test"), + ("plain", None), + ]