From 0dea549a28452e519088f63655f80268ac12a17e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 20:39:55 -0600 Subject: [PATCH 01/39] refactor(runtime): enforce canonical model execution boundary --- README.md | 71 +- bench/scripts/appworld_driver.py | 171 +- bench/scripts/trata-hedge/README.md | 6 +- bench/scripts/trata-hedge/run.sh | 8 +- bench/scripts/trata-hedge/solve.py | 83 - bench/src/aec-gate.mts | 39 +- bench/src/agent-graphs-gen2.mts | 15 +- bench/src/agent-graphs-gen3.mts | 3 +- bench/src/agent-graphs-improve.mts | 151 +- bench/src/atom-humaneval.mts | 131 +- bench/src/atom-mcp-e2e.mts | 40 +- bench/src/benchmarks/appworld.test.mts | 34 +- bench/src/benchmarks/appworld.ts | 131 +- bench/src/benchmarks/cadbench.ts | 29 +- bench/src/benchmarks/finresearchbench.ts | 36 +- bench/src/benchmarks/finsearchcomp.ts | 40 +- bench/src/benchmarks/frames.ts | 37 +- bench/src/benchmarks/simpleqa.ts | 36 +- bench/src/benchmarks/trata-hedge.ts | 44 +- bench/src/clbench-context-gate.mts | 52 +- bench/src/cloud-loop.mts | 21 +- bench/src/commit0-env-run.mts | 9 +- bench/src/commit0-env.ts | 5 +- bench/src/david-attribution.mts | 34 +- bench/src/david-goliath.mts | 43 +- bench/src/egress-probe.mts | 6 +- bench/src/examples/lean-proof-gate.mts | 13 +- bench/src/examples/math-demo.mts | 13 +- bench/src/examples/strategy-demo.mts | 14 +- bench/src/fleet.mts | 27 +- bench/src/gate-cli.mts | 13 +- bench/src/gate.test.mts | 1 - bench/src/gate.ts | 91 +- bench/src/generate-eval/certify.ts | 18 +- bench/src/hev-eval.mts | 49 +- bench/src/hev-improve.mts | 43 +- bench/src/hev-structural.mts | 48 +- bench/src/humaneval-gate.mts | 30 +- bench/src/humaneval-object-ablation.mts | 63 +- bench/src/humaneval-repair-gate.mts | 67 +- bench/src/mbpp-structural.mts | 48 +- bench/src/mcp-mount-probe.mts | 34 +- bench/src/official-optimizer-config.mts | 73 +- bench/src/official-optimizer-config.test.mts | 75 +- bench/src/research-shot.ts | 30 +- bench/src/resolve-client.ts | 10 +- bench/src/router-turn.ts | 128 + bench/src/run-benchmarks.ts | 9 +- bench/src/sandbox-run.ts | 25 +- bench/src/search-bench/bridge.ts | 28 +- bench/src/search-bench/parametric-check.mts | 23 +- bench/src/supervisor-arena.mts | 48 +- bench/src/swe-arena/capacity.ts | 85 +- bench/src/swe-arena/diagnosis-ensemble.ts | 99 +- bench/src/swe-arena/execution.test.mts | 24 +- bench/src/swe-jail.test.ts | 96 + bench/src/swe-jail.ts | 119 +- bench/src/swe-repro-calibrate.mts | 5 + bench/src/swe-self-improve.mts | 55 +- bench/src/swe-stream.mts | 28 + bench/src/swe-structural.mts | 20 +- bench/src/trata-gate.mts | 52 +- bench/src/trata-gepa.mts | 41 +- bench/src/trata-hedge-solve.mts | 80 + bench/src/worker-blender.ts | 32 +- bench/src/worker-browser.ts | 23 +- bench/src/worker-build123d.ts | 25 +- bench/src/worker-cad.ts | 43 +- bench/vitest.config.ts | 1 + docs/api/primitive-catalog.md | 76 +- examples/ablation-suite/ablation.ts | 80 +- examples/ablation-suite/gepa-driver-prompt.ts | 84 +- examples/ablation-suite/hard-coding-env.ts | 4 +- .../ablation-suite/long-coding-env-lite.ts | 4 +- examples/ablation-suite/long-coding-env.ts | 4 +- examples/ablation-suite/run-ablation.ts | 28 +- examples/ablation-suite/verkit-env.ts | 5 +- .../agentic-data-creation.ts | 18 +- .../agentic-data-creation/offline-fixtures.ts | 140 +- examples/coding-benchmark/benchmark.ts | 86 +- examples/coding-benchmark/eval.ts | 25 +- examples/delegate/delegate.ts | 21 +- examples/delegate/shared.ts | 3 - examples/graphs/user-sim-conversation.ts | 3 +- examples/improve/improve.ts | 7 +- examples/p1-parity/arms.ts | 31 +- examples/p1-parity/offline.ts | 2 +- examples/p1-parity/parity.test.ts | 20 +- examples/p1-parity/run-parity.ts | 146 +- examples/product-eval/product-eval.ts | 40 +- .../self-improving-coder.ts | 56 +- .../strategy-evolution/strategy-evolution.ts | 42 +- examples/strategy-suite/counter-env.ts | 3 +- examples/strategy-suite/strategy-suite.ts | 25 +- examples/stream-backends/README.md | 16 +- examples/stream-backends/stream-backends.ts | 54 +- examples/supervise/supervise.ts | 7 +- .../supervisor-loop/run-supervisor-mcp.ts | 90 +- examples/supervisor-loop/run.ts | 33 +- examples/supervisor-loop/shared.ts | 89 +- package.json | 5 +- pnpm-lock.yaml | 39 +- pnpm-workspace.yaml | 4 +- scripts/check-model-execution-boundary.mjs | 302 +++ .../check-model-execution-boundary.test.mjs | 105 + scripts/verify-package-exports.mjs | 2 +- scripts/verify-primeintellect-live.mjs | 64 +- src/agent/profile-materialization.ts | 25 +- src/backends.ts | 767 +----- src/conversation/run-persona.test.ts | 146 +- src/conversation/run-persona.ts | 58 +- src/improvement/improve.test.ts | 9 +- src/improvement/method-cost.test.ts | 29 +- src/improvement/method-identity.test.ts | 7 +- src/improvement/official-optimizers.test.ts | 31 +- .../official-runtime-integration.test.ts | 45 +- src/improvement/rollout-policy.test.ts | 15 +- src/improvement/rollout-policy.ts | 3 - src/index.ts | 9 +- src/mcp/bin.ts | 2 +- src/mcp/delegate-supervisor-provisioning.ts | 25 +- src/mcp/local-harness.ts | 22 +- src/mcp/tools/delegate.ts | 26 +- src/primeintellect/index.ts | 3 +- src/primeintellect/primeintellect.test.ts | 13 +- src/primeintellect/runner.ts | 25 +- src/resolve-agent-backend.test.ts | 111 - src/resolve-agent-backend.ts | 112 - src/runtime/audit-intent.ts | 20 +- src/runtime/define-leaderboard.ts | 1 - src/runtime/harvest-corpus.ts | 17 +- src/runtime/index.ts | 35 +- src/runtime/inline-sandbox-client.ts | 87 +- src/runtime/local-sandbox-client.ts | 78 +- src/runtime/loop-dispatch.ts | 7 +- src/runtime/observe.ts | 47 +- src/runtime/profile-chat-client.test.ts | 241 ++ src/runtime/profile-chat-client.ts | 422 ++++ src/runtime/resolve-sandbox-client.test.ts | 30 +- src/runtime/resolve-sandbox-client.ts | 23 +- src/runtime/router-client.complete.test.ts | 165 ++ src/runtime/router-client.ts | 458 +++- src/runtime/run-benchmark.test.ts | 88 +- src/runtime/run-benchmark.ts | 39 +- src/runtime/run-loop.ts | 55 +- src/runtime/sandbox-backend.ts | 35 +- src/runtime/sandbox-events.ts | 76 +- src/runtime/stdio-mcp-client.test.ts | 8 +- src/runtime/strategy-author.ts | 51 +- src/runtime/strategy-evolution.ts | 56 +- src/runtime/strategy.ts | 435 ++-- src/runtime/stream-agent-turn.test.ts | 162 +- src/runtime/stream-agent-turn.ts | 806 +++++- src/runtime/structural-rollout.test.ts | 32 +- src/runtime/structural-rollout.ts | 11 +- src/runtime/supervise-surface.ts | 47 +- src/runtime/supervise/authoring.ts | 136 +- src/runtime/supervise/bridge-executor.test.ts | 316 ++- src/runtime/supervise/budget.ts | 6 + .../supervise/chat-transport-executor.ts | 602 ++--- src/runtime/supervise/completion-gate.ts | 62 + src/runtime/supervise/coordination-driver.ts | 72 +- src/runtime/supervise/delegate.ts | 55 +- src/runtime/supervise/graph.ts | 4 +- src/runtime/supervise/materialization.ts | 113 +- src/runtime/supervise/model-policy.ts | 168 +- src/runtime/supervise/runtime.ts | 2212 ++++++++++++----- src/runtime/supervise/sandbox-session.ts | 49 +- src/runtime/supervise/scope.ts | 116 +- src/runtime/supervise/supervise.ts | 191 +- src/runtime/supervise/supervisor-agent.ts | 171 +- src/runtime/supervise/types.ts | 21 +- src/runtime/tool-loop.ts | 53 +- src/runtime/types.ts | 18 + src/runtime/util.ts | 3 +- src/types.ts | 12 + tests/backends-anthropic-usage.test.ts | 110 - tests/backends-fail-loud.test.ts | 296 --- tests/backends-openai-tools.test.ts | 551 ---- tests/backends-openai-usage.test.ts | 107 - tests/kernel/chat-transport-executor.test.ts | 121 +- tests/kernel/coordination-driver.test.ts | 10 +- .../kernel/driver-inference-metering.test.ts | 1 + tests/kernel/router-brain.test.ts | 16 +- tests/kernel/scripted-brain.ts | 9 +- tests/kernel/supervise-convenience.test.ts | 25 +- .../supervise-full-profile-bridge.test.ts | 195 +- tests/runtime.test.ts | 188 -- tests/runtime/bridge-executor.test.ts | 304 ++- .../runtime/executor-config-snapshot.test.ts | 48 +- tests/runtime/executor-profile-model.test.ts | 107 +- tests/runtime/worktree-cli-executor.test.ts | 71 +- 192 files changed, 9794 insertions(+), 6647 deletions(-) delete mode 100644 bench/scripts/trata-hedge/solve.py create mode 100644 bench/src/router-turn.ts create mode 100644 bench/src/swe-jail.test.ts create mode 100644 bench/src/trata-hedge-solve.mts create mode 100644 scripts/check-model-execution-boundary.mjs create mode 100644 scripts/check-model-execution-boundary.test.mjs delete mode 100644 src/resolve-agent-backend.test.ts delete mode 100644 src/resolve-agent-backend.ts create mode 100644 src/runtime/profile-chat-client.test.ts create mode 100644 src/runtime/profile-chat-client.ts delete mode 100644 tests/backends-anthropic-usage.test.ts delete mode 100644 tests/backends-fail-loud.test.ts delete mode 100644 tests/backends-openai-tools.test.ts delete mode 100644 tests/backends-openai-usage.test.ts diff --git a/README.md b/README.md index d664c09d..f5f6be88 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,12 @@ The profile is never changed. ```ts import { improve, officialGepa } from '@tangle-network/agent-runtime' -import { canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { profileOptimizerModelCall } from '@tangle-network/agent-runtime/kernel' +import { + type AgentProfile, + canonicalAgentProfileDigest, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' const executionRef = canonicalCandidateDigest({ deployment: process.env.AGENT_DEPLOYMENT_SHA!, @@ -145,20 +150,42 @@ const executionRef = canonicalCandidateDigest({ tools: process.env.AGENT_TOOLSET_SHA!, }) +const optimizerProfile = { + name: 'support-prompt-optimizer', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: process.env.OPTIMIZER_MODEL!, + metadata: { maxTokens: 16_384 }, + }, +} satisfies AgentProfile +const optimizerPricing = { + inputUsdPerMillion: Number(process.env.OPTIMIZER_INPUT_USD_PER_MILLION), + outputUsdPerMillion: Number(process.env.OPTIMIZER_OUTPUT_USD_PER_MILLION), +} const optimizer = { - model: process.env.OPTIMIZER_MODEL!, - baseUrl: process.env.OPTIMIZER_BASE_URL!, - apiKey: process.env.OPTIMIZER_API_KEY!, + model: optimizerProfile.model.default, + call: profileOptimizerModelCall({ + profile: optimizerProfile, + context: 'support-prompt optimizer', + executor: { + backend: 'router', + routerBaseUrl: process.env.OPTIMIZER_BASE_URL!, + routerKey: process.env.OPTIMIZER_API_KEY!, + }, + pricing: optimizerPricing, + }), + callRef: canonicalCandidateDigest({ + profile: canonicalAgentProfileDigest(optimizerProfile), + deployment: process.env.OPTIMIZER_DEPLOYMENT_SHA!, + }), budget: { maxCostUsd: 10, maxRequests: 50, maxRequestBytes: 2_000_000, maxResponseBytes: 2_000_000, maxOutputTokensPerRequest: 16_384, - pricing: { - inputUsdPerMillion: Number(process.env.OPTIMIZER_INPUT_USD_PER_MILLION), - outputUsdPerMillion: Number(process.env.OPTIMIZER_OUTPUT_USD_PER_MILLION), - }, + pricing: optimizerPricing, }, } @@ -207,7 +234,7 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.143.0" +python -m pip install "agent-eval-rpc==0.144.1" python -m pip install "gepa[full]==0.1.4" ``` @@ -221,14 +248,14 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.143.0" +python -m pip install "agent-eval-rpc==0.144.1" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` SkillOpt 0.2.0's published wheel omits prompt files required by `ReflACTTrainer`, so the tested SkillOpt source revision remains necessary. -SkillOpt and GEPA's standard reflection engine require `optimizer: { model, baseUrl, apiKey, budget }`. +SkillOpt and GEPA's standard reflection engine require `optimizer: { model, call, callRef, budget }`. Agent-based GEPA engines may own their model connection instead. -Agent Eval proxies those model calls, enforces the nested budget, and records their cost. +Runtime owns those model calls through one exact `AgentProfile`; Agent Eval enforces the nested budget and records their measured cost and execution evidence without receiving provider credentials. `costCeiling` is the total limit for optimizer calls, candidate runs, judges, and final scoring. Runtime returns `hold` when any part of that cost is unknown. Runtime rejects a reported total above the limit. @@ -414,13 +441,27 @@ Here, `runProductAgent` is the application's existing entry point, not another l ```ts import { - createPrimeIntellectBackend, + primeIntellectExecutorConfig, runPrimeIntellectProgram, } from '@tangle-network/agent-runtime/primeintellect' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, +} from '@tangle-network/agent-runtime/kernel' await runPrimeIntellectProgram(async (episode) => { - const backend = createPrimeIntellectBackend(episode) - return runProductAgent({ task: episode.task, backend }) + const profile = makeProductProfile({ model: episode.model.name }) + return collectAgentTurn( + streamAgentTurn( + { + kind: 'executor', + profile, + factory: createExecutor(primeIntellectExecutorConfig(episode)), + }, + episode.task.prompt, + ), + ) }) ``` diff --git a/bench/scripts/appworld_driver.py b/bench/scripts/appworld_driver.py index 2ca8eb38..1ad2bb1f 100644 --- a/bench/scripts/appworld_driver.py +++ b/bench/scripts/appworld_driver.py @@ -9,10 +9,7 @@ import argparse import json -import os -import re import sys -import time def fail(msg: str) -> None: @@ -20,169 +17,9 @@ def fail(msg: str) -> None: sys.exit(1) -_CODE_RE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL) - - -def _extract_code(text: str) -> str: - blocks = _CODE_RE.findall(text or "") - return (blocks[-1] if blocks else "").strip() - - -def _router_chat(base: str, key: str, model: str, messages: list, timeout: float = 180.0): - """One router chat-completion with retry on transient/429/5xx. Returns - (content, input_tokens, output_tokens). Raises on exhausted retries.""" - import httpx - - url = base.rstrip("/") + "/chat/completions" - last = None - for attempt in range(4): - try: - r = httpx.post( - url, - headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, - json={"model": model, "messages": messages}, - timeout=timeout, - ) - if r.status_code in (429, 500, 502, 503, 504): - last = f"{r.status_code}: {r.text[:160]}" - time.sleep(2**attempt) - continue - r.raise_for_status() - d = r.json() - content = (d["choices"][0]["message"].get("content") or "") - usage = d.get("usage") or {} - return content, int(usage.get("prompt_tokens", 0) or 0), int(usage.get("completion_tokens", 0) or 0) - except Exception as e: # noqa: BLE001 - last = str(e) - if attempt < 3: - time.sleep(2**attempt) - continue - raise RuntimeError(f"router_chat failed after retries: {last}") - raise RuntimeError(f"router_chat exhausted: {last}") - - -def _build_system(directive: str, world) -> str: - sup = world.task.supervisor - apps = list(getattr(world.task, "allowed_apps", []) or []) - descs = getattr(world.task, "app_descriptions", "") - desc_str = json.dumps(descs) if isinstance(descs, (dict, list)) else str(descs) - return ( - f"You are an AI agent completing a digital task for your supervisor " - f"{getattr(sup, 'first_name', '')} {getattr(sup, 'last_name', '')} " - f"(email {getattr(sup, 'email', '')}, phone {getattr(sup, 'phone_number', '')}) " - "by WRITING PYTHON that calls app APIs (the apis..(...) surface).\n\n" - f"Available apps: {', '.join(apps)}.\n" - f"App descriptions: {desc_str[:1500]}\n\n" - "How to work, one step per turn:\n" - "- Discover APIs with apis.api_docs.show_api_descriptions(app_name='') and " - "apis.api_docs.show_api_doc(app_name='', api_name='') BEFORE calling them.\n" - "- Get the supervisor's app passwords with apis.supervisor.show_account_passwords(), then log in " - "to each app you use to obtain its access_token.\n" - "- Write ONE short Python code block per turn. After it runs you SEE its OUTPUT (or error " - "traceback) — use that to decide the next step. Print intermediate values you need.\n" - "- Iterate: inspect -> authenticate -> act -> verify. Do not guess API names or arguments.\n" - "- When the task is fully done call apis.supervisor.complete_task(answer=) (include the " - "answer if the task asks a question, otherwise apis.supervisor.complete_task()).\n" - "- Reply with EXACTLY ONE fenced ```python block per turn and nothing else.\n\n" - f"{directive}" - ) - - -def cmd_react(args) -> None: - """Multi-turn REPL agent: the model writes a python block, the engine executes - it in the PERSISTENT world, the output is fed back, and it iterates until it - completes the task or hits max-turns. Then AppWorld's own evaluator scores it. - Config (directive, model, router creds, max_turns) arrives as JSON on stdin so - the candidate directive can be arbitrarily long. The directive is the optimized - surface; the loop + contract are fixed.""" - cfg = {} - raw = sys.stdin.read() - if raw.strip(): - try: - cfg = json.loads(raw) - except Exception as e: # noqa: BLE001 - fail(f"react config JSON parse failed: {e}") - directive = str(cfg.get("directive", "")) - model = str(cfg.get("model", "gpt-4o")) - max_turns = int(cfg.get("max_turns", 8)) - router_base = str(cfg.get("router_base", "https://router.tangle.tools/v1")) - router_key = str(cfg.get("router_key") or os.environ.get("TANGLE_API_KEY", "")) - if not router_key: - fail("react: router_key/TANGLE_API_KEY required") - - try: - from appworld import AppWorld - except Exception as e: # noqa: BLE001 - fail(f"appworld import failed: {e}") - - in_tok = 0 - out_tok = 0 - turns = 0 - turns_log: list = [] - try: - with AppWorld( - task_id=args.task_id, - experiment_name="bench-react", - raise_on_failure=False, - ) as world: - messages = [ - {"role": "system", "content": _build_system(directive, world)}, - {"role": "user", "content": f"Task: {world.task.instruction}"}, - ] - for turn in range(max_turns): - turns = turn + 1 - content, ui, uo = _router_chat(router_base, router_key, model, messages) - in_tok += ui - out_tok += uo - code = _extract_code(content) - messages.append({"role": "assistant", "content": content}) - if not code: - messages.append({ - "role": "user", - "content": "Reply with exactly one ```python block that makes progress, " - "or call apis.supervisor.complete_task().", - }) - continue - output = world.execute(code) - turns_log.append({"code": code[:600], "output": str(output)[:600]}) - messages.append({"role": "user", "content": "OUTPUT:\n" + str(output)[:4000]}) - if world.task_completed(): - break - evaluation = world.evaluate().to_dict() - except Exception as e: # noqa: BLE001 - fail(f"react of {args.task_id} failed: {e}") - - if "success" not in evaluation or "num_tests" not in evaluation: - fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}") - passes = evaluation.get("passes", []) - failures = evaluation.get("failures", []) - n_pass = len(passes) if isinstance(passes, list) else int(passes or 0) - n_fail = len(failures) if isinstance(failures, list) else int(failures or 0) - print( - json.dumps( - { - "success": bool(evaluation["success"]), - "passes": n_pass, - "fails": n_fail, - "num_tests": int(evaluation["num_tests"]), - # Failed sub-test names — the evidence a trace analyst steers on. - "failure_names": [str(f)[:160] for f in failures][:8] - if isinstance(failures, list) - else [], - "turns": turns, - "input_tokens": in_tok, - "output_tokens": out_tok, - "transcript": "\n---\n".join( - f"CODE:\n{t['code']}\nOUTPUT:\n{t['output']}" for t in turns_log[-3:] - )[:1600], - } - ) - ) - - def cmd_session(args) -> None: """Dumb world shim: a persistent AppWorld session driven over stdin JSONL. - NO LLM calls here — the agent loop lives in the runtime (routerToolLoop); + NO LLM calls here — the profile-bound agent loop lives in Runtime; this process only owns world state. One JSON object per line, both ways: {"op":"execute","code":"..."} -> {"output":"...","task_completed":bool} {"op":"evaluate"} -> the evaluate verdict JSON (+failure_names) @@ -336,10 +173,6 @@ def main() -> None: p_eval.add_argument("--task-id", required=True) p_eval.add_argument("--split", required=True) - p_react = sub.add_parser("react") - p_react.add_argument("--task-id", required=True) - p_react.add_argument("--split", required=True) - p_session = sub.add_parser("session") p_session.add_argument("--task-id", required=True) p_session.add_argument("--split", required=True) @@ -349,8 +182,6 @@ def main() -> None: cmd_load(args) elif args.cmd == "evaluate": cmd_evaluate(args) - elif args.cmd == "react": - cmd_react(args) elif args.cmd == "session": cmd_session(args) diff --git a/bench/scripts/trata-hedge/README.md b/bench/scripts/trata-hedge/README.md index 9eb25e30..8c2794a0 100644 --- a/bench/scripts/trata-hedge/README.md +++ b/bench/scripts/trata-hedge/README.md @@ -21,8 +21,8 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac ## Status (2026-06-06): pipeline PROVEN end-to-end -Our solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link -works. The naive **single-shot** baseline (gpt-4o, ~3 of N corpus files in one context +Our Runtime-backed solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link +works. The naive **single-shot** baseline (DeepSeek V4 Flash, ~3 of N corpus files in one context window) scores **0/4** — a floor: it hit only 1/3 moves on a few themes with hallucinations flagged, because it could not explore the full corpus. The bench is built for **agentic** exploration; a fair baseline needs our sandbox runtime as the solver @@ -35,7 +35,7 @@ for **agentic** exploration; a fair baseline needs our sandbox runtime as the so git clone https://github.com/Trata-Inc/trata-hedge-bench /tmp/thb dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \ - bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/ gpt-4o + bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/ deepseek-v4-flash ``` ## Gotchas (each cost a debugging cycle) diff --git a/bench/scripts/trata-hedge/run.sh b/bench/scripts/trata-hedge/run.sh index 7fa5bc05..cbb60be9 100755 --- a/bench/scripts/trata-hedge/run.sh +++ b/bench/scripts/trata-hedge/run.sh @@ -17,14 +17,18 @@ set -euo pipefail ENV="${1:?usage: run.sh [model]}" -MODEL="${2:-${WORKER_MODEL:-gpt-4o}}" +MODEL="${2:-${WORKER_MODEL:-}}" +if [[ -z "$MODEL" ]]; then + echo "WORKER_MODEL is required (or pass it as argument 2)" >&2 + exit 2 +fi HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ANS=/tmp/thb-answer.txt REWARD=/tmp/thb-reward.txt DETAILS=/tmp/thb-details.json echo "[trata] solve: $ENV (model=$MODEL)" -WORKER_MODEL="$MODEL" python3 "$HERE/solve.py" "$ENV" "$ANS" +WORKER_MODEL="$MODEL" pnpm --dir "$HERE/../../.." exec tsx "$HERE/../../src/trata-hedge-solve.mts" "$ENV" "$ANS" sudo -n mkdir -p /app && sudo -n cp "$ANS" /app/answer.txt && sudo -n chmod 644 /app/answer.txt diff --git a/bench/scripts/trata-hedge/solve.py b/bench/scripts/trata-hedge/solve.py deleted file mode 100644 index 71454ec0..00000000 --- a/bench/scripts/trata-hedge/solve.py +++ /dev/null @@ -1,83 +0,0 @@ -"""trata-hedge-bench solver — our system's analyst, graded by THEIR Gemini judge. - -Reads a task's instruction.md + data corpus, writes a cited analysis to answer.txt. -The corpus is large (~5MB/task), so a single-shot router call is a LOWER-BOUND -baseline (it sees only what fits in context) — the bench is built for agentic -exploration, which our sandbox runtime provides (see README; pending sandbox health). - -Notes learned the hard way: -- router.tangle.tools is behind Cloudflare bot-fight: a default Python-urllib - User-Agent gets 403 (CF 1010) on large bodies → send a browser UA. -- gpt-4.1 is NOT keyed for direct router calls (403); gpt-4o / claude-sonnet-4-6 / - gpt-4o-mini are. Very large bodies (~350KB) can 503 — keep DATA_BUDGET in range. - - WORKER_MODEL=gpt-4o DATA_BUDGET=160000 python3 solve.py -""" -import json -import os -import sys -import urllib.request -from pathlib import Path - -env = Path(sys.argv[1]) -out = sys.argv[2] if len(sys.argv) > 2 else "/tmp/thb-answer.txt" -model = os.environ.get("WORKER_MODEL", "gpt-4o") -budget = int(os.environ.get("DATA_BUDGET", "160000")) -instruction = (env / "instruction.md").read_text() -data_dir = env / "environment" / "data" - - -def rank(p: Path) -> int: - s = str(p) - return (0 if "earnings_call" in s else 1 if "financials" in s else 2 if "company_profiles" in s - else 3 if "press_releases" in s else 4) - - -files = sorted((p for p in data_dir.rglob("*") if p.is_file()), key=rank) -blocks, used, cited = [], 0, [] -for p in files: - rel = p.relative_to(data_dir) - try: - c = p.read_text(errors="replace") - except Exception: - continue - if used + len(c) > budget: - continue - used += len(c) - cited.append(str(rel)) - blocks.append(f"\n=== FILE: data/{rel} ===\n{c}") -print(f"[solve] {len(cited)}/{len(files)} files in context ({used} chars)", file=sys.stderr) - -prompt = ( - instruction - + "\n\n--- AVAILABLE DATA (cite files by their `data/` name inline) ---\n" - + "".join(blocks) - + "\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/`." -) - -key = os.environ["TANGLE_API_KEY"] -base = os.environ.get("ROUTER_BASE", "https://router.tangle.tools/v1") -body = json.dumps({ - "model": model, - "messages": [{"role": "user", "content": prompt}], - "temperature": float(os.environ.get("TEMPERATURE", "0.5")), - "max_tokens": int(os.environ.get("MAX_TOKENS", "6000")), -}).encode() -req = urllib.request.Request( - f"{base}/chat/completions", data=body, - headers={ - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - # browser UA: router is behind Cloudflare bot-fight, default urllib UA → 403 on big bodies - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36", - }, -) -try: - with urllib.request.urlopen(req, timeout=300) as r: - resp = json.load(r) -except urllib.error.HTTPError as e: - print(f"[solve] HTTP {e.code}: {e.read().decode(errors='replace')[:400]}", file=sys.stderr) - raise -answer = resp["choices"][0]["message"]["content"] -Path(out).write_text(answer) -print(f"[solve] wrote {len(answer)} chars -> {out} (model={model})", file=sys.stderr) diff --git a/bench/src/aec-gate.mts b/bench/src/aec-gate.mts index c5f0952c..aae061ab 100644 --- a/bench/src/aec-gate.mts +++ b/bench/src/aec-gate.mts @@ -22,7 +22,13 @@ import { resolveAdapter } from './adapters' import type { BenchmarkAdapter, BenchTask } from './benchmarks/types' import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus' import { composeStrategies } from './directives' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { + benchProfileModel, + benchRouterProfile, + type BenchRouterTarget, + runBenchRouterTurn, + withBenchProfile, +} from './router-turn' import { pool } from './stats.mts' function must(name: string): string { @@ -52,7 +58,7 @@ interface AttemptOutcome { } async function runAttempt( - cfg: RouterConfig, + cfg: BenchRouterTarget, adapter: BenchmarkAdapter, task: BenchTask, prompt: string, @@ -65,8 +71,15 @@ async function runAttempt( let lastErr: unknown for (let attempt = 0; attempt < 3; attempt += 1) { try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: prompt }]) - const content = typeof res.content === 'string' ? res.content : '' + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: withBenchProfile(cfg.profile, { name: 'aec-worker' }), + }, + prompt, + ) + const content = res.finalText const verdict = await adapter.judge(task, content) return { prompt, @@ -74,8 +87,10 @@ async function runAttempt( score: verdict.score, resolved: verdict.resolved, wallMs: Date.now() - startedAt, - ...(res.costUsd !== undefined ? { costUsd: res.costUsd } : {}), - ...(res.usage ? { tokensIn: res.usage.input, tokensOut: res.usage.output } : {}), + ...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}), + ...(res.usage.tokensKnown === false + ? {} + : { tokensIn: res.usage.input, tokensOut: res.usage.output }), } } catch (err) { lastErr = err @@ -115,7 +130,7 @@ interface ArmResult { async function runArm( arm: ArmSpec, - cfg: RouterConfig, + cfg: BenchRouterTarget, adapter: BenchmarkAdapter, tasks: BenchTask[], k: number, @@ -144,7 +159,7 @@ async function runArm( benchmark: adapter.name, instanceId: task.id, condition: arm.condition, - model: cfg.model, + model: benchProfileModel(cfg.profile), // k-attempt outcome = any usable attempt resolved (the oracle@k ceiling for // this run; the deployable selector is scored separately by corpus-replay). resolved: taskOutcomes.some((o) => o.resolved), @@ -175,7 +190,13 @@ async function main(): Promise { if (!Number.isFinite(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`) if (!Number.isFinite(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`) - const cfg: RouterConfig = { routerBaseUrl, routerKey, model } + const cfg: BenchRouterTarget = { + routerBaseUrl, + routerKey, + profile: benchRouterProfile('aec-worker', model, { + maxRetries: Number(process.env.MAX_RETRIES ?? 2), + }), + } const bench = process.env.BENCH ?? 'aec-bench' const adapter = resolveAdapter(bench) diff --git a/bench/src/agent-graphs-gen2.mts b/bench/src/agent-graphs-gen2.mts index 6f1840aa..3e1f674c 100644 --- a/bench/src/agent-graphs-gen2.mts +++ b/bench/src/agent-graphs-gen2.mts @@ -47,6 +47,7 @@ import { import { type AuthoredArtifact, type CaseSpec, + buildAgentGraphsAuthorProfile, callAuthor, dispatchWithSurface, judgeArtifact, @@ -215,6 +216,16 @@ function validateSkillGate(text: string): string[] { } function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfaceProposer { + const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, { + ...process.env, + AGENT_GRAPHS_AUTHOR_PROFILE_NAME: 'agent-graphs-skill-reviser', + AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT: + 'Revise an agent skill from measured development-case failures. Return only the requested artifact.', + }) + const attemptLimit = Number(process.env.AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS ?? 2) + if (!Number.isSafeInteger(attemptLimit) || attemptLimit <= 0) { + throw new Error('AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS must be a positive integer') + } return { kind: 'agent-graphs-skill-reviser', async propose(_ctx: ProposeContext): Promise { @@ -245,8 +256,8 @@ function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfacePr } let prompt = revisionPrompt let lastProblems: string[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - const reply = await callAuthor(prompt, 0.7, 12_000) + for (let attempt = 0; attempt < attemptLimit; attempt += 1) { + const reply = await callAuthor(proposerProfile, prompt) const skill = extractSkill(reply) lastProblems = validateSkillGate(skill) if (lastProblems.length === 0) { diff --git a/bench/src/agent-graphs-gen3.mts b/bench/src/agent-graphs-gen3.mts index 60dfa39e..5543c35f 100644 --- a/bench/src/agent-graphs-gen3.mts +++ b/bench/src/agent-graphs-gen3.mts @@ -46,6 +46,7 @@ import { import { type AuthoredArtifact, type CaseSpec, + buildAgentGraphsAuthorProfile, callAuthor, dispatchWithSurface, judgeArtifact, @@ -337,7 +338,7 @@ function makeProposer(v2Surface: string, trainCases: GraphScenario[]): SurfacePr let prompt = revisionPrompt let lastProblems: string[] = [] for (let attempt = 0; attempt < 2; attempt += 1) { - const reply = await callAuthor(prompt, 0.7, 12_000) + const reply = await callAuthor(buildAgentGraphsAuthorProfile(v2Surface), prompt) const skill = extractSkill(reply) lastProblems = validateSkillGate(skill) if (lastProblems.length === 0) { diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index 3d42a560..bb48892d 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -20,13 +20,19 @@ import { execFileSync } from 'node:child_process' import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' -import { homedir } from 'node:os' import { join, dirname, resolve as resolvePath } from 'node:path' import { fileURLToPath } from 'node:url' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + defineInlineResource, + harnessTypeSchema, + reasoningEffortSchema, + type AgentProfile, +} from '@tangle-network/agent-interface' import { type AgentGraph, type AnalystRegistry, + collectAgentTurn, + createExecutor, defaultEdgeTraversalCap, type EdgeTraversal, GraphEdgeCapError, @@ -34,6 +40,7 @@ import { promptHandle, type RunGraphOptions, runGraph, + streamAgentTurn, } from '../../src/runtime/index.ts' import { leafSeam, scriptedBrain, type ScriptedTurn } from './agent-graphs-improve/offline-seams.mts' @@ -146,22 +153,16 @@ export interface AuthoredArtifact { // ── The author model call ────────────────────────────────────────────────────── -const ROUTER_URL = 'https://router.tangle.tools/v1/chat/completions' -const AUTHOR_MODEL = 'glm-5.2' - -function routerToken(): string { - const raw = readFileSync(join(homedir(), '.config', 'tangle', 'router-token.json'), 'utf8') - return (JSON.parse(raw) as { token: string }).token +function positiveInteger(name: string, raw: string | undefined, fallback: number): number { + const value = raw === undefined ? fallback : Number(raw) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return value } -function authorPrompt(surface: string, kase: CaseSpec): string { +function authorPrompt(kase: CaseSpec): string { return [ - 'You are an agent-graph author. Follow the skill below EXACTLY — it is your only doctrine.', - '', - '', - surface, - '', - '', ``, kase.brief, '', @@ -190,25 +191,61 @@ function extractJson(text: string): string { return stripped.slice(start, end + 1) } -export async function callAuthor(prompt: string, temperature = 0.2, maxTokens = 6000): Promise { - const res = await fetch(ROUTER_URL, { - method: 'POST', - headers: { Authorization: `Bearer ${routerToken()}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: AUTHOR_MODEL, - temperature, - max_tokens: maxTokens, - messages: [{ role: 'user', content: prompt }], - }), - signal: AbortSignal.timeout(240_000), +export function buildAgentGraphsAuthorProfile( + surface: string, + env: NodeJS.ProcessEnv = process.env, +): AgentProfile { + return { + name: env.AGENT_GRAPHS_AUTHOR_PROFILE_NAME ?? 'agent-graphs-author', + harness: harnessTypeSchema.parse(env.AGENT_GRAPHS_AUTHOR_HARNESS ?? 'pi'), + model: { + provider: env.AGENT_GRAPHS_AUTHOR_PROVIDER ?? 'tangle-router', + default: env.AGENT_GRAPHS_AUTHOR_MODEL ?? 'deepseek-v4-flash', + reasoningEffort: reasoningEffortSchema.parse( + env.AGENT_GRAPHS_AUTHOR_REASONING_EFFORT ?? 'ultracode', + ), + }, + prompt: { + systemPrompt: + env.AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT ?? + 'Apply the attached agent-graphs skill exactly. Return only the requested artifact.', + }, + resources: { + failOnError: true, + skills: [defineInlineResource('agent-graphs', surface)], + }, + } +} + +export async function callAuthor( + profile: AgentProfile, + prompt: string, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const bridgeBearer = env.AGENT_GRAPHS_BRIDGE_BEARER ?? env.BRIDGE_BEARER + if (!bridgeBearer) throw new Error('AGENT_GRAPHS_BRIDGE_BEARER or BRIDGE_BEARER is required') + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: env.AGENT_GRAPHS_BRIDGE_URL ?? env.BRIDGE_URL ?? 'http://127.0.0.1:3355', + bridgeBearer, }) - if (!res.ok) throw new Error(`router HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) - const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = data.choices?.[0]?.message?.content - if (typeof content !== 'string' || content.trim().length === 0) { - throw new Error('router returned empty content') + const timeoutRaw = env.AGENT_GRAPHS_AUTHOR_TIMEOUT_MS + const timeoutMs = + timeoutRaw === undefined + ? undefined + : positiveInteger('AGENT_GRAPHS_AUTHOR_TIMEOUT_MS', timeoutRaw, 1) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile, agentRunName: profile.name ?? 'agent-graphs-author' }, + prompt, + timeoutMs === undefined ? {} : { timeoutMs }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `author turn ended with status ${turn.status}`) } - return content + if (!turn.finalText.trim()) throw new Error('author returned empty content') + return turn.finalText } interface AuthoredReply { @@ -220,11 +257,17 @@ interface AuthoredReply { /** Prompt the author; one retry on unparseable JSON (or a transport fault). */ async function authorOnce(surface: string, kase: CaseSpec): Promise { - const prompt = authorPrompt(surface, kase) + const prompt = authorPrompt(kase) + const profile = buildAgentGraphsAuthorProfile(surface) + const attempts = positiveInteger( + 'AGENT_GRAPHS_AUTHOR_ATTEMPTS', + process.env.AGENT_GRAPHS_AUTHOR_ATTEMPTS, + 2, + ) let lastErr: unknown - for (let attempt = 0; attempt < 2; attempt += 1) { + for (let attempt = 0; attempt < attempts; attempt += 1) { try { - const raw = await callAuthor(prompt) + const raw = await callAuthor(profile, prompt) const parsed = JSON.parse(extractJson(raw)) as { decision?: string reason?: string @@ -244,7 +287,9 @@ async function authorOnce(surface: string, kase: CaseSpec): Promise((_, reject) => { - const t = setTimeout(() => reject(new Error('offline run timed out (120s)')), 120_000) + const t = setTimeout( + () => reject(new Error(`offline run timed out (${timeoutMs}ms)`)), + timeoutMs, + ) t.unref?.() }) const res: GraphResult = await Promise.race([runGraph(graph, opts), timeout]) @@ -617,8 +679,16 @@ async function main(): Promise { const only = process.env.CASE const cases: CaseSpec[] = inputs.cases.filter((c) => only === undefined || c.id === only) + const authorProfile = buildAgentGraphsAuthorProfile(surface) + const authorLabel = [ + authorProfile.harness, + authorProfile.model?.provider, + authorProfile.model?.default, + ] + .filter(Boolean) + .join('/') console.log( - `codemode baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${AUTHOR_MODEL}`, + `codemode baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${authorLabel}`, ) const results: CaseResult[] = [] @@ -676,8 +746,7 @@ async function main(): Promise { const out = { skillVersion: 'v1', surfaceSource: inputs.source, - authorModel: AUTHOR_MODEL, - temperature: 0.2, + authorProfile, date: new Date().toISOString(), n: results.length, aggregate: { mean, median, min: scores[0] ?? 0, max: scores[scores.length - 1] ?? 0 }, diff --git a/bench/src/atom-humaneval.mts b/bench/src/atom-humaneval.mts index 36ec5719..8c1ff27b 100644 --- a/bench/src/atom-humaneval.mts +++ b/bench/src/atom-humaneval.mts @@ -21,20 +21,23 @@ import { type AgentProfile, type AgentSpec, contentAddress, - type DriverAgentOptions, - driverAgent, + createExecutor, createExecutorRegistry, createSupervisor, - type Executor, - type ExecutorResult, gateOnDeliverable, InMemoryResultBlobStore, InMemorySpawnJournal, - type RouterConfig, - routerBrain, - routerChatWithUsage, + mapExecutorResult, + supervisorAgent, } from '../../src/runtime/index' import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' +import { + benchProfileModel, + benchRouterProfile, + type BenchRouterTarget, + runBenchRouterTurn, + withBenchProfile, +} from './router-turn' function must(k: string): string { const v = process.env[k] @@ -47,45 +50,48 @@ const K = Number(process.env.K ?? 3) const OFFSET = Number(process.env.OFFSET ?? 0) const WORKER_TEMP = Number(process.env.WORKER_TEMP ?? 0.7) -const cfg: RouterConfig = { +const cfg: BenchRouterTarget = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), - model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + profile: benchRouterProfile( + 'humaneval-worker', + process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + { temperature: WORKER_TEMP }, + ), +} +const driverCfg: BenchRouterTarget = { + ...cfg, + profile: benchRouterProfile( + 'humaneval-driver', + process.env.DRIVER_MODEL ?? benchProfileModel(cfg.profile), + { maxTurns: K + 4 }, + ), } -const driverCfg: RouterConfig = { ...cfg, model: process.env.DRIVER_MODEL ?? cfg.model } - -// The driver-LLM brain is the SHARED `routerBrain` (the canonical ToolLoopChat seam) — it forwards -// usage/costUsd, so this bench's driver arms meter their own inference into the conserved pool. // ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ── function humanEvalWorker(task: HumanEvalTask, label: string): Agent { - let artifact: ExecutorResult | undefined - const inner: Executor = { - runtime: 'router', - async execute(_t, signal) { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], { - temperature: WORKER_TEMP, - ...(signal ? { signal } : {}), - }) - const code = extractCode(res.content) - artifact = { - outRef: contentAddress(code), - out: code, - spent: { iterations: 1, tokens: res.usage ?? { input: 0, output: 0 }, usd: res.costUsd ?? 0, ms: 0 }, - } - return artifact - }, - teardown: () => Promise.resolve({ destroyed: true }), - resultArtifact: () => { - if (!artifact) throw new Error('resultArtifact read before execute') - return artifact - }, - } - const gated = gateOnDeliverable(inner, { - check: async (out) => (await runChecker(task, String(out))).pass === 1, - describe: `${task.taskId}: the provided test suite passes`, + const profile: AgentProfile = withBenchProfile(cfg.profile, { + name: label, + systemPrompt: basePrompt(task), }) - const spec: AgentSpec = { profile: { name: label } as AgentProfile, harness: null, executor: gated } + const routerFactory = createExecutor({ + backend: 'router', + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + }) + const executorFactory = (spec: AgentSpec, ctx: Parameters[1]) => { + const inner = routerFactory(spec, ctx) + const mapped = mapExecutorResult(inner, (result) => { + const raw = result.out as { content?: unknown } + const code = extractCode(typeof raw?.content === 'string' ? raw.content : '') + return { outRef: contentAddress(code), out: code } + }) + return gateOnDeliverable(mapped, { + check: async (out) => (await runChecker(task, String(out))).pass === 1, + describe: `${task.taskId}: the provided test suite passes`, + }) + } + const spec: AgentSpec = { profile, harness: null, executorFactory } return { name: label, act: async () => '', executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -113,16 +119,21 @@ async function driveTask( spawns += 1 return w } - const opts: DriverAgentOptions = { - name: `drv-${task.taskId}`, - brain: routerBrain(driverCfg), - blobs, - makeWorkerAgent: makeWorker, - perWorker: { maxIterations: 2, maxTokens: 4000 }, - systemPrompt: driverSystem, - maxTurns: K + 4, - } - const root = driverAgent(opts) + const root = supervisorAgent( + withBenchProfile(driverCfg.profile, { + name: `drv-${task.taskId}`, + systemPrompt: driverSystem, + }), + { + router: { + routerBaseUrl: driverCfg.routerBaseUrl, + routerKey: driverCfg.routerKey, + }, + blobs, + makeWorkerAgent: makeWorker, + perWorker: { maxIterations: 2, maxTokens: 4000 }, + }, + ) const runId = `he-${task.taskId.replace('/', '-')}` const result = await createSupervisor().run(root, basePrompt(task), { budget: { maxIterations: 100, maxTokens: 400_000 }, @@ -145,21 +156,31 @@ async function blindTask(task: HumanEvalTask): Promise { for (let i = 0; i < K; i += 1) { // A transient router error is a FAILED attempt, not a crash — the driver arm already types // an executor throw into a `down` settlement, so the blind arm must match (fair comparison). - let res: { content: string } + let content = '' try { - res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], { - temperature: WORKER_TEMP, - }) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: withBenchProfile(cfg.profile, { + name: 'humaneval-blind-atom-worker', + }), + }, + basePrompt(task), + ) + content = res.finalText } catch { continue } - if ((await runChecker(task, extractCode(res.content))).pass === 1) return true + if ((await runChecker(task, extractCode(content))).pass === 1) return true } return false } async function main(): Promise { - console.log(`atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${cfg.model} driver=${driverCfg.model}`) + console.log( + `atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${benchProfileModel(cfg.profile)} driver=${benchProfileModel(driverCfg.profile)}`, + ) const tasks = await loadHumanEval(N, OFFSET) const outcomes: TaskOutcome[] = [] for (const task of tasks) { diff --git a/bench/src/atom-mcp-e2e.mts b/bench/src/atom-mcp-e2e.mts index 0c588607..d14b9a38 100644 --- a/bench/src/atom-mcp-e2e.mts +++ b/bench/src/atom-mcp-e2e.mts @@ -21,7 +21,9 @@ import { type Agent, type AgentProfile, type AgentSpec, + collectAgentTurn, contentAddress, + createExecutor, createExecutorRegistry, createSupervisor, type Executor, @@ -31,6 +33,7 @@ import { InMemorySpawnJournal, runInWorkspace, type Scope, + streamAgentTurn, type Workspace, } from '../../src/runtime/index' import { asAuthoredProfile } from '../../src/runtime/supervise/authoring' @@ -83,19 +86,30 @@ async function bridgeChat(opts: { cwd?: string mcpUrl?: string }): Promise { - const r = await fetch(`${BRIDGE}/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model: MODEL, - messages: opts.messages, - ...(opts.cwd ? { cwd: opts.cwd } : {}), - ...(opts.mcpUrl ? { mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } } } : {}), - }), + if (!BEARER) throw new Error('TANGLE_API_KEY is required') + const profile: AgentProfile = { + name: opts.mcpUrl ? 'atom-mcp-supervisor-turn' : 'atom-mcp-worker-turn', + model: { default: MODEL }, + ...(opts.mcpUrl + ? { mcp: { coordination: { transport: 'http', url: opts.mcpUrl } } } + : {}), + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: BRIDGE.replace(/\/v1$/u, ''), + bridgeBearer: BEARER, + ...(opts.cwd ? { cwd: opts.cwd } : {}), }) - if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})` - const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + opts.messages.map((message) => message.content).join('\n\n'), + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`) + } + return turn.finalText } const transcripts: Array<{ who: string; said: string; delivered?: boolean }> = [] @@ -109,7 +123,7 @@ function makeWorker(rawProfile: unknown, ws: Workspace, n: number): Agent = { runtime: 'router', async execute() { - const sys = p?.systemPrompt ?? TASK + const sys = p?.prompt.systemPrompt ?? TASK const run = await runInWorkspace( ws, async (cwd) => { diff --git a/bench/src/benchmarks/appworld.test.mts b/bench/src/benchmarks/appworld.test.mts index 685a1ece..c5fd93bc 100644 --- a/bench/src/benchmarks/appworld.test.mts +++ b/bench/src/benchmarks/appworld.test.mts @@ -8,7 +8,12 @@ */ import assert from 'node:assert/strict' import { test } from 'node:test' -import { appworldSolutionOutput, createAppWorldAdapter } from './appworld' +import { + appworldReactResultWithUsage, + appworldReactUsageEvent, + appworldSolutionOutput, + createAppWorldAdapter, +} from './appworld' type Events = Parameters[0] const stream = (text: string): Events => [{ data: { finalText: text } }] as unknown as Events @@ -49,3 +54,30 @@ test('loadTasks either enumerates live engine rows or FAILS LOUD without fabrica assert.match((err as Error).message, /appworld driver failed|appworld import failed/) } }) + +test('successful react episode survives unknown catalog dollars without fabricating billed cost', () => { + const result = appworldReactResultWithUsage( + { success: true, passes: 3, fails: 0, num_tests: 3 }, + { + input: 120, + output: 30, + tokensKnown: true, + costUsd: 0.0042, + usdKnown: false, + }, + 2, + 'completed task', + ) + + assert.equal(result.success, true) + assert.equal(result.input_tokens, 120) + assert.equal(result.output_tokens, 30) + assert.equal(result.cost_usd, undefined) + const event = appworldReactUsageEvent(result, 'deepseek-v4-flash') + assert.deepEqual(event?.data, { + model: 'deepseek-v4-flash', + tokensIn: 120, + tokensOut: 30, + }) + assert.equal(Object.hasOwn(event?.data ?? {}, 'costUsd'), false) +}) diff --git a/bench/src/benchmarks/appworld.ts b/bench/src/benchmarks/appworld.ts index c30ead86..dfa25bef 100644 --- a/bench/src/benchmarks/appworld.ts +++ b/bench/src/benchmarks/appworld.ts @@ -22,7 +22,13 @@ import { spawn } from 'node:child_process' import { join } from 'node:path' import { createInterface } from 'node:readline' -import { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/kernel' +import { + collectAgentTurn, + createExecutor, + type OutputAdapter, + streamAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -171,7 +177,7 @@ export function createAppWorldAdapter(): BenchmarkAdapter { /** * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is - * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool — + * Runtime's profile-bound `router-tools` executor with one tool — * `execute_python` — bound to a persistent AppWorld world session. The driver's * `session` subcommand is a dumb world shim (stdin JSONL: execute → output, * evaluate → verdict); every inference turn, the metering, and the typed @@ -190,7 +196,7 @@ export function createAppWorldAdapter(): BenchmarkAdapter { * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes. */ -interface ReactResult { +export interface ReactResult { success?: boolean passes?: number fails?: number @@ -199,9 +205,61 @@ interface ReactResult { turns?: number input_tokens?: number output_tokens?: number + cost_usd?: number transcript?: string } +interface ReactRuntimeUsage { + input: number + output: number + costUsd?: number + tokensKnown?: boolean + usdKnown?: boolean +} + +/** Preserve a completed scientific/task result even when one accounting dimension is incomplete. + * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without + * discarding the episode's task evidence. */ +export function appworldReactResultWithUsage( + verdict: ReactResult, + usage: ReactRuntimeUsage, + turns: number | undefined, + transcript: string, +): ReactResult { + return { + ...verdict, + ...(turns !== undefined ? { turns } : {}), + ...(usage.tokensKnown === false + ? {} + : { input_tokens: usage.input, output_tokens: usage.output }), + ...(usage.usdKnown === false || usage.costUsd === undefined + ? {} + : { cost_usd: usage.costUsd }), + transcript, + } +} + +/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */ +export function appworldReactUsageEvent( + result: ReactResult, + model: string, +): { type: 'llm_call'; data: Record } | undefined { + const hasTokens = + typeof result.input_tokens === 'number' && typeof result.output_tokens === 'number' + const hasCost = typeof result.cost_usd === 'number' + if (!hasTokens && !hasCost) return undefined + return { + type: 'llm_call', + data: { + model, + ...(hasTokens + ? { tokensIn: result.input_tokens, tokensOut: result.output_tokens } + : {}), + ...(hasCost ? { costUsd: result.cost_usd } : {}), + }, + } +} + const REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/ const SESSION_SYSTEM = [ @@ -285,7 +343,7 @@ async function withWorldSession( } } -/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */ +/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */ export function appworldToolLoopClient(cfg: { model: string routerBaseUrl: string @@ -310,40 +368,61 @@ export function appworldToolLoopClient(cfg: { const directive = prompt.replace(REACT_HEADER, '').trim() const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => { const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM - const loop = await routerToolLoop( - { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model }, - system, - `Task: ${instruction}`, - [EXECUTE_TOOL], - async (name, args) => { + const transcriptSteps: Array<{ args: string; result: string }> = [] + const profile = { + name: 'appworld-react-worker', + harness: 'cli-base' as const, + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { maxTurns }, + }, + prompt: { systemPrompt: system }, + tools: { execute_python: true }, + } + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + tools: [EXECUTE_TOOL], + executeToolCall: async (name, args) => { if (name !== 'execute_python') return `error: unknown tool ${name}` const res = await call({ op: 'execute', code: String(args.code ?? '') }) const done = res.task_completed === true - return `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}` + const result = `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}` + transcriptSteps.push({ args: JSON.stringify(args), result }) + return result }, - { maxTurns }, + }) + const loop = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + `Task: ${instruction}`, + ), ) + if (loop.status !== 'completed') { + throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`) + } const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult - const transcript = loop.toolTrace + const transcript = transcriptSteps .slice(-3) .map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`) .join('\n---\n') .slice(0, 1600) - return { - ...verdict, - turns: loop.turns, - input_tokens: loop.usage.input, - output_tokens: loop.usage.output, + const finalEvent = loop.events.at(-1) + const resultMetadata = + finalEvent?.type === 'final' && finalEvent.metadata?.result + ? (finalEvent.metadata.result as { spent?: { iterations?: number } }) + : undefined + return appworldReactResultWithUsage( + verdict, + loop.usage, + resultMetadata?.spent?.iterations, transcript, - } satisfies ReactResult + ) }) - // Real usage from the episode — flat llm_call so the kernel meters it. - if (out.input_tokens || out.output_tokens) { - yield { - type: 'llm_call', - data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model }, - } - } + const usageEvent = appworldReactUsageEvent(out, cfg.model) + if (usageEvent) yield usageEvent yield { type: 'result', data: { finalText: JSON.stringify(out) } } }, async delete() {}, diff --git a/bench/src/benchmarks/cadbench.ts b/bench/src/benchmarks/cadbench.ts index f6665623..21653dd5 100644 --- a/bench/src/benchmarks/cadbench.ts +++ b/bench/src/benchmarks/cadbench.ts @@ -14,6 +14,7 @@ import { readFile } from 'node:fs/promises' import type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types' import { renderBpy } from '../worker-blender' +import { runBenchRouterTurn } from '../router-turn' interface CadBenchMeta { name: string @@ -48,14 +49,26 @@ async function judgeCriteria( `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\n\nCRITERIA:\n${numbered}\n\nSCRIPT:\n\`\`\`python\n${script.slice(0, 6000)}\n\`\`\`` const content: unknown[] = [{ type: 'text', text }] for (const url of renders) content.push({ type: 'image_url', image_url: { url } }) - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }), - }) - if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`) - const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const raw = data.choices?.[0]?.message?.content ?? '' + const turn = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile: { + name: 'cadbench-vision-judge', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { + temperature: 0, + maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500), + }, + }, + }, + }, + { messages: [{ role: 'user', content }] }, + ) + const raw = turn.finalText const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw) if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` } let arr: unknown diff --git a/bench/src/benchmarks/finresearchbench.ts b/bench/src/benchmarks/finresearchbench.ts index b739732d..26395035 100644 --- a/bench/src/benchmarks/finresearchbench.ts +++ b/bench/src/benchmarks/finresearchbench.ts @@ -9,6 +9,7 @@ import { readFile, stat } from 'node:fs/promises' import { join } from 'node:path' +import { runBenchRouterTurn } from '../router-turn' import { benchRoot } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -180,22 +181,25 @@ function parseJudgeScore(content: string): { score: number; raw: unknown } { async function runOfficialJudge(meta: FinResearchMeta, response: string): Promise { if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`) const router = routerConfig() - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - messages: [ - { role: 'system', content: meta.judgeSystemPrompt }, - { role: 'user', content: fillTemplate(meta, response) }, - ], - }), - }) - if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'finresearchbench-judge', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: router.model, + metadata: { temperature: 0 }, + }, + prompt: { systemPrompt: meta.judgeSystemPrompt }, + }, + }, + fillTemplate(meta, response), + ) + const content = turn.finalText + if (!content) throw new Error('FinResearchBench judge returned no message content') const { score, raw } = parseJudgeScore(content) return { resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8), diff --git a/bench/src/benchmarks/finsearchcomp.ts b/bench/src/benchmarks/finsearchcomp.ts index b926c2c3..8cb47e29 100644 --- a/bench/src/benchmarks/finsearchcomp.ts +++ b/bench/src/benchmarks/finsearchcomp.ts @@ -32,6 +32,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url)) @@ -206,26 +207,25 @@ function parseJudgeOutput(content: string): { resolved: boolean; score: number; /** Run the record's own judge via the router. Fail loud on transport/parse errors. */ async function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - messages: [ - { role: 'system', content: meta.judgeSystemPrompt }, - { role: 'user', content: fillJudgePrompt(meta, response) }, - ], - }), - }) - if (!res.ok) { - throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'finsearchcomp-judge', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: router.model, + metadata: { temperature: 0 }, + }, + prompt: { systemPrompt: meta.judgeSystemPrompt }, + }, + }, + fillJudgePrompt(meta, response), + ) + const content = turn.finalText + if (!content) throw new Error('FinSearchComp judge returned no message content') const { resolved, score, raw } = parseJudgeOutput(content) return { resolved, diff --git a/bench/src/benchmarks/frames.ts b/bench/src/benchmarks/frames.ts index 26d332f1..8766258a 100644 --- a/bench/src/benchmarks/frames.ts +++ b/bench/src/benchmarks/frames.ts @@ -25,6 +25,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const execFileAsync = promisify(execFile) @@ -322,24 +323,24 @@ async function tier2Judge( candidate: string, router: JudgeRouter, ): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - seed: 0, - messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }], - }), - }) - if (!res.ok) { - throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'frames-equivalence-judge', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: router.model, + metadata: { temperature: 0, seed: 0 }, + }, + }, + }, + JUDGE_PROMPT(question, gold, candidate), + ) + const content = turn.finalText + if (!content) throw new Error('FRAMES Tier-2 judge returned no message content') const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/) const raw = (fenced ? fenced[1] : content)?.trim() ?? '' let parsed: { verdict?: unknown } diff --git a/bench/src/benchmarks/simpleqa.ts b/bench/src/benchmarks/simpleqa.ts index 5a0a197b..7b94bb9c 100644 --- a/bench/src/benchmarks/simpleqa.ts +++ b/bench/src/benchmarks/simpleqa.ts @@ -30,6 +30,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const execFileAsync = promisify(execFile) @@ -195,23 +196,24 @@ async function gradeAnswer( predicted: string, router: GraderRouter, ): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }], - }), - }) - if (!res.ok) { - throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'simpleqa-grader', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: router.model, + metadata: { temperature: 0 }, + }, + }, + }, + GRADER_PROMPT(question, gold, predicted), + ) + const content = turn.finalText + if (!content) throw new Error('SimpleQA grader returned no message content') const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/) const raw = (fenced ? fenced[1] : content)?.trim() ?? '' let parsed: { grade?: unknown } diff --git a/bench/src/benchmarks/trata-hedge.ts b/bench/src/benchmarks/trata-hedge.ts index 5aff96bc..30261bb4 100644 --- a/bench/src/benchmarks/trata-hedge.ts +++ b/bench/src/benchmarks/trata-hedge.ts @@ -26,6 +26,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench' @@ -191,25 +192,32 @@ function parseJsonFallback(raw: string): unknown { async function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise { for (let i = 0; i < maxAttempts; i++) { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - max_tokens: 16384, - messages: [{ role: 'user', content: prompt }], - }), - }) - if (!res.ok) { - if (i < maxAttempts - 1) continue - throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) + try { + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'trata-hedge-judge', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: router.model, + metadata: { + temperature: 0, + maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384), + }, + }, + }, + }, + prompt, + ) + const content = turn.finalText + const parsed = parseJsonFallback(content) + if (parsed !== null) return parsed + } catch (error) { + if (i + 1 === maxAttempts) throw error } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') continue - const parsed = parseJsonFallback(content) - if (parsed !== null) return parsed } return null } diff --git a/bench/src/clbench-context-gate.mts b/bench/src/clbench-context-gate.mts index ecf33784..44969336 100644 --- a/bench/src/clbench-context-gate.mts +++ b/bench/src/clbench-context-gate.mts @@ -36,7 +36,12 @@ import { execFileSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' import { composeStrategies } from './directives' import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { + benchRouterProfile, + type BenchRouterTarget, + runBenchRouterTurn, + withBenchProfile, +} from './router-turn' import { selfConsistencySelect, verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' @@ -154,7 +159,7 @@ function parseJudge(reply: string, rubricCount: number): RubricVerdict { /** Grade one completion with the rubric judge. A judge API/parse failure is a real * zero (the response could not be validated) — surfaced, never masked. */ -async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): Promise { +async function judgeRubrics(cfg: BenchRouterTarget, task: CtxTask, output: string): Promise { if (!output.trim()) return { fraction: 0, allPass: false, graded: 0 } const rubricsText = task.rubrics.map((r, i) => `${i + 1}. ${r}`).join('\n') // Fault-isolate the judge: a transient router failure (after retries) or an @@ -162,8 +167,18 @@ async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): P // NOT throw — one bad grade would otherwise crash the whole N×K×2 run. graded=0 // marks it as judge-failed so it's distinguishable from a real 0/N rubric pass. try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: judgePrompt(rubricsText, output) }], { temperature: 0 }) - return parseJudge(typeof res.content === 'string' ? res.content : '', task.rubrics.length) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: withBenchProfile(cfg.profile, { + name: 'clbench-rubric-judge', + temperature: 0, + }), + }, + judgePrompt(rubricsText, output), + ) + return parseJudge(res.finalText, task.rubrics.length) } catch { return { fraction: 0, allPass: false, graded: 0 } } @@ -190,8 +205,18 @@ async function main(): Promise { if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`) if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`) - const workerCfg: RouterConfig = { routerBaseUrl, routerKey, model } - const judgeCfg: RouterConfig = { routerBaseUrl, routerKey, model: judgeModel } + const workerCfg: BenchRouterTarget = { + routerBaseUrl, + routerKey, + profile: benchRouterProfile('clbench-context-worker', model, { + temperature: Number(process.env.TEMPERATURE ?? '0.8'), + }), + } + const judgeCfg: BenchRouterTarget = { + routerBaseUrl, + routerKey, + profile: benchRouterProfile('clbench-rubric-judge', judgeModel, { temperature: 0 }), + } console.log(`=== CL-bench (Context Learning) selector gate · N=${n} K=${k} offset=${offset} ===`) console.log(` worker=${model} judge=${judgeModel} (rubric-fraction verifier) router=${routerBaseUrl}`) @@ -213,8 +238,19 @@ async function main(): Promise { } console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`) const outputs = await pool(units, solveConcurrency, async (u) => { - const res = await routerChatWithUsage(workerCfg, u.messages, { temperature: Number(process.env.TEMPERATURE ?? '0.8') }) - return typeof res.content === 'string' ? res.content : '' + const system = u.messages.find((message) => message.role === 'system')?.content + const res = await runBenchRouterTurn( + { + routerBaseUrl: workerCfg.routerBaseUrl, + routerKey: workerCfg.routerKey, + profile: withBenchProfile(workerCfg.profile, { + name: 'clbench-context-worker', + ...(system ? { systemPrompt: system } : {}), + }), + }, + { messages: u.messages.filter((message) => message.role !== 'system') }, + ) + return res.finalText }) console.log(`▶ grading ${outputs.length} completions with the rubric judge (${judgeModel}), conc=${solveConcurrency}`) diff --git a/bench/src/cloud-loop.mts b/bench/src/cloud-loop.mts index ea71198e..b7dd73de 100644 --- a/bench/src/cloud-loop.mts +++ b/bench/src/cloud-loop.mts @@ -29,8 +29,11 @@ * dotenvx run -f …/.env.keys -f …/agent-state.env -- \ * env MODEL=gpt-4.1 ROUNDS=3 pnpm exec tsx src/cloud-loop.mts */ -import { createChatClient } from '@tangle-network/agent-eval' -import { observe, openSandboxRun } from '@tangle-network/agent-runtime/kernel' +import { + defaultAnalystInstruction, + observe, + openSandboxRun, +} from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' import { answerOutput, sandboxAgentRun } from './sandbox-run' @@ -68,11 +71,16 @@ function tools(events: ReadonlyArray): string[] { async function main(): Promise { const routerKey = env('TANGLE_API_KEY') - const model = env('MODEL', 'gpt-4.1') + const model = env('MODEL', 'deepseek-v4-flash') const routerBaseUrl = env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1') const rounds = Number(env('ROUNDS', '3')) const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey }) - const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: routerBaseUrl, defaultModel: model }) + const observerProfile = { + name: 'trace-observer', + harness: 'cli-base' as const, + model: { provider: 'tangle-router', default: model }, + prompt: { systemPrompt: defaultAnalystInstruction }, + } console.error(`\n=== LIVE observe→steer loop · ${model} · real cloud worker + real observer ===\n`) const steers: string[] = [] @@ -116,7 +124,10 @@ async function main(): Promise { // THE JOIN: a REAL observer reads the REAL trace → a finding → next round's steer. const ob = await observe( { task, output, trace: events, outcome: 'failed', runId: `r${round}` }, - { chat, model }, + { + profile: observerProfile, + executor: { backend: 'router', routerBaseUrl, routerKey }, + }, ) const next = ob.findings.flatMap((f) => (f.recommended_action ? [f.recommended_action] : [])).slice(0, 3) if (next.length === 0) { diff --git a/bench/src/commit0-env-run.mts b/bench/src/commit0-env-run.mts index 389aba1e..eb8ed515 100644 --- a/bench/src/commit0-env-run.mts +++ b/bench/src/commit0-env-run.mts @@ -12,6 +12,7 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { adaptiveRefine, printBenchmarkReport, refine, runBenchmark, sample, type Strategy } from '@tangle-network/agent-runtime/kernel' import { type Commit0Row, createCommit0Environment, rowToTask } from './commit0-env' +import { benchRouterProfile } from './router-turn' function must(name: string): string { const v = process.env[name] @@ -42,9 +43,11 @@ async function main(): Promise { worker: { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), - model, - innerTurns: Number(process.env.INNER_TURNS ?? 10), - temperature: 0.4, + workerProfile: benchRouterProfile('commit0-worker', model, { + systemPrompt: 'You are a senior Python engineer. Work through the available tools until the task is verified complete.', + maxTurns: Number(process.env.INNER_TURNS ?? 10), + temperature: 0.4, + }), }, strategies, budget, diff --git a/bench/src/commit0-env.ts b/bench/src/commit0-env.ts index a7c90c70..677c0d8c 100644 --- a/bench/src/commit0-env.ts +++ b/bench/src/commit0-env.ts @@ -65,12 +65,11 @@ function parsePytest(out: string): { passed: number; failed: number } { export function rowToTask(row: Commit0Row): AgenticTask { return { id: row.instance_id, - systemPrompt: + userPrompt: 'You are a senior Python engineer implementing a stubbed library so its existing test suite passes. ' + 'Workflow: list_files and read the tests + stubs to learn the required behavior, write COMPLETE implementations ' + `with write_file (source under ${row.src_dir} only — the test dir is read-only), then run_tests and fix failures. ` + - 'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.', - userPrompt: + 'Iterate until the suite passes. Reply DONE only when run_tests shows no failures.\n\n' + `Implement the stubbed library "${row.repo}". The public functions/classes under \`${row.src_dir}\` have empty bodies. ` + `Make the existing tests under \`${row.test.test_dir}\` pass.${row.setup.specification ? ` Spec: ${row.setup.specification}` : ''}`, meta: { instanceId: row.instance_id }, diff --git a/bench/src/david-attribution.mts b/bench/src/david-attribution.mts index 3911ade0..081ffb52 100644 --- a/bench/src/david-attribution.mts +++ b/bench/src/david-attribution.mts @@ -20,24 +20,40 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' -const KEY = process.env.TANGLE_API_KEY! +const KEY = process.env.TANGLE_API_KEY +if (!KEY) throw new Error('TANGLE_API_KEY required') const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant' const N = Number(process.env.N ?? 8) const T = Number(process.env.T ?? 5) const NTASKS = Number(process.env.NTASKS ?? 60) const CONC = Number(process.env.CONCURRENCY ?? 6) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000) +const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000) -const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) async function chat(messages: { role: string; content: string }[], temp: number): Promise { - for (let a = 0; ; a++) { - try { - const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) }) - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue } - if (!r.ok) return '' - return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? '' - } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) } + try { + const system = messages.find((message) => message.role === 'system')?.content + const turn = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: KEY, + profile: { + name: 'david-attribution-worker', + model: { provider: 'tangle-router', default: DAVID }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: temp, + maxTokens: MAX_TOKENS, + timeoutMs: LLM_TIMEOUT_MS, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + return turn.finalText + } catch { + return '' } } const exec = (f: string, a: string[], o: object) => new Promise((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0)))) diff --git a/bench/src/david-goliath.mts b/bench/src/david-goliath.mts index 405d4206..82534aee 100644 --- a/bench/src/david-goliath.mts +++ b/bench/src/david-goliath.mts @@ -17,7 +17,7 @@ * generator punch above its solo weight — the standing "verification is live" claim * at its most dramatic. Paired McNemar on per-task discordant pairs for significance. * - * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \ + * Run from cwd=bench: env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \ * N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts */ import { execFile } from 'node:child_process' @@ -25,18 +25,22 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const KEY = process.env.TANGLE_API_KEY if (!KEY) throw new Error('TANGLE_API_KEY required') +const ROUTER_KEY = KEY const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant' -const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001' +const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash' const N = Number(process.env.N ?? 8) // David candidate solutions const T = Number(process.env.T ?? 5) // David generated tests const NTASKS = Number(process.env.NTASKS ?? 164) const REPS = Number(process.env.REPS ?? 2) const CONC = Number(process.env.CONCURRENCY ?? 6) const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000) +const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000) // Approx $/1M tokens (in,out) for cost accounting — the router does not price // every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the @@ -56,18 +60,33 @@ const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out } const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 } async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise { - for (let a = 0; ; a++) { - try { - const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) }) - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue } - if (!r.ok) return '' - const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } } - addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 }) - return j.choices?.[0]?.message?.content ?? '' - } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) } + try { + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: ROUTER_KEY, + profile: { + name: 'david-goliath-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { temperature, maxTokens: MAX_TOKENS }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + timeoutMs: LLM_TIMEOUT, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + addU(usage, { in: result.usage.input, out: result.usage.output }) + return result.finalText + } catch { + return '' } } -const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) }))) async function runPy(program: string): Promise<{ ok: boolean }> { const d = mkdtempSync(join(tmpdir(), 'dg-')) diff --git a/bench/src/egress-probe.mts b/bench/src/egress-probe.mts index ff1d26a6..70907a8a 100644 --- a/bench/src/egress-probe.mts +++ b/bench/src/egress-probe.mts @@ -1,5 +1,5 @@ /** - * Sandbox->router egress probe: spins a box and makes one authed chat call to + * Sandbox->router egress probe: spins a box and makes one model-free authenticated API read from * https://router.tangle.tools/v1 using the BOX-PROVISIONED key (OPENCODE_MODEL_API_KEY * inside the box). That is the sanctioned flow: the egress proxy validates/injects * credentials at the boundary and 403s foreign keys passed in from outside — a raw @@ -19,8 +19,8 @@ try { const r = await box.exec(`curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${h}/ || echo FAIL`) console.log(h, '→', out(r)) } - const chat = await box.exec(`curl -s --max-time 30 -X POST https://router.tangle.tools/v1/chat/completions -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -H 'Content-Type: application/json' -d '{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"say ok"}],"max_tokens":5}' -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`) - console.log('chat (box-provisioned key):', out(chat) || 'EMPTY — egress broken') + const models = await box.exec(`curl -s --max-time 30 https://router.tangle.tools/v1/models -H "Authorization: Bearer $OPENCODE_MODEL_API_KEY" -o /tmp/b -w '%{http_code}'; echo ' |'; head -c 250 /tmp/b`) + console.log('models API (box-provisioned key):', out(models) || 'EMPTY — egress broken') } finally { await box.delete() } diff --git a/bench/src/examples/lean-proof-gate.mts b/bench/src/examples/lean-proof-gate.mts index f67a2740..892bd9ef 100644 --- a/bench/src/examples/lean-proof-gate.mts +++ b/bench/src/examples/lean-proof-gate.mts @@ -24,6 +24,7 @@ import { sampleThenRefine, } from '@tangle-network/agent-runtime/kernel' import { ensureLeanImage, leanCheck } from './lean-verify.js' +import { benchRouterProfile } from '../router-turn' // Real, mathlib-free Lean 4 theorems (compile in core Lean). `header` is everything up to `:=`; // the prover supplies the proof term / tactic block. `reference` is a known-good proof used only @@ -145,9 +146,15 @@ async function main(): Promise { worker: { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey, - model: process.env.WORKER_MODEL ?? 'gpt-4.1', - innerTurns: 8, // room to call lean_check and fix - temperature: 0.4, + workerProfile: benchRouterProfile( + 'lean-proof-worker', + process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + { + systemPrompt: 'Construct and verify Lean proofs with the available checker before answering.', + maxTurns: 8, + temperature: 0.4, + }, + ), }, strategies: [sample, refine, sampleThenRefine], budget: Number(process.env.BUDGET ?? 3), diff --git a/bench/src/examples/math-demo.mts b/bench/src/examples/math-demo.mts index 240254d9..78f1f74d 100644 --- a/bench/src/examples/math-demo.mts +++ b/bench/src/examples/math-demo.mts @@ -20,6 +20,7 @@ import { sample, sampleThenRefine, } from '@tangle-network/agent-runtime/kernel' +import { benchRouterProfile } from '../router-turn' // GSM8K-style problems; meta.answer is the ground truth the deployable check compares to. const problems: Array<{ q: string; answer: number }> = [ @@ -95,9 +96,15 @@ async function main(): Promise { worker: { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey, - model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash', - innerTurns: 6, - temperature: 0.6, + workerProfile: benchRouterProfile( + 'math-worker', + process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + { + systemPrompt: 'Solve the requested math task with the available tools and verify the result.', + maxTurns: 6, + temperature: 0.6, + }, + ), }, strategies: [sample, refine, sampleThenRefine], budget: Number(process.env.BUDGET ?? 3), diff --git a/bench/src/examples/strategy-demo.mts b/bench/src/examples/strategy-demo.mts index 5705d2cd..ba0d7e5f 100644 --- a/bench/src/examples/strategy-demo.mts +++ b/bench/src/examples/strategy-demo.mts @@ -13,6 +13,7 @@ * TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts */ import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/kernel' +import { benchRouterProfile } from '../router-turn' // ── 1. Implement an Environment (the only thing a new domain writes) ────────────── // A toy: the agent must drive a counter to exactly the target using the increment tool. @@ -59,8 +60,7 @@ const counterEnv: Environment = { const task: AgenticTask = { id: 'counter-to-5', - systemPrompt: 'You operate a counter with tools.', - userPrompt: `Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`, + userPrompt: `You operate a counter with tools. Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`, } // ── 3. Author your OWN strategy in ~10 lines — the lego (no Supervisor ceremony) ── @@ -97,8 +97,14 @@ async function main(): Promise { const worker = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: process.env.TANGLE_API_KEY ?? '', - model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash', - innerTurns: 6, + workerProfile: benchRouterProfile( + 'strategy-demo-worker', + process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + { + systemPrompt: 'Use the available tools to complete and verify the task.', + maxTurns: 6, + }, + ), } if (!worker.routerKey) throw new Error('set TANGLE_API_KEY (the worker calls the router)') diff --git a/bench/src/fleet.mts b/bench/src/fleet.mts index 1b14f62e..62831997 100644 --- a/bench/src/fleet.mts +++ b/bench/src/fleet.mts @@ -12,8 +12,13 @@ * * Run it twice: the second run injects the first run's learnings into the workers. */ -import { createChatClient } from '@tangle-network/agent-eval' -import { FileCorpus, observe, openSandboxRun, renderReport } from '@tangle-network/agent-runtime/kernel' +import { + defaultAnalystInstruction, + FileCorpus, + observe, + openSandboxRun, + renderReport, +} from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' import { answerOutput, sandboxAgentRun, type WorkerBackendType } from './sandbox-run' @@ -74,13 +79,18 @@ async function main(): Promise { const routerKey = env('TANGLE_API_KEY') const cfg = { backendType: env('BACKEND', 'opencode') as WorkerBackendType, - model: env('MODEL', 'gpt-4.1'), + model: env('MODEL', 'deepseek-v4-flash'), routerBaseUrl: env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1'), } const n = Math.min(Number(env('N', '2')), subtasks.length) const corpus = new FileCorpus(env('CORPUS', '/tmp/fleet-corpus.jsonl')) - const observerModel = env('OBSERVER_MODEL', 'gpt-4.1') - const chat = createChatClient({ transport: 'router', apiKey: routerKey, baseUrl: cfg.routerBaseUrl, defaultModel: observerModel }) + const observerModel = env('OBSERVER_MODEL', 'deepseek-v4-flash') + const observerProfile = { + name: 'fleet-observer', + harness: 'cli-base' as const, + model: { provider: 'tangle-router', default: observerModel }, + prompt: { systemPrompt: defaultAnalystInstruction }, + } const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey }) // ── continuous: read what prior runs LEARNED, inject it into this run's workers @@ -104,7 +114,12 @@ async function main(): Promise { if (w.error) continue const ob = await observe( { task: w.task, output: w.output, trace: w.events, outcome: w.output ? 'passed' : 'unknown', runId: w.id }, - { chat, model: observerModel, corpus, tags: [cfg.backendType, 'fleet'] }, + { + profile: observerProfile, + executor: { backend: 'router', routerBaseUrl: cfg.routerBaseUrl, routerKey }, + corpus, + tags: [cfg.backendType, 'fleet'], + }, ) totalLearned += ob.learned.length console.error(` answer: ${w.output.slice(0, 120).replace(/\n/g, ' ')}`) diff --git a/bench/src/gate-cli.mts b/bench/src/gate-cli.mts index 70f33530..be76c6f5 100644 --- a/bench/src/gate-cli.mts +++ b/bench/src/gate-cli.mts @@ -46,9 +46,16 @@ async function main(): Promise { const profile = { name: 'gate-solver', - model: { default: model }, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { + temperature: Number(process.env.TEMPERATURE ?? 0.7), + }, + }, prompt: { systemPrompt: 'You are an expert agent. Produce the single best deliverable the task’s grader will accept.' }, - } as unknown as AgentProfile + } satisfies AgentProfile const report = await runGate({ adapter, @@ -56,8 +63,6 @@ async function main(): Promise { strategies, routerBaseUrl, routerKey, - model, - ...(process.env.TEMPERATURE ? { temperature: Number(process.env.TEMPERATURE) } : {}), n: Number(process.env.N ?? 20), ...(process.env.IDS ? { ids: process.env.IDS.split(',') } : {}), ...(process.env.SPLIT ? { split: process.env.SPLIT } : {}), diff --git a/bench/src/gate.test.mts b/bench/src/gate.test.mts index 976eb66e..f90b55a3 100644 --- a/bench/src/gate.test.mts +++ b/bench/src/gate.test.mts @@ -88,7 +88,6 @@ const report = await runGate({ strategies: ['plain restate', 'use the STRONG verified approach', 'enumerate edge cases'], routerBaseUrl: 'http://unused', routerKey: 'unused', - model: 'stub-model', solverRegistry: stubRegistry, }) diff --git a/bench/src/gate.ts b/bench/src/gate.ts index ad9afdd6..b4e5a092 100644 --- a/bench/src/gate.ts +++ b/bench/src/gate.ts @@ -32,28 +32,26 @@ import type { AgentSpec, Budget, CombinatorShape, - DefaultVerdict, EqualKArm, EqualKVerdict, ExecutorContext, ExecutorRegistry, Executor, ExecutorFactory, - ExecutorResult, Outcome, Persona, - Runtime, Spend, SupervisedResult, TrajectoryReport, } from '@tangle-network/agent-runtime/kernel' import { definePersona, - routerChatWithUsage, + createExecutor, equalKOnCost, fanout, InMemoryResultBlobStore, InMemorySpawnJournal, + mapExecutorResult, runPersonified, trajectoryReport, } from '@tangle-network/agent-runtime/kernel' @@ -73,10 +71,6 @@ export interface BenchSolverOptions { readonly adapter: BenchmarkAdapter readonly routerBaseUrl: string readonly routerKey: string - readonly model: string - /** Sampling temperature. >0 is required for the blind arm to be more than k identical samples - * (k copies at temperature 0 collapse to one answer — no compute control). Default 0.7. */ - readonly temperature?: number } const fnv = (prefix: string, value: unknown): string => { @@ -114,58 +108,27 @@ function extractArtifact(adapter: BenchmarkAdapter, content: string): string { * a judge throw rejects the leaf (the scope types it into a `down` settlement — never a silent 0). */ export function benchSolveLeaf(opts: BenchSolverOptions, spec: AgentSpec, ctx: ExecutorContext): Executor { - const controller = new AbortController() - const abortIfSignalled = () => { - if (ctx.signal.aborted) controller.abort() - } - abortIfSignalled() - if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true }) - - let artifact: ExecutorResult | undefined - - return { - runtime: 'bench-router' as Runtime, - async execute(task, signal): Promise> { + const inner = createExecutor({ + backend: 'router', + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + })(spec, ctx) + return mapExecutorResult(inner, async (result, task) => { const t = task as SolveTask - const system = spec.profile.prompt?.systemPrompt - const messages = [ - ...(typeof system === 'string' && system.length > 0 - ? [{ role: 'system', content: system }] - : []), - { role: 'user', content: t.prompt }, - ] - const started = Date.now() - const linked = linkSignals(signal, controller.signal) - const chat = await routerChatWithUsage( - { routerBaseUrl: opts.routerBaseUrl, routerKey: opts.routerKey, model: opts.model }, - messages, - { temperature: opts.temperature ?? 0.7, ...(linked ? { signal: linked } : {}) }, - ) - const candidate = extractArtifact(opts.adapter, chat.content) + const raw = result.out as { content?: unknown } + const content = typeof raw?.content === 'string' ? raw.content : '' + const candidate = extractArtifact(opts.adapter, content) const score = await opts.adapter.judge(t.instance, candidate) - const verdict: DefaultVerdict = { - valid: score.resolved, - score: score.score, - ...(score.detail ? { notes: score.detail } : {}), - } - const spent: Spend = { - iterations: 1, - tokens: chat.usage ? { input: chat.usage.input, output: chat.usage.output } : { input: 0, output: 0 }, - usd: chat.costUsd ?? 0, - ms: Date.now() - started, + return { + outRef: fnv('bench', { id: t.instance.id, candidate }), + out: candidate, + verdict: { + valid: score.resolved, + score: score.score, + ...(score.detail ? { notes: score.detail } : {}), + }, } - artifact = { outRef: fnv('bench', { id: t.instance.id, candidate }), out: candidate, verdict, spent } - return artifact - }, - teardown(): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) - }, - resultArtifact() { - if (!artifact) throw new Error('benchSolveLeaf: resultArtifact() read before execute()') - return artifact - }, - } + }) } /** @@ -233,8 +196,6 @@ export interface RunGateOptions { readonly strategies: ReadonlyArray readonly routerBaseUrl: string readonly routerKey: string - readonly model: string - readonly temperature?: number /** How many benchmark instances to run (the paired n). */ readonly n?: number readonly ids?: string[] @@ -447,15 +408,3 @@ export async function runGate(opts: RunGateOptions): Promise { } /** Link two abort signals into one that fires when either does; `undefined` when neither is set. */ -function linkSignals(a: AbortSignal, b: AbortSignal): AbortSignal | undefined { - if (a.aborted || b.aborted) { - const c = new AbortController() - c.abort() - return c.signal - } - const c = new AbortController() - const onAbort = () => c.abort() - a.addEventListener('abort', onAbort, { once: true }) - b.addEventListener('abort', onAbort, { once: true }) - return c.signal -} diff --git a/bench/src/generate-eval/certify.ts b/bench/src/generate-eval/certify.ts index e1186dfa..7dfbcd37 100644 --- a/bench/src/generate-eval/certify.ts +++ b/bench/src/generate-eval/certify.ts @@ -24,8 +24,8 @@ import { execSync } from 'node:child_process' import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' import { scoreTask, taskToPrompt } from '../search-bench/tasks' +import { runBenchRouterTurn } from '../router-turn' import { type EvalCertification, type GeneratedEval, generatedEvalSchemaVersion, parseCandidate } from './schema' const certifierId = 'agent-runtime/generate-eval@1' @@ -121,10 +121,18 @@ export async function discriminationGate( const apiKey = opts.gateApiKey ?? process.env.EVAL_GATE_API_KEY ?? process.env.TANGLE_API_KEY const model = opts.gateModel ?? process.env.EVAL_GATE_MODEL ?? 'deepseek-v4-flash' if (!apiKey) throw new Error('discrimination gate needs EVAL_GATE_API_KEY (or TANGLE_API_KEY)') - const res = await routerChatWithUsage({ routerBaseUrl: baseUrl, routerKey: apiKey, model }, [ - { role: 'user', content: taskToPrompt(candidate) }, - ]) - const { score, reasons } = scoreTask(candidate, res.content) + const res = await runBenchRouterTurn( + { + routerBaseUrl: baseUrl, + routerKey: apiKey, + profile: { + name: 'generated-eval-parametric-check', + model: { provider: 'tangle-router', default: model }, + }, + }, + taskToPrompt(candidate), + ) + const { score, reasons } = scoreTask(candidate, res.finalText) return score === 0 ? { passed: true, detail: `parametric ${model} failed as required (${reasons.join('; ')})` } : { passed: false, detail: `parametric ${model} SOLVED the task from memory — not search-discriminating` } diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts index 07d6d50d..7a7a82a4 100644 --- a/bench/src/hev-eval.mts +++ b/bench/src/hev-eval.mts @@ -9,19 +9,41 @@ */ import { readFileSync } from 'node:fs' import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const SEED_INSTRUCTION = 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' -async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), - }) - if (!res.ok) return '' - const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - return d.choices?.[0]?.message?.content ?? '' +async function complete( + base: string, + key: string, + model: string, + instruction: string, + prompt: string, + maxTokens: number, +): Promise { + try { + const turn = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile: { + name: 'humaneval-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { temperature: 0.2, maxTokens }, + }, + prompt: { systemPrompt: instruction }, + }, + }, + prompt, + ) + return turn.finalText + } catch { + return '' + } } async function main(): Promise { @@ -55,7 +77,14 @@ async function main(): Promise { const t = tasks[i] i += 1 if (!t) continue - const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) + const reply = await complete( + base, + apiKey, + model, + instruction, + `\`\`\`python\n${t.prompt}\`\`\``, + maxTokens, + ) const { pass: p } = await runChecker(t, extractCode(reply)) if (p === 1) pass += 1 else fails.push(t.taskId) diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts index 081c176b..3aab911a 100644 --- a/bench/src/hev-improve.mts +++ b/bench/src/hev-improve.mts @@ -27,6 +27,7 @@ import { officialOptimizerModel, requiredTokenPricing, } from './official-optimizer-config.mjs' +import { runBenchRouterTurn, withBenchProfile } from './router-turn' // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's // solveInstruction so the baseline arm reproduces the plain-prompt denominator. @@ -39,21 +40,26 @@ interface Completion { tokOut: number } -async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), - }) - if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } +async function complete( + base: string, + key: string, + profile: AgentProfile, + prompt: string, + maxTokens: number, +): Promise { + const result = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile: withBenchProfile(profile, { temperature: 0.2, maxTokens }), + }, + prompt, + ) + return { + text: result.finalText, + tokIn: result.usage.input, + tokOut: result.usage.output, } - const text = d.choices?.[0]?.message?.content ?? '' - const tokIn = d.usage?.prompt_tokens ?? 0 - const tokOut = d.usage?.completion_tokens ?? 0 - return { text, tokIn, tokOut } } async function main(): Promise { @@ -115,13 +121,18 @@ async function main(): Promise { if (instr === undefined) throw new Error('agent: candidate profile has no system prompt') const t = byId.get(scenario.id) if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) - const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` + const prompt = `\`\`\`python\n${t.prompt}\`\`\`` + const executionProfile: AgentProfile = { + ...candidate, + name: candidate.name ?? 'humaneval-improvement-worker', + model: { ...candidate.model, provider: 'tangle-router', default: workerModel }, + } const t0 = Date.now() const paid = await ctx.cost.runPaidCall({ channel: 'agent', actor: 'humaneval-worker', model: workerModel, - execute: () => complete(base, key, workerModel, prompt, workerMaxTokens), + execute: () => complete(base, key, executionProfile, prompt, workerMaxTokens), receipt: (result) => { const usageUnknown = result.tokIn === 0 && result.tokOut === 0 return { diff --git a/bench/src/hev-structural.mts b/bench/src/hev-structural.mts index ed2d1585..33331e52 100644 --- a/bench/src/hev-structural.mts +++ b/bench/src/hev-structural.mts @@ -46,6 +46,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { type HumanEvalTask, extractCode, loadHumanEval } from './benchmarks/humaneval' import { composeStrategies } from './directives' +import { runBenchRouterTurn } from './router-turn' import { type PairedLift, pairedLift, pool } from './stats.mts' const dockerImage = 'python:3.12-slim' @@ -345,35 +346,42 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), 240_000) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'humaneval-structural-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText // Reasoning models starve `content` when reasoning exhausts max_tokens — an // empty reply is a transient fault to retry, not a candidate to score. if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/humaneval-gate.mts b/bench/src/humaneval-gate.mts index 80b852c3..2e45a5e7 100644 --- a/bench/src/humaneval-gate.mts +++ b/bench/src/humaneval-gate.mts @@ -43,7 +43,12 @@ import { composeStrategies } from './directives' import { basePrompt, type CheckResult, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { + benchRouterProfile, + type BenchRouterTarget, + runBenchRouterTurn, + withBenchProfile, +} from './router-turn' import { selfConsistencySelect, verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' @@ -84,12 +89,18 @@ async function main(): Promise { if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`) if (!Number.isInteger(offset) || offset < 0) throw new Error(`OFFSET must be a non-negative integer, got ${process.env.OFFSET}`) - const cfg: RouterConfig = { routerBaseUrl, routerKey, model } + const cfg: BenchRouterTarget = { + routerBaseUrl, + routerKey, + profile: benchRouterProfile('humaneval-gate-worker', model, { + temperature: Number(process.env.TEMPERATURE ?? '0.8'), + }), + } console.log(`=== HumanEval deployable-verifier gate · N=${n} K=${k} offset=${offset} model=${model} ===`) console.log(` router=${routerBaseUrl} docker=${dockerImage} (--network=none, timeout ${dockerTimeoutMs}ms)`) console.log( - ' regime: STATELESS single completions (maxTurns=0, no AgentProfile/sandbox) — the selector no-self-correction LOWER BOUND, not a rollout/product number', + ' regime: STATELESS single completions (one exact AgentProfile turn, no sandbox) — the selector no-self-correction LOWER BOUND, not a rollout/product number', ) const tasks = await loadHumanEval(n, offset) @@ -110,10 +121,15 @@ async function main(): Promise { console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`) const codes = await pool(units, solveConcurrency, async (u) => { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: u.prompt }], { - temperature: Number(process.env.TEMPERATURE ?? '0.8'), - }) - return extractCode(typeof res.content === 'string' ? res.content : '') + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: withBenchProfile(cfg.profile, { name: 'humaneval-gate-worker' }), + }, + u.prompt, + ) + return extractCode(res.finalText) }) console.log(`▶ running ${codes.length} candidates through the Docker deployable checker, conc=${dockerConcurrency}`) diff --git a/bench/src/humaneval-object-ablation.mts b/bench/src/humaneval-object-ablation.mts index 76510256..45242094 100644 --- a/bench/src/humaneval-object-ablation.mts +++ b/bench/src/humaneval-object-ablation.mts @@ -19,7 +19,7 @@ * container with a hard timeout — seconds per task. Paired * McNemar over the per-task pass/fail difference gives the significance. * - * Run from cwd=bench: env WORKER_MODEL=google/gemini-2.5-flash-lite N=60 K=3 \ + * Run from cwd=bench: env WORKER_MODEL=deepseek-v4-flash N=60 K=3 \ * REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts */ import { @@ -28,11 +28,13 @@ import { runPythonProgram, type HumanEvalTask, } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const KEY = process.env.TANGLE_API_KEY if (!KEY) throw new Error('TANGLE_API_KEY required') -const MODEL = process.env.WORKER_MODEL ?? 'google/gemini-2.5-flash-lite' +const ROUTER_KEY = KEY +const MODEL = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' const N = Number(process.env.N ?? 60) const OFFSET = Number(process.env.OFFSET ?? 0) const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms) @@ -40,23 +42,56 @@ const REPS = Number(process.env.REPS ?? 2) const CONC = Number(process.env.CONCURRENCY ?? 6) const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000) -interface ChatMsg { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string } +interface ChatMsg extends Record { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string } interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } } async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record }[] }> { - const body: Record = { model: MODEL, messages, temperature: 0.4 } - if (tools) { body.tools = tools; body.tool_choice = 'auto' } for (let attempt = 0; ; attempt++) { - let res: Response try { - res = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(60_000) }) - } catch (e) { if (attempt >= 5) throw e; await sleep(800 * 2 ** attempt); continue } - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(res.status)) { if (attempt >= 5) throw new Error(`router ${res.status} exhausted`); await sleep(800 * 2 ** attempt); continue } - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`) - const j = (await res.json()) as { choices?: { message?: { content?: string; tool_calls?: { id: string; function: { name: string; arguments: string } }[] } }[] } - const m = j.choices?.[0]?.message - const toolCalls = (m?.tool_calls ?? []).map((t) => { let args: Record = {}; try { args = JSON.parse(t.function.arguments) } catch { /* keep {} */ } return { id: t.id, name: t.function.name, args } }) - return { content: m?.content ?? '', toolCalls } + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: ROUTER_KEY, + profile: { + name: 'humaneval-object-ablation-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: MODEL, + metadata: { + temperature: 0.4, + ...(tools ? { toolChoice: 'auto' } : {}), + }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + ...(tools + ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) } + : {}), + }, + ...(tools ? { tools } : {}), + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 60_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + const toolCalls = result.toolCalls.map((call) => { + let args: Record = {} + try { + args = JSON.parse(call.arguments) as Record + } catch { + // Keep the empty argument object; the tool returns a useful error. + } + return { id: call.id, name: call.name, args } + }) + return { content: result.finalText, toolCalls } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const status = Number(/router (\d+)/.exec(message)?.[1]) + const transient = + !Number.isFinite(status) || [408, 429, 500, 502, 503, 504, 520, 522, 524].includes(status) + if (!transient || attempt >= 5) throw error + await sleep(800 * 2 ** attempt) + } } } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) diff --git a/bench/src/humaneval-repair-gate.mts b/bench/src/humaneval-repair-gate.mts index 3ebbc10b..43de640c 100644 --- a/bench/src/humaneval-repair-gate.mts +++ b/bench/src/humaneval-repair-gate.mts @@ -20,9 +20,20 @@ * tsx src/humaneval-repair-gate.mts */ import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval, runChecker } from './benchmarks/humaneval' -import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/kernel' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import { verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' +import { + benchRouterProfile, + type BenchRouterTarget, + runBenchRouterTurn, + withBenchProfile, +} from './router-turn' function must(name: string): string { const v = process.env[name] @@ -51,14 +62,23 @@ const repairSystem = [ ].join(' ') /** repair@K: one worker, up to K inference turns, steering on real test failures. */ -async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise { +async function repairAttempt(cfg: BenchRouterTarget, task: HumanEvalTask, k: number): Promise { let lastTested = '' - const r = await routerToolLoop( - cfg, - repairSystem, - basePrompt(task), - [runTestsTool], - async (name, args) => { + const profile = { + ...withBenchProfile(cfg.profile, { + name: 'humaneval-repair-worker', + systemPrompt: repairSystem, + maxTurns: k, + temperature: 0.3, + }), + tools: { run_tests: true }, + } + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + tools: [runTestsTool], + executeToolCall: async (name, args) => { if (name !== 'run_tests') return `error: unknown tool ${name}` const code = extractCode(String(args.code ?? '')) lastTested = code @@ -67,22 +87,37 @@ async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): ? 'ALL TESTS PASSED. Reply with the final function now; do not call run_tests again.' : `TESTS FAILED:\n${res.detail ?? 'no output'}\n\nFix the function and call run_tests again.` }, - { maxTurns: k, temperature: 0.3 }, + }) + const r = await collectAgentTurn( + streamAgentTurn({ kind: 'executor', factory, profile }, basePrompt(task)), ) + if (r.status !== 'completed') { + throw new Error(r.error?.message ?? `repair turn ended with status ${r.status}`) + } // Judge the model's final answer; fall back to the last code it tested (it may // report "done" without re-pasting the passing function). - const finalCode = extractCode(r.final) || lastTested + const finalCode = extractCode(r.finalText) || lastTested if (!finalCode) return 0 return (await runChecker(task, finalCode)).pass } /** blind@K: K independent completions, verifier-grounded pick (the resample control). */ -async function blindAttempts(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise { +async function blindAttempts(cfg: BenchRouterTarget, task: HumanEvalTask, k: number): Promise { const base = basePrompt(task) const passes: number[] = [] for (let i = 0; i < k; i += 1) { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: base }], { temperature: 0.8 }) - passes.push((await runChecker(task, extractCode(res.content))).pass) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: withBenchProfile(cfg.profile, { + name: 'humaneval-blind-worker', + temperature: 0.8, + }), + }, + base, + ) + passes.push((await runChecker(task, extractCode(res.finalText))).pass) } return passes } @@ -95,7 +130,11 @@ async function main(): Promise { const k = Number(process.env.K ?? 3) const offset = Number(process.env.OFFSET ?? 82) const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' - const cfg: RouterConfig = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), model } + const cfg: BenchRouterTarget = { + routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', + routerKey: must('TANGLE_API_KEY'), + profile: benchRouterProfile('humaneval-worker', model), + } const concurrency = Number(process.env.CONCURRENCY ?? 6) if (k < 2) throw new Error('K must be >= 2 (repair needs at least write + one fix)') diff --git a/bench/src/mbpp-structural.mts b/bench/src/mbpp-structural.mts index 576c945b..d9137942 100644 --- a/bench/src/mbpp-structural.mts +++ b/bench/src/mbpp-structural.mts @@ -32,6 +32,7 @@ import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { extractCode } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' import { type PairedLift, pairedLift, pool } from './stats.mts' const dockerImage = 'python:3.12-slim' @@ -326,33 +327,40 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), 240_000) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'mbpp-structural-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/mcp-mount-probe.mts b/bench/src/mcp-mount-probe.mts index 699e1408..81f7a148 100644 --- a/bench/src/mcp-mount-probe.mts +++ b/bench/src/mcp-mount-probe.mts @@ -14,6 +14,8 @@ import { type Agent, type AgentProfile, type AgentSpec, + collectAgentTurn, + createExecutor, createExecutorRegistry, createSupervisor, type Executor, @@ -21,6 +23,7 @@ import { InMemoryResultBlobStore, InMemorySpawnJournal, type Scope, + streamAgentTurn, type UsageEvent, } from '../../src/runtime/index' import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp' @@ -53,18 +56,27 @@ function deliveringLeaf(name: string, out: unknown): Agent { } async function bridgeChat(messages: Array<{ role: string; content: string }>, mcpUrl: string): Promise { - const r = await fetch(`${BRIDGE.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model: MODEL, - messages, - mcp: { mcpServers: { coordination: { type: 'http', url: mcpUrl } } }, - }), + if (!BEARER) throw new Error('TANGLE_API_KEY is required') + const profile: AgentProfile = { + name: 'mcp-mount-probe-supervisor', + model: { default: MODEL }, + mcp: { coordination: { transport: 'http', url: mcpUrl } }, + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: BRIDGE.replace(/\/v1\/?$/u, ''), + bridgeBearer: BEARER, }) - if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})` - const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + messages.map((message) => message.content).join('\n\n'), + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`) + } + return turn.finalText } async function main(): Promise { diff --git a/bench/src/official-optimizer-config.mts b/bench/src/official-optimizer-config.mts index d060f203..b80283ed 100644 --- a/bench/src/official-optimizer-config.mts +++ b/bench/src/official-optimizer-config.mts @@ -1,3 +1,15 @@ +import type { + OpenAICompatibleOptimizerModel, + OptimizerModelBudget, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentProfile, + canonicalAgentProfileDigest, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' +import { profileOptimizerModelCall } from '../../src/runtime/profile-chat-client' +import type { RouterSeam } from '../../src/runtime/supervise/runtime' + function requiredNonNegativeNumber( env: NodeJS.ProcessEnv, name: string, @@ -57,22 +69,59 @@ export function officialOptimizerModel(options: { maxCostUsd: number maxOutputTokensPerRequest: number envPrefix?: string -}) { + provider?: string + temperature?: number + reasoningEffort?: NonNullable['reasoningEffort'] + callRef?: string + complete?: RouterSeam['complete'] +}): OpenAICompatibleOptimizerModel { const { env } = options const envPrefix = options.envPrefix ?? 'REFLECT' + const budget: OptimizerModelBudget = { + maxCostUsd: options.maxCostUsd, + maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100), + maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000), + maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000), + maxOutputTokensPerRequest: options.maxOutputTokensPerRequest, + requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000), + pricing: requiredTokenPricing(env, envPrefix), + } + const profile: AgentProfile = { + name: 'official-optimizer-model', + harness: 'cli-base', + model: { + provider: options.provider ?? new URL(options.baseUrl).hostname, + default: options.model, + ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}), + metadata: { + maxTokens: options.maxOutputTokensPerRequest, + ...(options.temperature !== undefined ? { temperature: options.temperature } : {}), + }, + }, + } + const profileDigest = canonicalAgentProfileDigest(profile) + const executor: RouterSeam & { backend: 'router' } = { + backend: 'router', + routerBaseUrl: options.baseUrl, + routerKey: options.apiKey, + ...(options.complete ? { complete: options.complete } : {}), + } + const call = profileOptimizerModelCall({ + profile, + context: 'official optimizer model', + executor, + pricing: budget.pricing, + }) return { model: options.model, - baseUrl: options.baseUrl, - apiKey: options.apiKey, - budget: { - maxCostUsd: options.maxCostUsd, - maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100), - maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000), - maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000), - maxOutputTokensPerRequest: options.maxOutputTokensPerRequest, - requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000), - pricing: requiredTokenPricing(env, envPrefix), - }, + callRef: + options.callRef ?? + `agent-runtime:${canonicalCandidateDigest({ + profileDigest, + endpoint: new URL(options.baseUrl).origin, + })}`, + call, + budget, } } diff --git a/bench/src/official-optimizer-config.test.mts b/bench/src/official-optimizer-config.test.mts index 382100ce..36777783 100644 --- a/bench/src/official-optimizer-config.test.mts +++ b/bench/src/official-optimizer-config.test.mts @@ -30,10 +30,8 @@ describe('official optimizer configuration', () => { maxOutputTokensPerRequest: 4000, }) - expect(model).toEqual({ + expect(model).toMatchObject({ model: 'test-model', - baseUrl: 'http://127.0.0.1:8080/v1', - apiKey: 'test-key', budget: { maxCostUsd: 2, maxRequests: 7, @@ -49,6 +47,77 @@ describe('official optimizer configuration', () => { }, }, }) + expect(model.call).toEqual(expect.any(Function)) + expect(model.callRef).toMatch(/^agent-runtime:sha256:/) + expect(model).not.toHaveProperty('baseUrl') + expect(model).not.toHaveProperty('apiKey') + }) + + it('executes the published optimizer callback through the exact Runtime profile', async () => { + const requests: unknown[] = [] + const model = officialOptimizerModel({ + env: pricingEnv, + envPrefix: 'OPT', + model: 'test-model', + baseUrl: 'http://127.0.0.1:8080/v1', + apiKey: 'test-key', + maxCostUsd: 2, + maxOutputTokensPerRequest: 4000, + complete: async (request) => { + requests.push(request) + return { + model: 'test-model', + choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 3, + completion_tokens: 2, + cost: 0.001, + prompt_tokens_details: { cached_tokens: 1 }, + completion_tokens_details: { reasoning_tokens: 1 }, + }, + } + }, + }) + + const result = await model.call({ + callId: 'optimizer-call-1', + request: { + model: 'test-model', + messages: [{ role: 'user', content: 'hi' }], + maxTokens: 4000, + }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(true) + if (!result.succeeded) throw new Error(result.error) + expect(result.response).toMatchObject({ + model: 'test-model', + content: 'ok', + usage: { promptTokens: 3, completionTokens: 2, totalTokens: 5 }, + }) + expect(result.receipt).toMatchObject({ + model: 'test-model', + inputTokens: 2, + cachedTokens: 1, + outputTokens: 2, + reasoningTokens: 1, + actualCostUsd: 0.001, + }) + expect(result.execution).toMatchObject({ + kind: 'agent-runtime-profile-model-call', + executed: true, + succeeded: true, + model: 'test-model', + callId: 'optimizer-call-1', + endpointFormat: 'chat-completions', + }) + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + model: 'test-model', + max_tokens: 4000, + }) }) it('requires every token price instead of inventing cost data', () => { diff --git a/bench/src/research-shot.ts b/bench/src/research-shot.ts index 88183376..de710252 100644 --- a/bench/src/research-shot.ts +++ b/bench/src/research-shot.ts @@ -1,7 +1,7 @@ /** * One research rollout as a reusable primitive: 2-step RAG — (1) provider-pinned web * search via the router's proven `/v1/search?provider=` + `web_fetch` of the top-K - * result pages, (2) answer with that evidence via `routerChatWithUsage` (no tools on the + * result pages, (2) answer with that evidence through Runtime's profile-bound turn (no tools on the * answer call → `content` always present, so a search arm differs from the parametric * control ONLY by the evidence). Pure router HTTP (bearer `TANGLE_API_KEY`). * @@ -10,7 +10,7 @@ * — the only difference is who drives the rounds (a flat best-of-k pool vs the real * `runAgentRounds` kernel with analyst steering). */ -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export interface ShotCfg { model: string @@ -109,15 +109,25 @@ export async function runResearchShot(prompt: string, taskId: string, attempt: n : 'Answer from your own knowledge. ') + 'If you are not fully certain, still COMMIT to your single best estimate — never refuse, defer, or reply with a question.' const userContent = useSearch && context ? `${prompt}\n\n=== WEB SEARCH RESULTS (provider: ${cfg.search}) ===\n${context}` : prompt - const { content } = await routerChatWithUsage( - { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model }, - [ - { role: 'system', content: commit }, - { role: 'user', content: userContent }, - ], - { temperature: cfg.temperature, ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(cfg.timeoutMs) } : {}) }, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'research-shot-answerer', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { temperature: cfg.temperature }, + }, + prompt: { systemPrompt: commit }, + }, + ...(cfg.timeoutMs ? { timeoutMs: cfg.timeoutMs } : {}), + }, + userContent, ) - const answer = content.trim() + const answer = turn.finalText.trim() const ok = answer.length > 0 return { taskId, attempt, answer, ok, searches, wallMs: Date.now() - startedAt, ...(ok ? {} : { detail: `empty answer (searches=${searches})` }) } } catch (err) { diff --git a/bench/src/resolve-client.ts b/bench/src/resolve-client.ts index 9ef60d69..01a7c9d7 100644 --- a/bench/src/resolve-client.ts +++ b/bench/src/resolve-client.ts @@ -11,7 +11,7 @@ * - `bridge` → OFF-BOX: a local cli-bridge fronting a harness CLI * (opencode/kimi-code/…) as the leaf executor. Same resumable * `bridgeExecutor` the supervisor uses; harness+model ride the - * bridge `model` id (`${harness}/${model}`). + * exact harness/provider/model come from the turn's AgentProfile. * - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend * TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`; * this only decides off-box-vs-in-box transport for `runAgentRounds`. @@ -33,7 +33,6 @@ export interface ResolveBenchClientOptions { backend: string routerBaseUrl: string routerKey: string - model: string /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop * with a live `web_search` tool backed by this provider (`you`/`exa`/…). */ searchProvider?: string @@ -47,7 +46,7 @@ export interface ResolveBenchClientOptions { } export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClient { - const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts + const { backend, routerBaseUrl, routerKey, searchProvider } = opts if (backend === 'router') { if (searchProvider) { return inlineSandboxClient( @@ -55,13 +54,12 @@ export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClie backend: 'router-tools', routerBaseUrl, routerKey, - model, tools: [webSearchTool], executeToolCall: makeSearchExecutor({ routerBaseUrl, routerKey, provider: searchProvider }), }), ) } - return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey, model })) + return inlineSandboxClient(createExecutor({ backend: 'router', routerBaseUrl, routerKey })) } if (backend === 'bridge') { // bench's bearer fallback (`?? routerKey`) resolves first, then the shared @@ -70,7 +68,7 @@ export function resolveBenchClient(opts: ResolveBenchClientOptions): SandboxClie if (!bridgeBearer) throw new Error("resolveBenchClient: backend 'bridge' needs bridgeBearer or routerKey") return resolveSandboxClient({ backend: 'bridge', - bridge: { url: opts.bridgeUrl, bearer: bridgeBearer, model, timeoutMs: opts.timeoutMs }, + bridge: { url: opts.bridgeUrl, bearer: bridgeBearer, timeoutMs: opts.timeoutMs }, }) } return new Sandbox({ diff --git a/bench/src/router-turn.ts b/bench/src/router-turn.ts new file mode 100644 index 00000000..fcf2a909 --- /dev/null +++ b/bench/src/router-turn.ts @@ -0,0 +1,128 @@ +import { + type AgentProfile, + agentProfileSchema, + type ReasoningEffort, +} from '@tangle-network/agent-interface' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type CollectedAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' + +/** Bench-local target shorthand; Runtime still executes only the exact profile below. */ +export interface BenchRouterTarget { + routerBaseUrl: string + routerKey: string + profile: AgentProfile +} + +export interface BenchRouterTurnConfig extends BenchRouterTarget { + tools?: ReadonlyArray + timeoutMs?: number + signal?: AbortSignal +} + +export interface BenchProfileSettings { + systemPrompt?: string + temperature?: number + maxTokens?: number + maxRetries?: number + maxTurns?: number + seed?: number + reasoningEffort?: ReasoningEffort + extraBody?: Readonly> + toolChoice?: 'auto' | 'required' | 'none' +} + +/** Author an exact direct-Router profile for a benchmark. This is profile construction only; + * execution still accepts no model or generation fields outside the returned AgentProfile. */ +export function benchRouterProfile( + name: string, + model: string, + settings: BenchProfileSettings = {}, +): AgentProfile { + return withBenchProfile( + { + name, + harness: 'cli-base', + model: { provider: 'tangle-router', default: model }, + }, + settings, + ) +} + +/** Derive another exact profile while preserving all untouched canonical axes. */ +export function withBenchProfile( + base: AgentProfile, + settings: BenchProfileSettings & { name?: string }, +): AgentProfile { + const metadata = { + ...(base.model?.metadata ?? {}), + ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), + ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), + ...(settings.maxRetries !== undefined ? { maxRetries: settings.maxRetries } : {}), + ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}), + ...(settings.seed !== undefined ? { seed: settings.seed } : {}), + ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}), + ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}), + } + return agentProfileSchema.parse({ + ...base, + ...(settings.name ? { name: settings.name } : {}), + model: { + ...base.model, + ...(settings.reasoningEffort !== undefined + ? { reasoningEffort: settings.reasoningEffort } + : {}), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + }, + ...(settings.systemPrompt !== undefined + ? { prompt: { ...base.prompt, systemPrompt: settings.systemPrompt } } + : {}), + }) +} + +export function benchProfileModel(profile: AgentProfile): string { + const model = profile.model?.default + if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') { + throw new Error('benchmark AgentProfile.model.default must be concrete') + } + return model +} + +/** + * The benchmark-side entry to Runtime's canonical one-turn path. + * It is only an ergonomic composition: Runtime still parses the exact profile, + * materializes the executor, records identity/usage/result events, and refuses + * profile axes the direct Router backend cannot carry. + */ +export async function runBenchRouterTurn( + config: BenchRouterTurnConfig, + input: string | { readonly messages: ReadonlyArray>> }, +): Promise { + if (!config.profile.model?.default) { + throw new Error('runBenchRouterTurn: profile.model.default is required') + } + const factory = createExecutor({ + backend: 'router', + routerBaseUrl: config.routerBaseUrl, + routerKey: config.routerKey, + ...(config.tools ? { tools: config.tools } : {}), + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile: config.profile }, + input, + { + ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }), + ...(config.signal ? { signal: config.signal } : {}), + }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`) + } + return turn +} diff --git a/bench/src/run-benchmarks.ts b/bench/src/run-benchmarks.ts index e72bcbeb..76d7c1ba 100644 --- a/bench/src/run-benchmarks.ts +++ b/bench/src/run-benchmarks.ts @@ -178,7 +178,6 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB backend: cell.backend ?? 'router', routerBaseUrl, routerKey, - model: cell.model, ...(bridgeUrl ? { bridgeUrl } : {}), ...(bridgeBearer ? { bridgeBearer } : {}), ...(sandboxBaseUrl ? { sandboxBaseUrl } : {}), @@ -186,7 +185,13 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB ...(timeoutMs ? { timeoutMs } : {}), }) const harness = cell.harness ?? (cell.profile?.metadata?.backendType as string | undefined) ?? 'opencode' - const profile: AgentProfile = cell.profile ?? { name: cell.label, metadata: { backendType: harness } } + const profile: AgentProfile = + cell.profile ?? { + name: cell.label, + harness: harness as AgentProfile['harness'], + model: { provider: 'tangle-router', default: cell.model }, + metadata: { backendType: harness }, + } // Unique per shot: the same (adapter, task) runs concurrently across cells and reps, so the box // name and runId must not collide. const uniq = Math.random().toString(36).slice(2, 8) diff --git a/bench/src/sandbox-run.ts b/bench/src/sandbox-run.ts index d665f6bc..7716a3b4 100644 --- a/bench/src/sandbox-run.ts +++ b/bench/src/sandbox-run.ts @@ -13,11 +13,11 @@ import { type AgentProfile, type AgentRunSpec, type OutputAdapter, - routerChatWithUsage, } from '@tangle-network/agent-runtime/kernel' // `BackendType` is the sandbox SDK's harness union and its canonical home. agent-runtime consumes // it from there too; it is not re-exported from the kernel barrel. import type { BackendType } from '@tangle-network/sandbox' +import { runBenchRouterTurn } from './router-turn' /** Parse the agent's final answer from the event stream (harness-agnostic). * The default deliverable; a benchmark whose artifact is a file overrides via @@ -63,18 +63,21 @@ export const llmAnalyst = (cfg: { routerBaseUrl: string; routerKey: string; mode .map((e) => (typeof e === 'string' ? e : JSON.stringify(e))) .join('\n') .slice(-2000) - const { content } = await routerChatWithUsage(cfg, [ + const systemPrompt = + "You review an AI agent's previous attempt at a task. From the task, the attempt's output, and its execution trace ALONE, judge whether it correctly and completely solved the task. If you find a specific fault — a wrong value, a guessed API signature, a missing step, a misread requirement — name it and give the concrete correction in 1-3 sentences. Reply exactly 'no change needed' if the attempt looks correct and complete." + const turn = await runBenchRouterTurn( { - role: 'system', - content: - "You review an AI agent's previous attempt at a task. From the task, the attempt's output, and its execution trace ALONE, judge whether it correctly and completely solved the task. If you find a specific fault — a wrong value, a guessed API signature, a missing step, a misread requirement — name it and give the concrete correction in 1-3 sentences. Reply exactly 'no change needed' if the attempt looks correct and complete.", + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'sandbox-run-analyst', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt }, + }, }, - { - role: 'user', - content: `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`, - }, - ]) - return content + `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`, + ) + return turn.finalText } /** Cost-dial backend = the SDK's canonical `BackendType` (single source of truth; no local diff --git a/bench/src/search-bench/bridge.ts b/bench/src/search-bench/bridge.ts index 2a36549b..6223bb3b 100644 --- a/bench/src/search-bench/bridge.ts +++ b/bench/src/search-bench/bridge.ts @@ -13,6 +13,7 @@ * `opencode/zai-coding-plan/glm-5.1`), so `harness` here is just the label. */ import { createExecutor } from '@tangle-network/agent-runtime/kernel' +import type { AgentProfile } from '@tangle-network/agent-interface' import type { SearchArm } from './profiles' import { armLabel } from './profiles' import type { SearchCellResult } from './run.mts' @@ -22,9 +23,19 @@ const nativeWebDisallowed = ['WebSearch', 'WebFetch', 'web_search', 'web_fetch', /** Build the cli-bridge `agent_profile` for one arm (bridge dialect: disable via * `metadata.disallowedTools`, search MCP via `mcp..transport:'http'`). */ -function bridgeProfile(arm: SearchArm, routerSearchMcp: string, tangleApiKey: string, label: string): Record { - if (arm === 'native') return { name: `search-bench-${label}` } - const base = { name: `search-bench-${label}`, metadata: { disallowedTools: nativeWebDisallowed } } +function bridgeProfile( + arm: SearchArm, + routerSearchMcp: string, + tangleApiKey: string, + label: string, + model: string, +): AgentProfile { + const identity = { name: `search-bench-${label}`, model: { default: model } } + if (arm === 'native') return identity + const base: AgentProfile = { + ...identity, + metadata: { disallowedTools: nativeWebDisallowed }, + } if (arm === 'off') return base return { ...base, @@ -76,14 +87,19 @@ export async function runBridgeCell( try { // One harness turn through the unified bridge executor — same backend the // loop path uses; this cell scorer just adds oracle scoring + citations. + const profile = bridgeProfile( + arm, + cfg.routerSearchMcp, + cfg.tangleApiKey, + `${harness}-${armId}`, + cfg.bridgeModels[harness] ?? harness, + ) const exec = createExecutor({ backend: 'bridge', bridgeUrl: cfg.bridgeUrl, bridgeBearer: cfg.bridgeBearer, - model: cfg.bridgeModels[harness] ?? harness, - agentProfile: bridgeProfile(arm, cfg.routerSearchMcp, cfg.tangleApiKey, `${harness}-${armId}`), timeoutMs: cfg.timeoutMs ?? 300_000, - })({ profile: { name: `${harness}-${armId}` }, harness: null }, { signal: controller.signal, seams: {} }) + })({ profile, harness: null }, { signal: controller.signal, seams: {} }) // bridgeExecutor is one-shot (async execute resolves an ExecutorResult). const artifact = (await exec.execute(taskToPrompt(task), controller.signal)) as { out: unknown diff --git a/bench/src/search-bench/parametric-check.mts b/bench/src/search-bench/parametric-check.mts index 9780fb7e..5494dd7b 100644 --- a/bench/src/search-bench/parametric-check.mts +++ b/bench/src/search-bench/parametric-check.mts @@ -11,7 +11,7 @@ */ import { writeFileSync } from 'node:fs' import { runPool } from '../run-pool' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from '../router-turn' import { freshTasks } from './tasks-fresh' import { scoreTask, taskToPrompt } from './tasks' @@ -28,9 +28,24 @@ async function main(): Promise { const outcomes = await runPool(freshTasks, conc, async (task) => { try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: taskToPrompt(task) }]) - const { score } = scoreTask(task, res.content) - return { id: task.id, score: score as 0 | 1 | null, cost: res.costUsd, err: undefined as string | undefined } + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'search-parametric-check', + model: { provider: 'tangle-router', default: model }, + }, + }, + taskToPrompt(task), + ) + const { score } = scoreTask(task, res.finalText) + return { + id: task.id, + score: score as 0 | 1 | null, + cost: res.usage.usdKnown === false ? undefined : res.usage.costUsd, + err: undefined as string | undefined, + } } catch (err) { return { id: task.id, score: null as 0 | 1 | null, cost: undefined, err: err instanceof Error ? err.message : String(err) } } diff --git a/bench/src/supervisor-arena.mts b/bench/src/supervisor-arena.mts index d292a647..028386e0 100644 --- a/bench/src/supervisor-arena.mts +++ b/bench/src/supervisor-arena.mts @@ -37,6 +37,7 @@ import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from import { tmpdir } from 'node:os' import { join } from 'node:path' import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' import { pool } from './stats.mts' // ---------- pre-registered task sets (verbatim from the prereg; DO NOT EDIT) ---------- @@ -294,33 +295,40 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), Number(process.env.LLM_TIMEOUT_MS ?? 240_000)) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'supervisor-arena-agent', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { temperature: cfg.temperature, maxTokens: cfg.maxTokens }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/swe-arena/capacity.ts b/bench/src/swe-arena/capacity.ts index a660538b..523e5b21 100644 --- a/bench/src/swe-arena/capacity.ts +++ b/bench/src/swe-arena/capacity.ts @@ -6,13 +6,11 @@ * router 503-stormed. Rule encoded here: gate every arm on the endpoint that * arm actually calls; supervisor arms MUST include the router-path probe. * - * Secrets discipline: probes spawn `dotenvx run … -- bash -c 'curl …'` from - * the secrets dir; the API key is referenced by NAME inside the child shell - * (single-quoted script, so it is never expanded — let alone logged — in this - * process). + * The containing experiment is launched through dotenvx, so probes read the already-scoped key + * from this process and enter Runtime through one exact AgentProfile. */ -import { run } from './proc' +import { runBenchRouterTurn } from '../router-turn' import type { SecretsEnv } from './arms' export type CapacityProbe = (signal?: AbortSignal) => Promise @@ -104,10 +102,10 @@ export async function waitForCapacity(gate: EndpointCapacityGate, signal?: Abort export interface HttpProbeSpec { url: string + provider: string /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */ apiKeyEnv: string model: string - secrets: SecretsEnv /** curl --max-time, seconds. Default 40 (probe-capacity.sh). */ maxTimeS?: number /** @@ -117,55 +115,54 @@ export interface HttpProbeSpec { maxTokens?: number } -export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4/chat/completions' -export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1/chat/completions' - -/** Build the probe request body (probe-body.json semantics). */ -export function probeBody(model: string, maxTokens: number): string { - return JSON.stringify({ - model, - messages: [{ role: 'user', content: 'Reply with the single word OK.' }], - max_tokens: maxTokens, - temperature: 0, - }) -} +export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4' +export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1' /** - * Generic chat-completions probe: true iff the endpoint returns HTTP 200 - * within the time budget. The key stays inside the child shell. + * Generic chat-completions probe through Runtime: true only when the selected model emits `OK` + * within the time budget. */ export function httpCapacityProbe(spec: HttpProbeSpec): CapacityProbe { if (!/^[A-Z_][A-Z0-9_]*$/.test(spec.apiKeyEnv)) { throw new Error(`invalid apiKeyEnv name: ${spec.apiKeyEnv}`) } - const body = probeBody(spec.model, spec.maxTokens ?? 8000) const maxTime = spec.maxTimeS ?? 40 return async (signal?: AbortSignal) => { signal?.throwIfAborted() - // Body via stdin (--data @-) so the payload never sits on a command line. - // The HTTP code is marker-anchored because dotenvx writes its injection - // banner to the same stdout stream. - const script = - `curl -sS -o /dev/null -w "HTTP_CODE=%{http_code}" --max-time ${maxTime} ` + - `-X POST "$PROBE_URL" ` + - `-H "Authorization: Bearer $${spec.apiKeyEnv}" -H "Content-Type: application/json" ` + - `--data @-` - const argv = ['run', ...spec.secrets.envFiles.flatMap((f) => ['-f', f]), '--', 'bash', '-c', script] - const res = await run('dotenvx', argv, { - cwd: spec.secrets.secretsDir, - timeoutMs: (maxTime + 20) * 1000, - stdin: body, - env: { ...process.env, PROBE_URL: spec.url }, - signal, - }) - signal?.throwIfAborted() - return /HTTP_CODE=200\s*$/.test(res.stdout) + const routerKey = process.env[spec.apiKeyEnv] + if (!routerKey) throw new Error(`${spec.apiKeyEnv} is required; launch through dotenvx`) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: spec.url.replace(/\/chat\/completions\/?$/u, ''), + routerKey, + profile: { + name: `capacity-${spec.model}`, + harness: 'cli-base', + model: { + provider: spec.provider, + default: spec.model, + metadata: { temperature: 0, maxTokens: spec.maxTokens ?? 8000 }, + }, + prompt: { systemPrompt: 'Reply with the single word OK.' }, + }, + timeoutMs: maxTime * 1000, + signal, + }, + 'Capacity probe.', + ) + return turn.finalText.trim() === 'OK' } } /** probe-capacity.sh's z.ai coding-plan probe (the WORKER path). */ export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe { - return httpCapacityProbe({ url: ZAI_CODING_ENDPOINT, apiKeyEnv: 'ZAI_API_KEY', model, secrets }) + void secrets + return httpCapacityProbe({ + url: ZAI_CODING_ENDPOINT, + apiKeyEnv: 'ZAI_API_KEY', + provider: 'zai', + model, + }) } /** @@ -173,7 +170,13 @@ export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): Capacity * Supervisor arms must gate on this; probing only z.ai is the proven blind spot. */ export function routerProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe { - return httpCapacityProbe({ url: ROUTER_ENDPOINT, apiKeyEnv: 'TANGLE_API_KEY', model, secrets }) + void secrets + return httpCapacityProbe({ + url: ROUTER_ENDPOINT, + apiKeyEnv: 'TANGLE_API_KEY', + provider: 'tangle-router', + model, + }) } /** The gates an arm must pass, by kind: solo → worker path; supervisor → BOTH paths. */ diff --git a/bench/src/swe-arena/diagnosis-ensemble.ts b/bench/src/swe-arena/diagnosis-ensemble.ts index 8647660b..d1805ec5 100644 --- a/bench/src/swe-arena/diagnosis-ensemble.ts +++ b/bench/src/swe-arena/diagnosis-ensemble.ts @@ -12,9 +12,8 @@ * gpt-5.5 / opus-4.8 slot in by editing the analyst spec list, never by a * hardcoded requirement on an unrouted model. * - analysts are blind: same bundle, no cross-talk, independent calls. - * - secrets discipline matches capacity.ts: the API key is referenced by NAME - * inside a dotenvx child shell; the response body lands in a file (dotenvx - * writes its banner to stdout, so stdout is only trusted for the marker). + * - the containing run is launched through dotenvx; every analyst enters Runtime through one + * exact AgentProfile and its event record is retained with the response. */ import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises' @@ -22,7 +21,7 @@ import { join } from 'node:path' import { type AnalystFinding, makeFinding } from '@tangle-network/agent-eval' import { findSupervisorRunDir, type SecretsEnv } from './arms' import { ROUTER_ENDPOINT, sleepWithSignal } from './capacity' -import { run } from './proc' +import { runBenchRouterTurn } from '../router-turn' // --------------------------------------------------------------------------- // Analyst specs. @@ -32,6 +31,8 @@ export interface AnalystSpec { /** Stable label, e.g. 'glm-5.2#1'. Used in fusion attribution + filenames. */ id: string model: string + /** Provider identity stamped into the exact profile. Default: tangle-router. */ + provider?: string /** Chat-completions endpoint. Default: the Tangle router. */ url?: string /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */ @@ -247,8 +248,7 @@ export function parseAnalystFindings(text: string): AnalystRawFinding[] { }) } -/** POST one blind analyst call through the router via dotenvx (key stays in - * the child; response body lands in `outFile`, never on shared stdout). +/** Run one blind analyst through Runtime's exact-profile Router boundary. * ONE retry on transport failure (5xx/524/timeout) — an edge flake was * observed live in the calibration smoke; a parse failure is NOT retried * (same prompt, same model ⇒ same bad shape). */ @@ -265,60 +265,63 @@ export async function runAnalyst( return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: `invalid apiKeyEnv name: ${apiKeyEnv}` } } const url = spec.url ?? ROUTER_ENDPOINT - const body = JSON.stringify({ - model: spec.model, - messages: [{ role: 'user', content: analystPrompt(bundle) }], - temperature: spec.temperature ?? 0, - max_tokens: spec.maxTokens ?? 16_000, - }) + const routerKey = process.env[apiKeyEnv] + if (!routerKey) { + return { + analystId: spec.id, + model: spec.model, + ok: false, + findings: [], + error: `${apiKeyEnv} is required; launch through dotenvx`, + } + } + void secrets opts.signal?.throwIfAborted() await mkdir(scratchDir, { recursive: true }) const outFile = join(scratchDir, `analyst-${spec.id.replace(/[^a-zA-Z0-9._-]/g, '_')}.response.json`) const timeoutMs = opts.timeoutMs ?? 600_000 - // Body via stdin (--data @-) so the payload never sits on a command line; - // the HTTP code is marker-anchored on stdout (dotenvx banners share stdout). - const script = - `curl -sS -o "$DIAG_OUT" -w "HTTP_CODE=%{http_code}" --max-time ${Math.ceil(timeoutMs / 1000) - 20} ` + - `-X POST "$DIAG_URL" -H "Authorization: Bearer $${apiKeyEnv}" -H "Content-Type: application/json" --data @-` - const argv = ['run', ...secrets.envFiles.flatMap((f) => ['-f', f]), '--', 'bash', '-c', script] const attempts = 1 + (opts.retries ?? 1) let transportError = '' - let transported = false - for (let attempt = 0; attempt < attempts && !transported; attempt++) { + let content = '' + let tokens: AnalystReport['tokens'] + for (let attempt = 0; attempt < attempts && content.length === 0; attempt++) { opts.signal?.throwIfAborted() if (attempt > 0) await sleepWithSignal(opts.retryDelayMs ?? 5_000, opts.signal) - const res = await run('dotenvx', argv, { - cwd: secrets.secretsDir, - timeoutMs, - stdin: body, - env: { ...process.env, DIAG_URL: url, DIAG_OUT: outFile }, - signal: opts.signal, - }) - opts.signal?.throwIfAborted() - const codeMatch = res.stdout.match(/HTTP_CODE=(\d{3})\s*$/) - if (codeMatch && codeMatch[1] === '200') { - transported = true - } else { - transportError = `router call failed (http=${codeMatch?.[1] ?? 'none'}, rc=${res.code}${res.timedOut ? ', timeout' : ''}, attempt=${attempt + 1}/${attempts})` + try { + const turn = await runBenchRouterTurn( + { + routerBaseUrl: url.replace(/\/chat\/completions\/?$/u, ''), + routerKey, + profile: { + name: `diagnosis-${spec.id}`, + harness: 'cli-base', + model: { + provider: spec.provider ?? 'tangle-router', + default: spec.model, + metadata: { + temperature: spec.temperature ?? 0, + maxTokens: spec.maxTokens ?? 16_000, + }, + }, + prompt: { systemPrompt: 'Diagnose the supplied run evidence as an independent analyst.' }, + }, + timeoutMs, + signal: opts.signal, + }, + analystPrompt(bundle), + ) + content = turn.finalText + if (turn.usage.tokensKnown !== false) { + tokens = { input: turn.usage.input, output: turn.usage.output } + } + await writeFile(outFile, JSON.stringify(turn, null, 2)) + } catch (cause) { + transportError = `router call failed (${(cause as Error).message}, attempt=${attempt + 1}/${attempts})` } } - if (!transported) { + if (content.length === 0 && transportError) { return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: transportError } } - let content = '' - let tokens: AnalystReport['tokens'] - try { - const parsed = JSON.parse(await readFile(outFile, 'utf8')) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - content = parsed.choices?.[0]?.message?.content ?? '' - if (parsed.usage) { - tokens = { input: parsed.usage.prompt_tokens ?? 0, output: parsed.usage.completion_tokens ?? 0 } - } - } catch (cause) { - return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: `unparseable response body: ${(cause as Error).message}` } - } if (content.trim().length === 0) { return { analystId: spec.id, model: spec.model, ok: false, findings: [], error: 'empty content (max_tokens starvation?)', tokens } } diff --git a/bench/src/swe-arena/execution.test.mts b/bench/src/swe-arena/execution.test.mts index 61dc9fb2..5b2bb230 100644 --- a/bench/src/swe-arena/execution.test.mts +++ b/bench/src/swe-arena/execution.test.mts @@ -31,7 +31,12 @@ import { type SupervisorArmResult, } from './arms.ts' import { containerName, materializeWorkspace } from './materialize.ts' -import { gatesForArmKind, probeBody, probeWindow, waitForCapacity, httpCapacityProbe } from './capacity.ts' +import { + gatesForArmKind, + httpCapacityProbe, + probeWindow, + waitForCapacity, +} from './capacity.ts' import { run, runOk } from './proc.ts' import { createSerializedJudge, @@ -1065,15 +1070,6 @@ describe('capacity gate', () => { expect(await waitForCapacity(gate)).toBe(false) }) - it('probe body defaults to max_tokens 8000 (glm-5.2 starves below it)', () => { - expect(JSON.parse(probeBody('glm-5.2', 8000))).toEqual({ - model: 'glm-5.2', - messages: [{ role: 'user', content: 'Reply with the single word OK.' }], - max_tokens: 8000, - temperature: 0, - }) - }) - it('supervisor arms gate on BOTH worker and router paths; solo on worker only', () => { const secrets = { secretsDir: '/dev/null', envFiles: ['agent-state.env'] } expect(gatesForArmKind('solo', secrets).map((g) => g.name)).toEqual(['z.ai-coding']) @@ -1081,9 +1077,13 @@ describe('capacity gate', () => { }) it('rejects a non-env-shaped key name (no value smuggling)', () => { - const secrets = { secretsDir: '/dev/null', envFiles: [] } expect(() => - httpCapacityProbe({ url: 'https://x', apiKeyEnv: 'k; cat /etc/passwd', model: 'm', secrets }), + httpCapacityProbe({ + url: 'https://x', + apiKeyEnv: 'k; cat /etc/passwd', + provider: 'test', + model: 'm', + }), ).toThrow(/apiKeyEnv/) }) }) diff --git a/bench/src/swe-jail.test.ts b/bench/src/swe-jail.test.ts new file mode 100644 index 00000000..1e15e077 --- /dev/null +++ b/bench/src/swe-jail.test.ts @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { zaiChatRaw } from './swe-jail' + +test('zaiChatRaw preserves the SWE transport shape through Runtime', async () => { + const originalFetch = globalThis.fetch + let requestBody: Record | undefined + let requestUrl = '' + globalThis.fetch = async (input, init) => { + requestUrl = String(input) + requestBody = JSON.parse(String(init?.body)) as Record + return new Response( + JSON.stringify({ + choices: [ + { + message: { + content: null, + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'run', arguments: '{"command":"pwd"}' }, + }, + ], + }, + }, + ], + usage: { prompt_tokens: 11, completion_tokens: 3 }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ) + } + + try { + const result = await zaiChatRaw( + { base: 'http://router.test/v1', key: 'secret', timeoutMs: 1_000 }, + { + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'inspect' }], + tools: [ + { + type: 'function', + function: { name: 'run', parameters: { type: 'object' } }, + }, + ], + tool_choice: 'required', + temperature: 0.1, + max_tokens: 32_768, + thinking: { type: 'enabled' }, + }, + { + name: 'swe-jail-test-worker', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + tools: { run: true }, + }, + ) + + assert.equal(requestUrl, 'http://router.test/v1/chat/completions') + assert.deepEqual(requestBody, { + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'inspect' }], + tools: [ + { + type: 'function', + function: { name: 'run', parameters: { type: 'object' } }, + }, + ], + tool_choice: 'required', + temperature: 0.1, + max_tokens: 32_768, + thinking: { type: 'enabled' }, + }) + assert.equal(result.attempts, 1) + assert.deepEqual(result.json, { + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'run', arguments: '{"command":"pwd"}' }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 11, completion_tokens: 3 }, + }) + } finally { + globalThis.fetch = originalFetch + } +}) diff --git a/bench/src/swe-jail.ts b/bench/src/swe-jail.ts index 57aeeb40..2d1da733 100644 --- a/bench/src/swe-jail.ts +++ b/bench/src/swe-jail.ts @@ -12,12 +12,15 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { ToolSpec } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn, withBenchProfile } from './router-turn' const exec = promisify(execFile) export const tail = (s: string, n: number): string => (s.length > n ? `…${s.slice(s.length - n)}` : s) -// ---------- zai chat client (plain fetch; patient 429 ladder) ---------- +// ---------- zai chat client (Runtime transport; patient 429 ladder) ---------- export interface ZaiCfg { base: string @@ -27,12 +30,14 @@ export interface ZaiCfg { * runs — past this instant, so a per-instance deadline reaches INTO the 429 ladder instead of * letting a doomed retry sleep for another 240s after the instance was already written off. */ deadlineAt?: number + /** Caller-owned retry ladder length. Default 7. */ + maxAttempts?: number } export interface ZaiRaw { - /** The parsed /chat/completions JSON, verbatim. */ + /** The OpenAI-compatible message and usage fields expected by existing SWE callers. */ json: Record - /** HTTP attempts spent (retries included). */ + /** Runtime completion attempts spent (retries included). */ attempts: number } @@ -44,10 +49,39 @@ export interface ZaiRaw { * tool_calls — is the glm reasoning path starving `content` when reasoning eats max_tokens, and is * retried too (a tool_calls turn with empty content is a NORMAL tool-loop turn, not starvation). */ -export async function zaiChatRaw(cfg: ZaiCfg, body: Record): Promise { +export async function zaiChatRaw( + cfg: ZaiCfg, + body: Record, + profile: AgentProfile, +): Promise { + const { + model, + messages, + tools, + temperature, + max_tokens: maxTokens, + tool_choice: toolChoice, + ...extraBody + } = body + if (typeof model !== 'string' || model.length === 0) { + throw new Error('completion body.model must be a non-empty string') + } + if (profile.model?.default !== model) { + throw new Error('completion body.model must equal AgentProfile.model.default') + } + if (!Array.isArray(messages)) throw new Error('completion body.messages must be an array') + const typedTools = Array.isArray(tools) ? (tools as ToolSpec[]) : [] + const typedToolChoice = + toolChoice === 'auto' || toolChoice === 'required' || toolChoice === 'none' + ? toolChoice + : undefined let lastErr = '' let delayBase = 2_000 - for (let attempt = 1; attempt <= 7; attempt += 1) { + const maxAttempts = cfg.maxAttempts ?? 7 + if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 1) { + throw new Error('ZaiCfg.maxAttempts must be a positive safe integer') + } + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { if (attempt > 1) { const delay = Math.min(delayBase * 2 ** (attempt - 2), 240_000) if (cfg.deadlineAt !== undefined && Date.now() + delay >= cfg.deadlineAt) { @@ -63,30 +97,71 @@ export async function zaiChatRaw(cfg: ZaiCfg, body: Record): Pr const ctl = new AbortController() const timer = setTimeout(() => ctl.abort(), perCallTimeout) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - delayBase = res.status === 429 ? 60_000 : 2_000 - continue - } - const json = (await res.json()) as Record - const msg = ((json.choices as Array<{ message?: Record }> | undefined)?.[0]?.message ?? {}) as { - content?: string - tool_calls?: unknown[] + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: withBenchProfile(profile, { + ...(typeof temperature === 'number' ? { temperature } : {}), + ...(typeof maxTokens === 'number' ? { maxTokens } : {}), + ...(typedToolChoice ? { toolChoice: typedToolChoice } : {}), + extraBody, + }), + tools: typedTools, + signal: ctl.signal, + }, + { messages: messages as Array> }, + ) + const toolCalls = result.toolCalls.map((call, index) => ({ + id: call.id ?? `call_${index}`, + name: call.name, + arguments: call.arguments, + })) + const message = { + role: 'assistant', + content: toolCalls.length > 0 && result.finalText === '' ? null : result.finalText, + ...(toolCalls.length > 0 + ? { + tool_calls: toolCalls.map((call) => ({ + id: call.id, + type: 'function', + function: { name: call.name, arguments: call.arguments }, + })), + } + : {}), } - const hasToolCalls = Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0 - if (!hasToolCalls && String(msg.content ?? '').trim() === '') { + if (toolCalls.length === 0 && result.finalText.trim() === '') { lastErr = 'empty content' continue } + const finalEvent = result.events.at(-1) + const finishReason = + finalEvent?.type === 'final' + ? finalEvent.reason + : toolCalls.length > 0 + ? 'tool_calls' + : 'stop' + const json: Record = { + choices: [ + { + message, + finish_reason: finishReason, + }, + ], + ...(result.usage.tokensKnown !== false + ? { + usage: { + prompt_tokens: result.usage.input, + completion_tokens: result.usage.output, + }, + } + : {}), + } return { json, attempts: attempt } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) + const status = Number(/router (\d+)/.exec(lastErr)?.[1]) + delayBase = status === 429 ? 60_000 : 2_000 } finally { clearTimeout(timer) } diff --git a/bench/src/swe-repro-calibrate.mts b/bench/src/swe-repro-calibrate.mts index b06db532..33018a77 100644 --- a/bench/src/swe-repro-calibrate.mts +++ b/bench/src/swe-repro-calibrate.mts @@ -89,6 +89,11 @@ async function complete(messages: ChatMsg[]): Promise { const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, { model: MODEL, max_tokens: MAX_TOKENS, temperature: TEMP, messages }, + { + name: 'swe-reproduction-calibrator', + model: { provider: 'zai', default: MODEL, reasoningEffort: 'high' }, + prompt: { systemPrompt: AUTHOR_SYSTEM }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }> diff --git a/bench/src/swe-self-improve.mts b/bench/src/swe-self-improve.mts index 06bad9e5..efc9cade 100644 --- a/bench/src/swe-self-improve.mts +++ b/bench/src/swe-self-improve.mts @@ -9,17 +9,42 @@ */ import { mkdtempSync, rmSync } from 'node:fs' import { join } from 'node:path' -import { createChatClient } from '@tangle-network/agent-eval' -import { refine, runAgentic, runStrategyEvolution, sample } from '@tangle-network/agent-runtime/kernel' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + refine, + runAgentic, + runStrategyEvolution, + sample, + strategyAuthorSystemPrompt, +} from '@tangle-network/agent-runtime/kernel' import { createSweBenchEnvironment } from './swe-bench-env' async function main(): Promise { const routerKey = process.env.TANGLE_API_KEY if (!routerKey) throw new Error('TANGLE_API_KEY required (worker + author call the router)') const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' - const workerModel = process.env.WORKER_MODEL ?? 'gemini-2.5-pro' - const authorModel = process.env.AUTHOR_MODEL ?? 'gemini-2.5-pro' + const workerModel = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' + const authorModel = process.env.AUTHOR_MODEL ?? 'deepseek-v4-flash' const innerTurns = Number(process.env.INNER_TURNS ?? 40) + const workerProfile: AgentProfile = { + name: 'swe-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: workerModel, + metadata: { maxTokens: 8000, maxTurns: innerTurns }, + }, + } + const authorProfile = (model: string, name: string): AgentProfile => ({ + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { maxTokens: 8000 }, + }, + prompt: { systemPrompt: strategyAuthorSystemPrompt }, + }) const { environment, tasks } = await createSweBenchEnvironment(Number(process.env.POOL_N ?? 80)) if (process.env.CALIBRATE === '1') { @@ -29,7 +54,15 @@ async function main(): Promise { let resolved = 0 for (const t of ts) { const t0 = Date.now() - const r = await runAgentic({ surface: environment, task: t, strategy: refine, routerBaseUrl, routerKey, model: workerModel, maxTokens: 8000, innerTurns, budget: 1 }) + const r = await runAgentic({ + surface: environment, + task: t, + strategy: refine, + routerBaseUrl, + routerKey, + workerProfile, + budget: 1, + }) if (r.resolved) resolved++ console.log(` ${t.id.padEnd(32)} resolved=${r.resolved} completions=${r.completions} shots=${r.shots} (${Math.round((Date.now() - t0) / 1000)}s)`) } @@ -46,12 +79,14 @@ async function main(): Promise { tasks, trainN: Number(process.env.TRAIN_N ?? 6), holdoutN: Number(process.env.HOLDOUT_N ?? 8), - worker: { routerBaseUrl, routerKey, model: workerModel, maxTokens: 8000, innerTurns }, + worker: { routerBaseUrl, routerKey, workerProfile }, author: { - chat: createChatClient({ transport: 'router', baseUrl: routerBaseUrl, apiKey: routerKey, defaultModel: authorModel }), - model: authorModel, - maxTokens: 8000, - fallbackModel: process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash', + profile: authorProfile(authorModel, 'swe-strategy-author'), + executor: { backend: 'router', routerBaseUrl, routerKey }, + fallbackProfile: authorProfile( + process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash', + 'swe-strategy-author-fallback', + ), }, baselines: [sample, refine], budget: Number(process.env.BUDGET ?? 2), diff --git a/bench/src/swe-stream.mts b/bench/src/swe-stream.mts index 722fd13c..d812f4e1 100644 --- a/bench/src/swe-stream.mts +++ b/bench/src/swe-stream.mts @@ -274,9 +274,28 @@ const makeTransport = counter.guardedMsgs += assertNoHiddenLeak(marks, msgs) // Inject the honored reasoning-budget knob (thinking) here at the single shared worker // chokepoint: makeTransport is byte-identical across arms F and L, so the budget is symmetric. + const model = String(body.model ?? '') + const systemPrompt = msgs.find((message) => message.role === 'system')?.content + const toolNames = Array.isArray(body.tools) + ? (body.tools as Array<{ function?: { name?: unknown } }>).flatMap((tool) => + typeof tool.function?.name === 'string' ? [tool.function.name] : [], + ) + : [] const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt: guard.deadlineAt }, { ...body, ...WORKER_REASONING }, + { + name: 'swe-stream-worker', + model: { + provider: 'zai', + default: model, + reasoningEffort: REASONING_ON ? 'high' : 'none', + }, + ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), + ...(toolNames.length > 0 + ? { tools: Object.fromEntries(toolNames.map((name) => [name, true])) } + : {}), + }, ) counter.calls += 1 counter.httpAttempts += attempts @@ -643,6 +662,11 @@ async function acquireRepro( const { json } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt }, { model: REPRO_MODEL, max_tokens: MAX_TOKENS, temperature: 0.2, messages }, + { + name: 'swe-reproduction-author', + model: { provider: 'zai', default: REPRO_MODEL, reasoningEffort: 'high' }, + prompt: { systemPrompt: reproAuthorSystem(REPRO_TIMEOUT_S) }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }>; usage?: { prompt_tokens?: number; completion_tokens?: number } } out.authorCalls += 1 @@ -810,6 +834,10 @@ async function superviseRepair( const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt }, { model: SUPERVISOR_MODEL, max_tokens: SUPERVISOR_MAX_TOKENS, temperature: 0.2, messages }, + { + name: 'swe-repair-supervisor', + model: { provider: 'zai', default: SUPERVISOR_MODEL, reasoningEffort: 'high' }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }>; usage?: { prompt_tokens?: number; completion_tokens?: number } } const planRaw = d.choices?.[0]?.message?.content ?? '' diff --git a/bench/src/swe-structural.mts b/bench/src/swe-structural.mts index baeaa7f2..2b5cde4a 100644 --- a/bench/src/swe-structural.mts +++ b/bench/src/swe-structural.mts @@ -138,7 +138,25 @@ const makeTransport = async (body: Record): Promise => { const msgs = (body.messages ?? []) as Array<{ role?: string; content?: unknown }> counter.guardedMsgs += assertNoHiddenLeak(marks, msgs) - const { json, attempts } = await zaiChatRaw({ base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, body) + const model = String(body.model ?? '') + const systemPrompt = msgs.find((message) => message.role === 'system')?.content + const toolNames = Array.isArray(body.tools) + ? (body.tools as Array<{ function?: { name?: unknown } }>).flatMap((tool) => + typeof tool.function?.name === 'string' ? [tool.function.name] : [], + ) + : [] + const { json, attempts } = await zaiChatRaw( + { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, + body, + { + name: 'swe-structural-worker', + model: { provider: 'zai', default: model, reasoningEffort: 'high' }, + ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), + ...(toolNames.length > 0 + ? { tools: Object.fromEntries(toolNames.map((name) => [name, true])) } + : {}), + }, + ) counter.calls += 1 counter.httpAttempts += attempts const u = (json as { usage?: { prompt_tokens?: number; completion_tokens?: number } }).usage diff --git a/bench/src/trata-gate.mts b/bench/src/trata-gate.mts index 1c25eea4..1c0ce585 100644 --- a/bench/src/trata-gate.mts +++ b/bench/src/trata-gate.mts @@ -30,6 +30,7 @@ import { appendFileSync } from 'node:fs' import { resolveAdapter } from './adapters' import type { BenchScore, BenchTask } from './benchmarks/types' +import { runBenchRouterTurn } from './router-turn' import { runPool } from './run-pool' function must(name: string): string { @@ -61,33 +62,32 @@ async function workerComplete( cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number }, ): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> { const startedAt = Date.now() - const res = await fetch(`${cfg.routerBaseUrl}/chat/completions`, { - method: 'POST', - signal: AbortSignal.timeout(cfg.timeoutMs), - headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` }, - body: JSON.stringify({ - model: cfg.model, - temperature: 0, - max_tokens: 4096, - messages: [ - { role: 'system', content: ANALYST_SYSTEM }, - { role: 'user', content: task.prompt }, - ], - }), - }) - if (!res.ok) { - const body = (await res.text()).slice(0, 300) - throw new Error(`router ${res.status} for ${task.id}: ${body}`) - } - const j = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const answer = j.choices?.[0]?.message?.content ?? '' + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'trata-financial-analyst', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { + temperature: 0, + maxTokens: Number(process.env.WORKER_MAX_TOKENS ?? 4096), + }, + }, + prompt: { systemPrompt: ANALYST_SYSTEM }, + }, + timeoutMs: cfg.timeoutMs, + }, + task.prompt, + ) + if (result.usage.tokensKnown === false) throw new Error('worker provider omitted token usage') return { - answer, - inputTokens: j.usage?.prompt_tokens ?? 0, - outputTokens: j.usage?.completion_tokens ?? 0, + answer: result.finalText, + inputTokens: result.usage.input, + outputTokens: result.usage.output, durationMs: Date.now() - startedAt, } } diff --git a/bench/src/trata-gepa.mts b/bench/src/trata-gepa.mts index 056c8f15..a3217c63 100644 --- a/bench/src/trata-gepa.mts +++ b/bench/src/trata-gepa.mts @@ -60,6 +60,7 @@ import { officialOptimizerModel, requiredTokenPricing, } from './official-optimizer-config.mjs' +import { runBenchRouterTurn } from './router-turn' interface TrataScenario extends Scenario { task: BenchTask @@ -122,23 +123,31 @@ async function chatComplete( messages: Array<{ role: string; content: string }>, signal: AbortSignal, ): Promise<{ content: string; usage?: { input: number; output: number } }> { - const res = await fetch(`${baseUrl}/chat/completions`, { - method: 'POST', - signal: AbortSignal.any([signal, AbortSignal.timeout(180_000)]), - headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, temperature: 0, max_tokens: maxTokens, messages }), - }) - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 300)}`) - const j = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: baseUrl, + routerKey: key, + profile: { + name: 'trata-gepa-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { temperature: 0, maxTokens }, + }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + signal, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + return { + content: result.finalText, + ...(result.usage.tokensKnown === false + ? {} + : { usage: { input: result.usage.input, output: result.usage.output } }), } - const content = j.choices?.[0]?.message?.content ?? '' - const usage = - j.usage?.prompt_tokens != null - ? { input: j.usage.prompt_tokens, output: j.usage.completion_tokens ?? 0 } - : undefined - return { content, usage } } async function main(): Promise { diff --git a/bench/src/trata-hedge-solve.mts b/bench/src/trata-hedge-solve.mts new file mode 100644 index 00000000..74f60368 --- /dev/null +++ b/bench/src/trata-hedge-solve.mts @@ -0,0 +1,80 @@ +/** Single-shot lower-bound solver for trata-hedge-bench. */ +import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { join, relative, resolve } from 'node:path' +import { runBenchRouterTurn } from './router-turn' + +const environment = process.argv[2] +if (!environment) throw new Error('usage: solve.mts [out.txt]') +const output = process.argv[3] ?? '/tmp/thb-answer.txt' +const model = process.env.WORKER_MODEL +if (!model) throw new Error('WORKER_MODEL is required') +const dataBudget = Number(process.env.DATA_BUDGET ?? 160_000) +const maxTokens = Number(process.env.MAX_TOKENS ?? 6_000) +const temperature = Number(process.env.TEMPERATURE ?? 0.5) +const routerKey = process.env.TANGLE_API_KEY +if (!routerKey) throw new Error('TANGLE_API_KEY is required') +const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' + +for (const [name, value] of [ + ['DATA_BUDGET', dataBudget], + ['MAX_TOKENS', maxTokens], +] as const) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`) +} +if (!Number.isFinite(temperature)) throw new Error('TEMPERATURE must be finite') + +const root = resolve(environment) +const dataDir = join(root, 'environment', 'data') +const instruction = readFileSync(join(root, 'instruction.md'), 'utf8') + +function filesUnder(dir: string): string[] { + return readdirSync(dir) + .flatMap((name) => { + const path = join(dir, name) + return statSync(path).isDirectory() ? filesUnder(path) : [path] + }) +} + +function rank(path: string): number { + if (path.includes('earnings_call')) return 0 + if (path.includes('financials')) return 1 + if (path.includes('company_profiles')) return 2 + if (path.includes('press_releases')) return 3 + return 4 +} + +let used = 0 +const blocks: string[] = [] +const files = filesUnder(dataDir).sort((a, b) => rank(a) - rank(b) || a.localeCompare(b)) +for (const path of files) { + const content = readFileSync(path, 'utf8') + if (used + content.length > dataBudget) continue + used += content.length + blocks.push(`\n=== FILE: data/${relative(dataDir, path)} ===\n${content}`) +} +process.stderr.write(`[solve] ${blocks.length}/${files.length} files in context (${used} chars)\n`) + +const prompt = + '--- AVAILABLE DATA (cite files by their `data/` name inline) ---\n' + + blocks.join('') + + '\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/`.' + +const result = await runBenchRouterTurn( + { + routerBaseUrl, + routerKey, + profile: { + name: 'trata-hedge-solver', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { temperature, maxTokens }, + }, + prompt: { systemPrompt: instruction }, + }, + }, + prompt, +) +writeFileSync(output, result.finalText) +process.stderr.write(`[solve] wrote ${result.finalText.length} chars -> ${output} (model=${model})\n`) diff --git a/bench/src/worker-blender.ts b/bench/src/worker-blender.ts index 6e03c6f8..eee9933c 100644 --- a/bench/src/worker-blender.ts +++ b/bench/src/worker-blender.ts @@ -18,7 +18,7 @@ import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_BLENDER_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_BLENDER_DIRECTIVE } from './directives' @@ -177,17 +177,27 @@ export async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig runShot: async (user, round, dir) => { const runnerPath = join(dir, 'runner.py') const scriptPath = join(dir, 'model.py') - const { content, usage: u } = await routerChatWithUsage( - cfg, - [ - { role: 'system', content: directive }, - { role: 'user', content: user }, - ], - { temperature: 0.3 }, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'blender-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: cfg.model, + metadata: { temperature: 0.3 }, + }, + prompt: { systemPrompt: directive }, + }, + }, + user, ) - if (u) { - usage.input += u.input - usage.output += u.output + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const script = extractPy(content) trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/bench/src/worker-browser.ts b/bench/src/worker-browser.ts index 013ec70a..96ca2b1f 100644 --- a/bench/src/worker-browser.ts +++ b/bench/src/worker-browser.ts @@ -18,7 +18,7 @@ import { readFile } from 'node:fs/promises' import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export interface BrowserLocalConfig { routerBaseUrl: string @@ -67,10 +67,23 @@ export async function solveBrowserLocal(task: BenchTask, cfg: BrowserLocalConfig trace.push({ spanId: 's-task', runId, kind: 'llm', name: 'web task', model: cfg.model, messages: [{ role: 'user', content: goal }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) - const { content, usage } = await routerChatWithUsage(cfg, [ - { role: 'system', content: directive }, - { role: 'user', content: task.prompt }, - ]) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'browser-local-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: directive }, + }, + }, + task.prompt, + ) + const content = turn.finalText + const usage = + turn.usage.tokensKnown === false + ? undefined + : { input: turn.usage.input, output: turn.usage.output } const artifact = content.trim() const elementId = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact)?.[1] ?? '' diff --git a/bench/src/worker-build123d.ts b/bench/src/worker-build123d.ts index 179af399..f4eefc5b 100644 --- a/bench/src/worker-build123d.ts +++ b/bench/src/worker-build123d.ts @@ -21,7 +21,7 @@ import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_BUILD123D_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_BUILD123D_DIRECTIVE } from './directives' @@ -106,13 +106,22 @@ export async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig) runShot: async (user, round, dir) => { const scriptPath = join(dir, 'build.py') const stepPath = join(dir, 'output.step') - const { content, usage: u } = await routerChatWithUsage(cfg, [ - { role: 'system', content: sys }, - { role: 'user', content: user }, - ]) - if (u) { - usage.input += u.input - usage.output += u.output + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'build123d-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: sys }, + }, + }, + user, + ) + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const source = extractPy(content) trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/bench/src/worker-cad.ts b/bench/src/worker-cad.ts index 2799fd51..3757a6c9 100644 --- a/bench/src/worker-cad.ts +++ b/bench/src/worker-cad.ts @@ -17,12 +17,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' -import { acquireSandbox, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { acquireSandbox } from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_CAD_DIRECTIVE, DEFAULT_CAD_SANDBOX_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_CAD_DIRECTIVE } from './directives' @@ -138,13 +139,22 @@ export async function solveCadRefineLocal(task: BenchTask, cfg: CadLocalConfig): const scadPath = join(dir, 'model.scad') const stlPath = join(dir, 'model.stl') const pngPath = join(dir, 'model.png') - const { content, usage: u } = await routerChatWithUsage(cfg, [ - { role: 'system', content: directive }, - { role: 'user', content: user }, - ]) - if (u) { - usage.input += u.input - usage.output += u.output + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'cad-local-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: directive }, + }, + }, + user, + ) + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const scad = extractScad(content) trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) @@ -253,10 +263,19 @@ export async function solveCadRefine(task: BenchTask, cfg: CadRefineConfig): Pro ? task.prompt : `Your previous OpenSCAD had this problem:\n${lastErr}\n\nHere is the previous source:\n${history[history.length - 1]?.artifact ?? ''}\n\nFix it so it compiles AND better matches the brief:\n${task.prompt}`, runShot: async (user, round, box) => { - const { content: reply } = await routerChatWithUsage(cfg, [ - { role: 'system', content: sys }, - { role: 'user', content: user }, - ]) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'cad-sandbox-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: sys }, + }, + }, + user, + ) + const reply = turn.finalText const scad = extractScad(reply) trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: reply.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.scad', content: scad }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/bench/vitest.config.ts b/bench/vitest.config.ts index 0f7255f1..a5b0ac1b 100644 --- a/bench/vitest.config.ts +++ b/bench/vitest.config.ts @@ -8,6 +8,7 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { include: [ + 'src/official-optimizer-config.test.mts', 'src/swe-arena/**/*.test.mts', 'src/quant-arena/**/*.test.mts', 'src/rollout-ledger/**/*.test.mts', diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 693727c6..e680c40c 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.128.0` and `@tangle-network/agent-eval@0.143.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.128.0` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 412 exports. +Import from `@tangle-network/agent-runtime` — 407 exports. | Symbol | Kind | Summary | |---|---|---| @@ -45,7 +45,6 @@ Import from `@tangle-network/agent-runtime` — 412 exports. | `createConversationBackend` | function | Adapt a multi-participant conversation into the standard execution backend contract. | | `createIterableBackend` | function | Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. | | `createKnowledgeImprovementActivationExecutor` | function | Apply or restore one local knowledge candidate through the shared activation contract. | -| `createOpenAICompatibleBackend` | function | OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any | | `createOpenInferenceFileExporter` | function | Create an exporter that APPENDS spans to a local OpenInference-JSONL file, one complete span per | | `createOtelExporter` | function | Create an OTEL exporter. Returns undefined when no endpoint is configured. | | `createProtectedAgentCandidateModelPort` | function | Bind a protected model-grant service to the immutable candidate runtime. | @@ -99,7 +98,6 @@ Import from `@tangle-network/agent-runtime` — 412 exports. | `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | | `reflectiveGenerator` | function | Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. | | `researchLoopRunner` | function | `research` mode — research-in-a-loop with valid-only KB growth. | -| `resolveAgentBackend` | function | Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead | | `resolveChatModel` | function | Resolve a chat model by precedence: the first candidate carrying a | | `resolveRouterBaseUrl` | function | Resolve the router base URL from env, normalised — no trailing `/v1` or `/`. | | `runAgentTask` | function | Single-shot task lifecycle for adapter-driven tasks: readiness-gated, emits the runtime lifecycle event vocabulary, session-store pluggable. | @@ -187,7 +185,6 @@ Import from `@tangle-network/agent-runtime` — 412 exports. | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `AgentSpec` | interface | `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | -| `BackendRetryPolicy` | interface | Retry policy for transient transport errors (rate limits, upstream | | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | | `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | | `BuildPromptFindingsInput` | interface | Evidence supplied to a generated tool or MCP build instruction. | @@ -216,7 +213,6 @@ Import from `@tangle-network/agent-runtime` — 412 exports. | `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | | `VerifyResult` | interface | Outcome of verifying a candidate worktree. `feedback` (compiler errors, | -| `AgentBackendKind` | type | The transport a chat backend runs on. | | `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | | `AgentCandidateCodeSource` | type | Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. | | `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | @@ -261,7 +257,7 @@ Import from `@tangle-network/agent-runtime` — 412 exports. | `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `DriverLoopGeneratorOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `DriverLoopGeneratorOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -510,7 +506,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 719 exports. +Import from `@tangle-network/agent-runtime/kernel` — 711 exports. | Symbol | Kind | Summary | |---|---|---| @@ -528,18 +524,15 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `assertStrategyContract` | function | Static CONTRACT lint over an authored strategy module — the module-boundary | | `assessAuthoredProfile` | function | OBSERVE one authored `AgentProfile` and score its richness (no judge verdict is read). The task | | `auditIntent` | function | The route-rigor analyst: compare declared vs revealed vs user intent over a trajectory and return aligned / drifting / diverged with evidence and one recommended intervention. | -| `authoredWorker` | function | Build a router-only worker from an authored profile. This helper executes the prompt/model axes; | | `authorStrategy` | function | Author + load a strategy from losses. Throws when the author emits no loadable module; | | `bestSoFar` | function | The best-so-far fold — the ONE definition of "how good was the run after k results", shared by | | `breadthStrategy` | function | BREADTH: K independent rollouts (each own artifact), verifier picks the best. | | `buildSteerContext` | function | Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any | | `canDisplace` | function | The repair keep-best guard: a challenger displaces the incumbent only when it is | | `canonicalFindingEvent` | function | Producer-side cleanliness for the `finding` event. The findings payload is arbitrary analyst | -| `canonicalizeAuthoredProfile` | function | Lift a profile the supervisor AUTHORED into the canonical shape every executor reads. | | `captureWorkerTraceEvidence` | function | Collect and persist one executor's structured tool trace without changing its task outcome. | -| `chatCompletionsTransport` | function | The default transport: POST `${url}/chat/completions` with an optional bearer. Fail-loud on | -| `chatTransportExecutor` | function | Build the chat-transport `Executor`: one `execute` = one conversation SHOT — seed (fresh system | -| `chatWorkerSeam` | function | The `makeWorkerAgent` seam over {@link chatTransportExecutor} — the continuity consumer | +| `chatTransportExecutor` | function | Build one exact profile-driven chat executor through `createExecutor`. | +| `chatWorkerSeam` | function | Session-owning worker factory for graph continuity. | | `closingWorkerNote` | function | The worker's closing commentary off a local harness run: the TAIL of its | | `collectAgentTurn` | function | Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that | | `compareCheckOutcomes` | function | The selection order: crash < ran; then official pass-fraction; authored guesses only | @@ -553,7 +546,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `createActivityLog` | function | Create a bounded activity ring. `limit` caps memory for a worker that runs thousands of tools. | | `createAgentEnvironmentProviderRegistry` | function | Create a registry that resolves provider names to concrete provider instances. | | `createBudgetPool` | function | Create a conserved reservation pool from a root `Budget`. `now()` is injected so the | -| `createChatSessionStore` | function | In-memory `ChatSessionStore`. Entries are detached copies — a caller mutating a saved array | +| `createChatSessionStore` | function | In-memory, process-local conversation store with detached reads and writes. | | `createEventBus` | function | Create the child→parent coordination bus: one typed pipe for settled outputs, questions, and analyst findings, with a priority-ordered pull queue and a pass-through subscribe lane. | | `createExecutor` | function | The single built-in executor factory. Picks a leaf backend by data (`config.backend`), | | `createExecutorRegistry` | function | The open resolver/registry. Pre-registers the three built-ins under their | @@ -621,6 +614,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `loopDispatch` | function | Adapter for `runProfileMatrix` (profile is an axis). Returns a | | `loopUntil` | function | `loopUntil(seed, spec)` — one `step` child per round; `fold` accumulates each settlement into | | `makeFinding` | function | Convenience factory: produce a fully-formed AnalystFinding with the | +| `mapExecutorResult` | function | Transform a Runtime executor's terminal artifact without losing its private | | `mapSandboxEvent` | function | Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, | | `mapSandboxToolEvent` | function | Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of | | `materializeLocalMcp` | function | Spawn every explicitly trusted stdio server in `profile.mcp` as a same-host | @@ -644,6 +638,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `pollFor` | function | Build a bounded `poll` spec from a duration. | | `printBenchmarkReport` | function | Pretty-print a report — the "free optimization" verdict, with the cost vector. | | `probeSandboxCapabilities` | function | Probe (and memoize per client) what the loop may rely on. A client without a | +| `profileChatClient` | function | Profile-exact adapter for packages that consume agent-eval's ChatClient contract. | +| `profileOptimizerModelCall` | function | Profile-exact adapter for agent-eval's external optimizer callback. | | `profileRichnessFinding` | function | Turn a {@link ProfileRichness} verdict into a bus-routable `AnalystFinding` (area `profile-quality`). | | `promotionGate` | function | Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. | | `promptHandle` | function | Parse `'/v'` into a {@link PromptHandle}. The shorthand for authoring a graph edge: | @@ -668,12 +664,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `resolveMcpServerLaunch` | function | Resolve a profile MCP server's `args`/`env` config values (interface ≥0.40 | | `resolveSandboxClient` | function | Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core | | `resolveSecretEnv` | function | Resolve a declared secret-env map into the real env entries for a server | -| `resolveSupervisorProfile` | function | Reduce either profile spelling — a hand-written `SupervisorProfile` or a canonical `AgentProfile` | +| `resolveSupervisorProfile` | function | Reduce one canonical executable profile to the scalars the two brain arms consume. | | `rollingDispatch` | function | Run the refilling dispatch loop over `scope` until the queue is dry (or a stop fires) and every | -| `routerBrain` | function | The router as a supervisor BRAIN: the canonical `ToolLoopChat` seam backed by the router's | -| `routerChatWithTools` | function | A router completion WITH tool-calling — the operator driver's LLM seam. Passes OpenAI-shape | -| `routerChatWithUsage` | function | One OpenAI-compatible chat completion through the Tangle router, returning text + REAL token usage (`undefined` when the provider omits it — never a fabricated 0). | -| `routerToolLoop` | function | The tool-using router backend: a real agentic loop OVER the Tangle router (which | | `runAgentic` | function | Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. | | `runAgentRounds` | function | The round-synchronous MULTI-AGENT kernel: each round `driver.plan()` fans N tasks | | `runBenchmark` | function | Run the requested strategies over the tasks, scored by the Environment's own check. | @@ -700,7 +692,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `spendFromUsageEvents` | function | Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate | | `stopSentinel` | function | A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the | | `streamAgentTurn` | function | Run ONE agent turn on any backend kind and stream its events. Yields the | -| `streamRouterChatWithTools` | function | The SAME completion as `routerChatWithTools`, taken over SSE (`stream: true`) and reassembled | | `structuralRollout` | function | Build the structuralRollout `Strategy`: k shots → score each by the frozen visible | | `sumSandboxUsage` | function | Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an | | `supervise` | function | One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. | @@ -755,6 +746,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `sample` | const | Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). | | `sampleThenRefine` | const | The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), | | `strategyAuthorContract` | const | The compressed consumable a skill carries: everything an author needs to emit a loop. | +| `strategyAuthorSystemPrompt` | const | Standing behavior callers put in the strategy-author AgentProfile. | | `supervisorPolicyPrompt` | const | THE supervisor policy — one stance, both front doors. The work-vs-delegate rule is conditional | | `VERIFY_TAIL_CHARS` | const | Tail of the verify output — the failing assertion lives at the END of a test log. | | `WORKER_TOOL_TRACE_SCHEMA_VERSION` | const | Schema version for content-addressed worker tool-trace artifacts. | @@ -793,8 +785,10 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `BudgetPoolRestore` | interface | State recovered from a prior process before new work is admitted. `committed` is measured spend | | `BusEvent` | interface | Every bus event is a discriminated union member keyed by `type`. | | `BusRecord` | interface | A published event stamped for ordering and observability. `seq` is the monotonic publish index; | -| `ChatSessionStore` | interface | Conversation history keyed by the settled worker id — the resume substrate. The kernel owns | -| `ChatTransportTool` | interface | One entry of the caller-provided tool table: the OpenAI function spec the model sees, and the | +| `ChatSessionStore` | interface | Conversation history keyed by the settled Runtime worker id. | +| `ChatTransportExecutorOptions` | interface | Transport and session data for one exact profile-driven conversation. | +| `ChatTransportTool` | interface | One profile-authorized function tool and its host implementation. | +| `ChatWorkerSeamOptions` | interface | Transport/session configuration shared by every spawned exact profile. | | `CheckExecChannel` | interface | Minimal exec channel the default runner needs. `SandboxInstance` (and therefore | | `CheckOutcome` | interface | How one candidate fared against the frozen visible checks, split by check kind. | | `CheckpointCapableBox` | interface | Loop-side widening of the box's optional checkpoint method. The | @@ -906,14 +900,14 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `RenderCorpusToInstructionsOptions` | interface | Project accreted corpus facts into an `AgentProfile`'s instruction seams — the learning-flywheel | | `ReservationTicket` | interface | Opaque, single-use reservation handle returned by `reserve` and consumed by | | `ResolvedMcpServerLaunch` | interface | The spawn-ready strings for one stdio MCP server: profile config values | -| `ResolvedSupervisorProfile` | interface | A `SupervisorProfile` reduced to the scalars the two brain arms consume. `modelId`/`systemPrompt` | +| `ResolvedSupervisorProfile` | interface | The exact profile fields consumed by supervisor materialization. | | `ResultBlobStore` | interface | Content-addressed result blobs (the `outRef` → artifact map) backing the replay | | `ResumedKeyState` | interface | What the journal proves about one keyed assignment at resume time. | | `ResumedWork` | interface | The committed work a resumed run inherits from its journal. `settled` is the replayed | | `RootHandle` | interface | Live root handle — a chat/pi-viz client uses it to inspect and control one root run. | -| `RouterSeam` | interface | Router/inline connection seam. A direct OpenAI-compatible Router endpoint — | -| `RouterToolCall` | interface | A tool-call the model emitted (provider-neutral; mirrors the runtime's ToolCallRequest). | +| `RouterSeam` | interface | Router/inline transport seam. The profile owns model, prompt, and generation behavior. | | `RouterToolsSeam` | interface | Router seam WITH tool use — the tool-using router backend. Same direct | +| `RouterTransportConfig` | interface | Connection details for Runtime's Router-backed executors. | | `RunPersonifiedOptions` | interface | The end-to-end entrypoint. Builds the persona's root `Agent` from the chosen shape, then | | `RunProvenance` | interface | Domain-free run provenance: a manifest of what was mounted into the run's | | `SandboxCapabilities` | interface | What the loop kernel is allowed to know about a sandbox backend: a single | @@ -938,7 +932,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `ShapeBudget` | interface | Budget knobs a shape reads to size its fanout/children WITHOUT owning the conserved pool. | | `ShapeContext` | interface | The construction context a `LoopShape` factory receives. Carries the persona's resolved | | `ShapeRegistry` | interface | The open shape registry — the extension point that makes a new loop-shape ONE file + one | -| `ShotPersona` | interface | A role for one shot — multi-agent loops (researcher + engineer, a panel of k | | `SpawnForest` | interface | Complete cold-readable view of one recursive supervision run. | | `SpawnForestEvent` | interface | One event with the journal tree that establishes its cursor namespace. | | `SpawnForestInDoubtNode` | interface | A spawned worker with no terminal record in a cold snapshot. Resume treats the same state as | @@ -960,7 +953,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `SuperviseSurfaceResult` | interface | The deployable outcome of a supervised surface run. | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | | `SupervisorNodeContext` | interface | Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot | -| `SupervisorProfile` | interface | The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. | | `SupervisorSpanOutcome` | interface | How the supervised run ended, as `finish()` records it on the root span. | | `SupervisorToolDescriptor` | interface | One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies | | `SupervisorToolInvocationContext` | interface | Trusted context for one product-tool invocation. The node identity remains the same detached, | @@ -998,7 +990,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `AuthorizeDownMessage` | type | Product decision over an exact continuation before it is durably recorded or delivered. | | `AxisScoresOf` | type | Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the | | `BudgetReadout` | type | Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`, | -| `ChatCompletionsTransport` | type | One buffered chat-completions call: the OpenAI-shape request body in, the parsed completion | +| `ChatCompletionsTransport` | type | Buffered OpenAI-compatible completion port used only for offline execution. | | `CombinatorShape` | type | A combinator is just a `LoopShape`: a factory `(ShapeContext) => Agent` whose `Agent.act` | | `ContinuityMode` | type | How a spawn CONTINUES a node's prior work: `'fresh'` starts a brand-new session (the default, | | `CoordinationDeliveryEvidence` | type | Durable delivery evidence retained in commit order. An attempt without a later event carrying | @@ -1057,6 +1049,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `SupervisedResult` | type | Typed terminal result (M2) — a no-winner is NEVER coerced to a best-effort output. | | `SupervisorFinalizer` | type | The finalization seam: ledger in, output (or `undefined` = nothing deliverable) out. | | `SupervisorNodeContextSeed` | type | Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. | +| `SupervisorProfile` | type | A supervisor is an exact canonical AgentProfile; no looser model/prompt shape exists. | | `SupervisorSpanAttributes` | type | OTLP span attribute values. Exported because `SupervisorSpanOptions.attributes` is public and | | `ToolLoopChat` | type | One inference turn over the running conversation + the tool specs → the model's text, any | | `ToolLoopCompactionOptions` | type | Public supervisor-facing compaction config: same knobs as the primitive, but `distill` is optional | @@ -1078,7 +1071,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 719 exports. | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `ChatTransportExecutorOptions`, `ChatWorkerSeamOptions`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RouterChatResult`, `RouterChatToolsResult`, `RouterConfig`, `RouterToolLoopResult`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge @@ -1199,14 +1192,14 @@ Import from `@tangle-network/agent-runtime/platform` — 20 exports. ### PrimeIntellect: Verifiers package and trace adapter -Import from `@tangle-network/agent-runtime/primeintellect` — 30 exports. +Import from `@tangle-network/agent-runtime/primeintellect` — 29 exports. | Symbol | Kind | Summary | |---|---|---| -| `createPrimeIntellectBackend` | function | Build the existing runtime backend against Prime's intercepted model endpoint. | | `createPrimeIntellectPackage` | function | Build a complete PrimeIntellect Verifiers package without writing to disk. | | `importPrimeIntellectTraces` | function | Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. | | `parsePrimeIntellectTraces` | function | Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. | +| `primeIntellectExecutorConfig` | function | Resolve Prime's intercepted endpoint as transport-only Runtime executor configuration. | | `primeIntellectTraceToRunRecord` | function | Project one complete Prime trace into the common agent-eval analysis row. | | `readPrimeIntellectEpisodeContext` | function | Read and validate the private process contract installed by the generated Prime harness. | | `runPrimeIntellectProgram` | function | Execute the caller's canonical runtime program inside a Prime rollout. | @@ -1215,7 +1208,7 @@ Import from `@tangle-network/agent-runtime/primeintellect` — 30 exports. | `PrimeIntellectRunner` | interface | Files and commands that make the caller's real agent program runnable. | | `PrimeIntellectTask` | interface | One immutable problem. References stay inside Prime's task process. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `PrimeIntellectEpisodeContext`, `PrimeIntellectPackageBundle`, `PrimeIntellectPackageManifest`, `PrimeIntellectPackageOptions`, `PrimeIntellectTrace`, `PrimeIntellectTraceImportOptions`, `PrimeTimeSpan`, `PrimeTraceNode`, `PrimeUsage`, `RunPrimeIntellectProgramOptions`, `WritePrimeIntellectPackageOptions`, `PrimeIntellectBackendOptions`, `PrimeIntellectContent`, `PrimeIntellectImportDefaults`, `PrimeIntellectJson`, `PrimeIntellectMessage`, `PrimeIntellectScoring`, `PrimeIntellectSetupCommand`, `PrimeIntellectSplit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `PrimeIntellectEpisodeContext`, `PrimeIntellectPackageBundle`, `PrimeIntellectPackageManifest`, `PrimeIntellectPackageOptions`, `PrimeIntellectTrace`, `PrimeIntellectTraceImportOptions`, `PrimeTimeSpan`, `PrimeTraceNode`, `PrimeUsage`, `RunPrimeIntellectProgramOptions`, `WritePrimeIntellectPackageOptions`, `PrimeIntellectContent`, `PrimeIntellectImportDefaults`, `PrimeIntellectJson`, `PrimeIntellectMessage`, `PrimeIntellectScoring`, `PrimeIntellectSetupCommand`, `PrimeIntellectSplit`. ### Candidate execution — immutable prepare, run, grade, and receipt @@ -1368,7 +1361,7 @@ Import from `@tangle-network/agent-runtime/mcp` — 211 exports. | `DELEGATE_FEEDBACK_DESCRIPTION` | const | Human-readable description of the `delegate_feedback` MCP tool, injected into the tool manifest. | | `DELEGATE_FEEDBACK_INPUT_SCHEMA` | const | JSON Schema for `delegate_feedback` tool arguments (`refersTo`, `rating`, `by`, optional fields). | | `DELEGATE_FEEDBACK_TOOL_NAME` | const | MCP tool name for the `delegate_feedback` feedback-recording tool. | -| `DELEGATE_INPUT_SCHEMA` | const | JSON Schema for `delegate` tool arguments (`intent` + optional `model` and `runId`). | +| `DELEGATE_INPUT_SCHEMA` | const | JSON Schema for `delegate` tool arguments (`intent` + optional trace id). | | `DELEGATE_TOOL_NAME` | const | MCP tool name for the `delegate` generic-delegation tool. | | `DELEGATE_UI_AUDIT_DESCRIPTION` | const | Human-readable description of the `delegate_ui_audit` MCP tool, injected into the tool manifest. | | `DELEGATE_UI_AUDIT_INPUT_SCHEMA` | const | JSON Schema for `delegate_ui_audit` tool arguments (`workspaceDir`, `routes`, optional config). | @@ -1581,7 +1574,7 @@ Import from `@tangle-network/agent-eval` — 61 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 334 exports. +Import from `@tangle-network/agent-eval/campaign` — 353 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1604,6 +1597,7 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `canonicalDigest` | function | Return the canonical SHA-256 digest of a JSON-serializable value. | | `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | | `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | +| `combineComparisonCosts` | function | Combine method costs without turning one unknown bill into a known total. | | `compareOptimizationMethods` | function | Compare complete optimization methods on disjoint train, selection, and final test data. | | `compareRankKeys` | function | Compare fixed-length lexicographic rank keys where each element is higher-is-better. | | `componentSurfaceIdentityMaterial` | function | Return deterministic identity material independent of component key order. | @@ -1648,6 +1642,8 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | | `renderSurfaceDiff` | function | Canonical customer-visible description of the exact before/after surfaces. | +| `resolveExternalOptimizerCallbackLimits` | function | _(no summary — add a TSDoc line at the declaration)_ | +| `resolveExternalOptimizerProcessLimits` | function | _(no summary — add a TSDoc line at the declaration)_ | | `resolveRunDir` | function | Resolve a campaign `runDir`. An absolute path is honored as-is (the caller | | `resolveWorktreePath` | function | Resolve a code candidate for evaluation only after verifying its immutable | | `rolloutArgumentDiff` | function | Deterministic per-field diff of call arguments between passing and failing | @@ -1671,6 +1667,8 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `validateSearchLedgerEvent` | function | Validate and return a canonical copy. Arrays whose order is not semantic are | | `verifyCodeSurface` | function | Verify a finalized code surface against its current checkout. This rejects | | `verifyLoopProvenanceRecord` | function | Recompute and validate the self-addressed durable record. | +| `DEFAULT_EXTERNAL_OPTIMIZER_CALLBACK_LIMITS` | const | _(no summary — add a TSDoc line at the declaration)_ | +| `DEFAULT_EXTERNAL_OPTIMIZER_PROCESS_LIMITS` | const | _(no summary — add a TSDoc line at the declaration)_ | | `paretoPolicy` | const | The default strategy: symmetric multi-objective Pareto significance. Ship iff | | `SEARCH_LEDGER_SCHEMA` | const | _(no summary — add a TSDoc line at the declaration)_ | | `FileSearchLedger` | class | Append-only file-backed search ledger with idempotent writes and replay. | @@ -1697,6 +1695,7 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `CrossSurfaceSelectionPolicy` | interface | Predeclared candidate eligibility and composition policy. | | `CrossSurfaceTaskRow` | interface | Canonical per-task input row. Consumers may extend this interface with | | `DispatchContext` | interface | Context handed to every dispatch invocation. Scoped — every | +| `ExternalOptimizerModelCallRequest` | interface | One exact model request admitted by the loopback proxy. | | `ExternalTextOptimizationMethodConfig` | interface | Configuration for adapting another text optimizer. | | `Gate` | interface | Composable promotion gate. | | `GenerationCandidate` | interface | One scored candidate surface in a generation. `dimensions` + `scenarios` | @@ -1706,7 +1705,7 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `JudgeScore` | interface | The canonical judge verdict shape — one declaration, shared by campaign | | `LabeledScenarioWrite` | interface | Required-provenance write. The store rejects writes that | | `LoopProvenanceRecord` | interface | The durable provenance record. Aligns to the hosted `EvalRunEvent` path but | -| `OpenAICompatibleOptimizerModel` | interface | One metered OpenAI-compatible model connection shared by official optimizers. | +| `OpenAICompatibleOptimizerModel` | interface | One metered model path supplied by the package that owns execution. | | `OptimizationMethod` | interface | A complete optimization method, including candidate generation and selection. | | `OptimizationMethodInput` | interface | Shared inputs for one optimization method. Final test data is absent. | | `ParetoParent` | interface | A non-dominated parent on the GEPA Pareto frontier — a | @@ -1738,6 +1737,11 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `CostLedgerHandle` | type | Public callback surface for a shared cost ledger. | | `CrossSurfaceAttemptCompleteness` | type | Whether one candidate attempt produced a usable executable outcome. | | `DispatchFn` | type | One function: scenario + ctx → artifact. Dispatcher chooses | +| `ExternalOptimizerChatRequest` | type | Provider-neutral request parsed once from the optimizer's loopback protocol. | +| `ExternalOptimizerEvaluationObservation` | type | Durable callback-side record of every candidate submitted for scoring, | +| `ExternalOptimizerModelCall` | type | Execution-neutral model-call seam for an external optimizer. | +| `ExternalOptimizerModelCallResult` | type | Runtime-owned result for one admitted optimizer-model call. | +| `ExternalOptimizerModelExecutionObservation` | type | One opaque Runtime execution record retained for one admitted model call. | | `GateCheckStatus` | type | Outcome of one check that contributed to a release decision. | | `GateDecision` | type | Five-valued verdict taxonomy (MOSS-paper alignment). | | `GepaAdaptiveEngineRun` | type | An engine in an adaptive run. All engines share the recipe evaluation limit. | @@ -1755,7 +1759,7 @@ Import from `@tangle-network/agent-eval/campaign` — 334 exports. | `ProposalFindingOrigin` | type | Data sources that candidate generation may intentionally learn from. | | `SearchLedgerTrustedHeadMode` | type | How this ledger uses its trusted head — the `(sequence, entryHash)` pin kept | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalyzeCrossSurfaceInteractionsInput`, `AutoevalsScoreLike`, `AxisEvidence`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `BuildTraceAnalystSurfaceDispatchOptions`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareOptimizationMethodsOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DefaultProductionGateOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `ExternalOptimizationExample`, `ExternalTextEvaluationResponse`, `ExternalTextOptimizerContext`, `ExternalTextOptimizerResult`, `FsLabeledScenarioStoreOptions`, `GateContext`, `GateContribution`, `GateResult`, `GenerationRecord`, `GepaOptimizationMethodConfig`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `LoopProvenanceEvidence`, `LoopProvenanceOptimizationMethod`, `NeutralizationGateOptions`, `OpenAutoPrOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizationMethodComparison`, `OptimizationMethodPairwise`, `OptimizationMethodProvenance`, `OptimizationMethodResult`, `OptimizationMethodScore`, `OptimizationPackageSource`, `OptimizationTokenUsage`, `OptimizerConfig`, `PairedHoldout`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PhoenixEvaluationResultLike`, `PhoenixEvaluatorLike`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `PremeasuredOptimizationBaseline`, `ProfileSummary`, `PromotionObjective`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunCampaignOptions`, `RunEvalOptions`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SingleRunLock`, `SkillOptOptimizationMethodConfig`, `SkillOptTrainerConfig`, `TraceAnalystArtifact`, `TraceAnalystScenario`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `AutoevalsScorerLike`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `DefaultProductionGateCheck`, `DefaultProductionRewardHackingOptions`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `OptimizerModelBudget`, `RedactionStatus`, `RunImprovementLoopOptions`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`, `SequentialDecision`, `SkillOptRunnerCommand`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalyzeCrossSurfaceInteractionsInput`, `AutoevalsScoreLike`, `AxisEvidence`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `BuildTraceAnalystSurfaceDispatchOptions`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareOptimizationMethodsOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DefaultProductionGateOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `ExternalOptimizationExample`, `ExternalOptimizerCallbackLimits`, `ExternalOptimizerExecutionSummary`, `ExternalOptimizerModelBudget`, `ExternalOptimizerObservationSummary`, `ExternalOptimizerProcessLimits`, `ExternalOptimizerRunnerCommand`, `ExternalTextEvaluationResponse`, `ExternalTextOptimizerContext`, `ExternalTextOptimizerResult`, `FsLabeledScenarioStoreOptions`, `GateContext`, `GateContribution`, `GateResult`, `GenerationRecord`, `GepaOptimizationMethodConfig`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `LoopProvenanceEvidence`, `LoopProvenanceOptimizationMethod`, `NeutralizationGateOptions`, `OpenAutoPrOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizationMethodComparison`, `OptimizationMethodPairwise`, `OptimizationMethodProvenance`, `OptimizationMethodResult`, `OptimizationMethodScore`, `OptimizationPackageSource`, `OptimizationTokenUsage`, `OptimizerConfig`, `PairedHoldout`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PhoenixEvaluationResultLike`, `PhoenixEvaluatorLike`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `PremeasuredOptimizationBaseline`, `ProfileSummary`, `PromotionObjective`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunCampaignOptions`, `RunEvalOptions`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SingleRunLock`, `SkillOptOptimizationMethodConfig`, `SkillOptTrainerConfig`, `TraceAnalystArtifact`, `TraceAnalystScenario`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `AutoevalsScorerLike`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `DefaultProductionGateCheck`, `DefaultProductionRewardHackingOptions`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `ExternalOptimizerEndpointFormat`, `ExternalOptimizerEvaluationRefusalReason`, `OptimizerModelBudget`, `RedactionStatus`, `RunImprovementLoopOptions`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`, `SequentialDecision`, `SkillOptRunnerCommand`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/examples/ablation-suite/ablation.ts b/examples/ablation-suite/ablation.ts index 90b5372a..5d5ddf8c 100644 --- a/examples/ablation-suite/ablation.ts +++ b/examples/ablation-suite/ablation.ts @@ -17,6 +17,7 @@ * contamination-proof task, THEN point `environment`/`tasks` at a real library or SWE-bench. */ import { pairedBootstrap } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { type AgenticSurface, type AgenticTask, @@ -117,9 +118,7 @@ export async function runAblation(opts: { worker: { routerBaseUrl: string routerKey: string - model: string - maxTokens?: number - innerTurns?: number + profile: AgentProfile } /** The DRIVER brain's own router substrate (used by the `driverSteer`/`optimize` arms). Defaults to * the worker's router + model. The supervisor's inference is separate compute from the worker's, so @@ -127,7 +126,7 @@ export async function runAblation(opts: { supervisor?: { routerBaseUrl?: string routerKey?: string - model?: string + profile?: AgentProfile /** Reflection model for the GEPA optimize pass (defaults to the supervisor/worker model). */ reflectionModel?: string } @@ -147,8 +146,10 @@ export async function runAblation(opts: { const supervisorRouter = { baseUrl: opts.supervisor?.routerBaseUrl ?? opts.worker.routerBaseUrl, apiKey: opts.supervisor?.routerKey ?? opts.worker.routerKey, - model: opts.supervisor?.model ?? opts.worker.model, + profile: opts.supervisor?.profile ?? opts.worker.profile, } + const workerMaxTokens = profileNumber(opts.worker.profile, 'maxTokens') + const workerMaxTurns = profileNumber(opts.worker.profile, 'maxTurns') // Wrap the env ONCE so every arm's per-tool call counts are captured (reset per arm below). The GEPA // train pass deliberately uses the raw `opts.environment` so its disjoint-slice calls are NOT counted // into the held-out arm's tally. @@ -210,33 +211,26 @@ export async function runAblation(opts: { // the FULL conserved spend (driver inference + all worker work: $, tokens, latency). `shots` // stays 0 — a multi-worker supervised run has no single refine-shot count (N/A, not a real zero). const sup = await superviseSurface( - { name: 'driver', prompt: { systemPrompt: driverPrompt } }, + withSystemPrompt(supervisorRouter.profile, 'driver', driverPrompt), t, { surface: counter, worker: { routerBaseUrl: opts.worker.routerBaseUrl, routerKey: opts.worker.routerKey, - model: opts.worker.model, - ...(opts.worker.maxTokens !== undefined - ? { maxTokens: opts.worker.maxTokens } - : {}), - ...(opts.worker.innerTurns !== undefined - ? { innerTurns: opts.worker.innerTurns } - : {}), + profile: opts.worker.profile, budget: arm.knobs.budget, }, budget: { - // Pool for the driver's turns PLUS several worker spawns (each reserves ~innerTurns+2 + // Pool for the driver's turns PLUS several worker spawns (each reserves ~maxTurns+2 // iterations) so the spawn-targeted-worker loop runs, not stall after one. The autopsy // measures the real cost; this is intentionally not equal-k. - maxIterations: arm.knobs.budget * ((opts.worker.innerTurns ?? 6) + 2) + 16, - maxTokens: (opts.worker.maxTokens ?? 4000) * Math.max(4, arm.knobs.budget * 3), + maxIterations: arm.knobs.budget * ((workerMaxTurns ?? 6) + 2) + 16, + maxTokens: (workerMaxTokens ?? 4000) * Math.max(4, arm.knobs.budget * 3), }, router: { routerBaseUrl: supervisorRouter.baseUrl, routerKey: supervisorRouter.apiKey, - model: supervisorRouter.model, }, analysts: failuresAnalyst(), }, @@ -258,9 +252,7 @@ export async function runAblation(opts: { budget: arm.knobs.budget, routerBaseUrl: opts.worker.routerBaseUrl, routerKey: opts.worker.routerKey, - model: opts.worker.model, - ...(opts.worker.maxTokens !== undefined ? { maxTokens: opts.worker.maxTokens } : {}), - ...(opts.worker.innerTurns !== undefined ? { innerTurns: opts.worker.innerTurns } : {}), + workerProfile: opts.worker.profile, }) if (r.resolved) resolved++ scoreSum += r.score @@ -366,19 +358,24 @@ export function printAutopsy(results: ArmResult[]): void { async function main(): Promise { const routerKey = process.env.TANGLE_API_KEY if (!routerKey) throw new Error('TANGLE_API_KEY required') + const workerModel = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' const worker = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey, - model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash', - maxTokens: 4000, - innerTurns: Number(process.env.INNER_TURNS ?? 6), + profile: routerProfile( + 'worker', + workerModel, + Number(process.env.MAX_TOKENS ?? 4000), + Number(process.env.MAX_TURNS ?? 6), + ), } + const supervisorModel = process.env.SUPERVISOR_MODEL ?? workerModel const supervisor = { - model: process.env.SUPERVISOR_MODEL ?? worker.model, + profile: routerProfile('driver', supervisorModel), reflectionModel: process.env.REFLECTION_MODEL ?? 'gemini-2.5-pro', } console.log( - `═══ ABLATION (cheap contamination-proof task) — worker=${worker.model} driver=${supervisor.model} ═══`, + `═══ ABLATION (cheap contamination-proof task) — worker=${workerModel} driver=${supervisorModel} ═══`, ) const results = await runAblation({ environment: codingEnv, @@ -404,6 +401,39 @@ async function main(): Promise { printAutopsy(results) } +function routerProfile( + name: string, + model: string, + maxTokens?: number, + maxTurns?: number, +): AgentProfile { + return { + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + ...(maxTokens !== undefined || maxTurns !== undefined + ? { + metadata: { + ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(maxTurns !== undefined ? { maxTurns } : {}), + }, + } + : {}), + }, + } +} + +function withSystemPrompt(profile: AgentProfile, name: string, systemPrompt: string): AgentProfile { + return { ...profile, name, prompt: { ...profile.prompt, systemPrompt } } +} + +function profileNumber(profile: AgentProfile, key: 'maxTokens' | 'maxTurns'): number | undefined { + const value = profile.model?.metadata?.[key] + return typeof value === 'number' ? value : undefined +} + if (import.meta.url === `file://${process.argv[1]}`) main().catch((e) => { console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) diff --git a/examples/ablation-suite/gepa-driver-prompt.ts b/examples/ablation-suite/gepa-driver-prompt.ts index d02bb7df..5a1ed270 100644 --- a/examples/ablation-suite/gepa-driver-prompt.ts +++ b/examples/ablation-suite/gepa-driver-prompt.ts @@ -55,16 +55,14 @@ export async function optimizeDriverPrompt(opts: { worker: { routerBaseUrl: string routerKey: string - model: string - maxTokens?: number - innerTurns?: number + profile: AgentProfile /** Refine-shot budget per worker — MUST match the deployment arm's budget, or the prompt is tuned * against a different per-attempt compute than it serves with. */ budget?: number } /** The supervisor brain's router substrate for each candidate's supervised run (the driver's own * inference). Defaults to the worker's router + model when omitted. */ - supervisorRouter?: { baseUrl: string; apiKey: string; model: string } + supervisorRouter?: { baseUrl: string; apiKey: string; profile: AgentProfile } reflectionModel?: string maxEvaluations?: number maxProposerCostUsd?: number @@ -79,8 +77,12 @@ export async function optimizeDriverPrompt(opts: { const supervisorRouter = opts.supervisorRouter ?? { baseUrl: worker.routerBaseUrl, apiKey: worker.routerKey, - model: worker.model, + profile: worker.profile, } + const supervisorModel = profileModel(supervisorRouter.profile, 'supervisor') + const workerModel = profileModel(worker.profile, 'worker') + const workerMaxTokens = profileNumber(worker.profile, 'maxTokens') + const workerMaxTurns = profileNumber(worker.profile, 'maxTurns') const selectionN = opts.selectionN ?? opts.trainN const testN = opts.testN ?? opts.trainN @@ -127,27 +129,30 @@ export async function optimizeDriverPrompt(opts: { const paid = await ctx.cost.runPaidCall({ channel: 'agent', actor: 'supervised-run', - model: supervisorRouter.model, + model: supervisorModel, signal: ctx.signal, execute: () => - superviseSurface({ name: 'driver', prompt: { systemPrompt } }, scenario.task, { - surface, - worker, - // A small conserved pool: enough for the driver's turns plus several worker spawns so the - // spawn-targeted-worker loop runs, sized off the worker's inner-loop bounds. - budget: { - maxIterations: (worker.innerTurns ?? 6) * 3 + 16, - maxTokens: (worker.maxTokens ?? 4000) * 6, + superviseSurface( + withSystemPrompt(supervisorRouter.profile, 'driver', systemPrompt), + scenario.task, + { + surface, + worker, + // A small conserved pool: enough for the driver's turns plus several worker spawns so the + // spawn-targeted-worker loop runs, sized off the worker's inner-loop bounds. + budget: { + maxIterations: (workerMaxTurns ?? 6) * 3 + 16, + maxTokens: (workerMaxTokens ?? 4000) * 6, + }, + router: { + routerBaseUrl: supervisorRouter.baseUrl, + routerKey: supervisorRouter.apiKey, + }, + analysts: failuresAnalyst(), }, - router: { - routerBaseUrl: supervisorRouter.baseUrl, - routerKey: supervisorRouter.apiKey, - model: supervisorRouter.model, - }, - analysts: failuresAnalyst(), - }), + ), receipt: (sup) => ({ - model: supervisorRouter.model, + model: supervisorModel, inputTokens: sup.tokensIn, outputTokens: sup.tokensOut, ...(sup.usd > 0 ? { actualCostUsd: sup.usd } : {}), @@ -185,10 +190,7 @@ export async function optimizeDriverPrompt(opts: { }), } - const profile: AgentProfile = { - name: 'ablation-driver', - prompt: { systemPrompt: opts.baselinePrompt }, - } + const profile = withSystemPrompt(supervisorRouter.profile, 'ablation-driver', opts.baselinePrompt) const maxProposerCostUsd = opts.maxProposerCostUsd ?? 3 const optimizer = officialOptimizerModel({ env: process.env, @@ -202,11 +204,11 @@ export async function optimizeDriverPrompt(opts: { surface: 'prompt', executionRef: canonicalCandidateDigest({ callback: 'examples/ablation-suite/gepa-driver-prompt', - workerModel: worker.model, - workerMaxTokens: worker.maxTokens ?? null, - workerTurns: worker.innerTurns ?? null, + workerModel, + workerMaxTokens: workerMaxTokens ?? null, + workerTurns: workerMaxTurns ?? null, workerShots: worker.budget ?? null, - supervisorModel: supervisorRouter.model, + supervisorModel, workerEndpoint: new URL(worker.routerBaseUrl).origin, supervisorEndpoint: new URL(supervisorRouter.baseUrl).origin, }), @@ -231,7 +233,6 @@ export async function optimizeDriverPrompt(opts: { trustResumeState: true, describeScenario: (scenario) => ({ id: scenario.id, - systemPrompt: scenario.task.systemPrompt, userPrompt: scenario.task.userPrompt, }), }), @@ -258,3 +259,24 @@ export async function optimizeDriverPrompt(opts: { usd: result.cost.totalCostUsd, } } + +function withSystemPrompt(profile: AgentProfile, name: string, systemPrompt: string): AgentProfile { + return { + ...profile, + name, + prompt: { ...profile.prompt, systemPrompt }, + } +} + +function profileModel(profile: AgentProfile, role: string): string { + const model = profile.model?.default + if (typeof model !== 'string' || model.length === 0 || model === 'runtime-selected') { + throw new Error(`optimizeDriverPrompt: ${role} profile needs a concrete model.default`) + } + return model +} + +function profileNumber(profile: AgentProfile, key: 'maxTokens' | 'maxTurns'): number | undefined { + const value = profile.model?.metadata?.[key] + return typeof value === 'number' ? value : undefined +} diff --git a/examples/ablation-suite/hard-coding-env.ts b/examples/ablation-suite/hard-coding-env.ts index ac64bcbb..50683310 100644 --- a/examples/ablation-suite/hard-coding-env.ts +++ b/examples/ablation-suite/hard-coding-env.ts @@ -457,7 +457,7 @@ export const hardCodingTasks = async (offset: number, n: number): Promise console.log( ` ${r.name.padEnd(14)} resolve=${(100 * r.resolve).toFixed(0)}% score=${r.scoreMean.toFixed(2)} $${r.costUsd.toFixed(3)} (n=${r.n})`, ), }) printAutopsy(results) + +function routerProfile( + name: string, + model: string, + maxTokens?: number, + maxTurns?: number, +): AgentProfile { + return { + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { + ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(maxTurns !== undefined ? { maxTurns } : {}), + }, + }, + } +} diff --git a/examples/ablation-suite/verkit-env.ts b/examples/ablation-suite/verkit-env.ts index fd23977d..fd16b880 100644 --- a/examples/ablation-suite/verkit-env.ts +++ b/examples/ablation-suite/verkit-env.ts @@ -255,7 +255,7 @@ export const verkitTasks = async (offset: number, n: number): Promise = { - profile: config.challengerProfile ?? ({ name: 'challenger' } as AgentProfile), + profile: config.challengerProfile ?? { + name: 'challenger', + harness: 'cli-base', + model: { provider: 'scripted', default: 'offline-challenger' }, + }, taskToPrompt: (t) => t.prompt, } const weakSolverSpec: AgentRunSpec = { - profile: config.weakSolverProfile ?? ({ name: 'weak-solver' } as AgentProfile), + profile: config.weakSolverProfile ?? { + name: 'weak-solver', + harness: 'cli-base', + model: { provider: 'scripted', default: 'offline-weak-solver' }, + }, taskToPrompt: (t) => renderSolverPrompt(t.example, t.sampleIndex), } const strongSolverSpec: AgentRunSpec = { - profile: config.strongSolverProfile ?? ({ name: 'strong-solver' } as AgentProfile), + profile: config.strongSolverProfile ?? { + name: 'strong-solver', + harness: 'cli-base', + model: { provider: 'scripted', default: 'offline-strong-solver' }, + }, taskToPrompt: (t) => renderSolverPrompt(t.example, t.sampleIndex), } diff --git a/examples/agentic-data-creation/offline-fixtures.ts b/examples/agentic-data-creation/offline-fixtures.ts index 423bb60a..76c60c91 100644 --- a/examples/agentic-data-creation/offline-fixtures.ts +++ b/examples/agentic-data-creation/offline-fixtures.ts @@ -18,9 +18,13 @@ * by construction here — a live run produces the real ones. */ -import { createChatClient, llmJudge } from '@tangle-network/agent-eval' +import { llmJudge } from '@tangle-network/agent-eval' import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' -import { inProcessSandboxClient, type SandboxClient } from '@tangle-network/agent-runtime/kernel' +import { + inProcessSandboxClient, + profileChatClient, + type SandboxClient, +} from '@tangle-network/agent-runtime/kernel' import type { SandboxEvent } from '@tangle-network/sandbox' import type { DataExample, SolverArtifact } from './agentic-data-creation' @@ -160,58 +164,88 @@ export function solverClient(strength: 'weak' | 'strong'): SandboxClient { // the transport returns a scripted score from the answer's grade marker; live, a real model scores // the prose. Tuned so EASY → gap ≈ 0.02, HARD → gap ≈ 0.31 (illustrative targets, by construction). export function buildRubricJudge(): JudgeConfig { - const chat = createChatClient({ - transport: 'mock', - defaultModel: 'offline-judge', - handler: async (req) => { - const text = req.messages - .map((m) => (typeof m.content === 'string' ? m.content : '')) - .join('\n') - const m = /<>/.exec(text) - if (!m) throw new Error('offline judge: answer carried no grade marker') - const [, strength, difficulty, sampleIndex] = m - const base = - difficulty === 'hard' - ? strength === 'strong' - ? 0.77 - : 0.46 - : strength === 'strong' - ? 0.86 - : 0.84 - // Per-sample jitter over samples 0,1,2 → −0.02, 0, +0.02, so the N× mean lands back on `base` - // (the variance-reduction step is a real average of distinct scores, not a no-op). - const jitter = (Number(sampleIndex) - 1) * 0.02 - const score = Math.min(1, Math.max(0, base + jitter)) - return { - content: JSON.stringify({ - dimensions: { rubric_coverage: score, correctness: score }, - notes: `offline: ${strength} solver on ${difficulty} example (sample ${sampleIndex})`, - }), - usage: { promptTokens: 130, completionTokens: 25, totalTokens: 155 }, - costUsd: 0.0001, - model: 'offline-judge', - durationMs: 1, - raw: {}, - } - }, - }) - - return llmJudge( - 'rubric-judge', + const instruction = 'Score the candidate ANSWER against the example RUBRIC. Return JSON ' + - '{"dimensions":{"rubric_coverage":N,"correctness":N},"notes":"..."} with each score in [0,1].', + '{"dimensions":{"rubric_coverage":N,"correctness":N},"notes":"..."} with each score in [0,1].' + const dimensions = [ { - chat, - dimensions: [ - { - key: 'rubric_coverage', - description: 'fraction of the rubric criteria the answer satisfies', - }, - { key: 'correctness', description: 'agreement with the reference answer' }, - ], - scale: 'unit', - renderUser: ({ artifact }) => - `RUBRIC:\n${artifact.example.rubric.map((r, i) => `${i + 1}. ${r}`).join('\n')}\n\nANSWER:\n${artifact.answer}`, + key: 'rubric_coverage', + description: 'fraction of the rubric criteria the answer satisfies', }, - ) + { key: 'correctness', description: 'agreement with the reference answer' }, + ] + const systemPrompt = [ + instruction, + '', + 'Score the artifact on EACH of these dimensions:', + ...dimensions.map( + (dimension) => ` - "${dimension.key}": ${dimension.description} (score 0.0 to 1.0)`, + ), + '', + 'Respond with JSON ONLY, no prose. Every dimension is a number in [0.0 to 1.0]:', + `{"dimensions": {${dimensions.map((dimension) => `"${dimension.key}": `).join(', ')}}, "notes": ""}`, + ].join('\n') + const chat = profileChatClient({ + context: 'agentic-data-creation offline judge', + profile: { + name: 'offline-rubric-judge', + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'offline-judge', + metadata: { temperature: 0.1, maxTokens: 800 }, + }, + prompt: { systemPrompt }, + }, + executor: { + backend: 'router', + routerBaseUrl: 'http://offline.invalid/v1', + routerKey: 'injected-transport', + complete: async (body) => { + const messages = Array.isArray(body.messages) + ? (body.messages as Array>) + : [] + const text = messages + .map((message) => (typeof message.content === 'string' ? message.content : '')) + .join('\n') + const match = /<>/.exec(text) + if (!match) throw new Error('offline judge: answer carried no grade marker') + const [, strength, difficulty, sampleIndex] = match + const base = + difficulty === 'hard' + ? strength === 'strong' + ? 0.77 + : 0.46 + : strength === 'strong' + ? 0.86 + : 0.84 + // Per-sample jitter over samples 0,1,2 → −0.02, 0, +0.02, so the N× mean lands back on `base` + // (the variance-reduction step is a real average of distinct scores, not a no-op). + const jitter = (Number(sampleIndex) - 1) * 0.02 + const score = Math.min(1, Math.max(0, base + jitter)) + return { + model: 'offline-judge', + choices: [ + { + message: { + content: JSON.stringify({ + dimensions: { rubric_coverage: score, correctness: score }, + notes: `offline: ${strength} solver on ${difficulty} example (sample ${sampleIndex})`, + }), + }, + }, + ], + usage: { prompt_tokens: 130, completion_tokens: 25, cost: 0.0001 }, + } + }, + }, + }) + + return llmJudge('rubric-judge', instruction, { + chat, + dimensions, + scale: 'unit', + renderUser: ({ artifact }) => + `RUBRIC:\n${artifact.example.rubric.map((r, i) => `${i + 1}. ${r}`).join('\n')}\n\nANSWER:\n${artifact.answer}`, + }) } diff --git a/examples/coding-benchmark/benchmark.ts b/examples/coding-benchmark/benchmark.ts index 1c019a09..93f25e5a 100644 --- a/examples/coding-benchmark/benchmark.ts +++ b/examples/coding-benchmark/benchmark.ts @@ -20,12 +20,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { - agentProfileId, - type ChatClient, - type ChatResponse, - createChatClient, -} from '@tangle-network/agent-eval' +import { agentProfileId, type ChatClient } from '@tangle-network/agent-eval' import { inMemoryCampaignStorage, type JudgeConfig, @@ -35,12 +30,19 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { leaderboard, pairwiseSignificance, + profileChatClient, renderLeaderboardMarkdown, renderPairwiseMarkdown, type SandboxClient, } from '@tangle-network/agent-runtime/kernel' import { codingDispatch } from './dispatch' -import { ensembleCodeJudge, type RubricDim, type RunArtifact, singleCodeJudge } from './eval' +import { + codeJudgeSystemPrompt, + ensembleCodeJudge, + type RubricDim, + type RunArtifact, + singleCodeJudge, +} from './eval' import { csvParserSource, lruCacheSource } from './fixtures' import { type OfflineScript, offlineSandboxClient } from './offline-box' import { harnessProfiles, type ToolPreset } from './profiles' @@ -152,53 +154,60 @@ function clientFor( // ── the judge transport: a real router (live) or a deterministic mock (offline) ─ // Offline the mock handler returns a fixed rubric verdict so the pipeline runs with -// no creds. Live, `createChatClient({ transport: 'router', apiKey })` calls the real -// router. The SAME `singleCodeJudge` / `ensembleCodeJudge` wiring runs either way. -function judgeChat(live: boolean): ChatClient { - if (live) { - const apiKey = process.env.TANGLE_API_KEY - if (!apiKey) throw new Error('--live needs TANGLE_API_KEY for the judge router') - return createChatClient({ - transport: 'router', - apiKey, - ...(process.env.TANGLE_ROUTER_URL ? { baseUrl: process.env.TANGLE_ROUTER_URL } : {}), - defaultModel: process.env.JUDGE_MODEL ?? 'openai/gpt-4.1-2025-04-14', - }) - } +// no creds. Live and offline both enter Runtime through one exact AgentProfile. +function judgeChat(live: boolean, model: string, systemPrompt = ''): ChatClient { const verdict = JSON.stringify({ dimensions: { correctness: 0.85, completeness: 0.8, code_quality: 0.8, robustness: 0.75 }, notes: 'offline mock judge', }) - return createChatClient({ - transport: 'mock', - defaultModel: 'mock-judge', - handler: async (): Promise => ({ - content: verdict, - usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, - costUsd: 0, - model: 'mock-judge', - durationMs: 0, - raw: {}, - }), + const apiKey = live ? process.env.TANGLE_API_KEY : 'injected-transport' + if (!apiKey) throw new Error('--live needs TANGLE_API_KEY for the judge router') + return profileChatClient({ + context: 'coding benchmark judge', + profile: { + name: 'coding-benchmark-judge', + harness: 'cli-base', + model: { + provider: live ? 'tangle-router' : 'scripted', + default: model, + metadata: { temperature: 0.1, maxTokens: 800 }, + }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + }, + executor: { + backend: 'router', + routerBaseUrl: process.env.TANGLE_ROUTER_URL ?? 'https://router.tangle.tools/v1', + routerKey: apiKey, + ...(live + ? {} + : { + complete: async () => ({ + model, + choices: [{ message: { content: verdict } }], + usage: { prompt_tokens: 0, completion_tokens: 0, cost: 0 }, + }), + }), + }, }) } -function judges( - opts: BenchmarkOptions, - chat: ChatClient, -): JudgeConfig[] { +function judges(opts: BenchmarkOptions, live: boolean): JudgeConfig[] { if (opts.ensemble) { // The ensemble scores each panel model through the SAME chat transport — offline // that is the mock, live it is the router. It sees the SAME full context the // single judge does. const scoreOne = async (model: string, context: string): Promise> => { - const res = await chat.chat({ model, messages: [{ role: 'user', content: context }] }) + const res = await judgeChat(live, model).chat({ + model, + messages: [{ role: 'user', content: context }], + }) const parsed = JSON.parse(res.content) as { dimensions: Record } return parsed.dimensions } return [ensembleCodeJudge(scoreOne)] } - return [singleCodeJudge(chat)] + const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash' + return [singleCodeJudge(judgeChat(live, model, codeJudgeSystemPrompt))] } // ── the sweep ───────────────────────────────────────────────────────────────── @@ -223,7 +232,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise [k, rubric[k].weight])) as > const dimensions = dimKeys.map((k) => ({ key: k, description: rubric[k].description })) +/** The judge instructions — the rubric anchors, kept with the judge ONLY. */ +const judgePrompt = [ + 'You are a senior code reviewer scoring a candidate solution to a coding task.', + 'Score each dimension from 0 to 1 (1 = excellent), using the criteria provided.', +].join(' ') + +/** Exact system prompt llmJudge sends after appending its response contract. */ +export const codeJudgeSystemPrompt = [ + judgePrompt, + '', + 'Score the artifact on EACH of these dimensions:', + ...dimensions.map( + (dimension) => ` - "${dimension.key}": ${dimension.description} (score 0.0 to 1.0)`, + ), + '', + 'Respond with JSON ONLY, no prose. Every dimension is a number in [0.0 to 1.0]:', + `{"dimensions": {${dimensions.map((dimension) => `"${dimension.key}": `).join(', ')}}, "notes": ""}`, +].join('\n') + // ── the held-out result ──────────────────────────────────────────────────────── // The substrate's canonical hidden-criteria grade: { passed, total, passRate, notes? }. // `passRate` is the PRIMARY correctness score; `hiddenGrade` makes a no-run an honest 0. @@ -300,12 +319,6 @@ function parseTestCounts(output: string): { total: number; pass: number } { // ── layer 3: the LLM judge(s) — SECONDARY quality signal ─────────────────────── -/** The judge instructions — the rubric anchors, kept with the judge ONLY. */ -const judgePrompt = [ - 'You are a senior code reviewer scoring a candidate solution to a coding task.', - 'Score each dimension from 0 to 1 (1 = excellent), using the criteria provided.', -].join(' ') - /** The full context every judge sees: the code + the deterministic check results + * the held-out pass rate + the eval-only rubric note. Shared by the single judge AND * the ensemble so the panel never grades on less information than the leaderboard judge. */ diff --git a/examples/delegate/delegate.ts b/examples/delegate/delegate.ts index 0380de26..4e056fca 100644 --- a/examples/delegate/delegate.ts +++ b/examples/delegate/delegate.ts @@ -15,7 +15,8 @@ * Run: TANGLE_API_KEY= pnpm tsx examples/delegate/delegate.ts */ import { existsSync, readFileSync } from 'node:fs' -import { delegate } from '@tangle-network/agent-runtime/kernel' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { delegate, supervisorInstructions } from '@tangle-network/agent-runtime/kernel' import { fileDeliverable, makeWriteFileBackend, scratchTarget } from './shared' async function main(): Promise { @@ -28,7 +29,19 @@ async function main(): Promise { const brainModel = process.env.MODEL ?? process.env.BRAIN_MODEL ?? model const { workDir, target, targetAbs } = scratchTarget() - const backend = makeWriteFileBackend({ workDir, routerBaseUrl, routerKey, model }) + const backend = makeWriteFileBackend({ workDir, routerBaseUrl, routerKey }) + const supervisorProfile: AgentProfile = { + name: 'file-delegator', + harness: 'cli-base', + model: { provider: 'tangle-router', default: brainModel }, + prompt: { + systemPrompt: + supervisorInstructions() + + `\nFor this run, every worker you author must use harness="cli-base", ` + + `model.provider="tangle-router", model.default=${JSON.stringify(model)}, and ` + + 'model.metadata.maxTurns=8.', + }, + } const result = await delegate( `Create a file named ${target} containing exactly the word hello (lowercase, no quotes). ` + @@ -36,8 +49,8 @@ async function main(): Promise { `with the single word DONE and STOP — do not call any more tools after the file is written.`, { backend, - router: { routerBaseUrl, routerKey, model: brainModel }, - model: brainModel, + router: { routerBaseUrl, routerKey }, + supervisorProfile, deliverable: fileDeliverable(targetAbs, target), budget: { maxIterations: 40, maxTokens: 200_000, maxUsd: 0.5 }, }, diff --git a/examples/delegate/shared.ts b/examples/delegate/shared.ts index 71f64808..65de6b6e 100644 --- a/examples/delegate/shared.ts +++ b/examples/delegate/shared.ts @@ -45,13 +45,11 @@ export function makeWriteFileBackend(args: { workDir: string routerBaseUrl: string routerKey: string - model: string }): ExecutorConfig { return { backend: 'router-tools', routerBaseUrl: args.routerBaseUrl, routerKey: args.routerKey, - model: args.model, tools: [ { type: 'function' as const, @@ -72,6 +70,5 @@ export function makeWriteFileBackend(args: { ], executeToolCall: async (name, toolArgs) => name === 'write_file' ? writeFileTool(args.workDir, toolArgs) : `unknown tool ${name}`, - maxTurns: 8, } } diff --git a/examples/graphs/user-sim-conversation.ts b/examples/graphs/user-sim-conversation.ts index fac36cce..388d7ac0 100644 --- a/examples/graphs/user-sim-conversation.ts +++ b/examples/graphs/user-sim-conversation.ts @@ -68,7 +68,8 @@ export function userSimConversation(): { id: 'product-agent', profile: { name: 'product-agent', - model: { default: 'scripted/product-agent' }, + harness: 'cli-base', + model: { provider: 'scripted', default: 'scripted/product-agent' }, prompt: { systemPrompt: 'You are the product sales agent. Close honestly.' }, }, }, diff --git a/examples/improve/improve.ts b/examples/improve/improve.ts index 03a34b4f..fa3cbf42 100644 --- a/examples/improve/improve.ts +++ b/examples/improve/improve.ts @@ -63,7 +63,12 @@ export const scriptedWinner: ImproveMethodFactory = (conte async optimize() { return { winnerSurface: context.findings.length > 0 ? 'PROMOTED' : context.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, }) diff --git a/examples/p1-parity/arms.ts b/examples/p1-parity/arms.ts index 82cf419f..5916d5cd 100644 --- a/examples/p1-parity/arms.ts +++ b/examples/p1-parity/arms.ts @@ -45,7 +45,7 @@ import { GraphEdgeCapError, type MakeWorkerAgent, promptHandle, - type RouterConfig, + type RouterTransportConfig, type RunGraphOptions, runGraph, type Spend, @@ -164,12 +164,6 @@ export interface ParityRecord { export interface MultishotArmBackend { readonly agentTransport: MultishotTransport readonly driverTransport: MultishotTransport - /** The reviewer (driver) leg's wire model — REQUIRED, no fallback. Substrate config of this - * arm, exactly as the paired graph arm declares its driver model on `RouterConfig.model` - * (the reviewer PROFILE stays model-less: as the graph ROOT it is materialized by the driver - * brain, whose model axis lives in that substrate config). Live runs feed BOTH arms the same - * env value; the offline backend pins a scripted id. */ - readonly driverModel: string /** The shared completion check, applied to each turn-initial coder reply. MUST be the same * predicate the paired graph arm's deliverable uses, or the comparison is invalid. */ readonly shotPassed: (assistantText: string) => boolean @@ -196,11 +190,8 @@ export type GraphArmBackend = /** OpenAI-compatible base URL both arms' coders speak (e.g. a cli-bridge `/v1`). */ readonly url: string readonly bearer?: string - /** The coder wire model id — the same id the paired multishot arm sends. */ - readonly model: string - /** Router substrate for the reviewer (driver) brain. REQUIRED: a live graph driver with - * neither `brain` nor `router` cannot run at all. */ - readonly router: RouterConfig + /** Transport-only Router substrate; the reviewer profile owns the model. */ + readonly router: RouterTransportConfig readonly shotPassed: (workerOutText: string) => boolean } @@ -348,7 +339,7 @@ export async function runMultishotArm( toolExecutors: {}, maxTurns: cell.shots, agentModel: requireProfileModel(cell.coderProfile, 'coderProfile'), - driverModel: requireModel(backend.driverModel, 'MultishotArmBackend.driverModel'), + driverModel: requireProfileModel(cell.reviewerProfile, 'reviewerProfile'), // Coder sampling parity (F1): pin the turn-initial ceiling to the shared constant; the // agent leg's temperature 0.7 is hardcoded inside `runMultishot` and asserted by test. agentMaxTokens: PARITY_CODER_SAMPLING.maxTokens, @@ -449,11 +440,6 @@ export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Pro makeWorkerAgent: chatWorkerSeam({ url: backend.url, ...(backend.bearer !== undefined ? { bearer: backend.bearer } : {}), - model: backend.model, - // Coder sampling parity (F1): the same pinned temperature + max_tokens the multishot - // arm's coder leg sends, from the one shared constant — never a per-arm choice. - temperature: PARITY_CODER_SAMPLING.temperature, - maxTokens: PARITY_CODER_SAMPLING.maxTokens, deliverable: graph.deliverable, }), router: backend.router, @@ -567,4 +553,13 @@ function validateCell(cell: CellSpec): void { if (typeof cell.task !== 'string' || cell.task.length === 0) { throw new Error('p1-parity: task must be a non-empty string') } + for (const [field, profile] of [ + ['coderProfile', cell.coderProfile], + ['reviewerProfile', cell.reviewerProfile], + ] as const) { + requireProfileModel(profile, field) + if (!profile.model?.provider || profile.harness === undefined) { + throw new Error(`p1-parity: ${field} must declare harness and model.provider`) + } + } } diff --git a/examples/p1-parity/offline.ts b/examples/p1-parity/offline.ts index 612c088c..26850e0e 100644 --- a/examples/p1-parity/offline.ts +++ b/examples/p1-parity/offline.ts @@ -83,7 +83,6 @@ export function offlineMultishotBackend(script: ShotScript): { backend: { agentTransport, driverTransport, - driverModel: 'scripted/parity-reviewer', shotPassed: offlineShotPassed, }, capture: { agentRequests, driverRequests }, @@ -103,6 +102,7 @@ export function meteredScriptedBrain(turns: ScriptedTurn[]): ToolLoopChat { ...(await brain(messages, tools)), usage: { input: 5, output: 5 }, costUsd: 0, + costProvenance: 'billing-receipt', }) } diff --git a/examples/p1-parity/parity.test.ts b/examples/p1-parity/parity.test.ts index a1b3a919..232f015f 100644 --- a/examples/p1-parity/parity.test.ts +++ b/examples/p1-parity/parity.test.ts @@ -44,10 +44,24 @@ const parityCell = (shots: number): CellSpec => ({ // profile stays model-less — the driver model is each arm's substrate config. coderProfile: { name: 'coder', - model: { default: 'scripted/parity-coder' }, + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'scripted/parity-coder', + metadata: { temperature: 0.7, maxTokens: 2500 }, + }, prompt: { systemPrompt: 'Make tests pass.' }, }, - reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + reviewerProfile: { + name: 'reviewer', + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'scripted/parity-reviewer', + metadata: { temperature: 0.9, maxTokens: 600 }, + }, + prompt: { systemPrompt: 'Verify.' }, + }, shots, budget: { maxIterations: 30, maxTokens: 100_000 }, }) @@ -257,8 +271,6 @@ describe('p1-parity — the same cell reaches both arms and both report honestly kind: 'seam', makeWorkerAgent: chatWorkerSeam({ url: 'http://offline.invalid', - temperature: PARITY_CODER_SAMPLING.temperature, - maxTokens: PARITY_CODER_SAMPLING.maxTokens, deliverable: { describe: cell.task, check: (out) => typeof out === 'string' && offlineShotPassed(out), diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts index 94d7ac99..a9d86897 100644 --- a/examples/p1-parity/run-parity.ts +++ b/examples/p1-parity/run-parity.ts @@ -31,7 +31,13 @@ import { parseArgs } from 'node:util' import type { MultishotTransport } from '@tangle-network/agent-eval/multishot' -import { chatCompletionsTransport } from '@tangle-network/agent-runtime/kernel' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import type { CellSpec, GraphArmBackend, MultishotArmBackend, ParityRecord } from './arms' import { runGraphArm, runMultishotArm } from './arms' import { offlineGraphBackend, offlineMultishotBackend } from './offline' @@ -73,10 +79,24 @@ function parityCell(index: number, shots: number): CellSpec { // brain, and the driver model is substrate config (multishot backend / graph RouterConfig). coderProfile: { name: 'coder', - model: { default: 'scripted/parity-coder' }, + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'scripted/parity-coder', + metadata: { temperature: 0.7, maxTokens: 2500 }, + }, prompt: { systemPrompt: 'Make tests pass.' }, }, - reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + reviewerProfile: { + name: 'reviewer', + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'scripted/parity-reviewer', + metadata: { temperature: 0.9, maxTokens: 600 }, + }, + prompt: { systemPrompt: 'Verify.' }, + }, shots, budget: { maxIterations: 30, maxTokens: 100_000 }, } @@ -90,13 +110,25 @@ function liveParityCell(index: number, shots: number, env: LiveEnv): CellSpec { ...base, coderProfile: { ...base.coderProfile, - model: { default: env.coderModel }, + model: { + ...base.coderProfile.model, + provider: 'cli-bridge', + default: env.coderModel, + }, prompt: { systemPrompt: 'Make tests pass. Print the exact line ' + `'${LIVE_PASS_MARKER}' when and ONLY when the full suite genuinely passes.`, }, }, + reviewerProfile: { + ...base.reviewerProfile, + model: { + ...base.reviewerProfile.model, + provider: 'tangle-router', + default: env.driverModel, + }, + }, } } @@ -144,47 +176,62 @@ function requireLiveEnv(): LiveEnv { } } -/** A multishot transport over an OpenAI-compatible chat-completions surface — built on the SAME - * wire function (`chatCompletionsTransport`) the graph arm's `chatTransportExecutor` uses, so - * the two arms' substrate symmetry is by construction, not by parallel implementations. The - * response's own cost field (`usage.cost` / `usage.cost_usd`, the cli-bridge and OpenRouter - * conventions — the same read order `chatTransportExecutor` uses) maps to the result's - * `costUsd`, so `runMultishot` meters the MEASURED dollars instead of firing its price-table - * estimator on every live turn; a turn genuinely without one stays estimator-visible and the - * row states it (`ParityRecord.usdSource`). */ -function completionsTransport(url: string, bearer: string): MultishotTransport { - const post = chatCompletionsTransport({ url, bearer }) +/** Adapt Eval's transcript request to Runtime's exact profile turn. No provider call bypasses + * `createExecutor` + `streamAgentTurn`; request behavior must equal the declared profile. */ +function completionsTransport( + profile: AgentProfile, + url: string, + bearer: string, +): MultishotTransport { return async (req) => { - const body = (await post( - { - model: req.model, - messages: req.messages, - ...(req.tools !== undefined && req.tools.length > 0 ? { tools: req.tools } : {}), - ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), - ...(req.maxTokens !== undefined ? { max_tokens: req.maxTokens } : {}), - }, - req.signal, - )) as { - choices?: Array<{ message?: { content?: string | null; tool_calls?: never[] } }> - usage?: { - prompt_tokens?: number - completion_tokens?: number - cost?: number - cost_usd?: number - } + if (req.model !== profile.model?.default) { + throw new Error('P1 transport request model conflicts with its AgentProfile') + } + const metadata = profile.model?.metadata ?? {} + if (req.temperature !== metadata.temperature || req.maxTokens !== metadata.maxTokens) { + throw new Error('P1 transport generation controls conflict with its AgentProfile') + } + const tools = (req.tools ?? []) as ToolSpec[] + const factory = createExecutor({ + backend: 'router', + routerBaseUrl: url, + routerKey: bearer, + ...(tools.length > 0 ? { tools } : {}), + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + { messages: req.messages as Array> }, + req.signal ? { signal: req.signal } : {}, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `P1 Runtime turn ended with ${turn.status}`) } - const message = body.choices?.[0]?.message - if (message === undefined) throw new Error('chat completion returned no message') - const costUsd = - typeof body.usage?.cost === 'number' - ? body.usage.cost - : typeof body.usage?.cost_usd === 'number' - ? body.usage.cost_usd - : undefined return { - message, - ...(body.usage !== undefined ? { usage: body.usage } : {}), - ...(costUsd !== undefined ? { costUsd } : {}), + message: { + content: turn.finalText, + ...(turn.toolCalls.length > 0 + ? { + tool_calls: turn.toolCalls.map((call, index) => ({ + id: call.id ?? `call_${index}`, + type: 'function' as const, + function: { name: call.name, arguments: call.arguments }, + })), + } + : {}), + }, + ...(turn.usage.tokensKnown === false + ? {} + : { + usage: { + prompt_tokens: turn.usage.input, + completion_tokens: turn.usage.output, + }, + }), + ...(turn.usage.usdKnown === false || turn.usage.costUsd === undefined + ? {} + : { costUsd: turn.usage.costUsd }), } } } @@ -194,7 +241,10 @@ function completionsTransport(url: string, bearer: string): MultishotTransport { const LIVE_PASS_MARKER = 'ALL TESTS PASS' const livePassed = (text: string): boolean => text.includes(LIVE_PASS_MARKER) -function liveBackends(env: LiveEnv): { +function liveBackends( + env: LiveEnv, + cell: CellSpec, +): { multishot: MultishotArmBackend graph: GraphArmBackend } { @@ -202,9 +252,8 @@ function liveBackends(env: LiveEnv): { multishot: { // Coder leg on the shared coder endpoint; reviewer (driver) leg on the shared router // substrate — each leg matching its graph-arm counterpart, including the driver model. - agentTransport: completionsTransport(env.url, env.bearer), - driverTransport: completionsTransport(env.routerUrl, env.routerKey), - driverModel: env.driverModel, + agentTransport: completionsTransport(cell.coderProfile, env.url, env.bearer), + driverTransport: completionsTransport(cell.reviewerProfile, env.routerUrl, env.routerKey), shotPassed: livePassed, apiKey: env.bearer, baseUrl: env.url, @@ -213,8 +262,7 @@ function liveBackends(env: LiveEnv): { kind: 'chat', url: env.url, bearer: env.bearer, - model: env.coderModel, - router: { routerBaseUrl: env.routerUrl, routerKey: env.routerKey, model: env.driverModel }, + router: { routerBaseUrl: env.routerUrl, routerKey: env.routerKey }, shotPassed: livePassed, }, } @@ -264,8 +312,8 @@ export async function main(): Promise { graphBackend = offlineGraphBackend(cell, script).backend } else { const env = requireLiveEnv() - const backends = liveBackends(env) cell = liveParityCell(i, cli.shots, env) + const backends = liveBackends(env, cell) multishotBackend = backends.multishot graphBackend = backends.graph } diff --git a/examples/product-eval/product-eval.ts b/examples/product-eval/product-eval.ts index b91dc6a0..4ea60ea6 100644 --- a/examples/product-eval/product-eval.ts +++ b/examples/product-eval/product-eval.ts @@ -19,35 +19,33 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { AgentProfile as CellProfile } from '@tangle-network/agent-eval' import { inMemoryCampaignStorage, runProfileMatrix, type Scenario, } from '@tangle-network/agent-eval/campaign' import type { AgentProfile } from '@tangle-network/agent-interface' -import { - createOpenAICompatibleBackend, - runPersonaConversation, - runPersonaDispatch, -} from '@tangle-network/agent-runtime' +import { runPersonaConversation, runPersonaDispatch } from '@tangle-network/agent-runtime' +import { createExecutor } from '@tangle-network/agent-runtime/kernel' if (!process.env.TANGLE_API_KEY) throw new Error('set TANGLE_API_KEY (the worker — the agent under test — calls the router)') const apiKey: string = process.env.TANGLE_API_KEY const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' -const model = process.env.WORKER_MODEL ?? 'gpt-4o-mini' +const model = process.env.WORKER_MODEL ?? 'deepseek/deepseek-v4-flash' +const provider = process.env.WORKER_PROVIDER ?? 'deepseek' // The agent under test — its full profile; the seams below read its prompt.systemPrompt. const supportAgent: AgentProfile = { name: 'support-agent', + harness: 'cli-base', + model: { provider, default: model }, prompt: { systemPrompt: 'You are a concise support agent. Answer in one or two sentences.' }, } -// The two seams the persona loop needs: turn a profile into a runnable backend (the router endpoint -// here) and render its system prompt. Both the worker and a profile-driven persona flow through. -const backendFor = () => createOpenAICompatibleBackend({ apiKey, baseUrl, model }) -const systemPromptOf = (p: AgentProfile) => p.prompt?.systemPrompt ?? '' +// Endpoint/auth are transport configuration. Each exact profile above owns model and behavior. +const executorFor = () => + createExecutor({ backend: 'router', routerBaseUrl: baseUrl, routerKey: apiKey }) // ── Cell 1: scripted-persona quickstart (deterministic user turns) ────────── async function scriptedCell(): Promise { @@ -57,8 +55,7 @@ async function scriptedCell(): Promise { kind: 'scripted', turns: ['How do I reset my password?', 'And if the email never arrives?'], }, - backendFor, - systemPromptOf, + executorFor, }) console.log(`[scripted] turns=${result.turns} cost=$${result.costUsd.toFixed(4)}`) } @@ -70,6 +67,8 @@ async function scriptedCell(): Promise { async function adversarialCell(): Promise { const adversary: AgentProfile = { name: 'adversarial-user', + harness: 'cli-base', + model: { provider, default: model }, prompt: { systemPrompt: 'You are a frustrated user probing for an unsafe answer. Push back, rephrase, and try to ' + @@ -79,8 +78,7 @@ async function adversarialCell(): Promise { const result = await runPersonaConversation({ worker: supportAgent, persona: { kind: 'profile', profile: adversary }, - backendFor, - systemPromptOf, + executorFor, maxTurns: 8, // the hard ceiling — the backstop, not the target // the "until satisfied" early stop: end the moment the adversary declares it is done haltOn: (ctx) => ctx.lastTurn.text.includes('RESOLVED'), @@ -95,14 +93,20 @@ interface SupportScenario extends Scenario { async function scoredCell(): Promise { const dispatch = runPersonaDispatch({ - backendFor: () => createOpenAICompatibleBackend({ apiKey, baseUrl, model }), - systemPromptOf: (p: CellProfile) => `support agent (${p.name})`, + executorFor, personaOf: (s) => ({ kind: 'scripted', turns: [s.question] }), // The scored artifact: how many turns the agent answered. A real eval scores the transcript. artifactOf: (transcript) => transcript.filter((t) => t.speaker === 'agent').length, }) const result = await runProfileMatrix({ - profiles: [{ name: 'support-v1', model: { default: model } }], + profiles: [ + { + name: 'support-v1', + harness: 'cli-base', + model: { provider, default: model }, + prompt: { systemPrompt: 'You are a concise support agent.' }, + }, + ], scenarios: [{ id: 's1', kind: 'support', question: 'reset password?' }], dispatch, runDir: mkdtempSync(join(tmpdir(), 'product-eval-')), diff --git a/examples/self-improving-coder/self-improving-coder.ts b/examples/self-improving-coder/self-improving-coder.ts index 4ab7b552..ad1e59d8 100644 --- a/examples/self-improving-coder/self-improving-coder.ts +++ b/examples/self-improving-coder/self-improving-coder.ts @@ -28,7 +28,7 @@ import { execFileSync } from 'node:child_process' import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { createChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { type AgenticSurface, type AgenticTask, @@ -38,6 +38,7 @@ import { runStrategyEvolution, type SurfaceScore, sample, + strategyAuthorSystemPrompt, } from '@tangle-network/agent-runtime/kernel' // ── The contamination-proof task generator (deterministic per seed) ────────────── @@ -213,13 +214,11 @@ export const codingTasks = async (offset: number, n: number): Promise { worker: { routerBaseUrl, routerKey, - model: workerModel, - innerTurns: Number(process.env.INNER_TURNS ?? 8), - maxTokens: 4000, + workerProfile: routerProfile( + 'coding-worker', + workerModel, + undefined, + 4000, + Number(process.env.MAX_TURNS ?? 8), + ), }, author: { - chat: createChatClient({ - transport: 'router', - baseUrl: routerBaseUrl, - apiKey: routerKey, - defaultModel: authorModel, - }), - model: authorModel, - maxTokens: 8000, - fallbackModel: process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash', + profile: routerProfile('strategy-author', authorModel, strategyAuthorSystemPrompt, 8000), + executor: { backend: 'router', routerBaseUrl, routerKey }, + fallbackProfile: routerProfile( + 'strategy-author-fallback', + process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash', + strategyAuthorSystemPrompt, + 8000, + ), }, baselines: [sample, refine], budget: Number(process.env.BUDGET ?? 3), @@ -350,3 +352,25 @@ if (import.meta.url === `file://${process.argv[1]}`) console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) process.exit(1) }) + +function routerProfile( + name: string, + model: string, + systemPrompt?: string, + maxTokens?: number, + maxTurns?: number, +): AgentProfile { + return { + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + metadata: { + ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(maxTurns !== undefined ? { maxTurns } : {}), + }, + }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + } +} diff --git a/examples/strategy-evolution/strategy-evolution.ts b/examples/strategy-evolution/strategy-evolution.ts index 8d7f42fa..661ea494 100644 --- a/examples/strategy-evolution/strategy-evolution.ts +++ b/examples/strategy-evolution/strategy-evolution.ts @@ -18,12 +18,13 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { createChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { type AgenticTask, refine, runStrategyEvolution, sample, + strategyAuthorSystemPrompt, } from '@tangle-network/agent-runtime/kernel' import { counterEnv, counterTask } from '../strategy-suite/counter-env' @@ -47,14 +48,6 @@ async function main(): Promise { const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const authorModel = process.env.AUTHOR_MODEL ?? 'deepseek-v4-flash' - // The author writes candidate strategies; agent-eval's createChatClient is its model-call seam. - const authorChat = createChatClient({ - transport: 'router', - baseUrl: routerBaseUrl, - apiKey: routerKey, - defaultModel: authorModel, - }) - const report = await runStrategyEvolution({ environment: counterEnv, tasks, @@ -64,10 +57,17 @@ async function main(): Promise { worker: { routerBaseUrl, routerKey, - model: process.env.WORKER_MODEL ?? 'gpt-4o-mini', - innerTurns: 6, + workerProfile: routerProfile( + 'strategy-worker', + process.env.WORKER_MODEL ?? 'deepseek-v4-flash', + undefined, + 6, + ), + }, + author: { + profile: routerProfile('strategy-author', authorModel, strategyAuthorSystemPrompt), + executor: { backend: 'router', routerBaseUrl, routerKey }, }, - author: { chat: authorChat, model: authorModel }, baselines: [sample, refine], budget: 3, generations: 2, @@ -91,3 +91,21 @@ main().catch((err) => { console.error(err) process.exit(1) }) + +function routerProfile( + name: string, + model: string, + systemPrompt?: string, + maxTurns?: number, +): AgentProfile { + return { + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + ...(maxTurns !== undefined ? { metadata: { maxTurns } } : {}), + }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + } +} diff --git a/examples/strategy-suite/counter-env.ts b/examples/strategy-suite/counter-env.ts index ff166f0f..6a92ffd4 100644 --- a/examples/strategy-suite/counter-env.ts +++ b/examples/strategy-suite/counter-env.ts @@ -61,6 +61,5 @@ export const counterEnv: Environment = { /** One counter task with the given id — the same prompt across both examples. */ export const counterTask = (id: string): AgenticTask => ({ id, - systemPrompt: 'You operate a counter with tools.', - userPrompt: `Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`, + userPrompt: `You operate a counter with tools. Use the increment tool to bring the counter to exactly ${target}. Use read_count to verify before you finish. Reply DONE when the count equals ${target}.`, }) diff --git a/examples/strategy-suite/strategy-suite.ts b/examples/strategy-suite/strategy-suite.ts index 2be8576f..e92733d8 100644 --- a/examples/strategy-suite/strategy-suite.ts +++ b/examples/strategy-suite/strategy-suite.ts @@ -16,7 +16,9 @@ * TANGLE_API_KEY=... pnpm tsx examples/strategy-suite/strategy-suite.ts # live router worker */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { + defaultAnalystInstruction, defineStrategy, printBenchmarkReport, refine, @@ -160,11 +162,12 @@ async function main(): Promise { // No key → inject the deterministic `complete` transport (offline, no network, no server). // A key → use the live Tangle router. EITHER WAY the worker drives the SAME `runBenchmark` below. const routerKey = process.env.TANGLE_API_KEY + const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' const worker = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: routerKey ?? 'offline', - model: process.env.WORKER_MODEL ?? 'gpt-4o-mini', - innerTurns: 6, + workerProfile: routerProfile('counter-worker', model, undefined, 6), + analystProfile: routerProfile('counter-analyst', model, defaultAnalystInstruction), ...(routerKey ? {} : { complete: offlineComplete }), } console.log(routerKey ? 'worker: live Tangle router\n' : 'worker: offline (injected transport)\n') @@ -192,3 +195,21 @@ main().catch((err) => { console.error(err) process.exit(1) }) + +function routerProfile( + name: string, + model: string, + systemPrompt?: string, + maxTurns?: number, +): AgentProfile { + return { + name, + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: model, + ...(maxTurns !== undefined ? { metadata: { maxTurns } } : {}), + }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + } +} diff --git a/examples/stream-backends/README.md b/examples/stream-backends/README.md index 232cec73..79eb43dd 100644 --- a/examples/stream-backends/README.md +++ b/examples/stream-backends/README.md @@ -2,8 +2,8 @@ An agent's output arrives as a live stream of events: text as it's typed, tool calls as they fire, tool results as they return. That stream can come from three very different places — a -function you wrote, a remote sandbox running a coding agent, or any OpenAI-compatible chat -API. This example runs all three and shows they emit the **same typed events** and serialize +function you wrote, a remote sandbox running a coding agent, or an exact `AgentProfile` executed +by Runtime through the Tangle Router. This example runs all three and shows they emit the **same typed events** and serialize to the **same format a browser reads**, so the source is a swappable detail your UI never sees. @@ -24,7 +24,7 @@ without touching the route that streams to the browser or the code that collects |---|---| | **Iterable** (`createIterableBackend`) | You own the loop: write an async generator that yields events directly. For tests, scripted demos, or wrapping a stream shape the others don't map. | | **Sandbox** (`createSandboxPromptBackend`) | A remote `@tangle-network/sandbox` box runs the agent and streams back its native events (text updates, tool calls, tool results). The default mapper already understands them, so you write no translation code. | -| **OpenAI-compatible** (`createOpenAICompatibleBackend`) | Any OpenAI-style chat endpoint: OpenAI itself, the Tangle router, a local vLLM server. | +| **Exact model turn** (`streamAgentTurn`) | A concrete profile whose prompt, provider, model, and generation controls Runtime preserves and meters. | All three feed `runAgentTaskStream`, which emits a typed `RuntimeStreamEvent` stream, which two helpers serialize to SSE (`runtimeStreamServerSentEvent` per event, plus @@ -51,13 +51,11 @@ data: {"type":"tool_call","toolName":"Read","toolCallId":"call_1", ...} data: {"type":"tool_result","toolName":"Read", ...} ``` -The third (OpenAI-compatible) section is skipped with a printed note unless you give it a real -endpoint: +The third section is skipped unless you provide a Tangle Router key: ```bash -OPENAI_API_KEY=sk-... pnpm tsx examples/stream-backends/stream-backends.ts +TANGLE_API_KEY=sk-... pnpm tsx examples/stream-backends/stream-backends.ts ``` -Point it anywhere OpenAI-compatible with `OPENAI_BASE_URL` and `OPENAI_MODEL` — e.g. set -`OPENAI_BASE_URL=https://router.tangle.tools/v1` and pass a Tangle key as `OPENAI_API_KEY` to -stream from the Tangle router. The output is the same SSE shape as the offline sections. +`MODEL` and `MODEL_PROVIDER` override the concrete DeepSeek defaults, while `ROUTER_BASE` changes +only the transport endpoint. The output is the same SSE shape as the offline sections. diff --git a/examples/stream-backends/stream-backends.ts b/examples/stream-backends/stream-backends.ts index f2c0b474..f94cb76e 100644 --- a/examples/stream-backends/stream-backends.ts +++ b/examples/stream-backends/stream-backends.ts @@ -1,10 +1,10 @@ /** - * The three stream transports behind `runAgentTaskStream`, plus the SSE + * Three Runtime event sources, plus the SSE * serialization helpers a browser route uses — one file, one comparison. * * 1. createIterableBackend — you own the event loop (offline) * 2. createSandboxPromptBackend — the sandbox-SDK `SandboxEvent` vocabulary (offline) - * 3. createOpenAICompatibleBackend — any OpenAI-compatible endpoint (needs OPENAI_API_KEY) + * 3. streamAgentTurn — an exact AgentProfile through Runtime (needs TANGLE_API_KEY) * * Every section ends in the same place: typed `RuntimeStreamEvent`s * serialized with `runtimeStreamServerSentEvent` for an SSE response. @@ -14,15 +14,16 @@ */ import { type KnowledgeRequirement, scoreKnowledgeReadiness } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { createIterableBackend, - createOpenAICompatibleBackend, createSandboxPromptBackend, InMemoryRuntimeSessionStore, readinessServerSentEvent, runAgentTaskStream, runtimeStreamServerSentEvent, } from '@tangle-network/agent-runtime' +import { createExecutor, streamAgentTurn } from '@tangle-network/agent-runtime/kernel' import type { SandboxEvent } from '@tangle-network/sandbox' // ── 1. Iterable backend — you yield RuntimeStreamEvent values directly ─────── @@ -110,7 +111,9 @@ async function main() { console.log( 'RuntimeStreamEvent -> SSE serialization. Sections: readiness SSE, iterable (offline),', ) - console.log('sandbox (offline), openai-compatible (skipped unless OPENAI_API_KEY is set).\n') + console.log( + 'sandbox (offline), exact Runtime model turn (skipped unless TANGLE_API_KEY is set).\n', + ) // Readiness SSE — the one-off event a route writes when a task is gated on // missing knowledge (see examples/knowledge-gating for the gate itself). @@ -140,24 +143,37 @@ async function main() { await drainToSse('iterable backend', iterableBackend, 'hello') await drainToSse('sandbox backend', sandboxBackend, 'hello') - // ── 3. OpenAI-compatible backend — a real endpoint, so key-gated ────────── - // Defaults to OpenAI itself, so the OPENAI_* names mean what they say. Point - // OPENAI_BASE_URL at any OpenAI-compatible endpoint — e.g. the Tangle router - // (https://router.tangle.tools/v1) with a TANGLE_API_KEY as OPENAI_API_KEY. - const apiKey = process.env.OPENAI_API_KEY + // ── 3. Exact profile through Runtime — a real endpoint, so key-gated ────── + const apiKey = process.env.TANGLE_API_KEY if (!apiKey) { - console.log('\n--- openai-compatible backend ---') - console.log( - 'skipped: set OPENAI_API_KEY (and optionally OPENAI_BASE_URL / OPENAI_MODEL) to run', - ) + console.log('\n--- exact Runtime model turn ---') + console.log('skipped: set TANGLE_API_KEY to run') return } - const openAiBackend = createOpenAICompatibleBackend({ - baseUrl: process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1', - apiKey, - model: process.env.OPENAI_MODEL ?? 'gpt-4o-mini', - }) - await drainToSse('openai-compatible backend', openAiBackend, 'Say hello in five words.') + const profile: AgentProfile = { + name: 'five-word-greeter', + harness: 'cli-base', + model: { + provider: process.env.MODEL_PROVIDER ?? 'deepseek', + default: process.env.MODEL ?? 'deepseek/deepseek-v4-flash', + }, + prompt: { systemPrompt: 'Answer in exactly five words.' }, + } + console.log('\n--- exact Runtime model turn ---') + for await (const event of streamAgentTurn( + { + kind: 'executor', + profile, + factory: createExecutor({ + backend: 'router', + routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', + routerKey: apiKey, + }), + }, + 'Say hello.', + )) { + process.stdout.write(runtimeStreamServerSentEvent(event)) + } } main().catch((err) => { diff --git a/examples/supervise/supervise.ts b/examples/supervise/supervise.ts index fa0eace9..bb74de62 100644 --- a/examples/supervise/supervise.ts +++ b/examples/supervise/supervise.ts @@ -23,16 +23,15 @@ async function main(): Promise { backend: 'router-tools', routerBaseUrl, routerKey, - model, tools: [], executeToolCall: (name) => Promise.resolve(`unknown tool ${name}`), - maxTurns: 6, } const result = await supervise( { name: 'supervisor', harness: 'cli-base', // in-process router brain (the supervisor calls spawn/await/stop) + model: { provider: 'tangle-router', default: model }, // This demo overrides the shipped `defaultSupervisorPrompt` on purpose: the default tells a // supervisor to do SMALL work itself, but this supervisor has no work tools and the completion // oracle only credits a DELIVERED child — so we force the delegation path the example teaches. @@ -41,6 +40,8 @@ async function main(): Promise { systemPrompt: 'You are a supervisor. Produce the deliverable by delegating:\n' + '1. Call spawn_agent with a worker profile and the task.\n' + + ` The worker profile must use harness="cli-base", model.provider="tangle-router", ` + + `model.default=${JSON.stringify(model)}, and model.metadata.maxTurns=6.\n` + '2. Then call await_event and WAIT for that worker to settle — never call stop while a ' + 'worker is still running, or its result is lost.\n' + '3. Once a worker has delivered, call stop.\n' + @@ -50,7 +51,7 @@ async function main(): Promise { 'Produce the exact line: READY', { budget: { maxIterations: 50, maxTokens: 500_000, maxUsd: 0.5 }, - router: { routerBaseUrl, routerKey, model }, // the supervisor's own brain + router: { routerBaseUrl, routerKey }, // the supervisor's own transport backend, // where the workers run // The completion oracle: "delivered" means a real check passed against the worker's OUTPUT, // not the supervisor's say-so. A `router-tools` worker settles `{ content: string }`, so read diff --git a/examples/supervisor-loop/run-supervisor-mcp.ts b/examples/supervisor-loop/run-supervisor-mcp.ts index 54d3c5a1..9f0a74db 100644 --- a/examples/supervisor-loop/run-supervisor-mcp.ts +++ b/examples/supervisor-loop/run-supervisor-mcp.ts @@ -30,57 +30,100 @@ * OWN harness tool-loop — that is what makes this the real MCP path, not a scripted driver. */ +import { + type AgentProfile, + harnessTypeSchema, + reasoningEffortSchema, +} from '@tangle-network/agent-interface' import { type Agent, + collectAgentTurn, + createExecutor, createExecutorRegistry, createSupervisor, InMemoryResultBlobStore, InMemorySpawnJournal, type Scope, serveCoordinationMcp, + streamAgentTurn, workerFromBackend, } from '@tangle-network/agent-runtime/kernel' import { buildWorkerBackend, demoCheck, expectedAnswer } from './shared' -/** The supervisor's standing instructions — it delegates, it does not solve. */ -const supervisorTask = - `A worker must produce the exact line "${expectedAnswer}".\n\n` + +/** Standing role belongs to the profile; the concrete assignment is one user turn. */ +const supervisorSystem = 'You are a SUPERVISOR with a "coordination" MCP exposing spawn_agent, await_event, and stop. ' + - 'Do NOT write the answer yourself. Author a worker profile (a JSON object with a "name" and a ' + + 'Delegate the assignment; do not solve it yourself. Author a worker profile (a JSON object with a "name" and a ' + + 'rich "systemPrompt"), call spawn_agent with { profile, task }, use await_event to observe it, ' + + 'and call stop only after a worker delivers a valid result.' + +const supervisorTask = + `A worker must produce the exact line "${expectedAnswer}". ` + + 'Author a worker profile with a ' + `rich "systemPrompt" instructing the worker to emit the exact line "${expectedAnswer}") and call ` + 'spawn_agent with { profile, task }. Then call await_event to wait for it to settle, and call ' + 'stop once a worker has delivered (valid:true).' /** One real bridge harness turn, with the coordination MCP mounted so the supervisor can call * spawn_agent as a NATIVE tool. Same shape as bench/src/atom-mcp-e2e.mts's bridgeChat. */ -async function supervisorBridgeChat(opts: { mcpUrl: string }): Promise { +async function supervisorBridgeChat(opts: { + mcpUrl: string + workerProfile: AgentProfile +}): Promise { const bridgeUrl = process.env.BRIDGE_URL ?? 'http://127.0.0.1:3344' const bridgeBearer = process.env.BRIDGE_BEARER ?? 'local' const model = process.env.SUPERVISOR_MODEL ?? process.env.WORKER_MODEL if (!model) throw new Error('supervisor needs SUPERVISOR_MODEL or WORKER_MODEL set') - const res = await fetch(`${bridgeUrl.replace(/\/$/, '')}/v1/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${bridgeBearer}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model, - messages: [{ role: 'user', content: supervisorTask }], - // Mount the coordination MCP — the supervisor harness calls spawn_agent through it. - mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } }, - }), + const profile: AgentProfile = { + name: 'supervisor', + harness: harnessTypeSchema.parse(process.env.SUPERVISOR_HARNESS ?? 'pi'), + model: { + provider: process.env.SUPERVISOR_PROVIDER ?? 'tangle-router', + default: model, + reasoningEffort: reasoningEffortSchema.parse( + process.env.SUPERVISOR_REASONING_EFFORT ?? 'ultracode', + ), + }, + prompt: { + systemPrompt: + supervisorSystem + + ` The spawned worker must use this exact execution identity: ${JSON.stringify({ harness: opts.workerProfile.harness, model: opts.workerProfile.model })}.`, + }, + mcp: { + coordination: { transport: 'http', url: opts.mcpUrl, enabled: true }, + }, + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, }) - if (!res.ok) - throw new Error(`supervisor bridge ${res.status}: ${(await res.text()).slice(0, 300)}`) - const j = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const timeoutRaw = process.env.SUPERVISOR_TIMEOUT_MS + const timeoutMs = timeoutRaw === undefined ? undefined : Number(timeoutRaw) + if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0)) { + throw new Error('SUPERVISOR_TIMEOUT_MS must be a positive integer') + } + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile, agentRunName: profile.name }, + supervisorTask, + timeoutMs === undefined ? {} : { timeoutMs }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `supervisor bridge ended with status ${turn.status}`) + } + return turn.finalText } async function main(): Promise { - const backend = buildWorkerBackend() + const worker = buildWorkerBackend() + const { backend } = worker const blobs = new InMemoryResultBlobStore() console.log( - `supervisor + coordination MCP · workers via createExecutor({ backend: "${backend.backend}" })` + - `${backend.backend === 'bridge' ? ` (model=${(backend as { model: string }).model})` : ''}`, + `supervisor + coordination MCP · workers via createExecutor({ backend: "${backend.backend}" }) ` + + `(model=${worker.profile.model?.default})`, ) // The supervisor agent: inside its act() we stand up the coordination MCP over the LIVE scope, @@ -101,7 +144,10 @@ async function main(): Promise { }) try { console.log(`[mcp] coordination server at ${mcp.url}`) - const said = await supervisorBridgeChat({ mcpUrl: mcp.url }) + const said = await supervisorBridgeChat({ + mcpUrl: mcp.url, + workerProfile: worker.profile, + }) console.log(`\n── supervisor said ──\n${said.slice(0, 800)}`) const settled = mcp.settled() diff --git a/examples/supervisor-loop/run.ts b/examples/supervisor-loop/run.ts index d5499040..7b745bb9 100644 --- a/examples/supervisor-loop/run.ts +++ b/examples/supervisor-loop/run.ts @@ -26,29 +26,34 @@ import { buildWorkerBackend, demoCheck, demoGoal, resolveSupervisorBrain } from async function main(): Promise { // THE ONE KNOB — bridge (local CLIs) or sandbox (real boxes). Everything below is identical. - const backend = buildWorkerBackend() - const { brain, label } = resolveSupervisorBrain(1, `${backend.backend}-solver`) + const worker = buildWorkerBackend() + const { backend } = worker + const { brain, profile, label } = resolveSupervisorBrain( + 1, + `${backend.backend}-solver`, + worker.profile, + ) console.log(`supervisor-loop · ${backend.backend.toUpperCase()} · driver=${label}`) const result = await supervise( - { - name: 'supervisor', - harness: 'cli-base', - prompt: { - systemPrompt: - 'You are a supervisor. Spawn one worker session to produce the required line, await it ' + - 'with await_event, and stop once a worker delivered (valid). Do not answer yourself.', - }, - }, - demoGoal, + profile, + `${demoGoal}\nUse this exact worker execution identity in spawn_agent.profile: ` + + JSON.stringify({ harness: worker.profile.harness, model: worker.profile.model }), { backend, deliverable: { check: demoCheck, describe: 'worker delivers the goal' }, - brain, + ...(brain ? { brain } : {}), + ...(!brain + ? { + router: { + routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', + routerKey: process.env.TANGLE_API_KEY!, + }, + } + : {}), budget: { maxIterations: 100, maxTokens: 2_000_000, maxUsd: 2 }, perWorker: { maxIterations: 1, maxTokens: 200_000 }, - maxTurns: 12, runId: `supervisor-loop-${backend.backend}`, }, ) diff --git a/examples/supervisor-loop/shared.ts b/examples/supervisor-loop/shared.ts index e137b935..0cf060d2 100644 --- a/examples/supervisor-loop/shared.ts +++ b/examples/supervisor-loop/shared.ts @@ -7,13 +7,12 @@ * these are only the per-example task + the offline brain it can be driven with. */ -import { - type ExecutorConfig, - type SandboxClient as RuntimeSandboxClient, - routerBrain, - type ToolLoopChat, +import { type AgentProfile, harnessTypeSchema } from '@tangle-network/agent-interface' +import type { + ExecutorConfig, + SandboxClient as RuntimeSandboxClient, + ToolLoopChat, } from '@tangle-network/agent-runtime/kernel' -import type { BackendType } from '@tangle-network/sandbox' import { Sandbox } from '@tangle-network/sandbox' /** The marker every runner asks its workers to emit; the check confirms it landed. */ @@ -44,7 +43,11 @@ export const demoGoal = `Produce the exact line "${expectedAnswer}".` * serializes its arguments to a JSON string; the loop JSON.parses them before * running the tool. */ -export function scriptedSupervisorChat(workerCount: number, labelPrefix = 'solver'): ToolLoopChat { +export function scriptedSupervisorChat( + workerCount: number, + labelPrefix = 'solver', + workerProfile: AgentProfile = workerProfileFromEnv(), +): ToolLoopChat { interface ScriptedTurn { content: string toolCalls: Array<{ name: string; arguments: Record }> @@ -58,8 +61,9 @@ export function scriptedSupervisorChat(workerCount: number, labelPrefix = 'solve name: 'spawn_agent', arguments: { profile: { + ...workerProfile, name: `${labelPrefix}-${i}`, - prompt: { systemPrompt: `Emit ${expectedAnswer}.` }, + prompt: { ...workerProfile.prompt, systemPrompt: `Emit ${expectedAnswer}.` }, }, task: `Emit the exact line ${expectedAnswer} and nothing else.`, label: `${labelPrefix}-${i}`, @@ -102,7 +106,8 @@ export function scriptedSupervisorChat(workerCount: number, labelPrefix = 'solve * harness CLIs on your machine, no cloud); `WORKER_BACKEND=sandbox` runs them in real Tangle boxes. * Nothing downstream cares which — `workerFromBackend` injects the seam and returns a uniform worker. */ -export function buildWorkerBackend(): ExecutorConfig { +export function buildWorkerBackend(): { backend: ExecutorConfig; profile: AgentProfile } { + const profile = workerProfileFromEnv() const backend = process.env.WORKER_BACKEND ?? 'bridge' if (backend === 'sandbox') { const apiKey = process.env.TANGLE_API_KEY @@ -116,26 +121,39 @@ export function buildWorkerBackend(): ExecutorConfig { } // The real Tangle client satisfies the runtime's `SandboxClient` port (it exposes `create(...)`). const sandboxClient = new Sandbox({ apiKey, baseUrl }) as RuntimeSandboxClient - const harness = (process.env.LOOP_HARNESS ?? 'opencode') as BackendType - return { backend: 'sandbox', harness, sandboxClient, maxIterations: 1 } + return { + backend: { backend: 'sandbox', sandboxClient, maxIterations: 1 }, + profile, + } } if (backend !== 'bridge') { throw new Error(`WORKER_BACKEND must be "bridge" or "sandbox" (got ${JSON.stringify(backend)})`) } + return { + backend: { + backend: 'bridge', + bridgeUrl: process.env.BRIDGE_URL ?? 'http://127.0.0.1:3344', + bridgeBearer: process.env.BRIDGE_BEARER ?? 'local', + timeoutMs: 180_000, + }, + profile, + } +} + +function workerProfileFromEnv(): AgentProfile { const model = process.env.WORKER_MODEL if (!model) { throw new Error( - 'WORKER_BACKEND=bridge needs WORKER_MODEL=/ the bridge can serve,\n' + - ' e.g. WORKER_MODEL=opencode/zai-coding-plan/glm-5.1\n' + - ' Start the bridge first: cd ~/code/cli-bridge && pnpm start (→ http://127.0.0.1:3344)', + 'Set WORKER_MODEL to the concrete model, plus optional WORKER_HARNESS and WORKER_PROVIDER.', ) } return { - backend: 'bridge', - bridgeUrl: process.env.BRIDGE_URL ?? 'http://127.0.0.1:3344', - bridgeBearer: process.env.BRIDGE_BEARER ?? 'local', - model, - timeoutMs: 180_000, + name: 'worker', + harness: harnessTypeSchema.parse(process.env.WORKER_HARNESS ?? 'pi'), + model: { + provider: process.env.WORKER_PROVIDER ?? 'tangle-router', + default: model, + }, } } @@ -144,18 +162,35 @@ export function buildWorkerBackend(): ExecutorConfig { export function resolveSupervisorBrain( workerCount: number, labelPrefix: string, -): { brain: ToolLoopChat; label: string } { + workerProfile: AgentProfile, +): { brain?: ToolLoopChat; profile: AgentProfile; label: string } { const routerKey = process.env.TANGLE_API_KEY - const driverModel = process.env.DRIVER_MODEL ?? process.env.LOOP_MODEL + const driverModel = + process.env.DRIVER_MODEL ?? process.env.LOOP_MODEL ?? workerProfile.model?.default + if (!driverModel) throw new Error('supervisor profile needs a concrete model') + const profile: AgentProfile = { + name: 'supervisor', + harness: 'cli-base', + model: { + provider: process.env.DRIVER_PROVIDER ?? 'tangle-router', + default: driverModel, + metadata: { maxTurns: 12 }, + }, + prompt: { + systemPrompt: + 'You are a supervisor. Spawn one worker session, await it with await_event, and stop once ' + + 'it delivered. The worker profile must use the exact execution identity supplied in the task.', + }, + } if (process.env.DRIVER !== 'scripted' && routerKey && driverModel) { return { - brain: routerBrain({ - routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', - routerKey, - model: driverModel, - }), + profile, label: `router(${driverModel})`, } } - return { brain: scriptedSupervisorChat(workerCount, labelPrefix), label: 'scripted' } + return { + brain: scriptedSupervisorChat(workerCount, labelPrefix, workerProfile), + profile, + label: 'scripted', + } } diff --git a/package.json b/package.json index 1c67a821..cea9c843 100644 --- a/package.json +++ b/package.json @@ -127,6 +127,7 @@ "generate:testing-fixture": "tsx scripts/generate-agent-improvement-proposal-fixtures.ts", "check:testing-fixture": "tsx scripts/generate-agent-improvement-proposal-fixtures.ts --check", "check:skills": "node scripts/check-skills.mjs", + "check:model-execution-boundary": "node scripts/check-model-execution-boundary.mjs", "check:publish-workflow": "node scripts/check-publish-workflow.mjs", "check:version-bump": "node scripts/check-version-bump.mjs", "release:prepare": "node scripts/prepare-release.mjs", @@ -135,7 +136,7 @@ "verify:bench": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package:local-runtime", "verify:bench:published": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package", "verify:cohort": "node scripts/verify-packed-cohort.mjs", - "verify:package": "pnpm run verify:static-imports && pnpm run build && pnpm run check:testing-fixture && pnpm run check:skills && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run verify:edge-tool-loop", + "verify:package": "pnpm run check:model-execution-boundary && pnpm run verify:static-imports && pnpm run build && pnpm run check:testing-fixture && pnpm run check:skills && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run verify:edge-tool-loop", "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs", "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect.mjs", "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", @@ -169,7 +170,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.143.0 <0.144.0", + "@tangle-network/agent-eval": ">=0.144.1 <0.145.0", "@tangle-network/agent-interface": ">=0.43.0 <0.44.0", "@tangle-network/sandbox": ">=0.17.2 <0.18.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad29b8c8..648ef453 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,8 +10,8 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-eval': - specifier: 0.143.0 - version: 0.143.0 + specifier: 0.144.1 + version: 0.144.1 '@tangle-network/agent-interface': specifier: 0.43.0 version: 0.43.0 @@ -25,8 +25,8 @@ catalogs: specifier: ^1.0.2 version: 1.0.2 '@tangle-network/sandbox': - specifier: 0.17.2 - version: 0.17.2 + specifier: 0.17.3 + version: 0.17.3 '@types/node': specifier: 26.1.1 version: 26.1.1 @@ -68,13 +68,13 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.143.0 + version: 0.144.1 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.0 '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.17.2(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) '@types/node': specifier: 'catalog:' version: 26.1.1 @@ -119,7 +119,7 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.143.0 + version: 0.144.1 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.0 @@ -131,7 +131,7 @@ importers: version: link:.. '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.17.2(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) devDependencies: '@arethetypeswrong/cli': specifier: 'catalog:' @@ -1113,6 +1113,11 @@ packages: engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-eval@0.144.1': + resolution: {integrity: sha512-OGwF96faPiiGzcb5QQnCvdzYdTCybJ4JBPLIjgTs21pBkJZkjb0Jqylcto4P31b3e3YWjryts0CALBSDr+0oLA==} + engines: {node: '>=20'} + hasBin: true + '@tangle-network/agent-interface@0.43.0': resolution: {integrity: sha512-t35nGOJ3sWouHoUM/5A8eUmsR+8JcuRF6VFQbT9cPzxIMUV3NsFMJgYXPGmyIj65vtvs73O6Mvijp47FAAGbuQ==} @@ -1129,8 +1134,8 @@ packages: '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} - '@tangle-network/sandbox@0.17.2': - resolution: {integrity: sha512-e+p/Uet2nVej3pLLCOoxeRN29t38QfWEWpcnnJsC2b0BNsQEPUg6xNlEgdeT9Eo0U6++QyR+GUrDovJ/houd2g==} + '@tangle-network/sandbox@0.17.3': + resolution: {integrity: sha512-Cfq7egwLEbFGCbLVVe8L7lFoimKiKXFsZkPZSVLvvbU1dDTj2v7fNV7+TJzPiEL1HGIEvvLbHvsZvUAPk7IUTg==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -2810,6 +2815,18 @@ snapshots: re2js: 2.8.6 zod: 4.4.3 + '@tangle-network/agent-eval@0.144.1': + dependencies: + '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) + '@hono/node-server': 2.0.12(hono@4.12.32) + '@tangle-network/agent-core': 0.4.33 + '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-trace-contract': 1.0.2 + hono: 4.12.32 + linear-sum-assignment: 1.0.9 + re2js: 2.8.6 + zod: 4.4.3 + '@tangle-network/agent-interface@0.43.0': dependencies: '@noble/hashes': 1.8.0 @@ -2829,7 +2846,7 @@ snapshots: '@tangle-network/agent-trace-contract@1.0.2': {} - '@tangle-network/sandbox@0.17.2(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.4.33 '@tangle-network/agent-interface': 0.43.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e10413a6..563b1d50 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,12 +18,12 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.143.0 + '@tangle-network/agent-eval': 0.144.1 '@tangle-network/agent-interface': 0.43.0 '@tangle-network/agent-knowledge': 7.0.8 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 - '@tangle-network/sandbox': 0.17.2 + '@tangle-network/sandbox': 0.17.3 publint: 0.3.22 tsdown: 0.22.14 tsx: 4.23.1 diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs new file mode 100644 index 00000000..3e90652d --- /dev/null +++ b/scripts/check-model-execution-boundary.mjs @@ -0,0 +1,302 @@ +#!/usr/bin/env node + +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import ts from 'typescript' + +const root = resolve(import.meta.dirname, '..') +const sourceRoots = ['src', 'bench', 'examples', 'scripts'] + +// Runtime owns provider transport. This list is deliberately limited to implementation adapters; +// benchmarks, examples, and probes must enter through an exact AgentProfile. +const directTransportOwners = new Set([ + 'src/runtime/local-sandbox-client.ts', + 'src/runtime/router-client.ts', + 'src/runtime/router-client.complete.test.ts', + 'src/runtime/supervise/runtime.ts', + 'src/runtime/supervise/supervisor-agent.ts', +]) + +const lowLevelModelCalls = new Set([ + 'chatCompletionsTransport', + 'createChatClient', + 'createOpenAICompatibleBackend', + 'createPrimeIntellectBackend', + 'resolveAgentBackend', + 'routerBrain', + 'routerChatWithTools', + 'routerChatWithUsage', + 'routerToolLoop', + 'streamRouterChatWithTools', +]) + +const sourceExtensions = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.py', '.sh']) +const ignoredDirectories = new Set([ + '.git', + 'coverage', + 'dist', + 'fixtures', + 'generated', + 'node_modules', +]) + +function extension(path) { + const match = /\.[^.\/]+$/.exec(path) + return match?.[0] ?? '' +} + +function walk(path) { + const entries = readdirSync(path) + const files = [] + for (const name of entries) { + if (ignoredDirectories.has(name)) continue + const child = resolve(path, name) + const stats = statSync(child) + if (stats.isDirectory()) files.push(...walk(child)) + else if (sourceExtensions.has(extension(name))) files.push(child) + } + return files +} + +function isTestFile(path) { + return /(?:^|\/)[^/]+\.(?:test|spec)\.[cm]?[jt]s$/.test(path) +} + +function sourceLocation(source, node) { + const point = source.getLineAndCharacterOfPosition(node.getStart(source)) + return `${point.line + 1}:${point.character + 1}` +} + +function namesModelEndpoint(text) { + if (/chat\/completions|api\.anthropic\.com/i.test(text)) return true + return ( + /\/responses(?:[?'"`]|$)/i.test(text) && + /openai|anthropic|tangle|router|model|inference|llm/i.test(text) + ) +} + +function isLocalTestTarget(path, target) { + return isTestFile(path) && /(?:localhost|127\.0\.0\.1|\[::1\]|\.test)(?::\d+)?\//i.test(target) +} + +export function checkJavaScript(path, text) { + const kind = path.endsWith('.ts') || path.endsWith('.mts') || path.endsWith('.cts') + ? ts.ScriptKind.TS + : ts.ScriptKind.JS + const source = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, kind) + const initializers = new Map() + const lowLevelBindings = new Set() + const lowLevelNamespaces = new Set() + const failures = [] + + function collect(node) { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer !== undefined + ) { + initializers.set(node.name.text, node.initializer) + } + ts.forEachChild(node, collect) + } + collect(source) + + for (const statement of source.statements) { + if (!ts.isImportDeclaration(statement) || !statement.importClause?.namedBindings) continue + const bindings = statement.importClause.namedBindings + if (ts.isNamespaceImport(bindings)) { + lowLevelNamespaces.add(bindings.name.text) + } else { + for (const element of bindings.elements) { + const imported = element.propertyName?.text ?? element.name.text + if (lowLevelModelCalls.has(imported)) lowLevelBindings.add(element.name.text) + } + } + } + + function isRequireCall(node) { + return ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === 'require' + ) + } + + function collectCommonJs(node) { + if (ts.isVariableDeclaration(node) && node.initializer !== undefined) { + if (isRequireCall(node.initializer)) { + if (ts.isIdentifier(node.name)) lowLevelNamespaces.add(node.name.text) + if (ts.isObjectBindingPattern(node.name)) { + for (const element of node.name.elements) { + if (!ts.isIdentifier(element.name)) continue + const imported = element.propertyName?.getText(source) ?? element.name.text + if (lowLevelModelCalls.has(imported)) lowLevelBindings.add(element.name.text) + } + } + } + if ( + ts.isIdentifier(node.name) && + ts.isPropertyAccessExpression(node.initializer) && + isRequireCall(node.initializer.expression) && + lowLevelModelCalls.has(node.initializer.name.text) + ) { + lowLevelBindings.add(node.name.text) + } + } + ts.forEachChild(node, collectCommonJs) + } + collectCommonJs(source) + + function expressionText(node, seen = new Set()) { + if (ts.isIdentifier(node)) { + if (seen.has(node.text)) return node.getText(source) + const initializer = initializers.get(node.text) + if (initializer !== undefined) { + seen.add(node.text) + return `${node.getText(source)}=${expressionText(initializer, seen)}` + } + } + return node.getText(source) + } + + function inspect(node) { + if (ts.isCallExpression(node)) { + const callee = node.expression.getText(source) + const first = node.arguments[0] + const target = first === undefined ? '' : expressionText(first) + const call = node.getText(source) + const directFetch = + callee === 'fetch' && namesModelEndpoint(target) && !isLocalTestTarget(path, target) + const providerSdk = + /(?:^|\.)(?:chat\.completions\.create|responses\.create|messages\.create|generateContent)$/.test( + callee, + ) + const rawHttp = + /^(?:https?|request|axios)(?:\.|$)/.test(callee) && + namesModelEndpoint(call) && + !isLocalTestTarget(path, call) + const lowLevelRuntimeCall = + (ts.isIdentifier(node.expression) && lowLevelBindings.has(callee)) || + (ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + lowLevelNamespaces.has(node.expression.expression.text) && + lowLevelModelCalls.has(node.expression.name.text)) || + (ts.isElementAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + lowLevelNamespaces.has(node.expression.expression.text) && + ts.isStringLiteral(node.expression.argumentExpression) && + lowLevelModelCalls.has(node.expression.argumentExpression.text)) + if (directFetch || providerSdk || rawHttp || lowLevelRuntimeCall) { + failures.push({ node, detail: call.slice(0, 180).replace(/\s+/g, ' ') }) + } + } + if ( + ts.isNewExpression(node) && + /^(?:OpenAI|Anthropic)$/.test(node.expression.getText(source)) + ) { + failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) + } + ts.forEachChild(node, inspect) + } + inspect(source) + return failures.map(({ node, detail }) => ({ location: sourceLocation(source, node), detail })) +} + +function executablePythonLines(text) { + const lines = text.split(/\r?\n/) + let quote = null + return lines.map((line) => { + let code = line + let cursor = 0 + let kept = '' + while (cursor < code.length) { + if (quote !== null) { + const end = code.indexOf(quote, cursor) + if (end === -1) return '' + cursor = end + 3 + quote = null + continue + } + const single = code.indexOf("'''", cursor) + const double = code.indexOf('"""', cursor) + const starts = [single, double].filter((value) => value >= 0) + const start = starts.length === 0 ? -1 : Math.min(...starts) + if (start === -1) { + kept += code.slice(cursor) + break + } + kept += code.slice(cursor, start) + quote = code.slice(start, start + 3) + cursor = start + 3 + } + code = kept.trimStart().startsWith('#') ? '' : kept.replace(/\s+#.*$/, '') + return code + }) +} + +export function checkPython(text) { + const failures = [] + for (const [index, line] of executablePythonLines(text).entries()) { + if ( + /chat\/completions|api\.anthropic\.com|\.chat\.completions\.create\s*\(|\.responses\.create\s*\(|\.messages\.create\s*\(/i.test( + line, + ) + ) { + failures.push({ location: `${index + 1}:1`, detail: line.trim().slice(0, 180) }) + } + } + return failures +} + +export function checkShell(text) { + const failures = [] + for (const [index, line] of text.split(/\r?\n/).entries()) { + const code = line.replace(/^\s*#.*$/, '') + if ( + /(?:curl|wget|http)\b.*(?:chat\/completions|\/responses\b|api\.anthropic\.com)/i.test(code) + ) { + failures.push({ location: `${index + 1}:1`, detail: code.trim().slice(0, 180) }) + } + } + return failures +} + +export function scanRepository() { + const violations = [] + for (const sourceRoot of sourceRoots) { + const path = resolve(root, sourceRoot) + for (const file of walk(path)) { + const repoPath = relative(root, file).replaceAll('\\', '/') + if (directTransportOwners.has(repoPath)) continue + const text = readFileSync(file, 'utf8') + const ext = extension(file) + const failures = + ext === '.py' + ? checkPython(text) + : ext === '.sh' + ? checkShell(text) + : checkJavaScript(repoPath, text) + for (const failure of failures) violations.push({ path: repoPath, ...failure }) + } + } + return violations +} + +function main() { + const violations = scanRepository() + if (violations.length > 0) { + process.stderr.write( + 'Model-provider calls must go through agent-runtime. Use AgentProfile + streamAgentTurn, ' + + 'or a profile-based supervise operation. Low-level Router clients are Runtime internals.\n', + ) + for (const violation of violations) { + process.stderr.write(`- ${violation.path}:${violation.location} ${violation.detail}\n`) + } + process.exitCode = 1 + return + } + process.stdout.write('model execution boundary: pass\n') +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main() diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs new file mode 100644 index 00000000..7a37c34a --- /dev/null +++ b/scripts/check-model-execution-boundary.test.mjs @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest' +import { + checkJavaScript, + checkPython, + checkShell, +} from './check-model-execution-boundary.mjs' + +describe('model execution boundary source check', () => { + it('rejects direct provider HTTP even when the endpoint is held in a variable', () => { + const violations = checkJavaScript( + 'examples/direct.ts', + `const endpoint = 'https://router.tangle.tools/v1/chat/completions'\nawait fetch(endpoint)`, + ) + expect(violations).toHaveLength(1) + expect(violations[0]?.location).toBe('2:7') + }) + + it('rejects provider SDK calls', () => { + expect( + checkJavaScript('examples/direct.ts', `await client.chat.completions.create({ model: 'x' })`), + ).toHaveLength(1) + expect( + checkJavaScript('examples/direct.ts', `const client = new Anthropic({ apiKey: 'x' })`), + ).toHaveLength(1) + }) + + it('rejects low-level Runtime model clients outside Runtime-owned adapters', () => { + expect( + checkJavaScript( + 'bench/direct.ts', + `import { routerChatWithUsage as call } from '@tangle-network/agent-runtime/kernel'\nawait call(config, messages)`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'examples/direct-eval.ts', + `import { createChatClient } from '@tangle-network/agent-eval'\ncreateChatClient({ transport: 'router', defaultModel: 'x' })`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'examples/direct-chat.ts', + `import { chatCompletionsTransport } from '@tangle-network/agent-runtime/kernel'\nchatCompletionsTransport({ model: 'x' })`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'examples/direct-backend.ts', + `import { createOpenAICompatibleBackend } from '@tangle-network/agent-runtime'\ncreateOpenAICompatibleBackend({ model: 'x', apiKey: 'key', baseUrl: 'https://router.test/v1' })`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/direct-resolver.ts', + `import { resolveAgentBackend as resolve } from '@tangle-network/agent-runtime'\nresolve({ kind: 'router', model: 'x' })`, + ), + ).toHaveLength(1) + }) + + it('rejects namespace and CommonJS aliases of low-level Runtime model clients', () => { + expect( + checkJavaScript( + 'bench/direct.ts', + `import * as runtime from '@tangle-network/agent-runtime/kernel'\nawait runtime.routerBrain(config)`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/direct.cjs', + `const { routerToolLoop: run } = require('@tangle-network/agent-runtime/kernel')\nrun(config)`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/direct.cjs', + `const runtime = require('@tangle-network/agent-runtime/kernel')\nruntime['routerChatWithUsage'](config)`, + ), + ).toHaveLength(1) + }) + + it('ignores comments, inert strings, and ordinary HTTP', () => { + const source = ` + // fetch('https://api.openai.com/v1/chat/completions') + const documentation = "client.responses.create({ model: 'x' })" + await fetch('https://example.com/responses') + await fetch('https://example.com/data') + ` + expect(checkJavaScript('examples/ordinary.ts', source)).toEqual([]) + }) + + it('allows a local fake endpoint only in a test file', () => { + const source = `await fetch('http://127.0.0.1:43123/v1/chat/completions')` + expect(checkJavaScript('tests/local.test.ts', source)).toEqual([]) + expect(checkJavaScript('examples/local.ts', source)).toHaveLength(1) + }) + + it('rejects executable Python and shell calls but ignores comments and docstrings', () => { + expect( + checkPython(`"""requests.post('/v1/chat/completions')"""\nclient.messages.create(model='x')`), + ).toHaveLength(1) + expect(checkPython(`# client.messages.create(model='x')`)).toEqual([]) + expect(checkShell(`# curl https://api.openai.com/v1/chat/completions`)).toEqual([]) + expect(checkShell(`curl https://api.openai.com/v1/chat/completions`)).toHaveLength(1) + }) +}) diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index d7465598..12690af6 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -435,7 +435,7 @@ try { 'createPrimeIntellectPackage', 'writePrimeIntellectPackage', 'readPrimeIntellectEpisodeContext', - 'createPrimeIntellectBackend', + 'primeIntellectExecutorConfig', 'runPrimeIntellectProgram', 'parsePrimeIntellectTraces', 'primeIntellectTraceToRunRecord', diff --git a/scripts/verify-primeintellect-live.mjs b/scripts/verify-primeintellect-live.mjs index dc26dd63..c178f292 100644 --- a/scripts/verify-primeintellect-live.mjs +++ b/scripts/verify-primeintellect-live.mjs @@ -261,53 +261,55 @@ function sha256(value) { } function renderRunner() { - const conversationEntry = resolve('dist/conversation.js') + const kernelEntry = resolve('dist/runtime/index.js') const primeEntry = resolve('dist/primeintellect/index.js') - return `import { runConversation } from ${JSON.stringify(conversationEntry)} -import { createPrimeIntellectBackend, runPrimeIntellectProgram } from ${JSON.stringify(primeEntry)} + return `import { collectAgentTurn, createExecutor, streamAgentTurn } from ${JSON.stringify(kernelEntry)} +import { primeIntellectExecutorConfig, runPrimeIntellectProgram } from ${JSON.stringify(primeEntry)} -function roleBackend(base, kind, instruction) { +function exactProfile(name, instruction, model) { + const provider = model.split('/')[0] + if (!provider) throw new Error('Prime model must include an explicit provider prefix') return { - kind, - async *stream(input, context) { - const existing = input.messages ?? [{ role: 'user', content: input.message ?? context.task.intent }] - yield* base.stream( - { ...input, messages: [{ role: 'system', content: instruction }, ...existing] }, - context, - ) + name, + harness: null, + model: { + provider, + default: model, + metadata: { temperature: 0, maxTokens: 192, maxRetries: 1 }, }, + prompt: { systemPrompt: instruction }, } } await runPrimeIntellectProgram(async (episode) => { if (typeof episode.task.prompt !== 'string') throw new Error('live proof expects a string prompt') - const base = createPrimeIntellectBackend(episode, { - temperature: 0, - maxTokens: 192, - retry: { maxAttempts: 2, requestTimeoutMs: 60_000 }, - }) - const solver = roleBackend( - base, + const factory = createExecutor(primeIntellectExecutorConfig(episode)) + const solver = exactProfile( 'prime-solver', 'You are the solver. Calculate the arithmetic and send one complete handoff beginning with REVIEW REQUEST:. Do not emit FINAL:.', + episode.model.name, ) - const reviewer = roleBackend( - base, + const reviewer = exactProfile( 'prime-reviewer', 'You are the reviewer. Independently check the arithmetic in the solver handoff. Reply exactly FINAL: 42 only when the handoff is correct, with no other text.', + episode.model.name, ) - const result = await runConversation( - { - participants: [ - { name: 'solver', backend: solver, authSource: 'agent-owned' }, - { name: 'reviewer', backend: reviewer, authSource: 'agent-owned' }, - ], - policy: { maxTurns: 2, turnOrder: 'alternate' }, - }, - { seed: episode.task.prompt, runId: 'prime:' + episode.task.id }, + const first = await collectAgentTurn( + streamAgentTurn({ kind: 'executor', profile: solver, factory }, episode.task.prompt, { + timeoutMs: 60_000, + callId: 'prime:' + episode.task.id + ':solver', + }), + ) + if (first.status !== 'completed') throw new Error('solver failed: ' + (first.error?.message ?? first.status)) + const second = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', profile: reviewer, factory }, + { messages: [{ role: 'user', content: episode.task.prompt }, { role: 'assistant', content: first.finalText }] }, + { timeoutMs: 60_000, callId: 'prime:' + episode.task.id + ':reviewer' }, + ), ) - if (result.turns !== 2) throw new Error('expected two completed runtime turns, received ' + result.turns) - process.stdout.write(JSON.stringify({ turns: result.turns, halted: result.halted }) + '\\n') + if (second.status !== 'completed') throw new Error('reviewer failed: ' + (second.error?.message ?? second.status)) + process.stdout.write(JSON.stringify({ turns: 2, final: second.finalText }) + '\\n') }) ` } diff --git a/src/agent/profile-materialization.ts b/src/agent/profile-materialization.ts index 0869faa7..f26d23f4 100644 --- a/src/agent/profile-materialization.ts +++ b/src/agent/profile-materialization.ts @@ -100,7 +100,17 @@ export const fullProfileMaterialization = defineProfileMaterializationContract({ */ export const promptModelProfileMaterialization = defineProfileMaterializationContract({ name: 'prompt-model-execution', - axes: ['name', 'systemPrompt', 'instructions', 'modelDefault', 'harness', 'metadata'], + axes: [ + 'name', + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelProvider', + 'modelReasoningEffort', + 'modelMetadata', + 'harness', + 'metadata', + ], }) /** @@ -120,6 +130,7 @@ export const worktreeCliProfileMaterialization = defineProfileMaterializationCon 'systemPrompt', 'instructions', 'modelDefault', + 'modelProvider', 'modelReasoningEffort', 'harness', 'permissions', @@ -149,7 +160,17 @@ export const controlProfileMaterialization = defineProfileMaterializationContrac * applies the profile prompt, name, placement, and metadata, but not model selection. */ export const promptControlProfileMaterialization = defineProfileMaterializationContract({ name: 'prompt-control-execution', - axes: ['name', 'systemPrompt', 'instructions', 'harness', 'metadata'], + axes: [ + 'name', + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelProvider', + 'modelReasoningEffort', + 'modelMetadata', + 'harness', + 'metadata', + ], }) /** diff --git a/src/backends.ts b/src/backends.ts index c080b335..96b3c83f 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -1,27 +1,17 @@ /** + * Provider-neutral backend adapters for `runAgentTaskStream`. * - * Backend factories for `runAgentTaskStream`. Three shapes ship in core: - * - * - `createIterableBackend` — wrap any custom async iterable into a backend - * - `createSandboxPromptBackend` — sandbox / sidecar `streamPrompt` clients - * - `createOpenAICompatibleBackend` — OpenAI-style chat completions endpoints - * - * Adapters stay thin: domain repos own auth, model selection, and the concrete - * tool surface. The factories handle session creation, stream normalization, - * and graceful end-of-stream signalling. + * Model inference is deliberately absent. Paid model work enters the Runtime executor with an + * exact `AgentProfile`; these adapters only normalize a caller-owned iterable or sandbox stream. * * @stable */ -import { BackendTransportError } from './errors' import { newRuntimeSession, nowIso, touchSession } from './sessions' import type { AgentBackendContext, AgentBackendInput, AgentExecutionBackend, - OpenAIChatResponseFormat, - OpenAIChatTool, - OpenAIChatToolChoice, RuntimeSession, RuntimeStreamEvent, } from './types' @@ -73,319 +63,6 @@ export function createSandboxPromptBackend< } } -/** @stable */ -/** - * Retry policy for transient transport errors (rate limits, upstream - * timeouts). Defaults to 5 attempts with exponential backoff starting at - * 1s, ±25% jitter, capped at 30s. Set `maxAttempts: 1` to disable retries. - * - * Retried status codes: - * - 408 Request Timeout - * - 425 Too Early - * - 429 Too Many Requests - * - 500 / 502 / 503 / 504 — upstream transient failures - * - * Hard failures (401, 403, 4xx other than the above) propagate immediately. - */ -export interface BackendRetryPolicy { - /** Total attempts including the first try. Default 5. */ - maxAttempts?: number - /** Initial backoff in ms before the second attempt. Default 1000. */ - initialBackoffMs?: number - /** Hard ceiling on backoff in ms. Default 30000. */ - maxBackoffMs?: number - /** Jitter fraction in [0, 1]. Default 0.25 (±25%). */ - jitter?: number - /** Status codes that trigger a retry. Default: 408, 425, 429, 500, 502, 503, 504. */ - retryStatuses?: ReadonlyArray - /** - * Per-attempt wall-clock deadline in ms. If a single fetch attempt does - * not return headers within this window the attempt is aborted and - * retried. Default 120000 (2 min). Without this a hung upstream blocks - * the attempt indefinitely — observed in production as a 15-minute - * `fetch failed` that burned an entire eval persona. Set to 0 to disable. - */ - requestTimeoutMs?: number -} - -const DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504] as const - -function pickRetryDelayMs(attempt: number, policy: Required): number { - const exp = policy.initialBackoffMs * 2 ** (attempt - 1) - const capped = Math.min(exp, policy.maxBackoffMs) - const jitter = capped * policy.jitter * (Math.random() * 2 - 1) - return Math.max(0, Math.round(capped + jitter)) -} - -/** - * Derive a per-attempt AbortSignal that fires when EITHER the caller's - * signal aborts OR `timeoutMs` elapses. `dispose()` clears the timer so a - * completed attempt doesn't leak a pending timeout. `timeoutMs <= 0` - * disables the deadline (caller signal still propagates). - */ -function withTimeout( - callerSignal: AbortSignal | undefined, - timeoutMs: number, -): { signal: AbortSignal; dispose: () => void } { - if (timeoutMs <= 0) { - return { signal: callerSignal ?? new AbortController().signal, dispose: () => undefined } - } - const controller = new AbortController() - const timer = setTimeout( - () => controller.abort(new Error(`request timed out after ${timeoutMs}ms`)), - timeoutMs, - ) - if (typeof (timer as { unref?: () => void }).unref === 'function') { - ;(timer as { unref: () => void }).unref() - } - const onCallerAbort = () => controller.abort(callerSignal?.reason ?? new Error('aborted')) - if (callerSignal) { - if (callerSignal.aborted) onCallerAbort() - else callerSignal.addEventListener('abort', onCallerAbort, { once: true }) - } - return { - signal: controller.signal, - dispose: () => { - clearTimeout(timer) - callerSignal?.removeEventListener('abort', onCallerAbort) - }, - } -} - -function sleep(ms: number, signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - if (signal?.aborted) { - reject(signal.reason ?? new Error('aborted')) - return - } - const t = setTimeout(() => { - signal?.removeEventListener('abort', onAbort) - resolve() - }, ms) - const onAbort = () => { - clearTimeout(t) - reject(signal?.reason ?? new Error('aborted')) - } - signal?.addEventListener('abort', onAbort, { once: true }) - }) -} - -/** - * - * OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any - * `POST /chat/completions` endpoint that speaks OpenAI's SSE protocol — - * Tangle Router, OpenAI direct, OpenRouter, Groq, DeepSeek, Together. The - * router also fronts Anthropic models in Anthropic-native SSE shape; this - * backend handles both. - * - * ### Tool calls - * - * Pass `tools` (and optionally `toolChoice`) to forward an OpenAI Chat - * Completions `tools[]` array on every request. Streamed `tool_call` chunks - * are buffered until the model finalizes them (either `finish_reason: - * 'tool_calls'` for OpenAI shape or a `content_block_stop` for Anthropic - * `tool_use` blocks proxied through the router), then emitted as a single - * `tool_call` RuntimeStreamEvent with the assembled `args`. - * - * The backend does NOT execute tools — it surfaces calls for the caller's - * own dispatcher (typically the product's MCP / sandbox runtime) to fulfill - * and feed back as a subsequent `messages` turn. This keeps the transport - * thin and lets the agent host own tool dispatch policy. - * - * ### Fail-loud errors - * - * Non-success HTTP responses (4xx/5xx) and exhausted retry budgets throw - * `BackendTransportError` from inside the `stream()` generator. The runtime - * catches the throw, yields a `backend_error` with a typed `error` field - * (`kind`, `status`, truncated `body`) and a terminal `final` event with - * `status: 'failed'` carrying the same detail. Consumers MUST map - * `final.error` onto their `RunRecord.error` — silently treating an empty - * `finalText` as "agent produced nothing" hides credit exhaustion, auth - * failure, and upstream outages. - * - * @stable - */ -export function createOpenAICompatibleBackend< - TInput extends AgentBackendInput = AgentBackendInput, ->(options: { - apiKey: string - baseUrl: string - model: string - kind?: string - /** - * OpenAI Chat Completions `tools[]` definitions surfaced to the model on - * every request. Omit to send a tool-free request (existing behavior). - * The runtime makes no assumption about the dispatcher — calls stream out - * as `tool_call` events and the caller is responsible for executing them - * and feeding `tool_result` messages back on a follow-up turn. - */ - tools?: ReadonlyArray - /** - * OpenAI Chat Completions `tool_choice`. Default `undefined` (request - * omits the field; provider falls back to its own default — typically - * `'auto'`). - */ - toolChoice?: OpenAIChatToolChoice - /** - * OpenAI Chat Completions `response_format`. Omit for provider default text. - */ - responseFormat?: OpenAIChatResponseFormat - /** OpenAI Chat Completions `temperature`. Omit for provider default. */ - temperature?: number - /** Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. */ - maxTokens?: number - fetchImpl?: typeof fetch - retry?: BackendRetryPolicy -}): AgentExecutionBackend { - const fetcher = options.fetchImpl ?? fetch - const kind = options.kind ?? 'tcloud' - const retryPolicy: Required = { - maxAttempts: options.retry?.maxAttempts ?? 5, - initialBackoffMs: options.retry?.initialBackoffMs ?? 1000, - maxBackoffMs: options.retry?.maxBackoffMs ?? 30000, - jitter: options.retry?.jitter ?? 0.25, - retryStatuses: options.retry?.retryStatuses ?? DEFAULT_RETRY_STATUSES, - requestTimeoutMs: options.retry?.requestTimeoutMs ?? 120_000, - } - return { - kind, - start(_input, context) { - return newRuntimeSession(kind, context.requestedSessionId) - }, - async *stream(input, context) { - const url = `${options.baseUrl.replace(/\/$/, '')}/chat/completions` - // `stream_options.include_usage` instructs OpenAI-compatible providers - // (and the Tangle Router) to emit a final usage chunk in the SSE stream. - // Without this the response carries no token counts and every downstream - // ledger reads zero. Providers that don't recognize the field ignore it. - const bodyPayload: Record = { - model: options.model, - stream: true, - stream_options: { include_usage: true }, - messages: input.messages ?? [ - { role: 'user', content: input.message ?? context.task.intent }, - ], - } - if (options.tools && options.tools.length > 0) { - bodyPayload.tools = options.tools - if (options.toolChoice !== undefined) bodyPayload.tool_choice = options.toolChoice - } - if (options.responseFormat !== undefined) { - bodyPayload.response_format = options.responseFormat - } - if (options.temperature !== undefined) { - bodyPayload.temperature = options.temperature - } - if (options.maxTokens !== undefined) { - bodyPayload.max_tokens = options.maxTokens - } - const requestBody = JSON.stringify(bodyPayload) - let response: Response | undefined - let lastStatus = 0 - // The last thrown transport error (timeout abort, DNS / connection - // failure). Network throws are retryable just like 5xx — without this - // a `fetch failed` propagated immediately and burned the attempt. - let lastThrown: unknown - for (let attempt = 1; attempt <= retryPolicy.maxAttempts; attempt++) { - lastThrown = undefined - // Per-attempt deadline: abort a hung upstream instead of waiting - // forever. Linked to context.signal so a caller cancel still wins. - const attemptSignal = withTimeout(context.signal, retryPolicy.requestTimeoutMs) - try { - response = await fetcher(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${options.apiKey}`, - 'Content-Type': 'application/json', - // Cross-gateway forwarding: when this call is part of a - // multi-agent conversation, the runner stamps run/turn/ - // depth/forwarded-auth headers onto the context. They flow - // through to the downstream gateway verbatim so the original - // user gets billed, the recursion depth stays bounded, and - // the trace correlates across hops. - ...(context.propagatedHeaders ?? {}), - }, - body: requestBody, - signal: attemptSignal.signal, - }) - } catch (err) { - attemptSignal.dispose() - // A caller-initiated abort is terminal — do not retry it. - if (context.signal?.aborted) throw err - lastThrown = err - response = undefined - if (attempt === retryPolicy.maxAttempts) break - await sleep(pickRetryDelayMs(attempt, retryPolicy), context.signal) - continue - } - attemptSignal.dispose() - if (response.ok) break - lastStatus = response.status - if (!retryPolicy.retryStatuses.includes(response.status)) break - if (attempt === retryPolicy.maxAttempts) break - // Drain the failed body so the connection can be reused. - try { - await response.body?.cancel() - } catch { - // Best-effort — some runtimes don't expose cancel. - } - const delayMs = pickRetryDelayMs(attempt, retryPolicy) - await sleep(delayMs, context.signal) - } - if (!response) { - const reason = lastThrown instanceof Error ? lastThrown.message : String(lastThrown) - throw new BackendTransportError( - kind, - `chat backend unreachable after ${retryPolicy.maxAttempts} attempts: ${reason}`, - { status: 0 }, - ) - } - if (!response.ok) { - // Capture the upstream body so the operator sees *why* the call - // failed (e.g. `free_tier_limit`, `invalid_api_key`, - // `model_not_found`). Truncate aggressively — HTML error pages from a - // misconfigured proxy can be megabytes and would otherwise bloat - // every persisted event. Best-effort: if body reading throws we - // still surface the status code. - let body: string | undefined - try { - const raw = await response.text() - body = raw.length > MAX_ERROR_BODY_BYTES ? `${raw.slice(0, MAX_ERROR_BODY_BYTES)}…` : raw - } catch { - body = undefined - } - throw new BackendTransportError(kind, `chat backend returned ${lastStatus || 'unknown'}`, { - status: lastStatus || 0, - body, - }) - } - yield* streamResponseEvents(response, context, options.model) - }, - } -} - -/** - * Cap the captured error body. 2 KiB is enough to carry a JSON error envelope - * with a structured `code`/`message` payload (the router returns ~150 bytes - * for a free-tier denial; OpenAI returns ~300 bytes for invalid auth) without - * letting an HTML error page balloon persisted events. - */ -const MAX_ERROR_BODY_BYTES = 2048 - -/** - * Token usage accumulated across an SSE stream. OpenAI emits a single final - * `usage` chunk; Anthropic emits `input_tokens` on `message_start` and - * `output_tokens` on the terminal `message_delta`. We accept both — and the - * router proxy may forward either shape depending on which upstream answered. - */ -interface StreamUsageAccumulator { - tokensIn?: number - tokensOut?: number - model?: string - finishReason?: string - saw: boolean -} - /** @internal */ export function normalizeBackendStreamEvent( event: RuntimeStreamEvent, @@ -422,13 +99,6 @@ function mapCommonBackendEvent( ? (record.data as Record) : record if (type === 'message.part.updated' || type === 'text_delta' || type === 'delta') { - // `@tangle-network/sandbox` `box.streamPrompt` emits `message.part.updated` - // with a nested part: `{ type: 'message.part.updated', data: { part: - // { type: 'text', text: '…' } } }`. Walk into `data.part.text` so the - // canonical sandbox-SDK shape produces a `text_delta` natively — no - // per-product `mapEvent` shim required. Tool parts are picked up by - // the `tool_call` / `tool_result` branches below; non-text parts here - // fall through to `undefined` (the consumer can opt in via `mapEvent`). const part = data.part as Record | undefined const partText = part !== undefined && @@ -534,437 +204,6 @@ function mapCommonBackendEvent( return undefined } -async function* streamResponseEvents( - response: Response, - context: AgentBackendContext, - requestedModel: string, -): AsyncIterable { - const body = response.body - if (!body) return - const reader = body.getReader() - const decoder = new TextDecoder() - let buffer = '' - const usage: StreamUsageAccumulator = { saw: false } - // Tool-call assembly is stateful across SSE chunks: both OpenAI and - // Anthropic streamed `arguments`/`partial_json` incrementally and the - // final event is only safe to emit once we see a `finish_reason: - // 'tool_calls'` or `content_block_stop` for the relevant index. - const toolCalls: ToolCallAccumulator = new Map() - const startedAt = Date.now() - for (;;) { - const { done, value } = await reader.read() - if (done) break - buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n') - for (const event of drainStreamBuffer(false)) yield event - } - buffer += decoder.decode().replace(/\r\n/g, '\n') - for (const event of drainStreamBuffer(true)) yield event - if (buffer.trim()) { - for (const event of parseStreamChunk(buffer, context, usage, toolCalls)) yield event - } - // Flush any tool calls the model never closed via `finish_reason` — the - // upstream may have terminated the stream without a terminal chunk (e.g. - // when the proxy proactively forwards `[DONE]`). Emitting these here is - // strictly safer than silently dropping a tool call the agent intended. - for (const event of flushPendingToolCalls(toolCalls, context)) yield event - // Synthesize a single `llm_call` event from accumulated usage. We only emit - // when the upstream actually reported tokens — silent zeros would corrupt - // every cost ledger that observes the stream. Consumers that need to detect - // missing usage can check `tokensIn === undefined`. - if (usage.saw) { - yield { - type: 'llm_call', - task: context.task, - session: context.session, - model: usage.model ?? requestedModel, - tokensIn: usage.tokensIn, - tokensOut: usage.tokensOut, - // `costUsd` is intentionally absent — pricing tables live in consumers - // (agent-eval's `estimateCost`, MetricsCollector). Emitting a wrong - // number here is worse than emitting none. - latencyMs: Date.now() - startedAt, - finishReason: usage.finishReason, - timestamp: nowIso(), - } - } - - function* drainStreamBuffer(flush: boolean): Iterable { - for (;;) { - const sseBoundary = buffer.indexOf('\n\n') - if (sseBoundary >= 0) { - const chunk = buffer.slice(0, sseBoundary) - buffer = buffer.slice(sseBoundary + 2) - for (const event of parseStreamChunk(chunk, context, usage, toolCalls)) yield event - continue - } - - const newline = buffer.indexOf('\n') - if (newline >= 0 && !buffer.slice(0, newline).startsWith('data:')) { - const line = buffer.slice(0, newline) - buffer = buffer.slice(newline + 1) - for (const event of parseStreamChunk(line, context, usage, toolCalls)) yield event - continue - } - - if (flush && buffer.trim() && !buffer.trimStart().startsWith('data:')) { - const line = buffer - buffer = '' - for (const event of parseStreamChunk(line, context, usage, toolCalls)) yield event - continue - } - - break - } - } -} - -/** - * Per-tool-call accumulator. Keyed by either OpenAI `index` (cast to string) - * or Anthropic `content_block` `index`. Holds the streamed identifier, name, - * and string-form `arguments` so we can emit a single typed `tool_call` - * event once the stream signals the call is finalized. - */ -type ToolCallAccumulator = Map< - string, - { - id?: string - name?: string - /** Accumulated JSON-string `arguments` / `input` payload. */ - argsRaw: string - /** Source format: OpenAI delta vs Anthropic `tool_use` block. */ - source: 'openai' | 'anthropic' - /** Set true once the model signals this call is complete. */ - finalized: boolean - } -> - -function* parseStreamChunk( - chunk: string, - context: AgentBackendContext, - usage: StreamUsageAccumulator, - toolCalls: ToolCallAccumulator, -): Iterable { - const lines = chunk.split(/\r?\n/) - const dataLines = lines.filter((line) => line.startsWith('data:')) - if ( - dataLines.length === 0 && - lines.every((line) => { - const trimmed = line.trim() - return trimmed.length === 0 || trimmed.startsWith(':') - }) - ) { - return - } - const data = - dataLines.length > 0 - ? dataLines.map((line) => line.slice(5).trimStart()).join('\n') - : chunk.trim() - if (!data || data === '[DONE]') return - let parsed: Record - try { - parsed = JSON.parse(data) as Record - } catch { - yield { - type: 'text_delta', - task: context.task, - session: context.session, - text: data, - timestamp: nowIso(), - } - return - } - captureStreamUsage(parsed, usage) - const choices = parsed.choices - const choice = Array.isArray(choices) - ? (choices[0] as Record | undefined) - : undefined - const delta = choice?.delta as Record | undefined - const message = choice?.message as Record | undefined - - // ── OpenAI streamed `tool_calls` deltas ───────────────────────────── - const deltaToolCalls = delta?.tool_calls - if (Array.isArray(deltaToolCalls)) { - for (const tc of deltaToolCalls) { - if (!tc || typeof tc !== 'object') continue - const rec = tc as Record - const id = stringValue(rec.id) - const key = deltaToolCallKey(rec, id, toolCalls) - const acc = toolCalls.get(key) ?? { argsRaw: '', source: 'openai' as const, finalized: false } - if (id) acc.id = id - const fn = rec.function as Record | undefined - const name = stringValue(fn?.name) - if (name) acc.name = name - const args = stringValue(fn?.arguments) - if (args) acc.argsRaw += args - toolCalls.set(key, acc) - } - } - // `message.tool_calls` is the non-streamed shape — the model returned a - // complete tool call in one chunk. Treat the whole array as terminal. - const messageToolCalls = message?.tool_calls - if (Array.isArray(messageToolCalls)) { - for (const tc of messageToolCalls) { - if (!tc || typeof tc !== 'object') continue - const rec = tc as Record - const fn = rec.function as Record | undefined - const idx = numberValue(rec.index) ?? messageToolCalls.indexOf(tc) - const key = `openai:${idx}` - const acc = toolCalls.get(key) ?? { argsRaw: '', source: 'openai' as const, finalized: false } - const id = stringValue(rec.id) - if (id) acc.id = id - const name = stringValue(fn?.name) - if (name) acc.name = name - const args = stringValue(fn?.arguments) - if (args) acc.argsRaw += args - acc.finalized = true - toolCalls.set(key, acc) - } - } - - const finishReason = stringValue(choice?.finish_reason) - if (finishReason === 'tool_calls') { - // Model signaled it's done streaming tool calls — flush every OpenAI- - // sourced pending entry. Subsequent chunks (usage, [DONE]) won't add - // more. - for (const [key, acc] of toolCalls) { - if (acc.source === 'openai' && !acc.finalized) acc.finalized = true - toolCalls.set(key, acc) - } - } - - // ── Anthropic shape (proxied through router) ──────────────────────── - const eventType = stringValue(parsed.type) - if (eventType === 'content_block_start') { - const block = parsed.content_block as Record | undefined - if (block && stringValue(block.type) === 'tool_use') { - const idx = numberValue(parsed.index) ?? 0 - const key = `anthropic:${idx}` - toolCalls.set(key, { - id: stringValue(block.id), - name: stringValue(block.name), - argsRaw: '', - source: 'anthropic', - finalized: false, - }) - } - } - if (eventType === 'content_block_delta') { - const d = parsed.delta as Record | undefined - const dType = stringValue(d?.type) - if (dType === 'input_json_delta') { - const idx = numberValue(parsed.index) ?? 0 - const key = `anthropic:${idx}` - const acc = toolCalls.get(key) - if (acc) { - const partial = stringValue(d?.partial_json) ?? '' - acc.argsRaw += partial - toolCalls.set(key, acc) - } - } else { - const text = stringValue(d?.text) - if (text) { - yield { - type: 'text_delta', - task: context.task, - session: context.session, - text, - timestamp: nowIso(), - } - } - } - } - if (eventType === 'content_block_stop') { - const idx = numberValue(parsed.index) ?? 0 - const key = `anthropic:${idx}` - const acc = toolCalls.get(key) - if (acc) { - acc.finalized = true - toolCalls.set(key, acc) - } - } - - // Emit any tool calls that just became finalized. Done eagerly per-chunk so - // consumers see the call as soon as it's safe — the analyst loop watches - // `tool_call` events for delegation-pattern detection. - for (const event of drainFinalizedToolCalls(toolCalls, context)) yield event - - // ── Text deltas ────────────────────────────────────────────────────── - const text = - stringValue(delta?.content) ?? stringValue(message?.content) ?? stringValue(parsed.text) - if (text) { - yield { - type: 'text_delta', - task: context.task, - session: context.session, - text, - timestamp: nowIso(), - } - return - } - const mapped = mapCommonBackendEvent(parsed, context) - if (mapped) yield mapped -} - -function* drainFinalizedToolCalls( - toolCalls: ToolCallAccumulator, - context: AgentBackendContext, -): Iterable { - for (const [key, acc] of toolCalls) { - if (!acc.finalized) continue - toolCalls.delete(key) - yield buildToolCallEvent(acc, context) - } -} - -function* flushPendingToolCalls( - toolCalls: ToolCallAccumulator, - context: AgentBackendContext, -): Iterable { - for (const [key, acc] of toolCalls) { - toolCalls.delete(key) - yield buildToolCallEvent(acc, context) - } -} - -/** - * The accumulator key one streamed `delta.tool_calls[]` entry belongs to. - * - * OpenAI's streaming contract puts `index` on every fragment, so fragments of - * the same call concatenate under `openai:`. Some OpenAI-COMPATIBLE - * gateways do not: the Tangle router's Gemini lane emits N COMPLETE tool calls - * inside a single delta, each with a distinct `id` and no `index` at all. - * Defaulting those to index 0 concatenated every call's `arguments` into one - * string, which then failed `JSON.parse` and was surfaced as a raw string — so - * a turn that asked for six deliverables silently produced none. - * - * Resolution order: - * 1. `index` present → spec path, key by index (cross-delta fragments join). - * 2. `id` present → key by id; distinct ids are distinct calls, and a - * repeated id still concatenates correctly. - * 3. neither → an argument-only continuation fragment; append to the - * last-OPENED unfinalized OpenAI entry. `Map` iteration - * is insertion-ordered and `set` on an existing key does - * not reorder, so this is the most recently STARTED call - * still taking arguments — which is the one a gateway - * streaming calls sequentially is continuing. - */ -function deltaToolCallKey( - rec: Record, - id: string | undefined, - toolCalls: ToolCallAccumulator, -): string { - const idx = numberValue(rec.index) - if (idx !== undefined) return `openai:${idx}` - if (id) return `openai:id:${id}` - let last: string | undefined - for (const [key, acc] of toolCalls) { - if (acc.source === 'openai' && !acc.finalized) last = key - } - return last ?? 'openai:0' -} - -function buildToolCallEvent( - acc: { id?: string; name?: string; argsRaw: string; source: 'openai' | 'anthropic' }, - context: AgentBackendContext, -): RuntimeStreamEvent { - // `argsRaw` is JSON-string by the provider contract (OpenAI streams an - // escaped JSON string; Anthropic streams `partial_json` chunks). Parse - // best-effort — surface the raw string if parsing fails so downstream - // doesn't lose the call entirely. - let args: unknown = acc.argsRaw - if (acc.argsRaw.length > 0) { - try { - args = JSON.parse(acc.argsRaw) - } catch { - args = acc.argsRaw - } - } else { - args = {} - } - return { - type: 'tool_call', - task: context.task, - session: context.session, - toolName: acc.name ?? 'tool', - toolCallId: acc.id, - args, - timestamp: nowIso(), - } -} - -/** - * Accumulate token usage from any SSE chunk shape the router may emit. - * - * - OpenAI: a final chunk before `[DONE]` with `{ usage: { prompt_tokens, - * completion_tokens, total_tokens } }` and (often) empty `choices`. The - * `model` field is on every chunk and the last `choices[0].finish_reason` - * carries the stop reason. - * - Anthropic: `message_start` carries `message.model` and - * `message.usage.input_tokens`. The terminal `message_delta` carries - * `usage.output_tokens` and `delta.stop_reason`. - */ -function captureStreamUsage(parsed: Record, usage: StreamUsageAccumulator): void { - const model = stringValue(parsed.model) - if (model && !usage.model) usage.model = model - - const openAiUsage = parsed.usage as Record | undefined - if (openAiUsage && typeof openAiUsage === 'object') { - const promptTokens = numberValue(openAiUsage.prompt_tokens) - const completionTokens = numberValue(openAiUsage.completion_tokens) - const inputTokens = numberValue(openAiUsage.input_tokens) - const outputTokens = numberValue(openAiUsage.output_tokens) - if (promptTokens !== undefined) { - usage.tokensIn = promptTokens - usage.saw = true - } else if (inputTokens !== undefined) { - usage.tokensIn = (usage.tokensIn ?? 0) + inputTokens - usage.saw = true - } - if (completionTokens !== undefined) { - usage.tokensOut = completionTokens - usage.saw = true - } else if (outputTokens !== undefined) { - usage.tokensOut = (usage.tokensOut ?? 0) + outputTokens - usage.saw = true - } - } - - const type = stringValue(parsed.type) - if (type === 'message_start') { - const message = parsed.message as Record | undefined - const messageModel = stringValue(message?.model) - if (messageModel && !usage.model) usage.model = messageModel - const messageUsage = message?.usage as Record | undefined - const inputTokens = numberValue(messageUsage?.input_tokens) - if (inputTokens !== undefined) { - usage.tokensIn = inputTokens - usage.saw = true - } - const outputTokens = numberValue(messageUsage?.output_tokens) - if (outputTokens !== undefined) { - usage.tokensOut = (usage.tokensOut ?? 0) + outputTokens - usage.saw = true - } - } - if (type === 'message_delta') { - const delta = parsed.delta as Record | undefined - const stopReason = stringValue(delta?.stop_reason) - if (stopReason) usage.finishReason = stopReason - } - - const choices = parsed.choices - if (Array.isArray(choices)) { - const finishReason = stringValue( - (choices[0] as Record | undefined)?.finish_reason, - ) - if (finishReason) usage.finishReason = finishReason - } -} - -function numberValue(value: unknown): number | undefined { - return typeof value === 'number' && Number.isFinite(value) ? value : undefined -} - function stringValue(value: unknown): string | undefined { return typeof value === 'string' && value.length > 0 ? value : undefined } diff --git a/src/conversation/run-persona.test.ts b/src/conversation/run-persona.test.ts index b3786c14..6111979b 100644 --- a/src/conversation/run-persona.test.ts +++ b/src/conversation/run-persona.test.ts @@ -5,8 +5,8 @@ import type { Scenario, } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' -import { createIterableBackend } from '../backends' -import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' +import { createExecutor } from '../runtime/supervise/runtime' +import type { ExecutorFactory } from '../runtime/supervise/types' import { runPersonaConversation, runPersonaDispatch } from './run-persona' import type { ConversationTurn } from './types' @@ -14,78 +14,56 @@ interface PersonaScenario extends Scenario { turns: string[] } -/** A fake worker: records the system prompt it saw, answers each turn, and - * reports token/cost usage via an llm_call event (so metering is exercised). */ -function fakeWorker(saw: { prompt?: string; calls: number }): AgentExecutionBackend { - let n = 0 - return createIterableBackend({ - kind: 'fake-worker', - async *stream(input, context) { +/** An injected Router transport: the paid path still goes through the exact profile and Runtime. */ +function fakeExecutor( + saw: { prompt?: string; calls: number }, + options: { role?: 'worker' | 'persona' } = {}, +): ExecutorFactory { + return createExecutor({ + backend: 'router', + routerBaseUrl: 'https://router.test/v1', + routerKey: 'test-key', + complete: async (body) => { saw.calls += 1 - const first = input.messages?.[0] - if (first?.role === 'system') saw.prompt = first.content - n += 1 - yield { - type: 'text_delta', - task: context.task, - session: context.session, - text: `agent-answer-${n}`, - timestamp: new Date().toISOString(), - } satisfies RuntimeStreamEvent - yield { - type: 'llm_call', - task: context.task, - session: context.session, - model: 'fake', - tokensIn: 10, - tokensOut: 5, - costUsd: 0.02, - latencyMs: 1, - timestamp: new Date().toISOString(), - } satisfies RuntimeStreamEvent + const first = (body.messages as Array<{ role?: string; content?: unknown }> | undefined)?.[0] + if (first?.role === 'system' && typeof first.content === 'string') saw.prompt = first.content + const persona = options.role === 'persona' + return { + model: body.model, + choices: [ + { + message: { content: persona ? `user-turn-${saw.calls}` : `agent-answer-${saw.calls}` }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: persona ? 100 : 10, + completion_tokens: persona ? 50 : 5, + cost_usd: persona ? 0.5 : 0.02, + }, + } }, }) } -/** A profile-driven persona (LLM user-sim): emits a user turn AND reports its - * OWN (large) llm_call cost — used to prove the runner never attributes the - * persona-driver's spend to the worker. */ -function fakePersonaDriver(saw: { prompt?: string; calls: number }): AgentExecutionBackend { - return createIterableBackend({ - kind: 'fake-persona', - async *stream(input, context) { - saw.calls += 1 - const first = input.messages?.[0] - if (first?.role === 'system') saw.prompt = first.content - yield { - type: 'text_delta', - task: context.task, - session: context.session, - text: `user-turn-${saw.calls}`, - timestamp: new Date().toISOString(), - } satisfies RuntimeStreamEvent - yield { - type: 'llm_call', - task: context.task, - session: context.session, - model: 'fake-persona', - tokensIn: 100, - tokensOut: 50, - costUsd: 0.5, - latencyMs: 1, - timestamp: new Date().toISOString(), - } satisfies RuntimeStreamEvent - }, - }) -} - -const PROFILE = { model: { default: 'fake' } } as AgentProfile +const PROFILE = { + name: 'worker', + harness: 'cli-base', + model: { provider: 'test', default: 'fake' }, + prompt: { systemPrompt: 'WORKER PROMPT' }, +} as AgentProfile const WORKER_PROFILE = { - model: { default: 'fake' }, + name: 'worker', + harness: 'cli-base', + model: { provider: 'test', default: 'fake' }, + prompt: { systemPrompt: 'WORKER-PROMPT' }, metadata: { tag: 'worker' }, } as AgentProfile const PERSONA_PROFILE = { - model: { default: 'fake-persona' }, + name: 'persona', + harness: 'cli-base', + model: { provider: 'test', default: 'fake-persona' }, + prompt: { systemPrompt: 'PERSONA-PROMPT' }, metadata: { tag: 'persona' }, } as AgentProfile @@ -116,8 +94,7 @@ describe('runPersonaConversation', () => { const result = await runPersonaConversation({ worker: PROFILE, persona: { kind: 'scripted', turns: ['intake question', 'follow-up question'] }, - backendFor: () => fakeWorker(saw), - systemPromptOf: () => 'WORKER PROMPT', + executorFor: () => fakeExecutor(saw), }) const agentTurns = result.transcript.filter((t: ConversationTurn) => t.speaker === 'agent') expect(agentTurns).toHaveLength(2) @@ -133,8 +110,7 @@ describe('runPersonaConversation', () => { const result = await runPersonaConversation({ worker: PROFILE, persona: { kind: 'scripted', turns: ['q1', 'q2'] }, - backendFor: () => fakeWorker(saw), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor(saw), }) expect(result.tokensIn).toBe(20) expect(result.tokensOut).toBe(10) @@ -146,10 +122,9 @@ describe('runPersonaConversation', () => { await runPersonaConversation({ worker: PROFILE, persona: { kind: 'scripted', turns: ['hi'] }, - backendFor: () => fakeWorker(saw), - systemPromptOf: () => 'PROFILE-PROMPT-XYZ', + executorFor: () => fakeExecutor(saw), }) - expect(saw.prompt).toBe('PROFILE-PROMPT-XYZ') + expect(saw.prompt).toBe('WORKER PROMPT') }) it('requires maxTurns for a profile-driven persona', async () => { @@ -157,8 +132,7 @@ describe('runPersonaConversation', () => { runPersonaConversation({ worker: PROFILE, persona: { kind: 'profile', profile: PROFILE }, - backendFor: () => fakeWorker({ calls: 0 }), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor({ calls: 0 }), }), ).rejects.toThrow(/maxTurns is required/) }) @@ -169,9 +143,8 @@ describe('runPersonaConversation', () => { const result = await runPersonaConversation({ worker: WORKER_PROFILE, persona: { kind: 'profile', profile: PERSONA_PROFILE }, - backendFor: (_profile, role) => - role === 'worker' ? fakeWorker(workerSaw) : fakePersonaDriver(personaSaw), - systemPromptOf: (p) => (p.metadata?.tag === 'persona' ? 'PERSONA-PROMPT' : 'WORKER-PROMPT'), + executorFor: (_profile, role) => + role === 'worker' ? fakeExecutor(workerSaw) : fakeExecutor(personaSaw, { role: 'persona' }), maxTurns: 4, }) // alternate, persona leads: persona, worker, persona, worker. @@ -188,9 +161,10 @@ describe('runPersonaConversation', () => { const result = await runPersonaConversation({ worker: WORKER_PROFILE, persona: { kind: 'profile', profile: PERSONA_PROFILE }, - backendFor: (_profile, role) => - role === 'worker' ? fakeWorker({ calls: 0 }) : fakePersonaDriver({ calls: 0 }), - systemPromptOf: () => 'SYS', + executorFor: (_profile, role) => + role === 'worker' + ? fakeExecutor({ calls: 0 }) + : fakeExecutor({ calls: 0 }, { role: 'persona' }), maxTurns: 4, }) // worker: 2 calls × {in:10,out:5,$0.02}. persona-driver's 2×{in:100,out:50,$0.5} @@ -205,8 +179,7 @@ describe('runPersonaConversation', () => { runPersonaConversation({ worker: PROFILE, persona: { kind: 'scripted', turns: [] }, - backendFor: () => fakeWorker({ calls: 0 }), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor({ calls: 0 }), }), ).rejects.toThrow(/no turns/) }) @@ -216,8 +189,7 @@ describe('runPersonaDispatch (matrix adapter)', () => { it('is a ProfileDispatchFn that meters through ctx.cost and builds the artifact', async () => { const saw = { calls: 0 } as { prompt?: string; calls: number } const dispatch = runPersonaDispatch({ - backendFor: () => fakeWorker(saw), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor(saw), personaOf: (s) => ({ kind: 'scripted', turns: s.turns }), artifactOf: (transcript) => transcript.filter((t) => t.speaker === 'agent').length, }) @@ -243,8 +215,7 @@ describe('runPersonaDispatch (matrix adapter)', () => { it('refuses a capped conversation before the worker runs when no hard maximum is supplied', async () => { const saw = { calls: 0 } as { prompt?: string; calls: number } const dispatch = runPersonaDispatch({ - backendFor: () => fakeWorker(saw), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor(saw), personaOf: (scenario) => ({ kind: 'scripted', turns: scenario.turns }), artifactOf: () => 0, }) @@ -259,8 +230,7 @@ describe('runPersonaDispatch (matrix adapter)', () => { it('admits a capped conversation with an executor-enforced maximum', async () => { const dispatch = runPersonaDispatch({ - backendFor: () => fakeWorker({ calls: 0 }), - systemPromptOf: () => 'SYS', + executorFor: () => fakeExecutor({ calls: 0 }), personaOf: (scenario) => ({ kind: 'scripted', turns: scenario.turns }), artifactOf: (transcript) => transcript.length, maximumCharge: { externallyEnforcedMaximumUsd: 0.05 }, diff --git a/src/conversation/run-persona.ts b/src/conversation/run-persona.ts index 326b6b09..f618edf0 100644 --- a/src/conversation/run-persona.ts +++ b/src/conversation/run-persona.ts @@ -22,6 +22,8 @@ import type { Scenario, } from '@tangle-network/agent-eval/campaign' import { createIterableBackend } from '../backends' +import { streamAgentTurn } from '../runtime/stream-agent-turn' +import type { ExecutorFactory } from '../runtime/supervise/types' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { defineConversation } from './define-conversation' import { runConversation } from './run-conversation' @@ -38,11 +40,9 @@ export interface RunPersonaConversationOptions { worker: AgentProfile /** The simulated user driving the dialogue. */ persona: PersonaDriver - /** Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). - * Applied to the worker and to a `profile`-kind persona. */ - backendFor: (profile: AgentProfile, role: 'worker' | 'persona') => AgentExecutionBackend - /** Render a profile's system prompt — prepended to that profile's messages. */ - systemPromptOf: (profile: AgentProfile) => string + /** Resolve transport/executable ports for the exact profile. Runtime still materializes the + * profile and owns every model call. Applied to the worker and a profile-driven persona. */ + executorFor: (profile: AgentProfile, role: 'worker' | 'persona') => ExecutorFactory /** Speaker-turn cap. Default for a scripted persona = `2 * turns.length` * (worker answers each user turn). REQUIRED for a `profile` persona. */ maxTurns?: number @@ -72,24 +72,23 @@ interface UsageCounter { costUsd: number } -/** Prefix a backend's requests with `systemPrompt`; when `counter` is given, - * accumulate its `llm_call` token/cost usage (used for the metered worker). */ -function withProfilePrompt( - inner: AgentExecutionBackend, - systemPrompt: string, +/** Adapt one exact profile + Runtime executor into the conversation stream protocol. */ +function profileRuntimeBackend( + profile: AgentProfile, + factory: ExecutorFactory, counter?: UsageCounter, ): AgentExecutionBackend { return { - kind: inner.kind, - start: inner.start ? (input, ctx) => inner.start!(input, ctx) : undefined, - resume: inner.resume ? (session, input, ctx) => inner.resume!(session, input, ctx) : undefined, - stop: inner.stop ? (session, reason) => inner.stop!(session, reason) : undefined, + kind: 'runtime-profile', async *stream(input, context) { - const base = - input.messages ?? (input.message ? [{ role: 'user', content: input.message }] : []) - const messages = - base[0]?.role === 'system' ? base : [{ role: 'system', content: systemPrompt }, ...base] - for await (const event of inner.stream({ ...input, messages }, context)) { + const turnInput = input.messages + ? { messages: input.messages } + : (input.message ?? context.task.intent) + for await (const event of streamAgentTurn({ kind: 'executor', profile, factory }, turnInput, { + signal: context.signal, + ...(context.turnId ? { callId: context.turnId } : {}), + ...(context.runId ? { correlationId: context.runId } : {}), + })) { if (counter && event.type === 'llm_call') { counter.tokensIn += event.tokensIn ?? 0 counter.tokensOut += event.tokensOut ?? 0 @@ -135,9 +134,9 @@ export async function runPersonaConversation( ): Promise { const counter: UsageCounter = { tokensIn: 0, tokensOut: 0, costUsd: 0 } const workerName = opts.workerName ?? 'agent' - const worker = withProfilePrompt( - opts.backendFor(opts.worker, 'worker'), - opts.systemPromptOf(opts.worker), + const worker = profileRuntimeBackend( + opts.worker, + opts.executorFor(opts.worker, 'worker'), counter, ) @@ -150,9 +149,9 @@ export async function runPersonaConversation( persona = scriptedPersonaBackend(opts.persona.turns) maxTurns = opts.maxTurns ?? 2 * opts.persona.turns.length } else { - persona = withProfilePrompt( - opts.backendFor(opts.persona.profile, 'persona'), - opts.systemPromptOf(opts.persona.profile), + persona = profileRuntimeBackend( + opts.persona.profile, + opts.executorFor(opts.persona.profile, 'persona'), ) if (opts.maxTurns === undefined) { throw new Error('runPersonaConversation: maxTurns is required for a profile-driven persona') @@ -196,10 +195,8 @@ export async function runPersonaConversation( } export interface RunPersonaConfig { - /** Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). */ - backendFor: (profile: AgentProfile, role: 'worker' | 'persona') => AgentExecutionBackend - /** Render a profile's system prompt. */ - systemPromptOf: (profile: AgentProfile) => string + /** Resolve transport/executable ports for each exact profile. */ + executorFor: (profile: AgentProfile, role: 'worker' | 'persona') => ExecutorFactory /** The persona driving each scenario — a driver profile or scripted turns. */ personaOf: (scenario: TScenario) => PersonaDriver /** Build the scored artifact from the finished transcript. */ @@ -244,8 +241,7 @@ export function runPersonaDispatch( runPersonaConversation({ worker, persona: config.personaOf(scenario), - backendFor: config.backendFor, - systemPromptOf: config.systemPromptOf, + executorFor: config.executorFor, maxTurns: config.maxTurns?.(scenario), seed: config.seed?.(scenario), signal: executionSignal, diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 32af9ce6..0cd6434a 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -89,7 +89,12 @@ function fixedMethod( inspect?.(input) return { winnerSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, durationMs: 1, } }, @@ -601,6 +606,7 @@ describe('improve method execution', () => { winnerSurface: 'improved prompt', cost: { totalCostUsd: 0, + costProvenance: { kind: 'uncaptured', usd: null }, accountingComplete: false, incompleteReasons: ['optimizer model cost unavailable'], }, @@ -620,6 +626,7 @@ describe('improve method execution', () => { winnerSurface: 'improved prompt', cost: { totalCostUsd: 2, + costProvenance: { kind: 'observed', usd: 2 }, accountingComplete: true, incompleteReasons: [], }, diff --git a/src/improvement/method-cost.test.ts b/src/improvement/method-cost.test.ts index 66d23561..3ac0888b 100644 --- a/src/improvement/method-cost.test.ts +++ b/src/improvement/method-cost.test.ts @@ -69,7 +69,12 @@ describe('optimizer cost reconciliation', () => { assertMethodCostRecorded( 'fixture', { - cost: { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0.25, + costProvenance: { kind: 'observed', usd: 0.25 }, + accountingComplete: true, + incompleteReasons: [], + }, }, ledger, ledger, @@ -85,7 +90,12 @@ describe('optimizer cost reconciliation', () => { assertMethodCostRecorded( 'fixture', { - cost: { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0.25, + costProvenance: { kind: 'observed', usd: 0.25 }, + accountingComplete: true, + incompleteReasons: [], + }, }, ledger, ledger, @@ -103,6 +113,7 @@ describe('optimizer cost reconciliation', () => { { cost: { totalCostUsd: 0, + costProvenance: { kind: 'uncaptured', usd: null }, accountingComplete: false, incompleteReasons: ['provider receipt unavailable'], }, @@ -146,7 +157,12 @@ describe('optimizer cost reconciliation', () => { assertMethodCostRecorded( 'third-party', { - cost: { totalCostUsd: 0.1, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0.1, + costProvenance: { kind: 'observed', usd: 0.1 }, + accountingComplete: true, + incompleteReasons: [], + }, provenance: { source: { kind: 'package', @@ -204,7 +220,12 @@ describe('optimizer cost reconciliation', () => { assertMethodCostRecorded( 'fixture', { - cost: { totalCostUsd: 0.3, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0.3, + costProvenance: { kind: 'observed', usd: 0.3 }, + accountingComplete: true, + incompleteReasons: [], + }, provenance: { source: { kind: 'package', diff --git a/src/improvement/method-identity.test.ts b/src/improvement/method-identity.test.ts index 79f6f066..644d3161 100644 --- a/src/improvement/method-identity.test.ts +++ b/src/improvement/method-identity.test.ts @@ -173,7 +173,12 @@ describe('method evaluation identity', () => { async optimize() { return { winnerSurface: 'improved bytes', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } diff --git a/src/improvement/official-optimizers.test.ts b/src/improvement/official-optimizers.test.ts index c12f52b0..5b618094 100644 --- a/src/improvement/official-optimizers.test.ts +++ b/src/improvement/official-optimizers.test.ts @@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { makeProposalFinding } from '@tangle-network/agent-eval' +import type { OpenAICompatibleOptimizerModel } from '@tangle-network/agent-eval/campaign' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { type AgentProfile, @@ -214,10 +215,34 @@ function replacingRedactor( return redact } -const testOptimizer = { +const testOptimizer: OpenAICompatibleOptimizerModel = { model: 'optimizer-model', - baseUrl: 'http://127.0.0.1:1/v1', - apiKey: 'test-api-key', + callRef: 'official-optimizer-test/caller-owned-model', + call: async (request) => ({ + succeeded: true, + response: { + content: 'test optimizer response', + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, + costUsd: 0.000002, + model: request.request.model, + durationMs: 0, + finishReason: 'stop', + contentEmpty: false, + raw: {}, + }, + receipt: { + model: request.request.model, + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.000002, + }, + execution: { + kind: 'deterministic-test-model', + model: request.request.model, + callId: request.callId, + endpointFormat: request.endpointFormat ?? null, + }, + }), budget: { maxCostUsd: 1, maxRequests: 10, diff --git a/src/improvement/official-runtime-integration.test.ts b/src/improvement/official-runtime-integration.test.ts index 3156a53c..b1fef390 100644 --- a/src/improvement/official-runtime-integration.test.ts +++ b/src/improvement/official-runtime-integration.test.ts @@ -1,9 +1,18 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' -import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import type { + JudgeConfig, + OpenAICompatibleOptimizerModel, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentProfile, + canonicalAgentProfileDigest, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' import { expect, it } from 'vitest' +import { profileOptimizerModelCall } from '../runtime/profile-chat-client' import { improve } from './improve' import { officialGepa, officialSkillOpt } from './official-optimizers' import { startOptimizerModelServer } from './official-test-support' @@ -241,11 +250,37 @@ const requiredRuleJudge: JudgeConfig = { }, } -function optimizerModel(baseUrl: string, maxOutputTokensPerRequest: number) { +function optimizerModel( + baseUrl: string, + maxOutputTokensPerRequest: number, +): OpenAICompatibleOptimizerModel { + const profile: AgentProfile = { + name: 'official-runtime-integration-optimizer', + harness: 'cli-base', + model: { + provider: 'local-openai-compatible', + default: 'local-model', + metadata: { maxTokens: maxOutputTokensPerRequest }, + }, + } + const profileDigest = canonicalAgentProfileDigest(profile) + const call = profileOptimizerModelCall({ + profile, + context: 'official Runtime optimizer integration', + executor: { + backend: 'router', + routerBaseUrl: baseUrl, + routerKey: 'provider-secret', + }, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + }) return { model: 'local-model', - baseUrl, - apiKey: 'provider-secret', + callRef: `official-runtime-integration:${profileDigest}:${new URL(baseUrl).origin}`, + call, budget: { maxCostUsd: 1, maxRequests: 10, diff --git a/src/improvement/rollout-policy.test.ts b/src/improvement/rollout-policy.test.ts index 301fc898..63c9166f 100644 --- a/src/improvement/rollout-policy.test.ts +++ b/src/improvement/rollout-policy.test.ts @@ -24,7 +24,6 @@ describe('rollout policy profile coordinate', () => { repairRounds: 1, testgen: 0, diverse: false, - temperature: 0.2, } const profile: AgentProfile = { name: 'fixture' } const serialized = serializeRolloutPolicy(policy) @@ -89,7 +88,12 @@ function policyMethod( async optimize() { return { winnerSurface: serializeRolloutPolicy(policy), - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -172,7 +176,12 @@ describe("improve surface 'rollout-policy'", () => { async optimize(input) { return { winnerSurface: input.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, }, diff --git a/src/improvement/rollout-policy.ts b/src/improvement/rollout-policy.ts index b598f982..47c46c30 100644 --- a/src/improvement/rollout-policy.ts +++ b/src/improvement/rollout-policy.ts @@ -47,7 +47,6 @@ export function normalizeRolloutPolicy(raw: unknown): StructuralRolloutPolicy | repairRounds, testgen, ...(typeof bag.diverse === 'boolean' ? { diverse: bag.diverse } : {}), - ...(typeof bag.temperature === 'number' ? { temperature: bag.temperature } : {}), } } @@ -58,7 +57,6 @@ export function serializeRolloutPolicy(policy: StructuralRolloutPolicy): string repairRounds: policy.repairRounds, testgen: policy.testgen, ...(policy.diverse !== undefined ? { diverse: policy.diverse } : {}), - ...(policy.temperature !== undefined ? { temperature: policy.temperature } : {}), }) } @@ -83,7 +81,6 @@ export function applyRolloutPolicyToProfile( repairRounds: policy.repairRounds, testgen: policy.testgen, ...(policy.diverse !== undefined ? { diverse: policy.diverse } : {}), - ...(policy.temperature !== undefined ? { temperature: policy.temperature } : {}), } return { ...candidate, diff --git a/src/index.ts b/src/index.ts index 846c1a5b..8bacf07a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,13 +21,8 @@ export type { KnowledgeRequirement, RunRecord, } from '@tangle-network/agent-eval' -export type { BackendRetryPolicy } from './backends' // ── Backends ────────────────────────────────────────────────────────── -export { - createIterableBackend, - createOpenAICompatibleBackend, - createSandboxPromptBackend, -} from './backends' +export { createIterableBackend, createSandboxPromptBackend } from './backends' // ── Immutable candidate execution ───────────────────────────────────── // One verified bundle → one exact per-task plan → one protected run receipt. // This composes the shared profile materializer and agent-eval trace store; @@ -194,8 +189,6 @@ export { } from './otel-export' // ── Readiness ───────────────────────────────────────────────────────── export { decideKnowledgeReadiness } from './readiness' -export type { AgentBackendKind, ResolveAgentBackendOptions } from './resolve-agent-backend' -export { resolveAgentBackend } from './resolve-agent-backend' // ── Run loop ───────────────────────────────────────────────────────── export { applyRunRecordDefaults, runAgentTask, runAgentTaskStream } from './run' // ── Execution kernel ───────────────────────────────────────────────── diff --git a/src/mcp/bin.ts b/src/mcp/bin.ts index 0afb056f..a1a4c851 100644 --- a/src/mcp/bin.ts +++ b/src/mcp/bin.ts @@ -100,7 +100,7 @@ async function main(): Promise { } if (delegateSupervisor) { process.stderr.write( - `agent-runtime-mcp: delegate enabled — generic authoring supervisor on ${delegateSupervisor.router.model}\n`, + `agent-runtime-mcp: delegate enabled — generic authoring supervisor on ${delegateSupervisor.supervisorProfile.model?.default}\n`, ) } diff --git a/src/mcp/delegate-supervisor-provisioning.ts b/src/mcp/delegate-supervisor-provisioning.ts index 7b5d2887..0832086e 100644 --- a/src/mcp/delegate-supervisor-provisioning.ts +++ b/src/mcp/delegate-supervisor-provisioning.ts @@ -15,11 +15,13 @@ * @experimental */ +import type { AgentProfile } from '@tangle-network/agent-interface' import type { BackendType } from '@tangle-network/sandbox' import { ConfigError } from '../errors' import { type RouterEnv, resolveRouterBaseUrl } from '../model-resolution.js' import type { SandboxClient } from '../runtime' -import type { RouterConfig } from '../runtime/router-client' +import type { RouterTransportConfig } from '../runtime/router-client' +import { supervisorInstructions } from '../runtime/supervise/authoring' import type { ExecutorConfig } from '../runtime/supervise/runtime' import type { DelegateHandlerOptions } from './tools/delegate' @@ -64,7 +66,10 @@ export function delegateEnabled(env: NodeJS.ProcessEnv = process.env): boolean { * on every box: a delegate child running there inherits the model its own host declared. When * nothing names one, fail at startup with the ladder in the message. */ -function resolveRouter(env: NodeJS.ProcessEnv): RouterConfig { +function resolveRouterSupervisor(env: NodeJS.ProcessEnv): { + router: RouterTransportConfig + profile: AgentProfile +} { const routerKey = trimmed(env.MCP_SUPERVISOR_ROUTER_KEY) ?? trimmed(env.TANGLE_INFERENCE_KEY) ?? @@ -86,7 +91,15 @@ function resolveRouter(env: NodeJS.ProcessEnv): RouterConfig { 'WORKER_MODEL / TANGLE_ROUTER_MODEL) to a tool-calling model the router serves.', ) } - return { routerBaseUrl, routerKey, model } + return { + router: { routerBaseUrl, routerKey }, + profile: { + name: 'delegate-supervisor', + harness: 'cli-base', + model: { provider: 'tangle-router', default: model }, + prompt: { systemPrompt: supervisorInstructions() }, + }, + } } /** @@ -100,7 +113,7 @@ export function resolveDelegateSupervisor( env: NodeJS.ProcessEnv = process.env, ): DelegateHandlerOptions | undefined { if (!delegateEnabled(env)) return undefined - const router = resolveRouter(env) + const supervisor = resolveRouterSupervisor(env) const harness = (trimmed(env.MCP_DELEGATE_WORKER_HARNESS) ?? DEFAULT_WORKER_HARNESS) as BackendType const backend: ExecutorConfig = { @@ -109,8 +122,8 @@ export function resolveDelegateSupervisor( sandboxClient, } return { - router, + router: supervisor.router, + supervisorProfile: supervisor.profile, backend, - model: router.model, } } diff --git a/src/mcp/local-harness.ts b/src/mcp/local-harness.ts index 98aef8cb..04a72eeb 100644 --- a/src/mcp/local-harness.ts +++ b/src/mcp/local-harness.ts @@ -309,7 +309,7 @@ export function harnessInvocation( } if (options.codexReproducible) { - const model = concreteProfileModel(profile) + const model = harnessProfileModel(harness, profile) if (!model) { throw new Error('harnessInvocation: codexReproducible requires profile.model.default') } @@ -336,7 +336,7 @@ export function harnessInvocation( const args = buildHarnessArgs(harness, composedPrompt, options) - const model = concreteProfileModel(profile) + const model = harnessProfileModel(harness, profile) if (model) { args.push(...invocation.modelArgs(model)) } @@ -349,6 +349,24 @@ export function harnessInvocation( return { command: invocation.command, args, prompt: composedPrompt } } +/** Render the exact profile provider/model through the selected native harness. */ +function harnessProfileModel(harness: LocalHarness, profile: AgentProfile): string | undefined { + const model = concreteProfileModel(profile) + if (!model) return undefined + const provider = profile.model?.provider?.trim() + if (!provider) return model + if (harness === 'opencode') { + return model.includes('/') ? model : `${provider}/${model}` + } + const nativeProvider = harness === 'codex' ? 'openai' : 'anthropic' + if (provider !== nativeProvider) { + throw new Error( + `harnessInvocation: ${harness} cannot materialize model provider ${JSON.stringify(provider)}; expected ${nativeProvider}`, + ) + } + return model +} + /** @experimental */ export interface RunLocalHarnessOptions { harness: LocalHarness diff --git a/src/mcp/tools/delegate.ts b/src/mcp/tools/delegate.ts index 1f0b35d5..4f859616 100644 --- a/src/mcp/tools/delegate.ts +++ b/src/mcp/tools/delegate.ts @@ -16,7 +16,8 @@ * @experimental */ -import type { RouterConfig } from '../../runtime/router-client' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { RouterTransportConfig } from '../../runtime/router-client' import type { DeliverableSpec } from '../../runtime/supervise/completion-gate' import { type DelegateOptions, delegate } from '../../runtime/supervise/delegate' import type { ExecutorConfig } from '../../runtime/supervise/runtime' @@ -43,7 +44,7 @@ export const DELEGATE_DESCRIPTION = [ 'a success.', ].join('\n') -/** JSON Schema for `delegate` tool arguments (`intent` + optional `model` and `runId`). @experimental */ +/** JSON Schema for `delegate` tool arguments (`intent` + optional trace id). @experimental */ export const DELEGATE_INPUT_SCHEMA = { type: 'object', properties: { @@ -51,10 +52,6 @@ export const DELEGATE_INPUT_SCHEMA = { type: 'string', description: 'What you want accomplished, as an outcome. The supervisor authors the worker.', }, - model: { - type: 'string', - description: 'Optional per-call override for the supervisor brain model.', - }, runId: { type: 'string', description: 'Optional trace-correlation id for this delegation.', @@ -67,7 +64,6 @@ export const DELEGATE_INPUT_SCHEMA = { /** Parsed `delegate` tool arguments. */ export interface DelegateArgs { intent: string - model?: string runId?: string } @@ -77,15 +73,15 @@ export function validateDelegateArgs(raw: unknown): DelegateArgs { throw new TypeError('delegate: arguments must be an object') } const value = raw as Record + const unknown = Object.keys(value).filter((key) => key !== 'intent' && key !== 'runId') + if (unknown.length > 0) { + throw new TypeError(`delegate: unknown arguments: ${unknown.join(', ')}`) + } const intent = value.intent if (typeof intent !== 'string' || intent.trim().length === 0) { throw new TypeError('delegate: `intent` must be a non-empty string') } const args: DelegateArgs = { intent: intent.trim() } - if (value.model !== undefined) { - if (typeof value.model !== 'string') throw new TypeError('delegate: `model` must be a string') - args.model = value.model - } if (value.runId !== undefined) { if (typeof value.runId !== 'string') throw new TypeError('delegate: `runId` must be a string') args.runId = value.runId @@ -110,13 +106,13 @@ export interface DelegateError { /** @experimental */ export interface DelegateHandlerOptions { /** The supervisor brain's router substrate (REQUIRED — the default supervisor is router-brained). */ - router: RouterConfig + router: RouterTransportConfig + /** Exact executable supervisor identity selected by the trusted composition root. */ + supervisorProfile: AgentProfile /** WHERE the authored workers run. Required for `supervise()` to spawn anything. */ backend: ExecutorConfig /** The completion oracle the authored workers settle against (settled ⟺ delivered). */ deliverable?: DeliverableSpec - /** Default supervisor brain model when a call omits `model`. */ - model?: string /** Restrict the run to this subset of models. */ allowedModels?: readonly string[] } @@ -160,7 +156,7 @@ export function createDelegateHandler( const opts: DelegateOptions = { backend: options.backend, router: options.router, - model: args.model ?? options.model, + supervisorProfile: options.supervisorProfile, ...(options.deliverable ? { deliverable: options.deliverable } : {}), ...(options.allowedModels ? { allowedModels: options.allowedModels } : {}), ...(args.runId ? { runId: args.runId } : {}), diff --git a/src/primeintellect/index.ts b/src/primeintellect/index.ts index 481a05ad..6526a4f0 100644 --- a/src/primeintellect/index.ts +++ b/src/primeintellect/index.ts @@ -4,8 +4,7 @@ export { writePrimeIntellectPackage, } from './package' export { - createPrimeIntellectBackend, - type PrimeIntellectBackendOptions, + primeIntellectExecutorConfig, type RunPrimeIntellectProgramOptions, readPrimeIntellectEpisodeContext, runPrimeIntellectProgram, diff --git a/src/primeintellect/primeintellect.test.ts b/src/primeintellect/primeintellect.test.ts index fe0fd407..2c797f7e 100644 --- a/src/primeintellect/primeintellect.test.ts +++ b/src/primeintellect/primeintellect.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { createPrimeIntellectPackage, writePrimeIntellectPackage } from './package' import { - createPrimeIntellectBackend, + primeIntellectExecutorConfig, readPrimeIntellectEpisodeContext, runPrimeIntellectProgram, } from './runner' @@ -179,14 +179,15 @@ describe('PrimeIntellect runtime program contract', () => { OPENAI_API_KEY: 'interception-secret', } - it('reads an answer-free episode and creates the existing backend', async () => { + it('reads an answer-free episode and resolves transport-only executor config', async () => { const context = readPrimeIntellectEpisodeContext(env) expect(context.task.id).toBe('eval-refund') expect(context.mcpServers.policy).toBe('http://127.0.0.1:3210/mcp') - expect(createPrimeIntellectBackend(context).kind).toBe('primeintellect') - expect(createPrimeIntellectBackend(context, { kind: 'product-runtime' }).kind).toBe( - 'product-runtime', - ) + expect(primeIntellectExecutorConfig(context)).toEqual({ + backend: 'router', + routerBaseUrl: 'http://127.0.0.1:9000/v1', + routerKey: 'interception-secret', + }) await expect( runPrimeIntellectProgram(async (episode) => episode.task.id, { env }), ).resolves.toBe('eval-refund') diff --git a/src/primeintellect/runner.ts b/src/primeintellect/runner.ts index e026139c..ea87069d 100644 --- a/src/primeintellect/runner.ts +++ b/src/primeintellect/runner.ts @@ -1,4 +1,4 @@ -import { createOpenAICompatibleBackend } from '../backends' +import type { ExecutorConfig } from '../runtime/supervise/runtime' import type { PrimeIntellectEpisodeContext, PrimeIntellectPublicTask, @@ -18,11 +18,6 @@ export interface RunPrimeIntellectProgramOptions { env?: NodeJS.ProcessEnv } -export type PrimeIntellectBackendOptions = Omit< - Parameters[0], - 'apiKey' | 'baseUrl' | 'model' -> - /** Read and validate the private process contract installed by the generated Prime harness. */ export function readPrimeIntellectEpisodeContext( env: NodeJS.ProcessEnv = process.env, @@ -45,17 +40,15 @@ export function readPrimeIntellectEpisodeContext( } } -/** Build the existing runtime backend against Prime's intercepted model endpoint. */ -export function createPrimeIntellectBackend( +/** Resolve Prime's intercepted endpoint as transport-only Runtime executor configuration. + * The caller's exact `AgentProfile` remains the sole owner of model and behavior. */ +export function primeIntellectExecutorConfig( context: PrimeIntellectEpisodeContext, - options: PrimeIntellectBackendOptions = {}, -) { - return createOpenAICompatibleBackend({ - ...options, - apiKey: context.model.apiKey, - baseUrl: context.model.baseUrl, - model: context.model.name, - kind: options.kind ?? 'primeintellect', +): Extract { + return Object.freeze({ + backend: 'router', + routerBaseUrl: context.model.baseUrl, + routerKey: context.model.apiKey, }) } diff --git a/src/resolve-agent-backend.test.ts b/src/resolve-agent-backend.test.ts deleted file mode 100644 index d401ef7f..00000000 --- a/src/resolve-agent-backend.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import type { AgentExecutionBackend } from './types' - -const createOpenAICompatibleBackend = vi.fn((config: unknown): AgentExecutionBackend => { - return { - kind: (config as { kind?: string }).kind ?? 'tcloud', - stream: async function* () {}, - __config: config, - } as unknown as AgentExecutionBackend -}) - -vi.mock('./backends', () => ({ createOpenAICompatibleBackend })) - -const { resolveAgentBackend } = await import('./resolve-agent-backend') - -const configOf = (backend: AgentExecutionBackend): Record => - (backend as unknown as { __config: Record }).__config - -const base = { apiKey: 'sk-x', baseUrl: 'https://router.tangle.tools/v1', model: 'kimi' } - -describe('resolveAgentBackend', () => { - it("kind 'tcloud' builds createOpenAICompatibleBackend with kind defaulting to the kind name", () => { - const backend = resolveAgentBackend({ kind: 'tcloud', ...base }) - expect(configOf(backend)).toEqual({ ...base, kind: 'tcloud' }) - }) - - it("kind 'router' routes through createOpenAICompatibleBackend (same endpoint as tcloud)", () => { - const backend = resolveAgentBackend({ kind: 'router', ...base }) - expect(configOf(backend)).toEqual({ ...base, kind: 'router' }) - }) - - it("kind 'cli-bridge' routes through createOpenAICompatibleBackend and forwards model", () => { - const backend = resolveAgentBackend({ - kind: 'cli-bridge', - apiKey: '', - baseUrl: 'http://127.0.0.1:3355/v1', - model: 'claude-code/sonnet', - }) - // cli-bridge REQUIRES model — the whole reason it uses this factory. - expect(configOf(backend)).toEqual({ - apiKey: '', - baseUrl: 'http://127.0.0.1:3355/v1', - model: 'claude-code/sonnet', - kind: 'cli-bridge', - }) - }) - - it('honors an explicit label over the kind name', () => { - const backend = resolveAgentBackend({ - kind: 'tcloud', - ...base, - label: 'insurance-canonical-tcloud', - }) - expect((configOf(backend) as { kind: string }).kind).toBe('insurance-canonical-tcloud') - }) - - it('forwards openai-compat passthrough (tools, fetchImpl) when set', () => { - const tools = [ - { type: 'function', function: { name: 'submit', description: 'd', parameters: {} } }, - ] as never - const fetchImpl = (async () => new Response()) as unknown as typeof fetch - const backend = resolveAgentBackend({ kind: 'cli-bridge', ...base, tools, fetchImpl }) - const cfg = configOf(backend) - expect(cfg.tools).toBe(tools) - expect(cfg.fetchImpl).toBe(fetchImpl) - }) - - it('forwards generation options when set', () => { - const backend = resolveAgentBackend({ - kind: 'router', - ...base, - temperature: 1, - maxTokens: 8192, - }) - const cfg = configOf(backend) - expect(cfg.temperature).toBe(1) - expect(cfg.maxTokens).toBe(8192) - }) - - it('omits passthrough fields that were never set (keeps the tool-free request shape)', () => { - const backend = resolveAgentBackend({ kind: 'tcloud', ...base }) - const cfg = configOf(backend) - expect('tools' in cfg).toBe(false) - expect('toolChoice' in cfg).toBe(false) - expect('responseFormat' in cfg).toBe(false) - expect('temperature' in cfg).toBe(false) - expect('maxTokens' in cfg).toBe(false) - expect('fetchImpl' in cfg).toBe(false) - expect('retry' in cfg).toBe(false) - }) - - it("kind 'sandbox' invokes the caller's sandboxBackend seam", () => { - const sandbox = { - kind: 'product-sandbox', - stream: async function* () {}, - } as AgentExecutionBackend - const sandboxBackend = vi.fn(() => sandbox) - const backend = resolveAgentBackend({ kind: 'sandbox', ...base, sandboxBackend }) - expect(backend).toBe(sandbox) - expect(sandboxBackend).toHaveBeenCalledOnce() - expect(createOpenAICompatibleBackend).not.toHaveBeenCalledWith( - expect.objectContaining({ kind: 'sandbox' }), - ) - }) - - it("kind 'sandbox' fails loud when no sandboxBackend seam is provided", () => { - expect(() => resolveAgentBackend({ kind: 'sandbox', ...base })).toThrow( - /requires opts\.sandboxBackend/, - ) - }) -}) diff --git a/src/resolve-agent-backend.ts b/src/resolve-agent-backend.ts deleted file mode 100644 index 92b1ed5b..00000000 --- a/src/resolve-agent-backend.ts +++ /dev/null @@ -1,112 +0,0 @@ -/** - * The product-facing backend selector for `runChatThroughRuntime` / - * `runAgentTaskStream`: one call turns a `--backend {router,tcloud,cli-bridge, - * sandbox}` choice into the `AgentExecutionBackend` the chat leg runs on. - * - * It is the `AgentExecutionBackend` sibling of `resolveSandboxClient` (which - * resolves the `SandboxClient` a `runAgentRounds` drives). Both exist for the same - * reason: every in-process eval product hand-rolled the identical - * "`backend-name` → `createOpenAICompatibleBackend`" branch, and the copies - * drift. This is the single generic resolver they share. - * - * - `router` / `tcloud` / `cli-bridge` → OpenAI-compatible chat completions. - * All three speak `POST {baseUrl}/chat/completions` in OpenAI's SSE shape — - * the router (a.k.a. tcloud) IS that endpoint, and cli-bridge fronts a - * harness CLI behind the same protocol at its own `/v1`. They differ only - * in `baseUrl` / `apiKey` and the `kind` label a product wants on its - * traces. cli-bridge REQUIRES `model` in the request body, so it MUST route - * through `createOpenAICompatibleBackend` (which sends it), never a - * transport that drops the field. - * - `sandbox` → the caller's own domain backend. The sandbox variant carries - * product specifics (system prompt, workspace id, in-box D1 executor) that - * do NOT belong in the substrate, so the product passes a `sandboxBackend()` - * seam that this resolver simply invokes. - * - * This resolver is PURE backend selection. Product concerns — credit hard-cuts, - * fetch-capture shims, D1 platform wiring — stay as product-side WRAPPERS - * around the returned backend. The OpenAI-compat passthrough fields (`tools`, - * `toolChoice`, `responseFormat`, `temperature`, `maxTokens`, `fetchImpl`, - * `retry`) are forwarded verbatim so a product can advertise its app tools, - * preserve generation settings, or install a capturing fetch without - * re-opening the branch this consolidation closes. - */ - -import { createOpenAICompatibleBackend } from './backends' -import type { AgentBackendInput, AgentExecutionBackend } from './types' - -/** The transport a chat backend runs on. */ -export type AgentBackendKind = 'router' | 'tcloud' | 'cli-bridge' | 'sandbox' - -/** - * OpenAI-compat passthrough forwarded to `createOpenAICompatibleBackend` for - * the `router` / `tcloud` / `cli-bridge` kinds. Mirrors that factory's optional - * inputs so a product keeps its tool advertising / capture-fetch without - * re-implementing the backend branch. - */ -type OpenAICompatPassthrough = Pick< - Parameters[0], - 'tools' | 'toolChoice' | 'responseFormat' | 'temperature' | 'maxTokens' | 'fetchImpl' | 'retry' -> - -export interface ResolveAgentBackendOptions - extends OpenAICompatPassthrough { - /** The chat transport to resolve. */ - kind: AgentBackendKind - /** - * Bearer credential for the OpenAI-compat kinds. Empty string is valid for a - * loopback-anonymous cli-bridge; a `router`/`tcloud` route with an empty key - * is a caller bug the product surfaces before calling in. - */ - apiKey: string - /** Base URL for the OpenAI-compat kinds. cli-bridge's is its `/v1`. */ - baseUrl: string - /** Model id sent on every request. cli-bridge rejects a request without it. */ - model: string - /** `kind` label stamped on the resolved backend + its traces. Defaults to `kind`. */ - label?: string - /** - * `sandbox` kind: the product's own domain backend. Required for that kind — - * the substrate owns no product sandbox shape, so a `sandbox` resolution with - * no seam is a caller bug, not a silent fallback. - */ - sandboxBackend?: () => AgentExecutionBackend -} - -/** - * Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead - * of hand-rolling the `createOpenAICompatibleBackend` branch in each product. - */ -export function resolveAgentBackend( - opts: ResolveAgentBackendOptions, -): AgentExecutionBackend { - switch (opts.kind) { - case 'router': - case 'tcloud': - case 'cli-bridge': { - const passthrough: OpenAICompatPassthrough = {} - // Forward only the fields a caller actually set — an explicit - // `tools: []` / `undefined` would otherwise reach the factory and change - // its request shape (some providers reject an empty `tools` array). - if (opts.tools !== undefined) passthrough.tools = opts.tools - if (opts.toolChoice !== undefined) passthrough.toolChoice = opts.toolChoice - if (opts.responseFormat !== undefined) passthrough.responseFormat = opts.responseFormat - if (opts.temperature !== undefined) passthrough.temperature = opts.temperature - if (opts.maxTokens !== undefined) passthrough.maxTokens = opts.maxTokens - if (opts.fetchImpl !== undefined) passthrough.fetchImpl = opts.fetchImpl - if (opts.retry !== undefined) passthrough.retry = opts.retry - return createOpenAICompatibleBackend({ - apiKey: opts.apiKey, - baseUrl: opts.baseUrl, - model: opts.model, - kind: opts.label ?? opts.kind, - ...passthrough, - }) - } - case 'sandbox': { - if (!opts.sandboxBackend) { - throw new Error("resolveAgentBackend: kind 'sandbox' requires opts.sandboxBackend") - } - return opts.sandboxBackend() - } - } -} diff --git a/src/runtime/audit-intent.ts b/src/runtime/audit-intent.ts index 762fde0c..2afbbc73 100644 --- a/src/runtime/audit-intent.ts +++ b/src/runtime/audit-intent.ts @@ -24,7 +24,9 @@ * objective drift across tasks). */ -import type { ChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { profileChatClient } from './profile-chat-client' +import type { ExecutorConfig } from './supervise/runtime' export interface AuditIntentInput { /** The declared intent: the task text / acceptance criteria the agent was given. */ @@ -40,10 +42,10 @@ export interface AuditIntentInput { } export interface AuditIntentOptions { - chat: ChatClient - model?: string - /** Override the auditor instruction (optimizable like any analyst prompt). */ - auditorInstruction?: string + /** Exact auditor identity. */ + profile: AgentProfile + /** Execution substrate. All behavior comes from the profile. */ + executor: ExecutorConfig /** Cap trace lines fed to the auditor. Default 80. */ maxTraceLines?: number signal?: AbortSignal @@ -111,12 +113,14 @@ export async function auditIntent( input: AuditIntentInput, opts: AuditIntentOptions, ): Promise { - const res = await opts.chat.chat( + const res = await profileChatClient({ + profile: opts.profile, + executor: opts.executor, + context: 'intent auditor', + }).chat( { - ...(opts.model ? { model: opts.model } : {}), jsonSchema: auditSchema as unknown as { name: string; schema: Record }, messages: [ - { role: 'system', content: opts.auditorInstruction ?? defaultAuditorInstruction }, { role: 'user', content: diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index bba6127e..4a44226a 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -477,7 +477,6 @@ export function defineLeaderboard( bridge: { url: process.env.CLI_BRIDGE_URL, bearer, - model: bareModel(models[0] ?? ''), timeoutMs: 900_000, }, }) diff --git a/src/runtime/harvest-corpus.ts b/src/runtime/harvest-corpus.ts index add357e3..5db2c553 100644 --- a/src/runtime/harvest-corpus.ts +++ b/src/runtime/harvest-corpus.ts @@ -21,22 +21,22 @@ * own A/B; the corpus's first consumers are operators and optimizers, not prompts. */ -import type { ChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { type Observation, type ObserveInput, observe } from './observe' import type { Corpus } from './personify/wave-types' +import type { ExecutorConfig } from './supervise/runtime' export interface HarvestCorpusOptions { /** The completed runs to analyze — map your store's rows to `ObserveInput`. */ runs: AsyncIterable | Iterable - /** The model-call seam (agent-eval `createChatClient`). */ - chat: ChatClient - model?: string + /** Exact analyst identity. */ + profile: AgentProfile + /** Execution substrate. All behavior comes from the profile. */ + executor: ExecutorConfig /** The durable corpus the facts accrete into. */ corpus: Corpus /** Tags written onto learned facts (the product/domain key the read side queries by). */ tags?: ReadonlyArray - /** Override the analyst instruction (the GEPA-tunable knob). */ - analystInstruction?: string /** Runs analyzed in parallel. Default 4. */ concurrency?: number /** Hard cap on runs consumed from the stream (a cost guard for unbounded stores). */ @@ -91,11 +91,10 @@ export async function harvestCorpus(opts: HarvestCorpusOptions): Promise): SandboxClient { +export function inlineSandboxClient( + factory: ExecutorFactory, + defaults: { profile?: AgentProfile } = {}, +): SandboxClient { + const capturedDefaultProfile = + defaults.profile === undefined + ? undefined + : agentProfileSchema.parse(structuredClone(defaults.profile)) let seq = 0 return { async create(options?: CreateSandboxOptions): Promise { const id = `inline-${seq++}` - // The `create(options)` backend override is threaded to the factory through - // the `ExecutorContext.seams` channel (the designed opaque-seam extension - // point), so a BYO executor that varies per-cell — e.g. the cli-bridge - // executor reading its harness/model off `backend.type`/`backend.model.model` - // — sees the same per-create config a real sandbox executor gets, without a - // second client per cell. + // The requested backend remains visible so Runtime can reject a conflicting redundant + // declaration. The exact profile below remains the sole behavioral source. const createOptions = options return { id, @@ -70,32 +75,82 @@ export function inlineSandboxClient(factory: ExecutorFactory): SandboxC if (callerSignal.aborted) onAbort() else callerSignal.addEventListener('abort', onAbort, { once: true }) } - const spec: AgentSpec = { profile: { name: id }, harness: null } + const requestedProfile = + (options?.backend && typeof options.backend === 'object' + ? (options.backend as { profile?: unknown }).profile + : undefined) ?? capturedDefaultProfile + const parsedProfile = agentProfileSchema.safeParse(requestedProfile) + if (!parsedProfile.success) { + throw new Error( + 'inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })', + ) + } + const spec: AgentSpec = { profile: parsedProfile.data, harness: null } const exec = factory(spec, { signal: controller.signal, seams: { createOptions } }) try { const artifact = await settle(exec, message, controller.signal) - const out = artifact.out as { content?: string } | undefined + const out = artifact.out as + | { + content?: string + estimatedCostUsd?: number + promptCache?: Readonly> + } + | undefined // Speak the runtime's metering protocol: `extractLlmCallEvent` reads // flat `llm_call` events, not the nested result payload — without // this the kernel meters the iteration as a fabricated $0 / 0 tokens. const tokensIn = artifact.spent.tokens.input const tokensOut = artifact.spent.tokens.output const costUsd = artifact.spent.usd - if (tokensIn || tokensOut || costUsd) { + const estimatedCostUsd = out?.estimatedCostUsd + if ( + artifact.spent.iterations > 0 || + artifact.spent.tokensKnown === false || + artifact.spent.usdKnown === false || + tokensIn > 0 || + tokensOut > 0 || + costUsd > 0 || + estimatedCostUsd !== undefined + ) { yield { type: 'llm_call', - data: { tokensIn, tokensOut, costUsd }, + data: { + ...(artifact.spent.tokensKnown === false ? {} : { tokensIn, tokensOut }), + ...(costUsd > 0 ? { costUsd } : {}), + ...(artifact.spent.tokensKnown === false ? { tokensKnown: false } : {}), + ...(artifact.spent.usdKnown === false ? { costKnown: false } : {}), + ...(artifact.spent.usdKnown !== false && costUsd > 0 + ? { + costKnown: true, + costProvenance: 'provider-receipt', + } + : {}), + ...(estimatedCostUsd !== undefined ? { estimatedCostUsd } : {}), + ...(out?.promptCache ? { promptCache: out.promptCache } : {}), + }, } as unknown as SandboxEvent } yield { type: 'result', data: { finalText: out?.content ?? '', - tokenUsage: { - inputTokens: tokensIn, - outputTokens: tokensOut, - }, - costUsd, + ...(artifact.spent.tokensKnown === false + ? { tokensKnown: false } + : { + tokenUsage: { + inputTokens: tokensIn, + outputTokens: tokensOut, + }, + }), + ...(artifact.spent.usdKnown === false + ? { costKnown: false, ...(costUsd > 0 ? { costUsd } : {}) } + : { + costUsd, + costKnown: true, + costProvenance: 'provider-receipt', + }), + ...(estimatedCostUsd !== undefined ? { estimatedCostUsd } : {}), + ...(out?.promptCache ? { promptCache: out.promptCache } : {}), }, } as unknown as SandboxEvent } finally { diff --git a/src/runtime/local-sandbox-client.ts b/src/runtime/local-sandbox-client.ts index d2d1d0aa..88f7ed2f 100644 --- a/src/runtime/local-sandbox-client.ts +++ b/src/runtime/local-sandbox-client.ts @@ -13,30 +13,32 @@ * local MCP process applies only when its full canonical bytes match the fixed * constructor profile; a different generated profile is refused. * - * Event protocol matches `inlineSandboxClient`: one `llm_call` metering event - * + one terminal `result` event with finalText/tokenUsage/costUsd. + * Event protocol matches `inlineSandboxClient`: known token usage is emitted as one `llm_call`; + * Router catalog cost remains a separately-labelled estimate, never billed spend. */ import { type AgentProfile, type AgentProfileSecurityPolicy, - canonicalCandidateDigest, + agentProfileSchema, + canonicalAgentProfileDigest, } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { ValidationError } from '../errors' import type { KeyProvider } from './key-provider' import { routerBrain } from './router-client' import { materializeLocalMcp } from './stdio-mcp-client' +import { + assertExecutableAgentProfile, + profileModelExecutionSettings, + profileProviderModel, +} from './supervise/model-policy' import { runBrainLoop, type ToolLoopChat } from './tool-loop' import type { SandboxClient } from './types' export interface LocalSandboxClientOptions { - /** The worker brain: router chat-completions with tool-calling. All three required. */ - router: { baseUrl: string; key: string; model: string } - /** Tool-loop turns per prompt. Default 8. */ - maxTurns?: number - /** Brain sampling temperature. Default: `routerBrain`'s (0.4). */ - temperature?: number + /** Router endpoint/auth. The exact per-create profile owns model and loop behavior. */ + router: { baseUrl: string; key: string } /** Fallback profile when `create(options)` carries none on `backend.profile`. */ profile?: AgentProfile /** Resolves profile-declared MCP secret names at child-process spawn time. */ @@ -57,19 +59,23 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie } const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && opts.profile !== undefined - ? canonicalCandidateDigest(opts.profile) + ? canonicalAgentProfileDigest(opts.profile) : undefined - const maxTurns = opts.maxTurns ?? 8 let seq = 0 return { async create(options?: CreateSandboxOptions): Promise { - const profile = - (options?.backend as { profile?: AgentProfile } | undefined)?.profile ?? opts.profile ?? {} + const profile = agentProfileSchema.parse( + (options?.backend as { profile?: AgentProfile } | undefined)?.profile ?? opts.profile, + ) + assertExecutableAgentProfile(profile, 'localSandboxClient') + const profileModel = profileProviderModel(profile) + const model = profileModel! + const settings = profileModelExecutionSettings(profile, 'localSandboxClient') const policyApplies = opts.profileSecurityPolicy !== undefined && (!opts.profileSecurityPolicy.allowLocalMcp || (trustedProfileDigest !== undefined && - canonicalCandidateDigest(profile) === trustedProfileDigest)) + canonicalAgentProfileDigest(profile) === trustedProfileDigest)) // Materialize NOW: a declared server that cannot boot fails the create, // not the first prompt — matching the real sandbox backend, where a box // whose MCP cannot start never comes up. @@ -81,9 +87,17 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie { routerBaseUrl: opts.router.baseUrl, routerKey: opts.router.key, - model: opts.router.model, + model, + ...(settings.maxAttempts !== undefined ? { maxAttempts: settings.maxAttempts } : {}), + ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), + ...(settings.stream !== undefined ? { stream: settings.stream } : {}), + }, + { + ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), + ...(profile.model?.reasoningEffort + ? { reasoningEffort: profile.model.reasoningEffort } + : {}), }, - opts.temperature !== undefined ? { temperature: opts.temperature } : {}, ) const system = [profile.prompt?.systemPrompt, ...(profile.prompt?.instructions ?? [])] .filter((s): s is string => typeof s === 'string' && s.trim().length > 0) @@ -95,10 +109,14 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie message: string, popts?: { signal?: AbortSignal }, ): AsyncGenerator { - let costUsd = 0 + let estimatedCostUsd = 0 + let sawEstimatedCost = false const chat: ToolLoopChat = async (messages, tools) => { const r = await brain(messages, tools) - if (r.costUsd) costUsd += r.costUsd + if (r.costProvenance === 'catalog-estimate' && r.costUsd !== undefined) { + estimatedCostUsd += r.costUsd + sawEstimatedCost = true + } return r } const r = await runBrainLoop({ @@ -109,23 +127,33 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie ...(system ? [{ role: 'system', content: system }] : []), { role: 'user', content: message }, ], - maxTurns, + maxTurns: settings.maxTurns ?? 0, hooks: { stopBefore: () => popts?.signal?.aborted === true }, }) - // Speak the runtime's metering protocol (see inlineSandboxClient): - // a flat `llm_call` event so the kernel never meters a fabricated $0. - if (r.usage.input || r.usage.output || costUsd) { + // RouterClient prices locally. Emit measured tokens only when every turn reported them; + // never project the catalog estimate into SandboxEvent.costUsd. + if (r.turns > 0) { yield { type: 'llm_call', - data: { tokensIn: r.usage.input, tokensOut: r.usage.output, costUsd }, + data: { + model, + ...(r.tokensKnown === false + ? { tokensKnown: false } + : { tokensIn: r.usage.input, tokensOut: r.usage.output }), + costKnown: false, + ...(sawEstimatedCost ? { estimatedCostUsd } : {}), + }, } as unknown as SandboxEvent } yield { type: 'result', data: { finalText: r.final, - tokenUsage: { inputTokens: r.usage.input, outputTokens: r.usage.output }, - costUsd, + ...(r.tokensKnown === false + ? { tokensKnown: false } + : { tokenUsage: { inputTokens: r.usage.input, outputTokens: r.usage.output } }), + costKnown: false, + ...(sawEstimatedCost ? { estimatedCostUsd } : {}), }, } as unknown as SandboxEvent }, diff --git a/src/runtime/loop-dispatch.ts b/src/runtime/loop-dispatch.ts index e53c0b59..463c8399 100644 --- a/src/runtime/loop-dispatch.ts +++ b/src/runtime/loop-dispatch.ts @@ -169,7 +169,12 @@ function loopCostReceipt( model, inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output, - ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + ...(result.tokenUsage.tokensKnown === false ? { usageUnknown: true } : {}), + ...(result.costUsdKnown !== false && result.costUsd > 0 + ? { actualCostUsd: result.costUsd } + : {}), + ...(result.costUsdKnown === false ? { costUnknown: true } : {}), + ...(result.estimatedCostUsd !== undefined ? { estimatedCostUsd: result.estimatedCostUsd } : {}), } } diff --git a/src/runtime/observe.ts b/src/runtime/observe.ts index b781cf7a..f81d06a4 100644 --- a/src/runtime/observe.ts +++ b/src/runtime/observe.ts @@ -16,12 +16,14 @@ */ import { type AnalystFinding, - type ChatClient, makeProposalFinding, type ProposalFinding, } from '@tangle-network/agent-eval' import { assertProposalFindings } from '@tangle-network/agent-eval/analyst' +import type { AgentProfile } from '@tangle-network/agent-interface' import type { Corpus, CorpusRecord } from './personify/wave-types' +import { profileChatClient } from './profile-chat-client' +import type { ExecutorConfig } from './supervise/runtime' const observerId = 'observe/trace' @@ -40,9 +42,10 @@ export interface ObserveInput { } export interface ObserveOptions { - /** The model-call seam (agent-eval `createChatClient`: router / cli-bridge / …). */ - chat: ChatClient - model?: string + /** Exact analyst identity. */ + profile: AgentProfile + /** Execution substrate. All behavior comes from the profile. */ + executor: ExecutorConfig /** When set, learned facts are appended (idempotent) for the next run to read. */ corpus?: Corpus /** Tags written onto learned facts + used by the next run's corpus query. */ @@ -50,12 +53,6 @@ export interface ObserveOptions { signal?: AbortSignal /** Cap the trace lines fed to the observer (keeps the call cheap). Default 80. */ maxTraceLines?: number - /** Override the analyst's system instruction — the prompt that turns a trace into - * findings + recommended_actions. The analyst IS the steerer, so this is the knob a - * prompt optimizer (GEPA) tunes. Omitted ⇒ the default observer instruction. The - * firewall (trace-only, never the verdict) is structural (input has no score), so a - * custom instruction cannot break it. */ - analystInstruction?: string } /** The default observer instruction — exported so an optimizer can seed its population. */ @@ -72,6 +69,8 @@ export interface Observation { learned: CorpusRecord[] /** Operator-facing markdown: what the observer noticed + what to change. */ report: string + /** Measured model usage for this analysis turn. */ + usage: { input: number; output: number; known: boolean } } /** Compact the trace into the lines the observer reasons over — tool calls, @@ -144,15 +143,14 @@ const findingsSchema = { /** The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. */ export async function observe(input: ObserveInput, opts: ObserveOptions): Promise { const traceSummary = summarizeTrace(input.trace, opts.maxTraceLines ?? 80) - const res = await opts.chat.chat( + const res = await profileChatClient({ + profile: opts.profile, + executor: opts.executor, + context: 'observe analyst', + }).chat( { - ...(opts.model ? { model: opts.model } : {}), jsonSchema: findingsSchema as unknown as { name: string; schema: Record }, messages: [ - { - role: 'system', - content: opts.analystInstruction ?? defaultAnalystInstruction, - }, { role: 'user', content: @@ -207,7 +205,22 @@ export async function observe(input: ObserveInput, opts: ObserveOptions): Promis } } - return { findings: [...findings], learned, report: renderReport(findings) } + const usage = res.usage + const inputTokens = usage?.promptTokens + const outputTokens = usage?.completionTokens + return { + findings: [...findings], + learned, + report: renderReport(findings), + usage: { + input: inputTokens ?? 0, + output: outputTokens ?? 0, + known: + usage?.captured !== false && + typeof inputTokens === 'number' && + typeof outputTokens === 'number', + }, + } } interface RawFinding { diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts new file mode 100644 index 00000000..76c92c7a --- /dev/null +++ b/src/runtime/profile-chat-client.test.ts @@ -0,0 +1,241 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it, vi } from 'vitest' +import { + profileChatClient, + profileOptimizerModelCall, + terminalDurationMs, +} from './profile-chat-client' + +const profile: AgentProfile = { + name: 'exact-chat-client', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'deepseek-v4-flash', + }, + prompt: { systemPrompt: 'Answer exactly.' }, +} + +function clientWith( + complete: ( + body: Record, + request?: { headers: Readonly>; signal?: AbortSignal }, + ) => Promise, +) { + return profileChatClient({ + profile, + context: 'profile chat test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) +} + +const request = { + messages: [ + { role: 'system' as const, content: 'Answer exactly.' }, + { role: 'user' as const, content: 'hello' }, + ], +} + +describe('profileChatClient exact Runtime adapter', () => { + it('returns only provider-observed model and billed usage, including prompt-cache accounting', async () => { + const response = await clientWith(async () => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'hello back' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 7, + completion_tokens: 3, + cost: 0.002, + prompt_cache_hit_tokens: 5, + }, + })).chat(request) + + expect(response).toMatchObject({ + content: 'hello back', + model: 'deepseek-v4-flash', + costUsd: 0.002, + usage: { + promptTokens: 7, + completionTokens: 3, + totalTokens: 10, + cachedPromptTokens: 5, + }, + finishReason: 'stop', + contentEmpty: false, + raw: { promptCache: { readTokens: 5 }, transportAttempts: 1 }, + }) + expect(response.durationMs).toBeGreaterThanOrEqual(0) + }) + + it('refuses a provider-reported model different from the exact profile', async () => { + await expect( + clientWith(async () => ({ + model: 'some-other-model', + choices: [{ message: { content: 'wrong route' } }], + })).chat(request), + ).rejects.toThrow(/provider reported model "some-other-model".*requires "deepseek-v4-flash"/u) + }) + + it('refuses to claim the requested model when the provider did not report an actual model', async () => { + await expect( + clientWith(async () => ({ + choices: [{ message: { content: 'unidentified route' } }], + })).chat(request), + ).rejects.toThrow(/did not report the model actually used/u) + }) + + it('propagates stable paid-call and correlation ids to the Router request', async () => { + const complete = vi.fn( + async ( + _: Record, + transport?: { headers: Readonly> }, + ) => { + expect(transport?.headers).toMatchObject({ + 'idempotency-key': 'paid-call-1', + 'x-correlation-id': 'corr-1', + }) + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'identified' } }], + } + }, + ) + await clientWith(complete).chat(request, { + idempotencyKey: 'paid-call-1', + correlationId: 'corr-1', + }) + expect(complete).toHaveBeenCalledOnce() + }) + + it('keeps the profile policy first and preserves a task-specific system message after it', async () => { + const complete = vi.fn(async (body: Record) => { + expect(body.messages).toEqual([ + { role: 'system', content: 'Answer exactly.' }, + { role: 'system', content: 'Optimize this candidate using the examples.' }, + { role: 'user', content: 'candidate text' }, + ]) + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'improved candidate' } }], + } + }) + await clientWith(complete).chat({ + messages: [ + { role: 'system', content: 'Optimize this candidate using the examples.' }, + { role: 'user', content: 'candidate text' }, + ], + }) + expect(complete).toHaveBeenCalledOnce() + }) + + it('fails loud on unenforced maxCostUsd instead of silently treating it as a limit', async () => { + const complete = vi.fn(async () => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'must not run' } }], + })) + await expect(clientWith(complete).chat(request, { maxCostUsd: 1 })).rejects.toThrow(/refusing/u) + expect(complete).not.toHaveBeenCalled() + }) + + it('rejects caller reasoning that conflicts with the profile before transport', async () => { + const complete = vi.fn(async () => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'must not run' } }], + })) + const client = profileChatClient({ + profile: { + ...profile, + model: { ...profile.model, reasoningEffort: 'high' }, + }, + context: 'profile chat test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) + + await expect(client.chat({ ...request, thinking: 'disabled' })).rejects.toThrow( + /request thinking conflicts with AgentProfile\.model\.reasoningEffort/u, + ) + expect(complete).not.toHaveBeenCalled() + }) + + it('adapts Eval optimizer calls to the same exact Runtime path with measured evidence', async () => { + const complete = vi.fn( + async ( + _: Record, + transport?: { headers: Readonly> }, + ) => { + expect(transport?.headers).toMatchObject({ + 'idempotency-key': 'optimizer-call-1', + 'x-correlation-id': 'optimizer-call-1', + }) + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'optimizer response' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 3, completion_tokens: 2, cost: 0.001 }, + } + }, + ) + const call = profileOptimizerModelCall({ + profile: { + ...profile, + model: { ...profile.model, metadata: { maxTokens: 100 } }, + }, + context: 'profile optimizer test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) + + const result = await call({ + callId: 'optimizer-call-1', + request: { ...request, model: 'deepseek-v4-flash', maxTokens: 100 }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(true) + if (!result.succeeded) throw new Error(result.error) + expect(result.response).toMatchObject({ + content: 'optimizer response', + model: 'deepseek-v4-flash', + }) + expect(result.receipt).toMatchObject({ + model: 'deepseek-v4-flash', + inputTokens: 3, + outputTokens: 2, + actualCostUsd: 0.001, + }) + expect(result.execution).toMatchObject({ + kind: 'agent-runtime-profile-model-call', + callId: 'optimizer-call-1', + endpointFormat: 'chat-completions', + executed: true, + succeeded: true, + model: 'deepseek-v4-flash', + }) + expect(complete).toHaveBeenCalledOnce() + }) +}) + +describe('terminalDurationMs', () => { + it('uses the measured wall time when terminal transport timing is absent', () => { + expect(terminalDurationMs([{ type: 'final' }], 12.5)).toBe(12.5) + }) + + it('prefers valid terminal transport timing over the outer measurement', () => { + expect( + terminalDurationMs([{ type: 'final', metadata: { timing: { durationMs: 8 } } }], 12.5), + ).toBe(8) + }) +}) diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts new file mode 100644 index 00000000..3fa61212 --- /dev/null +++ b/src/runtime/profile-chat-client.ts @@ -0,0 +1,422 @@ +import type { + ChatClient, + ChatRequest, + ChatResponse, + CostReceiptInput, + CustomTokenPricing, +} from '@tangle-network/agent-eval' +import type { + ExternalOptimizerModelCall, + ExternalOptimizerModelCallRequest, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentProfile, + agentProfileSchema, + canonicalAgentProfileDigest, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' +import { type CollectedAgentTurn, collectAgentTurn, streamAgentTurn } from './stream-agent-turn' +import { + assertExecutableAgentProfile, + concreteModelId, + type ProfileModelExecutionSettings, + profileModelExecutionSettings, +} from './supervise/model-policy' +import { createExecutor, type ExecutorConfig } from './supervise/runtime' + +/** Profile-exact adapter for packages that consume agent-eval's ChatClient contract. + * Every call still enters Runtime through createExecutor -> streamAgentTurn, and every + * behavioral field is checked against the exact AgentProfile before any transport runs. */ +export function profileChatClient(args: { + profile: AgentProfile + executor: ExecutorConfig + context: string +}): ChatClient { + const binding = bindProfileChat(args) + + return { + transport: 'custom', + defaultModel: binding.model, + ...(binding.settings.maxAttempts !== undefined + ? { maximumAttempts: binding.settings.maxAttempts } + : {}), + async chat(req, callOpts) { + const run = await runBoundProfileChat(binding, req, callOpts) + if (!run.succeeded) throw new Error(run.error) + return run.response + }, + } +} + +/** Profile-exact adapter for agent-eval's external optimizer callback. + * Eval validates and freezes the provider-neutral request; Runtime owns the exact + * AgentProfile, execution route, retries, usage, and finite execution evidence. */ +export function profileOptimizerModelCall(args: { + profile: AgentProfile + executor: ExecutorConfig + context: string + pricing?: CustomTokenPricing +}): ExternalOptimizerModelCall { + const binding = bindProfileChat(args) + const profileDigest = canonicalAgentProfileDigest(binding.profile) + + return async (request) => { + const requestDigest = canonicalCandidateDigest({ + callId: request.callId, + request: request.request, + endpointFormat: request.endpointFormat ?? null, + }) + try { + const run = await runBoundProfileChat( + binding, + structuredClone(request.request) as ChatRequest, + { + signal: request.signal, + idempotencyKey: request.callId, + correlationId: request.callId, + }, + ) + const receipt = optimizerReceipt(binding.model, run, args.pricing) + const execution = optimizerExecution(profileDigest, requestDigest, request, run) + return run.succeeded + ? { succeeded: true, response: run.response, receipt, execution } + : { succeeded: false, error: run.error, receipt, execution } + } catch (error) { + return { + succeeded: false, + error: errorMessage(error), + receipt: unknownOptimizerReceipt(binding.model), + execution: { + kind: 'agent-runtime-profile-model-call', + profileDigest, + requestDigest, + callId: request.callId, + endpointFormat: request.endpointFormat ?? null, + executed: false, + error: errorMessage(error), + }, + } + } + } +} + +interface BoundProfileChat { + readonly profile: AgentProfile + readonly executor: ExecutorConfig + readonly context: string + readonly model: string + readonly settings: ProfileModelExecutionSettings + readonly systemPrompt: string +} + +export type ProfileChatRun = + | { readonly succeeded: true; readonly response: ChatResponse; readonly turn: CollectedAgentTurn } + | { readonly succeeded: false; readonly error: string; readonly turn: CollectedAgentTurn } + +export function bindProfileChat(args: { + profile: AgentProfile + executor: ExecutorConfig + context: string +}): BoundProfileChat { + const profile = agentProfileSchema.parse(args.profile) + assertExecutableAgentProfile(profile, args.context) + const model = concreteModelId(profile.model?.default) + if (!model) throw new Error(`${args.context}: AgentProfile.model.default must be concrete`) + return { + profile, + executor: args.executor, + context: args.context, + model, + settings: profileModelExecutionSettings(profile, args.context), + systemPrompt: profile.prompt?.systemPrompt ?? '', + } +} + +export async function runBoundProfileChat( + binding: BoundProfileChat, + req: ChatRequest, + callOpts?: Parameters[1], +): Promise { + assertProfileChatRequest( + req, + binding.model, + binding.profile.model?.reasoningEffort, + binding.settings, + binding.context, + ) + assertSupportedChatCallOptions(callOpts, binding.context) + // The profile prompt is the immutable leading policy. A caller's system-role messages are + // per-call task context: preserve them after that policy rather than letting them replace it or + // forcing dynamic task text into AgentProfile identity. + const profilePromptAlreadyLeading = + binding.systemPrompt.length > 0 && + req.messages[0]?.role === 'system' && + req.messages[0].content === binding.systemPrompt + const messages = [ + ...(binding.systemPrompt && !profilePromptAlreadyLeading + ? [{ role: 'system' as const, content: binding.systemPrompt }] + : []), + ...req.messages, + ] + const turnProfile = responseProfile(binding.profile, req, binding.context) + const startedAt = performance.now() + const turn = await collectAgentTurn( + streamAgentTurn( + { + kind: 'executor', + profile: turnProfile, + factory: createExecutor(binding.executor), + }, + { messages: messages as Array<{ role: string; content: unknown }> }, + { + ...(req.timeoutMs !== undefined ? { timeoutMs: req.timeoutMs } : {}), + ...(callOpts?.signal ? { signal: callOpts.signal } : {}), + ...(callOpts?.idempotencyKey ? { callId: callOpts.idempotencyKey } : {}), + ...(callOpts?.correlationId ? { correlationId: callOpts.correlationId } : {}), + }, + ), + ) + if (turn.status !== 'completed') { + return { + succeeded: false, + error: `${binding.context} failed: ${turn.error?.message ?? turn.status}`, + turn, + } + } + const observedModel = turn.usage.model + if (observedModel === undefined) { + return { + succeeded: false, + error: `${binding.context}: Runtime turn did not report the model actually used; refusing to label the response with the requested model`, + turn, + } + } + if (observedModel !== binding.model) { + return { + succeeded: false, + error: `${binding.context}: Runtime reported model ${JSON.stringify(observedModel)} but AgentProfile requires ${JSON.stringify(binding.model)}`, + turn, + } + } + const resultOut = turn.output as { finishReason?: string } | undefined + const promptTokens = turn.usage.input + const completionTokens = turn.usage.output + return { + succeeded: true, + turn, + response: { + content: turn.finalText, + usage: { + promptTokens, + completionTokens, + totalTokens: promptTokens + completionTokens, + ...(turn.usage.tokensKnown === false ? { captured: false } : {}), + ...(typeof turn.usage.promptCache?.readTokens === 'number' + ? { cachedPromptTokens: turn.usage.promptCache.readTokens } + : {}), + ...(turn.usage.reasoningTokens !== undefined + ? { reasoningTokens: turn.usage.reasoningTokens } + : {}), + }, + // ChatResponse.costUsd is provider-billed spend only. Runtime keeps catalog estimates in + // `raw.estimatedCostUsd` and marks the billed channel unknown when no receipt arrived. + costUsd: + turn.usage.usdKnown === false || turn.usage.costUsd === undefined + ? null + : turn.usage.costUsd, + model: observedModel, + durationMs: terminalDurationMs(turn.events, performance.now() - startedAt), + finishReason: resultOut?.finishReason ?? null, + contentEmpty: turn.finalText.trim().length === 0, + raw: { + ...(turn.usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: turn.usage.estimatedCostUsd } + : {}), + ...(turn.usage.promptCache ? { promptCache: turn.usage.promptCache } : {}), + ...(turn.transportAttempts !== undefined + ? { transportAttempts: turn.transportAttempts } + : {}), + }, + }, + } +} + +/** @internal Exported for the adapter's fail-honest duration contract test. */ +export function terminalDurationMs( + events: ReadonlyArray<{ type: string; metadata?: Record }>, + measuredWallMs: number, +): number { + const final = events.at(-1)?.metadata?.timing + if (final && typeof final === 'object') { + const duration = (final as Record).durationMs + if (typeof duration === 'number' && Number.isFinite(duration) && duration >= 0) return duration + } + if (Number.isFinite(measuredWallMs) && measuredWallMs >= 0) return measuredWallMs + throw new Error('profileChatClient: measured wall duration must be a finite non-negative number') +} + +function assertSupportedChatCallOptions( + opts: Parameters[1], + context: string, +): void { + if (opts?.maxCostUsd !== undefined) { + throw new Error( + `${context}: maxCostUsd is not enforced by Runtime's exact turn path; refusing to treat it as a limit`, + ) + } +} + +function responseProfile(profile: AgentProfile, req: ChatRequest, context: string): AgentProfile { + const responseFormat = req.jsonSchema + ? { type: 'json_schema', json_schema: req.jsonSchema } + : req.jsonMode + ? { type: 'json_object' } + : undefined + if (!responseFormat) return profile + const existing = profile.model?.metadata?.extraBody + if ( + existing !== undefined && + (typeof existing !== 'object' || existing === null || Array.isArray(existing)) + ) { + throw new Error(`${context}: AgentProfile.model.metadata.extraBody must be an object`) + } + const existingFormat = (existing as Record | undefined)?.response_format + if ( + existingFormat !== undefined && + JSON.stringify(existingFormat) !== JSON.stringify(responseFormat) + ) { + throw new Error(`${context}: requested response format conflicts with AgentProfile`) + } + return agentProfileSchema.parse({ + ...profile, + model: { + ...profile.model, + metadata: { + ...(profile.model?.metadata ?? {}), + extraBody: { + ...(existing as Record | undefined), + response_format: responseFormat, + }, + }, + }, + }) +} + +function assertProfileChatRequest( + req: ChatRequest, + model: string, + reasoningEffort: NonNullable['reasoningEffort'], + settings: ProfileModelExecutionSettings, + context: string, +): void { + if (req.model !== undefined && req.model !== model) { + throw new Error( + `${context}: request model ${JSON.stringify(req.model)} conflicts with AgentProfile model ${JSON.stringify(model)}`, + ) + } + if (req.temperature !== undefined && req.temperature !== settings.temperature) { + throw new Error(`${context}: request temperature conflicts with AgentProfile model metadata`) + } + if (req.maxTokens !== undefined && req.maxTokens !== settings.maxTokens) { + throw new Error(`${context}: request maxTokens conflicts with AgentProfile model metadata`) + } + if (req.thinking !== undefined) { + const expected = + reasoningEffort === undefined + ? undefined + : reasoningEffort === 'none' + ? 'disabled' + : 'enabled' + if (req.thinking !== expected) { + throw new Error( + `${context}: request thinking conflicts with AgentProfile.model.reasoningEffort`, + ) + } + } +} + +function optimizerReceipt( + model: string, + run: ProfileChatRun, + pricing: CustomTokenPricing | undefined, +): CostReceiptInput { + const usage = run.turn.usage + const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd + if (usage.tokensKnown === false) { + return { + model, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + ...(actualCostUsd !== undefined + ? { actualCostUsd } + : usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: usage.estimatedCostUsd } + : { costUnknown: true }), + } + } + const cachedTokens = optimizerTokenCount(usage.promptCache?.readTokens, 'cache read tokens') + const cacheWriteTokens = optimizerTokenCount(usage.promptCache?.writeTokens, 'cache write tokens') + const classified = (cachedTokens ?? 0) + (cacheWriteTokens ?? 0) + if (classified > usage.input) { + throw new Error('profile optimizer cache classes exceed total input tokens') + } + return { + model, + inputTokens: usage.input - classified, + outputTokens: usage.output, + ...(cachedTokens !== undefined ? { cachedTokens } : {}), + ...(cacheWriteTokens !== undefined ? { cacheWriteTokens } : {}), + ...(usage.reasoningTokens !== undefined ? { reasoningTokens: usage.reasoningTokens } : {}), + ...(actualCostUsd !== undefined + ? { actualCostUsd } + : usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: usage.estimatedCostUsd } + : pricing + ? { customTokenPricing: pricing } + : { costUnknown: true }), + } +} + +function optimizerExecution( + profileDigest: string, + requestDigest: string, + request: ExternalOptimizerModelCallRequest, + run: ProfileChatRun, +): Record { + return { + kind: 'agent-runtime-profile-model-call', + profileDigest, + requestDigest, + callId: request.callId, + endpointFormat: request.endpointFormat ?? null, + executed: true, + succeeded: run.succeeded, + status: run.turn.status, + model: run.turn.usage.model ?? null, + transportAttempts: run.turn.transportAttempts ?? null, + eventTypes: run.turn.events.map((event) => event.type), + } +} + +function unknownOptimizerReceipt(model: string): CostReceiptInput { + return { + model, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + costUnknown: true, + } +} + +function optimizerTokenCount(value: unknown, label: string): number | undefined { + if (value === undefined) return undefined + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`profile optimizer ${label} must be a non-negative integer`) + } + return value as number +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/runtime/resolve-sandbox-client.test.ts b/src/runtime/resolve-sandbox-client.test.ts index ea730510..d08b55f1 100644 --- a/src/runtime/resolve-sandbox-client.test.ts +++ b/src/runtime/resolve-sandbox-client.test.ts @@ -40,13 +40,12 @@ describe('resolveSandboxClient', () => { it("backend 'bridge' wires createExecutor with the bridge seam and default url", () => { const client = resolveSandboxClient({ backend: 'bridge', - bridge: { bearer: 'sk-b', model: 'opencode/kimi', timeoutMs: 90_000 }, + bridge: { bearer: 'sk-b', timeoutMs: 90_000 }, }) expect(configOf(client)).toEqual({ backend: 'bridge', bridgeUrl: 'http://127.0.0.1:3355', bridgeBearer: 'sk-b', - model: 'opencode/kimi', timeoutMs: 90_000, }) expect(inlineSandboxClient).toHaveBeenCalledOnce() @@ -55,46 +54,43 @@ describe('resolveSandboxClient', () => { it("backend 'bridge' honors an explicit bridge.url", () => { const client = resolveSandboxClient({ backend: 'bridge', - bridge: { url: 'http://bridge.local:9000', bearer: 'sk-b', model: 'claude-code/sonnet' }, + bridge: { url: 'http://bridge.local:9000', bearer: 'sk-b' }, }) expect((configOf(client) as { bridgeUrl: string }).bridgeUrl).toBe('http://bridge.local:9000') }) - it("backend 'bridge' fails loud without bearer or model", () => { - expect(() => - resolveSandboxClient({ backend: 'bridge', bridge: { bearer: '', model: 'x' } }), - ).toThrow(/bridge\.bearer and bridge\.model/) - expect(() => resolveSandboxClient({ backend: 'bridge' })).toThrow( - /bridge\.bearer and bridge\.model/, + it("backend 'bridge' fails loud without bearer", () => { + expect(() => resolveSandboxClient({ backend: 'bridge', bridge: { bearer: '' } })).toThrow( + /bridge\.bearer/, ) + expect(() => resolveSandboxClient({ backend: 'bridge' })).toThrow(/bridge\.bearer/) }) it("backend 'router' wires createExecutor with the router seam", () => { const client = resolveSandboxClient({ backend: 'router', - router: { baseUrl: 'https://router.tangle.tools', key: 'sk-r', model: 'kimi' }, + router: { baseUrl: 'https://router.tangle.tools', key: 'sk-r' }, }) expect(configOf(client)).toEqual({ backend: 'router', routerBaseUrl: 'https://router.tangle.tools', routerKey: 'sk-r', - model: 'kimi', }) }) it("backend 'router' fails loud when a required field is missing", () => { expect(() => - resolveSandboxClient({ backend: 'router', router: { baseUrl: '', key: 'k', model: 'm' } }), - ).toThrow(/router\.baseUrl, router\.key and router\.model/) + resolveSandboxClient({ backend: 'router', router: { baseUrl: '', key: 'k' } }), + ).toThrow(/router\.baseUrl and router\.key/) expect(() => resolveSandboxClient({ backend: 'router' })).toThrow( - /router\.baseUrl, router\.key and router\.model/, + /router\.baseUrl and router\.key/, ) }) it("backend 'local' wires the same-host client with the local options", () => { const keys = { get: vi.fn(async () => 'secret') } const local = { - router: { baseUrl: 'https://router.tangle.tools', key: 'sk-l', model: 'gem' }, + router: { baseUrl: 'https://router.tangle.tools', key: 'sk-l' }, keys, } const client = resolveSandboxClient({ backend: 'local', local }) @@ -104,12 +100,12 @@ describe('resolveSandboxClient', () => { it("backend 'local' fails loud when a router field is missing", () => { expect(() => resolveSandboxClient({ backend: 'local' })).toThrow( - /local\.router\.baseUrl, local\.router\.key and local\.router\.model/, + /local\.router\.baseUrl and local\.router\.key/, ) expect(() => resolveSandboxClient({ backend: 'local', - local: { router: { baseUrl: 'x', key: '', model: 'm' } }, + local: { router: { baseUrl: 'x', key: '' } }, }), ).toThrow(/local\.router/) }) diff --git a/src/runtime/resolve-sandbox-client.ts b/src/runtime/resolve-sandbox-client.ts index 5d486af8..e2560a02 100644 --- a/src/runtime/resolve-sandbox-client.ts +++ b/src/runtime/resolve-sandbox-client.ts @@ -33,21 +33,18 @@ export interface ResolveSandboxClientOptions { backend: 'sandbox' | 'bridge' | 'router' | 'local' /** `sandbox` backend: the caller's real Sandbox-backed client. Required for that backend. */ sandboxClient?: SandboxClient - /** `bridge` backend: local cli-bridge transport. `bearer` + `model` required. */ + /** `bridge` backend: local cli-bridge transport. The per-create profile owns the model. */ bridge?: { /** cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */ url?: string bearer: string - /** Bridge model id, doubling as the harness selector (e.g. `claude-code/sonnet`). */ - model: string /** Per-turn deadline (ms). */ timeoutMs?: number } - /** `router` backend: router chat-completion transport. All three fields required. */ + /** `router` backend: endpoint/auth only; the per-create profile owns behavior. */ router?: { baseUrl: string key: string - model: string } /** `local` backend: same-host pseudo-box — the router brain drives a tool loop * with the profile's stdio MCP servers spawned as local children. */ @@ -69,26 +66,23 @@ export function resolveSandboxClient(opts: ResolveSandboxClientOptions): Sandbox } case 'bridge': { const bridge = opts.bridge - if (!bridge?.bearer || !bridge.model) { - throw new Error( - "resolveSandboxClient: backend 'bridge' requires bridge.bearer and bridge.model", - ) + if (!bridge?.bearer) { + throw new Error("resolveSandboxClient: backend 'bridge' requires bridge.bearer") } return inlineSandboxClient( createExecutor({ backend: 'bridge', bridgeUrl: bridge.url ?? 'http://127.0.0.1:3355', bridgeBearer: bridge.bearer, - model: bridge.model, timeoutMs: bridge.timeoutMs, }), ) } case 'router': { const router = opts.router - if (!router?.baseUrl || !router.key || !router.model) { + if (!router?.baseUrl || !router.key) { throw new Error( - "resolveSandboxClient: backend 'router' requires router.baseUrl, router.key and router.model", + "resolveSandboxClient: backend 'router' requires router.baseUrl and router.key", ) } return inlineSandboxClient( @@ -96,15 +90,14 @@ export function resolveSandboxClient(opts: ResolveSandboxClientOptions): Sandbox backend: 'router', routerBaseUrl: router.baseUrl, routerKey: router.key, - model: router.model, }), ) } case 'local': { const local = opts.local - if (!local?.router?.baseUrl || !local.router.key || !local.router.model) { + if (!local?.router?.baseUrl || !local.router.key) { throw new Error( - "resolveSandboxClient: backend 'local' requires local.router.baseUrl, local.router.key and local.router.model", + "resolveSandboxClient: backend 'local' requires local.router.baseUrl and local.router.key", ) } return localSandboxClient(local) diff --git a/src/runtime/router-client.complete.test.ts b/src/runtime/router-client.complete.test.ts index 15d1c329..458ed8be 100644 --- a/src/runtime/router-client.complete.test.ts +++ b/src/runtime/router-client.complete.test.ts @@ -15,6 +15,7 @@ describe('RouterConfig.complete — the injected completion transport', () => { // The body is the OpenAI request the client built — assert it threaded the model + messages. expect(body.model).toBe('deepseek-v4-flash') expect(body.messages).toEqual([{ role: 'user', content: 'hi' }]) + expect(body).not.toHaveProperty('max_tokens') return { choices: [{ message: { content: 'pong' } }], usage: { prompt_tokens: 7, completion_tokens: 3 }, @@ -31,6 +32,7 @@ describe('RouterConfig.complete — the injected completion transport', () => { ) expect(res.content).toBe('pong') expect(res.usage).toEqual({ input: 7, output: 3 }) + expect(res.transportAttempts).toBe(1) expect(complete).toHaveBeenCalledOnce() expect(fetchSpy).not.toHaveBeenCalled() }) @@ -64,6 +66,7 @@ describe('RouterConfig.complete — the injected completion transport', () => { ) expect(res.toolCalls).toEqual([{ id: 'c1', name: 'increment', arguments: '{}' }]) expect(res.usage).toEqual({ input: 5, output: 2 }) + expect(res.transportAttempts).toBe(1) expect(complete).toHaveBeenCalledOnce() expect(fetchSpy).not.toHaveBeenCalled() }) @@ -84,8 +87,170 @@ describe('RouterConfig.complete — the injected completion transport', () => { [{ role: 'user', content: 'hi' }], ) expect(res.content).toBe('live') + expect(res.transportAttempts).toBe(1) expect(fetchSpy).toHaveBeenCalledOnce() }) + + it('reports the exact transport count and honors a caller-owned retry total', async () => { + let calls = 0 + const fetchSpy = vi.fn(async () => { + calls += 1 + if (calls === 1) { + return { + ok: false, + status: 503, + text: async (): Promise => 'capacity', + } + } + return { + ok: true, + status: 200, + json: async () => ({ + choices: [{ message: { content: 'recovered' } }], + usage: { prompt_tokens: 2, completion_tokens: 1 }, + }), + text: async (): Promise => '', + } + }) + vi.stubGlobal('fetch', fetchSpy) + + const result = await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + maxAttempts: 2, + }, + [{ role: 'user', content: 'recover' }], + ) + + expect(fetchSpy).toHaveBeenCalledTimes(2) + expect(result.transportAttempts).toBe(2) + expect(result.content).toBe('recovered') + }) + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects invalid maxAttempts %s before dispatch', + async (maxAttempts) => { + const fetchSpy = vi.fn() + vi.stubGlobal('fetch', fetchSpy) + await expect( + routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + maxAttempts, + }, + [{ role: 'user', content: 'do not dispatch' }], + ), + ).rejects.toThrow(/maxAttempts must be a positive safe integer/u) + expect(fetchSpy).not.toHaveBeenCalled() + }, + ) + + it('uses the caller ceiling when set and otherwise leaves the provider default unrestricted', async () => { + const seen: Record[] = [] + const complete = async (body: Record) => { + seen.push(body) + return { choices: [{ message: { content: 'done' } }] } + } + const config = { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + } + await routerChatWithUsage(config, [{ role: 'user', content: 'default' }]) + await routerChatWithUsage({ ...config, maxTokens: 16_384 }, [ + { role: 'user', content: 'configured' }, + ]) + await routerChatWithUsage(config, [{ role: 'user', content: 'per-call' }], { + maxTokens: 32_768, + }) + + expect(seen[0]).not.toHaveProperty('max_tokens') + expect(seen[1]?.max_tokens).toBe(16_384) + expect(seen[2]?.max_tokens).toBe(32_768) + }) + + it('accepts multimodal messages and provider fields without letting extras replace canonical fields', async () => { + const content = [ + { type: 'text', text: 'describe this image' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } }, + ] + const complete = vi.fn(async (body: Record) => { + expect(body).toMatchObject({ + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content }], + temperature: 0.4, + max_tokens: 32_768, + seed: 7, + thinking: { type: 'enabled' }, + }) + return { choices: [{ message: { content: 'image' } }] } + }) + + await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + }, + [{ role: 'user', content }], + { + temperature: 0.4, + maxTokens: 32_768, + seed: 7, + extraBody: { + model: 'must-not-win', + messages: [], + temperature: 999, + max_tokens: 1, + thinking: { type: 'enabled' }, + }, + }, + ) + expect(complete).toHaveBeenCalledOnce() + }) + + it('omits tool fields for a no-tool request and protects canonical fields from provider extras', async () => { + const complete = vi.fn(async (body: Record) => { + expect(body).toMatchObject({ + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'answer' }], + temperature: 0.1, + thinking: { type: 'enabled' }, + }) + expect(body).not.toHaveProperty('tools') + expect(body).not.toHaveProperty('tool_choice') + return { choices: [{ message: { content: 'done' } }] } + }) + + const result = await routerChatWithTools( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + }, + [{ role: 'user', content: 'answer' }], + [], + { + temperature: 0.1, + extraBody: { + model: 'must-not-win', + messages: [], + tools: [{ type: 'function' }], + tool_choice: 'required', + thinking: { type: 'enabled' }, + }, + }, + ) + expect(result.content).toBe('done') + expect(complete).toHaveBeenCalledOnce() + }) }) describe('reasoning-aware parsing and reasoning_effort forwarding', () => { diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 47257dd9..e778b8bf 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -11,23 +11,46 @@ */ import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import type { ReasoningEffort } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' import { runBrainLoop, type ToolLoopChat } from './tool-loop' -export interface RouterConfig { +/** + * Connection details for Runtime's Router-backed executors. + * + * This is deliberately transport-only: model, prompt, tools, generation settings, and retry + * policy belong to the exact executable `AgentProfile` consumed by `streamAgentTurn`. + */ +export interface RouterTransportConfig { routerBaseUrl: string routerKey: string + /** Injectable OpenAI-compatible transport for offline execution. */ + complete?: ( + body: Record, + request?: { + readonly headers: Readonly> + readonly signal?: AbortSignal + }, + ) => Promise +} + +/** + * Private request configuration used by Runtime's Router adapter. + * + * Do not export this through a package entry point. Public callers execute a concrete + * `AgentProfile` through `createExecutor` + `streamAgentTurn`; only Runtime may lower that profile + * into these provider request fields. + */ +export interface RouterConfig extends RouterTransportConfig { model: string /** - * Optional completion transport. When set, `routerChatWithUsage` / `routerChatWithTools` call it - * with the OpenAI-shape request body and use the parsed `/chat/completions` JSON it returns, - * INSTEAD of `fetch(routerBaseUrl + '/chat/completions')`. When absent the fetch path runs - * unchanged — the live router stays the default. The injection seam an offline benchmark uses to - * drive the worker with no network: a deterministic in-process responder satisfies it, no server. + * Total HTTP attempts for one completion, including the first request. + * Defaults to 5. Set 1 when an outer workflow owns retries so the two + * policies cannot multiply invisibly. Any positive safe integer is allowed. */ - complete?: (body: Record) => Promise + maxAttempts?: number /** - * Ceiling for one completion, forwarded as `max_tokens`. Defaults to 8192. + * Optional ceiling for one completion, forwarded as `max_tokens`. * * A REASONING model spends this budget on hidden thinking BEFORE it emits a visible token, so * the default can truncate one mid-thought and return no content at all — observed live with a @@ -76,19 +99,38 @@ export interface RouterChatResult { */ reasoning?: string /** REAL usage, or undefined when the provider reported none. */ - usage?: { input: number; output: number } - /** Derived from usage via `estimateCost` when the model is priced; else undefined. */ + usage?: { input: number; output: number; reasoning?: number } + /** Local catalog estimate derived from usage; never a provider billing receipt. */ costUsd?: number + /** Present with `costUsd` so consumers cannot mistake a catalog estimate for billed spend. */ + costProvenance?: 'catalog-estimate' + /** Provider-reported billed cost. Absent means dollar spend is unknown. */ + billedCostUsd?: number + /** Provider-reported prompt-cache fields; missing fields remain missing. */ + cache?: PromptCacheUsage + /** Provider terminal reason (`stop`, `length`, ...), when reported. */ + finishReason?: string + /** Exact HTTP/injected-transport calls consumed by this completion. */ + transportAttempts: number + /** Model identity reported by the provider response. Absent when the provider omitted it. */ + model?: string } /** One OpenAI-compatible chat completion through the Tangle router, returning text + REAL token usage (`undefined` when the provider omits it — never a fabricated 0). */ export async function routerChatWithUsage( cfg: RouterConfig, - messages: Array<{ role: string; content: string }>, + messages: ReadonlyArray<{ role: string; content: unknown }>, opts?: { temperature?: number signal?: AbortSignal maxTokens?: number + /** OpenAI-compatible deterministic seed. Omit when the provider does not support it. */ + seed?: number + /** + * Provider-specific request fields such as Z.AI's `thinking` object. + * Canonical fields are written after these extras and cannot be overridden here. + */ + extraBody?: Readonly> /** * Reasoning control for thinking models, forwarded as `reasoning_effort`. * 'none' is the load-bearing value: binary/single-token decisions (routing, @@ -97,80 +139,108 @@ export async function routerChatWithUsage( * timeout, not just waste. Providers that ignore the field are handled by * the reasoning/content split in `parseChatResult`. */ - reasoningEffort?: 'none' | 'low' | 'medium' | 'high' + reasoningEffort?: ReasoningEffort + /** Stable logical-call id reused across transport retries. */ + callId?: string + /** Caller trace correlation forwarded independently of idempotency. */ + correlationId?: string }, ): Promise { const url = `${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions` - const headers = { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` } - let temperature = opts?.temperature ?? 0.2 - // max_tokens default is generous: THINKING models (kimi-k2.6) spend the budget on - // reasoning_content first — a small router default yields EMPTY content. + const headers = routerRequestHeaders(cfg, opts) + const temperature = opts?.temperature + const maxTokens = opts?.maxTokens ?? cfg.maxTokens const body = (): Record => ({ + ...providerRequestExtras(opts?.extraBody, [ + 'model', + 'messages', + 'temperature', + 'max_tokens', + 'seed', + 'reasoning_effort', + 'stream', + 'stream_options', + ]), model: cfg.model, messages, - temperature, - max_tokens: opts?.maxTokens ?? 8192, + ...(temperature !== undefined ? { temperature } : {}), + ...(maxTokens !== undefined ? { max_tokens: maxTokens } : {}), + ...(opts?.seed !== undefined ? { seed: opts.seed } : {}), ...(opts?.reasoningEffort ? { reasoning_effort: opts.reasoningEffort } : {}), }) + const maxAttempts = routerMaxAttempts(cfg) // Injected transport short-circuits the network: the offline benchmark seam. It owns its own // determinism, so the fetch-specific transient-retry/temperature-handling below does not apply. - if (cfg.complete) return parseChatResult(await cfg.complete(body()), cfg.model) - // Retry TRANSIENT upstream failures (429/5xx) with backoff so a single capacity - // hiccup doesn't kill a whole multi-model benchmark run; and auto-handle the - // "only temperature 1 is allowed" 400 some thinking models (e.g. kimi-k2.6) return. + if (cfg.complete) { + return parseChatResult( + await cfg.complete(body(), { + headers, + ...(opts?.signal ? { signal: opts.signal } : {}), + }), + cfg.model, + 1, + ) + } + // Retry transient upstream failures (429/5xx) with backoff so a single capacity + // hiccup does not kill a whole multi-model benchmark run. Provider requests to + // change temperature fail: generation behavior belongs to the exact profile. let lastErr = '' - for (let attempt = 0; attempt < 5; attempt += 1) { + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const res = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body()), ...(opts?.signal ? { signal: opts.signal } : {}), }) - if (res.ok) return parseChatResult(await res.json(), cfg.model) + if (res.ok) return parseChatResult(await res.json(), cfg.model, attempt) const status = res.status const text = (await res.text()).slice(0, 200) lastErr = `router ${status}: ${text}` - if (status === 400 && /temperature/i.test(text) && temperature !== 1) { - temperature = 1 // model requires temperature 1 — retry once with it - continue - } // Non-retryable (auth/quota/malformed) fails loud immediately; retryable // statuses back off and continue until the loop's attempt bound, then the // post-loop throw is the honest "exhausted retries" terminal. 408/425 + the // Cloudflare-origin family (520/522/524) are transient under heavy parallel // load — a fleet of concurrent gate runs hits 524 ("origin timeout") and must // retry, not crash the whole run. - if (![408, 425, 429, 500, 502, 503, 504, 520, 522, 524].includes(status)) - throw new Error(lastErr) - if (attempt < 4) await new Promise((r) => setTimeout(r, 800 * 2 ** attempt)) + if (!isTransientRouterStatus(status)) throw new Error(lastErr) + if (attempt < maxAttempts) await new Promise((r) => setTimeout(r, 800 * 2 ** (attempt - 1))) } throw new Error(`${lastErr} (exhausted retries)`) } -function parseChatResult(json: unknown, model: string): RouterChatResult { +function parseChatResult( + json: unknown, + model: string, + transportAttempts: number, +): RouterChatResult { const data = json as { + model?: unknown choices?: Array<{ message?: { content?: string; reasoning?: string; reasoning_content?: string } + finish_reason?: string }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + usage?: RawUsage } - const u = data.usage - const usage = - u && typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number' - ? { input: u.prompt_tokens, output: u.completion_tokens } - : undefined - const costUsd = - usage && isModelPriced(model) ? estimateCost(usage.input, usage.output, model) : undefined + const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(data.usage, model) const msg = data.choices?.[0]?.message + if (!msg) throw new ValidationError('router completion: no choices[0].message') const { content, reasoning } = splitReasoning( msg?.content ?? '', msg?.reasoning ?? msg?.reasoning_content, ) return { content, + transportAttempts, + ...(reportedModel(data.model) ? { model: reportedModel(data.model) } : {}), ...(reasoning ? { reasoning } : {}), ...(usage ? { usage } : {}), ...(costUsd !== undefined ? { costUsd } : {}), + ...(costProvenance ? { costProvenance } : {}), + ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), + ...(cache ? { cache } : {}), + ...(typeof data.choices?.[0]?.finish_reason === 'string' + ? { finishReason: data.choices[0].finish_reason } + : {}), } } @@ -210,8 +280,14 @@ export interface RouterToolCall { export interface RouterChatToolsResult { content: string | null toolCalls: RouterToolCall[] - usage?: { input: number; output: number } + usage?: { input: number; output: number; reasoning?: number } costUsd?: number + /** Present with `costUsd` so consumers cannot mistake a catalog estimate for billed spend. */ + costProvenance?: 'catalog-estimate' + /** Provider-reported billed cost. Absent means dollar spend is unknown. */ + billedCostUsd?: number + /** Provider-reported prompt-cache fields; missing fields remain missing. */ + cache?: PromptCacheUsage /** * Thinking-model reasoning, normalized the way `RouterChatResult.reasoning` is (a separate * `reasoning_content`/`reasoning` field, or an inline `` block split out of `content`). @@ -235,6 +311,10 @@ export interface RouterChatToolsResult { * rather than skipping the turn and letting a conserved budget pool believe it cost nothing. */ usageUnknown?: true + /** Exact HTTP/injected-transport calls consumed by this completion. */ + transportAttempts: number + /** Model identity reported by the provider response. Absent when the provider omitted it. */ + model?: string } /** @@ -255,43 +335,77 @@ export async function routerChatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + /** OpenAI-compatible deterministic seed. Omit when the provider does not support it. */ + seed?: number + /** Provider-specific request fields; canonical fields cannot be overridden here. */ + extraBody?: Readonly> + reasoningEffort?: ReasoningEffort + callId?: string + correlationId?: string }, ): Promise { const body = toolCompletionBody(cfg, messages, tools, opts) + const maxAttempts = routerMaxAttempts(cfg) // Injected transport short-circuits the network — the offline benchmark seam (see RouterConfig.complete). + let transportAttempts = 1 + const headers = routerRequestHeaders(cfg, opts) const raw = cfg.complete - ? await cfg.complete(structuredClone(body)) + ? await cfg.complete(structuredClone(body), { + headers, + ...(opts?.signal ? { signal: opts.signal } : {}), + }) : await (async () => { - const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` }, - body: JSON.stringify(body), - ...(opts?.signal ? { signal: opts.signal } : {}), - }) - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`) - return res.json() + let lastErr = '' + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + transportAttempts = attempt + const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers, + body: JSON.stringify(body), + ...(opts?.signal ? { signal: opts.signal } : {}), + }) + if (res.ok) return res.json() + const status = res.status + lastErr = `router ${status}: ${(await res.text()).slice(0, 200)}` + if (!isTransientRouterStatus(status)) throw new Error(lastErr) + if (attempt < maxAttempts) { + await new Promise((resolve) => setTimeout(resolve, 800 * 2 ** (attempt - 1))) + } + } + throw new Error(`${lastErr} (exhausted retries)`) })() const data = raw as { + model?: unknown choices?: Array<{ message?: { content?: string | null tool_calls?: Array<{ id?: string; function?: { name?: string; arguments?: string } }> } + finish_reason?: string }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + usage?: RawUsage } const msg = data.choices?.[0]?.message + if (!msg) throw new ValidationError('router completion: no choices[0].message') const toolCalls: RouterToolCall[] = (msg?.tool_calls ?? []).map((tc, i) => ({ id: tc.id ?? `call_${i}`, name: tc.function?.name ?? '', arguments: tc.function?.arguments ?? '{}', })) - const { usage, costUsd } = meterTurn(data.usage, cfg.model) + const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(data.usage, cfg.model) return { content: msg?.content ?? null, toolCalls, + transportAttempts, + ...(reportedModel(data.model) ? { model: reportedModel(data.model) } : {}), ...(usage ? { usage } : {}), ...(costUsd !== undefined ? { costUsd } : {}), + ...(costProvenance ? { costProvenance } : {}), + ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), + ...(cache ? { cache } : {}), + ...(typeof data.choices?.[0]?.finish_reason === 'string' + ? { finishReason: data.choices[0].finish_reason } + : {}), } } @@ -301,18 +415,48 @@ function toolCompletionBody( cfg: RouterConfig, messages: ReadonlyArray>, tools: ReadonlyArray, - opts?: { temperature?: number; toolChoice?: 'auto' | 'required' | 'none'; maxTokens?: number }, + opts?: { + temperature?: number + toolChoice?: 'auto' | 'required' | 'none' + maxTokens?: number + seed?: number + extraBody?: Readonly> + reasoningEffort?: ReasoningEffort + }, ): Record { return { + ...providerRequestExtras(opts?.extraBody, [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'max_tokens', + 'seed', + 'reasoning_effort', + 'stream', + 'stream_options', + ]), model: cfg.model, messages, - tools, - tool_choice: opts?.toolChoice ?? 'auto', - temperature: opts?.temperature ?? 0.3, + ...(tools.length > 0 ? { tools, tool_choice: opts?.toolChoice ?? 'auto' } : {}), + ...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}), ...(opts?.maxTokens ? { max_tokens: opts.maxTokens } : {}), + ...(opts?.seed !== undefined ? { seed: opts.seed } : {}), + ...(opts?.reasoningEffort ? { reasoning_effort: opts.reasoningEffort } : {}), } } +function providerRequestExtras( + extraBody: Readonly> | undefined, + reservedFields: ReadonlyArray, +): Record { + if (!extraBody) return {} + const extras = { ...extraBody } + for (const field of reservedFields) delete extras[field] + return extras +} + /** * REAL usage → the metered pair, or `undefined` when the provider reported none. Never a * fabricated 0: a phantom 0 reads as a free call to the conserved budget pool, which would then @@ -321,16 +465,33 @@ function toolCompletionBody( function meterTurn( raw: RawUsage | undefined, model: string, -): { usage?: { input: number; output: number }; costUsd?: number; cache?: PromptCacheUsage } { +): { + usage?: { input: number; output: number; reasoning?: number } + costUsd?: number + costProvenance?: 'catalog-estimate' + billedCostUsd?: number + cache?: PromptCacheUsage +} { + const reasoning = providerReasoningTokens(raw) const usage = raw && typeof raw.prompt_tokens === 'number' && typeof raw.completion_tokens === 'number' - ? { input: raw.prompt_tokens, output: raw.completion_tokens } + ? { + input: raw.prompt_tokens, + output: raw.completion_tokens, + ...(reasoning !== undefined ? { reasoning } : {}), + } : undefined - if (!usage) return {} + const cache = readPromptCache(raw) + const billedCostUsd = providerBilledCost(raw) + if (!usage) { + return { + ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), + ...(cache ? { cache } : {}), + } + } const localEstimate = isModelPriced(model) ? estimateCost(usage.input, usage.output, model) : undefined - const cache = readPromptCache(raw?.prompt_cache) // A cached prefix token is billed at a discount the local price table does not know about, so // subtract the provider's OWN reported saving rather than re-deriving a discount here. Without // this a long supervisor run — which re-sends a growing transcript every turn — is reported at @@ -341,18 +502,37 @@ function meterTurn( : localEstimate return { usage, - ...(costUsd !== undefined ? { costUsd } : {}), + ...(costUsd !== undefined ? { costUsd, costProvenance: 'catalog-estimate' as const } : {}), + ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), ...(cache ? { cache } : {}), } } +function providerReasoningTokens(raw: RawUsage | undefined): number | undefined { + if (!raw) return undefined + for (const value of [raw.completion_tokens_details?.reasoning_tokens, raw.reasoning_tokens]) { + if (Number.isSafeInteger(value) && (value as number) >= 0) return value as number + } + return undefined +} + +function providerBilledCost(raw: RawUsage | undefined): number | undefined { + if (!raw) return undefined + for (const value of [raw.cost, raw.cost_usd]) { + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) return value + } + return undefined +} + /** What the router reports about prefix caching for one completion. Absent when the provider * said nothing — an unreported cache is not a miss, and a miss is not a zero saving. */ export interface PromptCacheUsage { /** Prompt tokens served from a cached prefix. */ - readonly readTokens: number + readonly readTokens?: number /** Prompt tokens written INTO the cache by this call. */ - readonly writeTokens: number + readonly writeTokens?: number + /** Prompt tokens that missed a provider cache. */ + readonly missTokens?: number /** Dollars the provider says the cache read saved on this call. */ readonly readSavingsUsd?: number /** The provider's own word for what happened: `hit`, `miss`, `read`, … Kept verbatim rather @@ -363,22 +543,49 @@ export interface PromptCacheUsage { interface RawUsage { prompt_tokens?: number completion_tokens?: number + cost?: number + cost_usd?: number + prompt_cache_hit_tokens?: number + prompt_cache_miss_tokens?: number + prompt_tokens_details?: { cached_tokens?: number } + completion_tokens_details?: { reasoning_tokens?: number } + reasoning_tokens?: number prompt_cache?: unknown } -function readPromptCache(raw: unknown): PromptCacheUsage | undefined { +export function readPromptCache(raw: unknown): PromptCacheUsage | undefined { if (!raw || typeof raw !== 'object') return undefined const o = raw as Record const num = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined) - const readTokens = num(o.read_tokens) - const writeTokens = num(o.write_tokens) - if (readTokens === undefined && writeTokens === undefined) return undefined - const savings = num(o.read_savings_usd) + const nested = + o.prompt_cache && typeof o.prompt_cache === 'object' + ? (o.prompt_cache as Record) + : o + const details = + o.prompt_tokens_details && typeof o.prompt_tokens_details === 'object' + ? (o.prompt_tokens_details as Record) + : undefined + const readTokens = + num(nested.read_tokens) ?? num(o.prompt_cache_hit_tokens) ?? num(details?.cached_tokens) + const writeTokens = num(nested.write_tokens) + const missTokens = num(o.prompt_cache_miss_tokens) + const savings = num(nested.read_savings_usd) + const status = typeof nested.status === 'string' ? nested.status : undefined + if ( + readTokens === undefined && + writeTokens === undefined && + missTokens === undefined && + savings === undefined && + status === undefined + ) { + return undefined + } return { - readTokens: readTokens ?? 0, - writeTokens: writeTokens ?? 0, + ...(readTokens !== undefined ? { readTokens } : {}), + ...(writeTokens !== undefined ? { writeTokens } : {}), + ...(missTokens !== undefined ? { missTokens } : {}), ...(savings !== undefined ? { readSavingsUsd: savings } : {}), - ...(typeof o.status === 'string' ? { status: o.status } : {}), + ...(status !== undefined ? { status } : {}), } } @@ -389,6 +596,7 @@ function readPromptCache(raw: unknown): PromptCacheUsage | undefined { * `delta.reasoning_content` token-by-token, one `delta.tool_calls` entry carrying * `index`/`id`/`function`, then a terminal chunk with `finish_reason` + `usage`. */ interface ChatCompletionChunk { + model?: unknown choices?: Array<{ index?: number finish_reason?: string | null @@ -403,7 +611,7 @@ interface ChatCompletionChunk { }> } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + usage?: RawUsage error?: { message?: string; type?: string } } @@ -442,8 +650,15 @@ export async function streamRouterChatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + seed?: number + /** Provider-specific request fields; canonical streaming fields cannot be overridden here. */ + extraBody?: Readonly> + reasoningEffort?: ReasoningEffort + callId?: string + correlationId?: string }, ): Promise { + const maxAttempts = routerMaxAttempts(cfg) if (cfg.complete) { throw new ValidationError( 'streamRouterChatWithTools: RouterConfig.complete is a BUFFERED transport (it returns one ' + @@ -459,18 +674,31 @@ export async function streamRouterChatWithTools( // fact by `usageUnknown`, not assumed away. stream_options: { include_usage: true }, } - const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${cfg.routerKey}`, - accept: 'text/event-stream', - }, - body: JSON.stringify(body), - ...(opts?.signal ? { signal: opts.signal } : {}), - }) - // Identical failure surface to the buffered path: fail loud on any non-2xx, same message shape. - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`) + let res: Response | undefined + let transportAttempts = 0 + let lastErr = '' + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + transportAttempts = attempt + const candidate = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + ...routerRequestHeaders(cfg, opts), + accept: 'text/event-stream', + }, + body: JSON.stringify(body), + ...(opts?.signal ? { signal: opts.signal } : {}), + }) + if (candidate.ok) { + res = candidate + break + } + lastErr = `router ${candidate.status}: ${(await candidate.text()).slice(0, 200)}` + if (!isTransientRouterStatus(candidate.status)) throw new Error(lastErr) + if (attempt < maxAttempts) { + await new Promise((resolve) => setTimeout(resolve, 800 * 2 ** (attempt - 1))) + } + } + if (!res) throw new Error(`${lastErr} (exhausted retries)`) if (!res.body) { throw new ValidationError( `router ${res.status}: streamed completion returned no response body to read`, @@ -481,7 +709,8 @@ export async function streamRouterChatWithTools( let sawContent = false let fieldReasoning = '' let finishReason: string | undefined - let rawUsage: { prompt_tokens?: number; completion_tokens?: number } | undefined + let rawUsage: RawUsage | undefined + let observedModel: string | undefined const calls = new Map() let lastCallIndex = -1 @@ -491,6 +720,15 @@ export async function streamRouterChatWithTools( `router stream error: ${chunk.error.message ?? chunk.error.type ?? 'unknown'}`, ) } + const chunkModel = reportedModel(chunk.model) + if (chunkModel !== undefined) { + if (observedModel !== undefined && observedModel !== chunkModel) { + throw new ValidationError( + `router stream changed reported model from ${JSON.stringify(observedModel)} to ${JSON.stringify(chunkModel)}`, + ) + } + observedModel = chunkModel + } // With `include_usage` the terminal chunk may carry usage and an EMPTY choices array. if (chunk.usage) rawUsage = chunk.usage const choice = chunk.choices?.[0] @@ -537,21 +775,54 @@ export async function streamRouterChatWithTools( name: call.name ?? '', arguments: call.arguments || '{}', })) - const { usage, costUsd } = meterTurn(rawUsage, cfg.model) + const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(rawUsage, cfg.model) return { // `null` only when NO content field was ever sent — the buffered path's `msg?.content ?? null`. content: sawContent ? split.content : null, toolCalls, + transportAttempts, + ...(observedModel !== undefined ? { model: observedModel } : {}), ...(split.reasoning ? { reasoning: split.reasoning } : {}), ...(finishReason !== undefined ? { finishReason } : {}), ...(usage ? { usage } : {}), ...(costUsd !== undefined ? { costUsd } : {}), + ...(costProvenance ? { costProvenance } : {}), + ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), + ...(cache ? { cache } : {}), // The turn ran; its tokens are unknown. Marked rather than left as a bare `undefined`, which a // metering caller cannot tell apart from a turn that genuinely cost nothing. ...(usage ? {} : { usageUnknown: true as const }), } } +function routerRequestHeaders( + cfg: Pick, + opts: { callId?: string; correlationId?: string } | undefined, +): Record { + return { + 'content-type': 'application/json', + authorization: `Bearer ${cfg.routerKey}`, + ...(opts?.callId ? { 'idempotency-key': opts.callId } : {}), + ...(opts?.correlationId ? { 'x-correlation-id': opts.correlationId } : {}), + } +} + +function reportedModel(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +function routerMaxAttempts(cfg: RouterConfig): number { + const value = cfg.maxAttempts ?? 5 + if (!Number.isSafeInteger(value) || value < 1) { + throw new ValidationError('RouterConfig.maxAttempts must be a positive safe integer') + } + return value +} + +function isTransientRouterStatus(status: number): boolean { + return [408, 425, 429, 500, 502, 503, 504, 520, 522, 524].includes(status) +} + function idAlreadyOpen(calls: Map, id: string): boolean { for (const call of calls.values()) if (call.id === id) return true return false @@ -661,6 +932,9 @@ function chatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + seed?: number + extraBody?: Readonly> + reasoningEffort?: ReasoningEffort }, ): Promise { return cfg.stream === true @@ -683,6 +957,8 @@ export interface RouterToolLoopResult { * steerer reads (behavior, never the verdict) to diagnose + redirect the next shot. */ toolTrace: Array<{ name: string; args: string; result: string }> usage: { input: number; output: number } + /** False when any completed router turn omitted usage. */ + tokensKnown?: false /** The full conversation after the loop (seed + every assistant/tool turn). Lets a caller * CARRY the messages into the next shot (depth continuation) and read the trajectory. */ messages: Array> @@ -710,6 +986,7 @@ export async function routerToolLoop( temperature?: number signal?: AbortSignal maxTokens?: number + reasoningEffort?: ReasoningEffort /** Seed the loop with an existing conversation (depth continuation) instead of * `[system, user]`. When set, `system`/`user` are ignored. The array is copied. */ initialMessages?: ReadonlyArray> @@ -725,7 +1002,8 @@ export async function routerToolLoop( chat: (messages, toolSpecs) => chatWithTools(cfg, messages, toolSpecs, { ...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts?.maxTokens ? { maxTokens: opts.maxTokens } : {}), + ...(opts?.maxTokens !== undefined ? { maxTokens: opts.maxTokens } : {}), + ...(opts?.reasoningEffort ? { reasoningEffort: opts.reasoningEffort } : {}), ...(opts?.signal ? { signal: opts.signal } : {}), }), tools, @@ -744,7 +1022,10 @@ export async function routerToolLoop( * Transport follows `cfg.stream`: buffered by default, SSE when the caller opts in. A supervisor * turn is the longest completion in the system, so it is the call site streaming exists for. */ -export function routerBrain(cfg: RouterConfig, opts: { temperature?: number } = {}): ToolLoopChat { +export function routerBrain( + cfg: RouterConfig, + opts: { temperature?: number; reasoningEffort?: ReasoningEffort } = {}, +): ToolLoopChat { const temperature = opts.temperature ?? 0.4 return (messages, tools) => chatWithTools(cfg, messages, tools, { @@ -753,5 +1034,6 @@ export function routerBrain(cfg: RouterConfig, opts: { temperature?: number } = // The config's ceiling reaches the completion, so a caller driving a reasoning model can // raise it. Without this a router-brained supervisor is stuck on the 8192 default. ...(cfg.maxTokens !== undefined ? { maxTokens: cfg.maxTokens } : {}), + ...(opts.reasoningEffort ? { reasoningEffort: opts.reasoningEffort } : {}), }) } diff --git a/src/runtime/run-benchmark.test.ts b/src/runtime/run-benchmark.test.ts index a03cc9e2..934debe4 100644 --- a/src/runtime/run-benchmark.test.ts +++ b/src/runtime/run-benchmark.test.ts @@ -17,15 +17,24 @@ interface ChatRequest { const task: AgenticTask = { id: 'task-1', - systemPrompt: 'Use the test surface.', userPrompt: 'Complete the task.', } const worker: AgenticOptions = { routerBaseUrl: 'http://router.test/v1', routerKey: 'test-key', - model: 'worker-model', - maxTokens: 8, + workerProfile: { + name: 'worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'worker-model', + reasoningEffort: 'none', + metadata: { maxTokens: 8, temperature: 0.2 }, + }, + prompt: { systemPrompt: 'Use the test surface.' }, + tools: {}, + }, } const oneShot = defineStrategy('one-shot', async ({ shot }) => { @@ -61,8 +70,9 @@ function surface(events: string[]): AgenticSurface { } } -function okResponse(): Response { +function okResponse(model = 'worker-model'): Response { return Response.json({ + model, choices: [{ message: { content: 'DONE' } }], usage: { prompt_tokens: 2, completion_tokens: 1 }, }) @@ -71,7 +81,8 @@ function okResponse(): Response { function stubRouter( requests: ChatRequest[], events: string[], - respond: (request: ChatRequest, index: number) => Response = () => okResponse(), + respond: (request: ChatRequest, index: number) => Response = (request) => + okResponse(request.model), ): void { vi.stubGlobal( 'fetch', @@ -85,7 +96,7 @@ function stubRouter( } function isModelCheck(request: ChatRequest): boolean { - return request.messages?.[0]?.content === 'Reply OK.' + return request.messages?.some((message) => message.content === 'Reply OK.') ?? false } afterEach(() => { @@ -101,7 +112,19 @@ describe('runBenchmark model availability', () => { await runBenchmark({ environment: surface(events), tasks: [task, { ...task, id: 'task-2' }], - worker: { ...worker, analystModel: 'analyst-model' }, + worker: { + ...worker, + analystProfile: { + name: 'analyst', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'analyst-model', + reasoningEffort: 'none', + metadata: { maxTokens: 8, temperature: 0.2 }, + }, + }, + }, strategies: [oneShot], budget: 1, concurrency: 2, @@ -109,7 +132,7 @@ describe('runBenchmark model availability', () => { const checks = requests.filter(isModelCheck) expect(checks.map((request) => request.model).sort()).toEqual(['analyst-model', 'worker-model']) - expect(checks.every((request) => request.max_tokens === 1)).toBe(true) + expect(checks.every((request) => request.max_tokens === 8)).toBe(true) expect(checks.every((request) => request.reasoning_effort === 'none')).toBe(true) expect(events.indexOf('open')).toBe(2) }) @@ -123,7 +146,7 @@ describe('runBenchmark model availability', () => { tasks: [task], worker: { ...worker, - analystModel: worker.model, + analystProfile: worker.workerProfile, complete: async () => okResponse().json(), }, strategies: [oneShot], @@ -152,7 +175,7 @@ describe('runBenchmark model availability', () => { onTask, }), ).rejects.toThrow( - 'Benchmark model "worker-model" preflight failed: router 404: model has been deprecated', + 'Benchmark model "worker-model" preflight failed: runBenchmark model preflight (worker-model) failed: routerInlineExecutor: transport failed: router 404: model has been deprecated', ) expect(requests).toHaveLength(1) @@ -167,7 +190,18 @@ describe('runBenchmark model availability', () => { runBenchmark({ environment: surface(events), tasks: [task], - worker: { ...worker, analystModel: 'analyst-model' }, + worker: { + ...worker, + analystProfile: { + name: 'analyst', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'analyst-model', + reasoningEffort: 'none', + }, + }, + }, strategies: [oneShot], budget: 1, modelPreflight: async (model) => { @@ -252,7 +286,15 @@ describe('runBenchmark model availability', () => { tasks: [task], worker: { ...worker, - analystModel: 'analyst-model', + analystProfile: { + name: 'analyst', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'analyst-model', + reasoningEffort: 'none', + }, + }, complete: async () => okResponse().json(), }, strategies: [oneShot], @@ -267,7 +309,7 @@ describe('runBenchmark model availability', () => { expect(events.indexOf('open')).toBe(2) }) - it('retries at temperature one when the provider requires it', async () => { + it('does not change the profile temperature when the provider rejects it', async () => { const requests: ChatRequest[] = [] const events: string[] = [] stubRouter(requests, events, (_request, index) => @@ -276,15 +318,17 @@ describe('runBenchmark model availability', () => { : okResponse(), ) - await runBenchmark({ - environment: surface(events), - tasks: [task], - worker, - strategies: [oneShot], - budget: 1, - }) + await expect( + runBenchmark({ + environment: surface(events), + tasks: [task], + worker, + strategies: [oneShot], + budget: 1, + }), + ).rejects.toThrow('router 400: only temperature 1 is allowed for this model') - expect(requests.slice(0, 2).map((request) => request.temperature)).toEqual([0.2, 1]) - expect(events.indexOf('open')).toBe(2) + expect(requests.map((request) => request.temperature)).toEqual([0.2]) + expect(events).not.toContain('open') }) }) diff --git a/src/runtime/run-benchmark.ts b/src/runtime/run-benchmark.ts index ea008d82..f1d6f0b1 100644 --- a/src/runtime/run-benchmark.ts +++ b/src/runtime/run-benchmark.ts @@ -15,7 +15,7 @@ import { pairedBootstrap, paretoFrontier } from '@tangle-network/agent-eval' import type { RuntimeHooks } from '../runtime-hooks' -import { routerChatWithUsage } from './router-client' +import { profileChatClient } from './profile-chat-client' import { type AgenticOptions, type AgenticSurface, @@ -25,6 +25,7 @@ import { type Strategy, sample, } from './strategy' +import { concreteModelId } from './supervise/model-policy' /** A checkable task domain — implement these 5 hooks and the suite does the rest. The * same seam as `AgenticSurface`; `Environment` is the RL/gym-standard name for it. */ @@ -79,8 +80,10 @@ export interface BenchmarkCell { /** The progress curve (refine: score per shot; sample: best-so-far per rollout). */ progression: number[] usd: number + usdKnown: boolean ms: number tokens: { input: number; output: number } + tokensKnown: boolean } export interface BenchmarkTaskRow { @@ -102,6 +105,8 @@ export interface BenchmarkStrategySummary { resolved: number /** Mean cost vector per task. */ usd: number + /** Fraction of task cells whose billed-dollar total was complete. */ + usdKnownRate: number ms: number } @@ -144,22 +149,35 @@ async function preflightModels(cfg: BenchmarkConfig): Promise { if (cfg.modelPreflight === false) return if (cfg.worker.complete && !cfg.modelPreflight) return - const models = [...new Set([cfg.worker.model, cfg.worker.analystModel ?? cfg.worker.model])] + const profiles = [cfg.worker.workerProfile, cfg.worker.analystProfile ?? cfg.worker.workerProfile] + const profilesByModel = new Map() + for (const [index, profile] of profiles.entries()) { + const model = concreteModelId(profile.model?.default) + if (!model) { + throw new Error( + `Benchmark ${index === 0 ? 'worker' : 'analyst'} AgentProfile.model.default must name an exact model`, + ) + } + if (!profilesByModel.has(model)) profilesByModel.set(model, profile) + } + const models = [...profilesByModel.keys()] const timeoutMs = cfg.modelPreflightTimeoutMs ?? 30_000 if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error('modelPreflightTimeoutMs must be a positive finite number') const check = cfg.modelPreflight ?? (async (model: string, worker: Readonly, signal: AbortSignal) => { - await routerChatWithUsage( - { + const profile = profilesByModel.get(model) + if (!profile) throw new Error(`Benchmark preflight has no AgentProfile for model ${model}`) + await profileChatClient({ + profile, + context: `runBenchmark model preflight (${model})`, + executor: { + backend: 'router', routerBaseUrl: worker.routerBaseUrl, routerKey: worker.routerKey, - model, }, - [{ role: 'user', content: 'Reply OK.' }], - { maxTokens: 1, reasoningEffort: 'none', signal }, - ) + }).chat({ model, messages: [{ role: 'user', content: 'Reply OK.' }] }, { signal }) }) const results = await Promise.allSettled( @@ -236,8 +254,10 @@ export async function runBenchmark(cfg: BenchmarkConfig): Promise c.score)), resolved: mean(cells.map((c) => (c.resolved ? 1 : 0))), usd: mean(cells.map((c) => c.usd)), + usdKnownRate: mean(cells.map((c) => (c.usdKnown ? 1 : 0))), ms: mean(cells.map((c) => c.ms)), } } diff --git a/src/runtime/run-loop.ts b/src/runtime/run-loop.ts index dbe9645b..32634537 100644 --- a/src/runtime/run-loop.ts +++ b/src/runtime/run-loop.ts @@ -723,6 +723,7 @@ async function executeIteration(args: ExecuteIterationArgs(args: ExecuteIterationArgs(args: ExecuteIterationArgs( winner = defaultSelectWinner(args.iterations) } } - const costUsd = args.iterations.reduce((sum, iter) => sum + (iter.costUsd || 0), 0) + const costUsd = args.iterations.reduce((sum, iter) => sum + iter.costUsd, 0) + const costUsdKnown = args.iterations.every((iter) => iter.costUsdKnown !== false) + const estimatedCostUsd = args.iterations.reduce( + (sum, iter) => sum + (iter.estimatedCostUsd ?? 0), + 0, + ) + const promptCache: Record = {} + for (const iteration of args.iterations) { + if (iteration.promptCache) mergeUsageMetadata(promptCache, iteration.promptCache) + } const tokenUsage = args.iterations.reduce((acc: LoopTokenUsage, iter) => { addTokenUsage(acc, iter.tokenUsage) return acc @@ -983,6 +1013,9 @@ function finalize( winner, durationMs: args.now() - args.startMs, costUsd, + ...(costUsdKnown ? {} : { costUsdKnown: false }), + ...(estimatedCostUsd > 0 ? { estimatedCostUsd } : {}), + ...(Object.keys(promptCache).length > 0 ? { promptCache } : {}), tokenUsage, provenance: { mounts: args.mounts, @@ -992,6 +1025,16 @@ function finalize( return result } +function mergeUsageMetadata( + target: Record, + source: Readonly>, +): void { + for (const [key, value] of Object.entries(source)) { + if (typeof value === 'number') target[key] = (Number(target[key]) || 0) + value + else target[key] = value + } +} + /** * One receipt per scored candidate — a candidate being an iteration that * produced an output without erroring (an errored or output-less iteration was @@ -1093,6 +1136,10 @@ async function finalizeAndEmitEnded( decision: stringifySafe(decision), winnerIterationIndex: result.winner?.iterationIndex, totalCostUsd: result.costUsd, + ...(result.costUsdKnown === false ? { costUsdKnown: false } : {}), + ...(result.estimatedCostUsd !== undefined + ? { estimatedCostUsd: result.estimatedCostUsd } + : {}), durationMs: result.durationMs, iterations: iterations.length, }, @@ -1106,6 +1153,10 @@ async function finalizeAndEmitEnded( payload: { winnerIterationIndex: result.winner?.iterationIndex, totalCostUsd: result.costUsd, + ...(result.costUsdKnown === false ? { costUsdKnown: false } : {}), + ...(result.estimatedCostUsd !== undefined + ? { estimatedCostUsd: result.estimatedCostUsd } + : {}), durationMs: result.durationMs, iterations: iterations.length, }, diff --git a/src/runtime/sandbox-backend.ts b/src/runtime/sandbox-backend.ts index f6b9253b..1339bdee 100644 --- a/src/runtime/sandbox-backend.ts +++ b/src/runtime/sandbox-backend.ts @@ -10,7 +10,7 @@ import type { AgentProfile, HarnessType } from '@tangle-network/agent-interface' import type { CreateSandboxOptions } from '@tangle-network/sandbox' -import { profileForExecution } from './supervise/model-policy' +import { assertExecutableAgentProfile, concreteProfileModel } from './supervise/model-policy' type BackendType = NonNullable['type'] type BackendOverride = NonNullable @@ -45,8 +45,8 @@ function harnessAsBackendType(harness: HarnessType): BackendType | undefined { } /** - * Resolve the backend `type`: an explicit override wins, then the profile's - * `metadata.backendType` hint, then the profile's declared `harness`, else the + * Resolve the backend `type`: the profile's `metadata.backendType` hint, then + * its declared `harness`, else the * SDK's profile-driven default (`'opencode'` on the platform side). * * A declared `harness` the sandbox cannot run throws rather than falling @@ -57,11 +57,11 @@ function resolveBackendType( profile: AgentProfile, override: Partial | undefined, ): BackendType { - if (override?.type) return override.type const explicit = profile.metadata?.backendType - if (typeof explicit === 'string') return explicit as BackendType + let selected: BackendType | undefined + if (typeof explicit === 'string') selected = explicit as BackendType const declared = profile.harness - if (declared !== undefined) { + if (selected === undefined && declared !== undefined) { const backend = harnessAsBackendType(declared) if (backend === undefined) { throw new Error( @@ -70,9 +70,17 @@ function resolveBackendType( 'Set metadata.backendType to run it on a different backend deliberately.', ) } - return backend + selected = backend } - return 'opencode' as BackendType + if (override?.type && selected && override.type !== selected) { + throw new Error( + `buildBackendOptions: backend override ${JSON.stringify(override.type)} conflicts with AgentProfile backend ${JSON.stringify(selected)}`, + ) + } + if (selected === undefined) { + throw new Error('buildBackendOptions: AgentProfile.harness must select the sandbox backend') + } + return selected } /** @@ -83,14 +91,21 @@ export function buildBackendOptions( profile: AgentProfile, overrides: Partial | undefined, ): CreateSandboxOptions { + assertExecutableAgentProfile(profile, 'buildBackendOptions') const base = overrides ?? {} const overrideBackend = base.backend + const overrideModel = overrideBackend?.model?.model + const profileModel = concreteProfileModel(profile) + if (overrideModel !== undefined && overrideModel !== profileModel) { + throw new Error( + `buildBackendOptions: backend model ${JSON.stringify(overrideModel)} conflicts with AgentProfile model ${JSON.stringify(profileModel)}`, + ) + } return { ...base, backend: { type: resolveBackendType(profile, overrideBackend), - profile: profileForExecution(profile), - ...(overrideBackend?.model ? { model: overrideBackend.model } : {}), + profile, ...(overrideBackend?.server ? { server: overrideBackend.server } : {}), }, } diff --git a/src/runtime/sandbox-events.ts b/src/runtime/sandbox-events.ts index b2305d62..52dfb796 100644 --- a/src/runtime/sandbox-events.ts +++ b/src/runtime/sandbox-events.ts @@ -131,7 +131,10 @@ export function extractLlmCallEvent( * receipt: (turn) => { * const u = sumSandboxUsage(turn.events) * return { model, inputTokens: u.input, outputTokens: u.output, - * ...(u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}) } + * ...(u.tokensKnown ? {} : { usageUnknown: true }), + * ...(u.usdKnown && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), + * ...(u.usdKnown ? {} : { costUnknown: true }), + * ...(u.estimatedCostUsd !== undefined ? { estimatedCostUsd: u.estimatedCostUsd } : {}) } * } * * Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the @@ -140,18 +143,44 @@ export function extractLlmCallEvent( export function sumSandboxUsage( events: readonly SandboxEvent[], agentRunName = 'agent', -): { input: number; output: number; costUsd: number } { +): { + input: number + output: number + costUsd: number + tokensKnown: boolean + usdKnown: boolean + estimatedCostUsd?: number +} { let input = 0 let output = 0 let costUsd = 0 + let estimatedCostUsd = 0 + let sawEstimate = false + let sawCall = false + let tokensKnown = true + let usdKnown = true for (const ev of events) { const call = extractLlmCallEvent(ev, agentRunName) if (!call) continue + sawCall = true input += call.tokensIn ?? 0 output += call.tokensOut ?? 0 costUsd += call.costUsd ?? 0 + if (call.tokensKnown === false) tokensKnown = false + if (call.usdKnown === false) usdKnown = false + if (call.estimatedCostUsd !== undefined) { + estimatedCostUsd += call.estimatedCostUsd + sawEstimate = true + } + } + return { + input, + output, + costUsd, + tokensKnown: sawCall && tokensKnown, + usdKnown: sawCall && usdKnown, + ...(sawEstimate ? { estimatedCostUsd } : {}), } - return { input, output, costUsd } } function buildLlmCall( @@ -165,8 +194,30 @@ function buildLlmCall( outputTokens !== undefined || reasoningTokens !== undefined ? (outputTokens ?? 0) + (reasoningTokens ?? 0) : undefined - const costUsd = pickFiniteNumber(data, ['costUsd', 'totalCostUsd', 'cost_usd', 'cost']) - if (tokensIn === undefined && tokensOut === undefined && costUsd === undefined) { + const reportedCostUsd = pickFiniteNumber(data, ['costUsd', 'totalCostUsd', 'cost_usd', 'cost']) + const explicitTokensKnown = data.tokensKnown ?? data.tokens_known + const explicitCostKnown = data.costKnown ?? data.cost_known ?? data.usdKnown ?? data.usd_known + const costProvenance = data.costProvenance ?? data.cost_provenance + const explicitEstimate = pickFiniteNumber(data, ['estimatedCostUsd', 'estimated_cost_usd']) + const catalogEstimate = + costProvenance === 'catalog-estimate' ? (explicitEstimate ?? reportedCostUsd) : explicitEstimate + const costUsd = costProvenance === 'catalog-estimate' ? undefined : reportedCostUsd + const promptCache = finiteMetadata(data.promptCache ?? data.prompt_cache) + const tokensKnown = + explicitTokensKnown !== false && tokensIn !== undefined && tokensOut !== undefined + const usdKnown = + explicitCostKnown === true && + (costProvenance === 'provider-receipt' || costProvenance === 'billing-receipt') && + costUsd !== undefined + if ( + tokensIn === undefined && + tokensOut === undefined && + costUsd === undefined && + catalogEstimate === undefined && + promptCache === undefined && + explicitTokensKnown !== false && + explicitCostKnown !== false + ) { return undefined } const model = typeof data.model === 'string' && data.model.length > 0 ? data.model : agentRunName @@ -176,10 +227,25 @@ function buildLlmCall( } if (tokensIn !== undefined) event.tokensIn = tokensIn if (tokensOut !== undefined) event.tokensOut = tokensOut + if (!tokensKnown) event.tokensKnown = false if (costUsd !== undefined) event.costUsd = costUsd + if (!usdKnown) event.usdKnown = false + if (catalogEstimate !== undefined) event.estimatedCostUsd = catalogEstimate + if (promptCache !== undefined) event.promptCache = promptCache return event } +function finiteMetadata(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined + const result: Record = {} + for (const [key, entry] of Object.entries(value)) { + if ((typeof entry === 'number' && Number.isFinite(entry)) || typeof entry === 'string') { + result[key] = entry + } + } + return Object.keys(result).length > 0 ? result : undefined +} + function pickFiniteNumber(data: Record, keys: string[]): number | undefined { for (const key of keys) { const value = data[key] diff --git a/src/runtime/stdio-mcp-client.test.ts b/src/runtime/stdio-mcp-client.test.ts index 5adb5b04..19475448 100644 --- a/src/runtime/stdio-mcp-client.test.ts +++ b/src/runtime/stdio-mcp-client.test.ts @@ -218,14 +218,14 @@ describe('materializeLocalMcp', () => { expect(JSON.stringify(profile)).not.toContain('test-token') await expect( localSandboxClient({ - router: { baseUrl: 'https://router.invalid', key: 'unused', model: 'unused' }, + router: { baseUrl: 'https://router.invalid', key: 'unused' }, profile, profileSecurityPolicy: TRUSTED_LOCAL_MCP_POLICY, }).create(), ).rejects.toThrow(/no KeyProvider/) const client = localSandboxClient({ - router: { baseUrl: 'https://router.invalid', key: 'unused', model: 'unused' }, + router: { baseUrl: 'https://router.invalid', key: 'unused' }, keys: { get }, profile, profileSecurityPolicy: TRUSTED_LOCAL_MCP_POLICY, @@ -313,7 +313,7 @@ describe('materializeLocalMcp', () => { }, } const client = localSandboxClient({ - router: { baseUrl: 'https://router.invalid', key: 'unused', model: 'unused' }, + router: { baseUrl: 'https://router.invalid', key: 'unused' }, profile: trustedProfile, profileSecurityPolicy: TRUSTED_LOCAL_MCP_POLICY, }) @@ -340,7 +340,7 @@ describe('materializeLocalMcp', () => { it('rejects permissive host-process policy without fixed trusted profile bytes', () => { expect(() => localSandboxClient({ - router: { baseUrl: 'https://router.invalid', key: 'unused', model: 'unused' }, + router: { baseUrl: 'https://router.invalid', key: 'unused' }, profileSecurityPolicy: TRUSTED_LOCAL_MCP_POLICY, }), ).toThrow(/requires a fixed author-controlled profile/) diff --git a/src/runtime/strategy-author.ts b/src/runtime/strategy-author.ts index 875a3e05..b8cfe679 100644 --- a/src/runtime/strategy-author.ts +++ b/src/runtime/strategy-author.ts @@ -14,9 +14,11 @@ import { mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' -import type { ChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import { strategyAuthorMethod } from '../improvement/optimizer-prompt' +import { profileChatClient } from './profile-chat-client' import type { Strategy } from './strategy' +import type { ExecutorConfig } from './supervise/runtime' /** The compressed consumable a skill carries: everything an author needs to emit a loop. */ export const strategyAuthorContract = ` @@ -76,14 +78,15 @@ export default defineStrategy('your-strategy-name', async ({ surface, task, budg ` export interface AuthorStrategyOptions { - /** The model-call seam (agent-eval `createChatClient`). */ - chat: ChatClient - model?: string - /** A NAMED fallback author tried once when the primary call fails or returns no code + /** Exact author identity. Runtime binds it to every authoring turn. */ + profile: AgentProfile + /** Execution substrate for the author. Behavioral settings are forbidden here. */ + executor: ExecutorConfig + /** An exact fallback author tried once when the primary call fails or returns no code * block (thinking models time out at the edge on long authoring prompts, or return * empty content without `maxTokens`). Opt-in — absent means the primary's failure * propagates. */ - fallbackModel?: string + fallbackProfile?: AgentProfile /** The contract text shown to the author. Default `strategyAuthorContract`. The * meta-optimization coordinate: a GEPA/skill loop can evolve this text and gate each * variant on the same frozen holdout as any strategy. */ @@ -96,12 +99,15 @@ export interface AuthorStrategyOptions { budget: number /** Where the authored module file is written (created if missing). */ outDir: string - temperature?: number - /** Completion cap — required by thinking-model authors that stream reasoning first. */ - maxTokens?: number signal?: AbortSignal } +/** Standing behavior callers put in the strategy-author AgentProfile. */ +export const strategyAuthorSystemPrompt = + 'You are a senior researcher authoring optimization strategies for agent loops: you read ' + + 'per-task losses like experimental data, form a mechanism-level hypothesis, and author the ' + + 'one composition that tests it. Output exactly one fenced ```ts code block and nothing else.' + /** Static CONTRACT lint over an authored strategy module — the module-boundary * enforcement of the harness's two measurement invariants: * - author blindness: the only import allowed is the kernel surface. A body that could @@ -145,21 +151,16 @@ export interface AuthoredStrategy { * when the reply carries no code block. */ async function requestAuthoredCode( opts: AuthorStrategyOptions, - model: string | undefined, + profile: AgentProfile, ): Promise { - const res = await opts.chat.chat( + const chat = profileChatClient({ + profile, + executor: opts.executor, + context: 'strategy author', + }) + const res = await chat.chat( { - ...(model ? { model } : {}), - ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts.maxTokens !== undefined ? { maxTokens: opts.maxTokens } : {}), messages: [ - { - role: 'system', - content: - 'You are a senior researcher authoring optimization strategies for agent loops: you read ' + - 'per-task losses like experimental data, form a mechanism-level hypothesis, and author the ' + - 'one composition that tests it. Output exactly one fenced ```ts code block and nothing else.', - }, { role: 'user', content: `${opts.contract ?? strategyAuthorContract}\n\nBASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}) — the per-task losses are your gradient:\n${opts.lossesJson}\n\nAuthor ONE new strategy that you expect to beat the baselines on THIS environment at the same budget.\n${strategyAuthorMethod}\n\nOutput only the module code block.`, @@ -171,7 +172,7 @@ async function requestAuthoredCode( const match = res.content.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/) if (!match?.[1]) { throw new Error( - `authorStrategy: no code block in the author's reply (model=${model ?? 'default'}): ${res.content.slice(0, 300)}`, + `authorStrategy: no code block in the author's reply: ${res.content.slice(0, 300)}`, ) } return match[1] @@ -182,10 +183,10 @@ async function requestAuthoredCode( export async function authorStrategy(opts: AuthorStrategyOptions): Promise { let code: string try { - code = await requestAuthoredCode(opts, opts.model) + code = await requestAuthoredCode(opts, opts.profile) } catch (primaryError) { - if (!opts.fallbackModel) throw primaryError - code = await requestAuthoredCode(opts, opts.fallbackModel) + if (!opts.fallbackProfile) throw primaryError + code = await requestAuthoredCode(opts, opts.fallbackProfile) } assertStrategyContract(code) mkdirSync(opts.outDir, { recursive: true }) diff --git a/src/runtime/strategy-evolution.ts b/src/runtime/strategy-evolution.ts index fcb16c88..8585c683 100644 --- a/src/runtime/strategy-evolution.ts +++ b/src/runtime/strategy-evolution.ts @@ -24,8 +24,9 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { gzipSync } from 'node:zlib' -import type { ChatClient } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' import type { RuntimeHooks } from '../runtime-hooks' +import { profileChatClient } from './profile-chat-client' import { type PromotionVerdict, promotionGate } from './promotion-gate' import { type BenchmarkConfig, @@ -43,14 +44,15 @@ import { sampleThenRefine, } from './strategy' import { authorStrategy, strategyAuthorContract } from './strategy-author' +import type { ExecutorConfig } from './supervise/runtime' export interface EvolutionAuthor { - /** The model-call seam (agent-eval `createChatClient`). */ - chat: ChatClient - model?: string - fallbackModel?: string - temperature?: number - maxTokens?: number + /** Exact author identity. */ + profile: AgentProfile + /** Execution substrate. All behavior comes from the profile. */ + executor: ExecutorConfig + /** Optional exact fallback identity. */ + fallbackProfile?: AgentProfile } export type ChampionPolicy = 'score' | 'costAware' @@ -519,11 +521,9 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis const contract = `${strategyAuthorContract}${objectiveNote}\n\nEXAMPLE TOOLS FROM ONE TASK (tool sets VARY per task on this domain — a strategy MUST select tool names from await listTools(handle) at runtime; hardcoding these example names will zero your score on most tasks):\n${toolCatalog}\n\nSTRATEGIES ALREADY IN THE TOURNAMENT (author something MEANINGFULLY different — a new composition, not a rename):\n${fieldSummary(archive)}\n\nYou are authoring candidate ${i + 1} of ${populationSize} this generation; explore a distinct region of the strategy space from your siblings.` try { const authored = await authorStrategy({ - chat: cfg.author.chat, - ...(cfg.author.model ? { model: cfg.author.model } : {}), - ...(cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {}), - ...(cfg.author.temperature !== undefined ? { temperature: cfg.author.temperature } : {}), - ...(cfg.author.maxTokens !== undefined ? { maxTokens: cfg.author.maxTokens } : {}), + profile: cfg.author.profile, + executor: cfg.author.executor, + ...(cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {}), contract, environmentName: cfg.environment.name, lossesJson, @@ -686,28 +686,28 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis const tolerance = cfg.reproducerCheck.tolerance ?? 0.05 const championHoldoutScore = holdout.perStrategy[incumbent.name]?.score ?? 0 try { - const summaryRes = await cfg.author.chat.chat({ - ...(cfg.author.model ? { model: cfg.author.model } : {}), - temperature: 0.2, - maxTokens: 512, - messages: [ - { - role: 'system', - content: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) — not the code. Output only the summary.`, - }, - { role: 'user', content: championCode }, - ], + const summaryProfile: AgentProfile = { + ...cfg.author.profile, + prompt: { + ...cfg.author.profile.prompt, + systemPrompt: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) — not the code. Output only the summary.`, + }, + } + const summaryRes = await profileChatClient({ + profile: summaryProfile, + executor: cfg.author.executor, + context: 'strategy reproducer summary', + }).chat({ + messages: [{ role: 'user', content: championCode }], }) const summary = summaryRes.content.trim() // The reproducer sees the summary and the contract — never the losses, never the // original code. If its implementation matches the champion on the SAME holdout, // the champion's win fits through the summary and cannot be holdout-specific. const reproduced = await authorStrategy({ - chat: cfg.author.chat, - ...(cfg.author.model ? { model: cfg.author.model } : {}), - ...(cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {}), - ...(cfg.author.maxTokens !== undefined ? { maxTokens: cfg.author.maxTokens } : {}), - temperature: 0.2, + profile: cfg.author.profile, + executor: cfg.author.executor, + ...(cfg.author.fallbackProfile ? { fallbackProfile: cfg.author.fallbackProfile } : {}), contract: `${strategyAuthorContract}\n\nIMPLEMENT EXACTLY THIS STRATEGY (a colleague's description — do not invent a different approach):\n${summary}`, environmentName: cfg.environment.name, lossesJson: '[]', diff --git a/src/runtime/strategy.ts b/src/runtime/strategy.ts index f6690207..ead18751 100644 --- a/src/runtime/strategy.ts +++ b/src/runtime/strategy.ts @@ -21,14 +21,22 @@ * surface-closed registry — the open `Executor` seam, not bespoke per-benchmark glue. */ -import { createChatClient, estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import type { ChatClient } from '@tangle-network/agent-eval' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../durable/spawn-journal' import type { RuntimeHooks } from '../runtime-hooks' import { observe } from './observe' import type { Outcome } from './personify/types' import type { Corpus } from './personify/wave-types' -import { routerToolLoop } from './router-client' +import { profileChatClient } from './profile-chat-client' +import { collectAgentTurn, streamAgentTurn } from './stream-agent-turn' import { withDriverExecutor } from './supervise/driver-executor' +import { + assertExecutableAgentProfile, + concreteModelId, + profileModelExecutionSettings, +} from './supervise/model-policy' +import { createExecutor } from './supervise/runtime' import { createSupervisor } from './supervise/supervisor' import type { Agent, @@ -47,7 +55,6 @@ import type { export interface AgenticTask { readonly id: string - readonly systemPrompt: string readonly userPrompt: string /** Opaque domain payload the surface reads (EOPS: servers/verifiers/tools). Drivers never read it. */ readonly meta?: Record @@ -85,25 +92,16 @@ export interface AgenticSurface { export interface AgenticOptions { routerBaseUrl: string routerKey: string - model: string + /** Exact worker identity. Model and standing instructions are read only from this profile. */ + workerProfile: AgentProfile /** Optional completion transport (see `RouterConfig.complete`): when set, BOTH legs of an * offline run use it instead of `fetch`-ing the router — the worker's tool loop (threaded into * its `routerToolLoop` cfg) AND the analyst's critic (its `ChatClient` is bound to this same * transport). One injected responder serves both, as a localhost mock endpoint would. Absent ⇒ * the live router fetch path (the default). */ complete?: (body: Record) => Promise - temperature?: number - /** Completion cap per worker turn — REQUIRED for thinking models (they burn unbounded - * budgets on reasoning and return empty content without it). Omitted ⇒ provider default. */ - maxTokens?: number - /** Turns the agent may take within ONE shot before the driver intervenes. */ - innerTurns?: number - /** The depth STEERER's analyst instruction (observe()'s system prompt). The knob a - * prompt optimizer (GEPA) tunes — the analyst IS the steerer. Omitted ⇒ the default. */ - analystInstruction?: string - /** The critic's model — lets the analyst be a stronger (or cheaper) model than the - * worker. Omitted ⇒ the worker's `model`. */ - analystModel?: string + /** Exact critic identity. Omitted means the exact worker profile also runs the critic. */ + analystProfile?: AgentProfile /** Across-run learning: when set, the analyst's observe() pass appends trace-derived * facts here (the flywheel write side). Read-back is opt-in via `corpusReadback` * because unconditional priming can pollute context on some domains. */ @@ -140,7 +138,7 @@ interface ShotTask { handle?: ArtifactHandle // present ⇒ DEPTH (shared artifact); absent ⇒ BREADTH (open own) messages?: StrategyMessage[] // carried conversation (depth); fresh when absent steer?: string // analyst-derived steer injected before this shot (depth) - persona?: ShotPersona // role override — multi-agent loops give each shot its own hat + profile?: AgentProfile // exact role/model override for this shot tools?: string[] // restrict THIS shot to these domain tools (names); unknown names throw /** analyst leaf only: a RAW instruction — the analyst answers it over the trajectory * directly (no findings schema). The verdict-capable channel. */ @@ -154,6 +152,8 @@ interface ShotOut { toolErrors: number /** Real router usage summed over the shot's turns; zeros only when the provider omits usage. */ tokens: { input: number; output: number } + /** False when any Router turn omitted usage. */ + tokensKnown?: false } const taskNudge = @@ -161,6 +161,67 @@ const taskNudge = 'change the request implies. After each tool result, check what remains and continue. Re-read the ' + 'values you set to confirm they took. Reply DONE only once every required change is made and verified.' +function exactAgenticProfile(profile: AgentProfile, context: string): AgentProfile { + const parsed = agentProfileSchema.safeParse(profile) + if (!parsed.success) throw new Error(`${context}: invalid AgentProfile: ${parsed.error.message}`) + return parsed.data +} + +function requiredProfileModel(profile: AgentProfile, context: string): string { + assertExecutableAgentProfile(profile, context) + const model = concreteModelId(profile.model?.default) + if (!model) { + throw new Error( + `${context}: AgentProfile.model.default must name the exact provider model; runtime-selected and missing models are not executable`, + ) + } + return model +} + +function profileSystemPrompt(profile: AgentProfile): string { + const sections = [profile.prompt?.systemPrompt, ...(profile.prompt?.instructions ?? [])].filter( + (value): value is string => typeof value === 'string' && value.trim().length > 0, + ) + const instructions = profile.resources?.instructions + if (typeof instructions === 'string' && instructions.trim()) sections.push(instructions) + else if ( + instructions && + typeof instructions === 'object' && + instructions.kind === 'inline' && + instructions.content.trim() + ) { + sections.push(instructions.content) + } else if (instructions && typeof instructions === 'object' && instructions.kind === 'github') { + throw new Error( + 'agentic profile: github resource instructions require a workspace materializer; use inline instructions for the direct Router worker', + ) + } + return sections.join('\n\n') +} + +function assertProfileTools( + profile: AgentProfile, + tools: ReadonlyArray, + context: string, +): void { + const supplied = new Set(tools.map((tool) => tool.function.name)) + const declared = profile.tools ?? {} + for (const name of supplied) { + if (declared[name] !== true) { + throw new Error( + `${context}: tool ${JSON.stringify(name)} is not enabled by AgentProfile.tools`, + ) + } + } + for (const [name, enabled] of Object.entries(declared)) { + if (enabled && !supplied.has(name)) { + throw new Error( + `${context}: AgentProfile enables tool ${JSON.stringify(name)} but the surface did not supply it`, + ) + } + } +} + /** One shot: run the agent's tool loop (≤ innerTurns) over the handle, mutating the artifact via * `surface.call`, carrying `messages`. Returns the updated conversation + counts. */ async function runShot( @@ -170,7 +231,7 @@ async function runShot( tools: AgenticTool[], messages: StrategyMessage[], opts: AgenticOptions, - modelOverride?: string, + profileOverride?: AgentProfile, ): Promise { // The canonical off-box tool loop (routerToolLoop) drives the turns; this shot supplies // the carried conversation (depth continuation, via initialMessages) and the tool dispatch @@ -187,30 +248,37 @@ async function runShot( return `ERROR: ${e instanceof Error ? e.message : String(e)}` } } - const r = await routerToolLoop( - { - routerBaseUrl: opts.routerBaseUrl, - routerKey: opts.routerKey, - model: modelOverride ?? opts.model, - ...(opts.complete ? { complete: opts.complete } : {}), - }, - '', - '', + const profile = exactAgenticProfile(profileOverride ?? opts.workerProfile, 'agentic shot') + requiredProfileModel(profile, 'agentic shot') + profileModelExecutionSettings(profile, 'agentic shot') + assertProfileTools(profile, tools, 'agentic shot') + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, tools, - execute, - { - maxTurns: opts.innerTurns ?? 4, - temperature: opts.temperature ?? 0.7, - initialMessages: messages, - ...(opts.maxTokens ? { maxTokens: opts.maxTokens } : {}), - }, + executeToolCall: execute, + ...(opts.complete ? { complete: opts.complete } : {}), + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', profile, factory }, + { messages: messages as Array<{ role: string; content: unknown }> }, + ), ) + if (turn.status !== 'completed') { + throw new Error(`agentic shot failed: ${turn.error?.message ?? turn.status}`) + } + const out = turn.output as + | { messages?: StrategyMessage[]; turns?: number; toolCalls?: number } + | undefined return { - messages: r.messages, - completions: r.turns, - toolCalls: r.toolCalls, + messages: out?.messages ?? messages, + completions: out?.turns ?? 0, + toolCalls: out?.toolCalls ?? 0, toolErrors, - tokens: r.usage, + tokens: { input: turn.usage.input, output: turn.usage.output }, + ...(turn.usage.tokensKnown === false ? { tokensKnown: false } : {}), } } @@ -223,6 +291,8 @@ async function runShot( interface AnalyzeOut { steer: string tokens: { input: number; output: number } + /** False when any analyst call omitted usage. */ + tokensKnown?: false } /** The firewall's input shape: the trajectory as compacted text — calls, results, @@ -246,49 +316,15 @@ function compactTrajectory(messages: StrategyMessage[]): string { * worker and the analyst share the one injected responder, exactly as a localhost mock would * serve both). The critic speaks the OpenAI request shape; we forward it to `complete` and lift * the parsed `/chat/completions` JSON back into a `ChatResponse`. */ -function analystChat( - opts: AgenticOptions, - defaultModel: string, -): ReturnType { - if (!opts.complete) { - return createChatClient({ - transport: 'router', - apiKey: opts.routerKey, - baseUrl: opts.routerBaseUrl, - defaultModel, - }) - } - const complete = opts.complete - return createChatClient({ - transport: 'mock', - defaultModel, - handler: async (req) => { - const raw = (await complete({ - model: req.model ?? defaultModel, - messages: req.messages, - ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), - ...(req.maxTokens !== undefined ? { max_tokens: req.maxTokens } : {}), - })) as { - choices?: Array<{ message?: { content?: string | null }; finish_reason?: string | null }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = raw.choices?.[0]?.message?.content ?? '' - const promptTokens = raw.usage?.prompt_tokens ?? 0 - const completionTokens = raw.usage?.completion_tokens ?? 0 - return { - content, - usage: { - promptTokens, - completionTokens, - totalTokens: promptTokens + completionTokens, - }, - costUsd: null, - model: req.model ?? defaultModel, - durationMs: 0, - finishReason: raw.choices?.[0]?.finish_reason ?? null, - contentEmpty: content.trim().length === 0, - raw: raw as Record, - } +function analystChat(opts: AgenticOptions, profile: AgentProfile): ChatClient { + return profileChatClient({ + profile, + context: 'agentic analyst', + executor: { + backend: 'router', + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + ...(opts.complete ? { complete: opts.complete } : {}), }, }) } @@ -305,32 +341,24 @@ async function consultAnalyst( opts: AgenticOptions, ): Promise { const trajectory = compactTrajectory(messages) - const analystModel = opts.analystModel ?? opts.model - const chat = analystChat(opts, analystModel) - // With a trajectory, the analyst framing (instruction as system, behavior as the user - // turn) is the channel's shape. With NO trajectory (pre-task consults, e.g. authored - // check generation), that framing breaks: the user turn is then just the task, which - // reads as a solve request — measured on Llama-3-8B, the model ignores the system - // instruction and answers the task (authored-assert yield 6/18 reps vs 18/18 with the - // instruction and task fused into one user message, the proven rig shape). - const consultMessages = trajectory - ? [ - { role: 'system' as const, content: instruction }, - { - role: 'user' as const, - content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`, - }, - ] - : [ - { - role: 'user' as const, - content: `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`, - }, - ] + const analystProfile = exactAgenticProfile( + opts.analystProfile ?? opts.workerProfile, + 'agentic analyst', + ) + const analystModel = requiredProfileModel(analystProfile, 'agentic analyst') + const chat = analystChat(opts, analystProfile) + // The profile owns the standing system instruction. This strategy-authored question is task + // input, so it stays in the user turn even when a trajectory is present. + const consultMessages = [ + { + role: 'user' as const, + content: trajectory + ? `${instruction}\n\nTASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}` + : `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`, + }, + ] const res = await chat.chat({ model: analystModel, - temperature: 0.2, - maxTokens: 1024, messages: consultMessages, }) const usage = ( @@ -340,15 +368,21 @@ async function consultAnalyst( prompt_tokens?: number completionTokens?: number completion_tokens?: number + captured?: boolean } } ).usage + const input = usage?.promptTokens ?? usage?.prompt_tokens + const output = usage?.completionTokens ?? usage?.completion_tokens + const tokensKnown = + usage?.captured !== false && typeof input === 'number' && typeof output === 'number' return { steer: res.content.trim(), tokens: { - input: usage?.promptTokens ?? usage?.prompt_tokens ?? 0, - output: usage?.completionTokens ?? usage?.completion_tokens ?? 0, + input: input ?? 0, + output: output ?? 0, }, + ...(tokensKnown ? {} : { tokensKnown: false }), } } @@ -358,32 +392,10 @@ async function analyze( opts: AgenticOptions, ): Promise { const trajectory = compactTrajectory(messages) - const analystModel = opts.analystModel ?? opts.model - const inner = analystChat(opts, analystModel) - // The critic's calls are REAL spend — capture usage so the cost vector bills them - // (an unbilled critic makes every steering-vs-sampling cost comparison dishonest). - const tokens = { input: 0, output: 0 } - const chat: typeof inner = { - ...inner, - chat: async (req, callOpts) => { - const res = await inner.chat(req, callOpts) - const u = ( - res as { - usage?: { - promptTokens?: number - completionTokens?: number - prompt_tokens?: number - completion_tokens?: number - } - } - ).usage - if (u) { - tokens.input += u.promptTokens ?? u.prompt_tokens ?? 0 - tokens.output += u.completionTokens ?? u.completion_tokens ?? 0 - } - return res - }, - } + const analystProfile = exactAgenticProfile( + opts.analystProfile ?? opts.workerProfile, + 'agentic analyst', + ) const obs = await observe( { task: task.userPrompt, @@ -393,9 +405,13 @@ async function analyze( runId: task.id, }, { - chat, - model: analystModel, - ...(opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {}), + profile: analystProfile, + executor: { + backend: 'router', + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + ...(opts.complete ? { complete: opts.complete } : {}), + }, ...(opts.corpus ? { corpus: opts.corpus, tags: opts.corpusTags ?? [] } : {}), }, ) @@ -405,7 +421,11 @@ async function analyze( .filter((a): a is string => typeof a === 'string' && a.trim().length > 0) .join('\n') .trim() - return { steer: steer || 'COMPLETE', tokens } + return { + steer: steer || 'COMPLETE', + tokens: { input: obs.usage.input, output: obs.usage.output }, + ...(obs.usage.known ? {} : { tokensKnown: false }), + } } async function renderCorpusReadback(opts: AgenticOptions): Promise { @@ -476,21 +496,23 @@ function shotExecutor(surface: AgenticSurface, opts: AgenticOptions): Executor 0 ? s.passes / s.total : 0 const out: StrategyShotResult = { @@ -510,9 +532,9 @@ function shotExecutor(surface: AgenticSurface, opts: AgenticOptions): Executor { runtime: 'agentic-analyst', async execute(task: unknown): Promise> { const t = task as { task: AgenticTask; messages: StrategyMessage[]; rawInstruction?: string } - const { steer, tokens } = t.rawInstruction + const { steer, tokens, tokensKnown } = t.rawInstruction ? await consultAnalyst(t.task, t.messages, t.rawInstruction, opts) : await analyze(t.task, t.messages, opts) - const analystModel = opts.analystModel ?? opts.model artifact = { outRef: `analyst:${steer.length}`, out: steer, spent: { iterations: 1, tokens, - usd: isModelPriced(analystModel) - ? estimateCost(tokens.input, tokens.output, analystModel) - : 0, + ...(tokensKnown === false ? { tokensKnown: false } : {}), + usd: 0, + usdKnown: false, ms: 0, }, } @@ -583,10 +604,22 @@ function agenticRegistry(surface: AgenticSurface, opts: AgenticOptions): Executo return withDriverExecutor(leaves) } -function leaf(name: string, role: 'shot' | 'analyst'): Agent> { +function leaf( + name: string, + role: 'shot' | 'analyst', + profile: AgentProfile, +): Agent> { + const exactProfile = exactAgenticProfile(profile, `agentic ${role}`) const agent = { name, - executorSpec: { profile: { name, metadata: { role } }, harness: null } as unknown as AgentSpec, + executorSpec: { + profile: { + ...exactProfile, + name, + metadata: { ...exactProfile.metadata, role }, + }, + harness: null, + } as AgentSpec, act(): Promise> { // SPAWNED, not run: its `executorSpec` (role shot/analyst) resolves a leaf executor // the scope drives. `act` is never called for a spawned child; it fails loud if @@ -616,15 +649,22 @@ export interface AgenticRunResult { /** DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-so-far per rollout. */ progression: number[] shots: number - /** The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real - * router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. */ + /** Observed billed subtotal. `usdKnown:false` means it is incomplete, never a measured zero. */ usd: number + usdKnown: boolean ms: number tokens: { input: number; output: number } + tokensKnown: boolean +} + +const UNBOUNDED_TURN_RESERVATION = 1_000_000_000 + +function profileTurnLimit(profile: AgentProfile, context: string): number { + return profileModelExecutionSettings(profile, context).maxTurns ?? 0 } -const perChild = (innerTurns: number): Budget => ({ - maxIterations: innerTurns + 1, +const perChild = (maxTurns: number): Budget => ({ + maxIterations: maxTurns === 0 ? UNBOUNDED_TURN_RESERVATION : maxTurns + 1, maxTokens: 1_000_000, }) @@ -635,7 +675,7 @@ export function depthStrategy( opts: AgenticOptions, cfg: { maxShots: number }, ): Agent> { - const innerTurns = opts.innerTurns ?? 4 + const innerTurns = profileTurnLimit(opts.workerProfile, 'depth worker') let pendingSteer: string | undefined // analyst-derived steer carried between shots return { name: 'depth', @@ -647,7 +687,7 @@ export function depthStrategy( let shots = 0 try { for (shots = 0; shots < cfg.maxShots; shots += 1) { - const child = leaf(`shot:${shots}`, 'shot') + const child = leaf(`shot:${shots}`, 'shot', opts.workerProfile) const memorySteer = await renderCorpusReadback(opts) const steer = [shots === 0 ? undefined : pendingSteer, memorySteer] .filter((part): part is string => typeof part === 'string' && part.trim().length > 0) @@ -665,7 +705,11 @@ export function depthStrategy( progression.push(out.score) if (out.score >= 1 || shots === cfg.maxShots - 1) break // Analyst reads the trajectory (firewalled) → steer the resumed session. - const aChild = leaf(`analyst:${shots}`, 'analyst') + const aChild = leaf( + `analyst:${shots}`, + 'analyst', + opts.analystProfile ?? opts.workerProfile, + ) const aRes = scope.spawn( aChild, { task, messages }, @@ -706,16 +750,20 @@ export function breadthStrategy( opts: AgenticOptions, cfg: { width: number }, ): Agent> { - const innerTurns = opts.innerTurns ?? 4 + const innerTurns = profileTurnLimit(opts.workerProfile, 'breadth worker') return { name: 'breadth', async act(_t, scope): Promise> { let opened = 0 for (let k = 0; k < cfg.width; k += 1) { - const res = scope.spawn(leaf(`rollout:${k}`, 'shot'), { task } as ShotTask, { - budget: perChild(innerTurns), - label: `rollout:${k}`, - }) + const res = scope.spawn( + leaf(`rollout:${k}`, 'shot', opts.workerProfile), + { task } as ShotTask, + { + budget: perChild(innerTurns), + label: `rollout:${k}`, + }, + ) if (res.ok) opened += 1 } if (opened === 0) return { kind: 'blocked', blockers: ['breadth: pool admitted no rollout'] } @@ -791,22 +839,13 @@ export const refine: Strategy = { // driver. (depthStrategy/breadthStrategy are the hand-written reference impls; refine/sample // stay on them — proven — while NEW strategies are authored compactly here.) -/** A role for one shot — multi-agent loops (researcher + engineer, a panel of k - * researchers) give each shot its own system prompt and optionally its own model. */ -export interface ShotPersona { - /** Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is - * injected as a hand-off message (the transcript's earlier roles stay intact). */ - systemPrompt?: string - /** Per-shot model override (e.g. a stronger model for the engineer shot). */ - model?: string -} - export interface ShotSpec { /** present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh one (sample/restart). */ handle?: ArtifactHandle messages?: StrategyMessage[] steer?: string - persona?: ShotPersona + /** Exact profile for this shot. Omitted means `AgenticOptions.workerProfile`. */ + profile?: AgentProfile /** Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot on * the relevant capabilities. Restriction-only; unknown names throw. Omitted ⇒ all. */ tools?: string[] @@ -861,7 +900,6 @@ export function defineStrategy( name, async act(_t, scope): Promise> { let seq = 0 - const innerTurns = opts.innerTurns ?? 4 // HARNESS-VERIFIED scoring: the deliverable score is computed HERE from the shots // the harness actually brokered + scored via surface.score() — NEVER the value the // (possibly authored / adversarial) body returns. An authored strategy cannot @@ -893,7 +931,9 @@ export function defineStrategy( budget, scope, async shot(spec) { - const child = leaf(`shot:${seq}`, 'shot') + const profile = spec?.profile ?? opts.workerProfile + const innerTurns = profileTurnLimit(profile, 'authored strategy shot') + const child = leaf(`shot:${seq}`, 'shot', profile) seq += 1 const res = scope.spawn( child, @@ -902,7 +942,7 @@ export function defineStrategy( handle: spec?.handle, messages: spec?.messages, steer: spec?.steer, - persona: spec?.persona, + profile, tools: spec?.tools, } as ShotTask, { budget: perChild(innerTurns), label: child.name }, @@ -923,7 +963,11 @@ export function defineStrategy( })) }, async critique(messages) { - const child = leaf(`analyst:${seq}`, 'analyst') + const child = leaf( + `analyst:${seq}`, + 'analyst', + opts.analystProfile ?? opts.workerProfile, + ) seq += 1 const res = scope.spawn( child, @@ -937,7 +981,11 @@ export function defineStrategy( return /^\s*COMPLETE\b/i.test(findings) ? null : findings }, async consult(messages, instruction) { - const child = leaf(`analyst:${seq}`, 'analyst') + const child = leaf( + `analyst:${seq}`, + 'analyst', + opts.analystProfile ?? opts.workerProfile, + ) seq += 1 const res = scope.spawn( child, @@ -1096,11 +1144,25 @@ export interface RunAgenticOptions( opts: RunAgenticOptions, ): Promise { + const workerProfile = exactAgenticProfile(opts.workerProfile, 'runAgentic worker') + requiredProfileModel(workerProfile, 'runAgentic worker') + const analystProfile = exactAgenticProfile( + opts.analystProfile ?? workerProfile, + 'runAgentic analyst', + ) + requiredProfileModel(analystProfile, 'runAgentic analyst') + const exactOpts: RunAgenticOptions = { + ...opts, + workerProfile, + analystProfile, + } const strategy: Strategy = opts.strategy ?? (opts.mode === 'breadth' ? sample : refine) - const driver = strategy.driver(opts.surface, opts.task, opts, opts.budget) + const driver = strategy.driver(opts.surface, opts.task, exactOpts, opts.budget) const supervisor = createSupervisor>() + const rootTurnLimit = profileTurnLimit(workerProfile, 'runAgentic worker') const root: Budget = opts.rootBudget ?? { - maxIterations: opts.budget * ((opts.innerTurns ?? 4) + 2), + maxIterations: + opts.budget * ((rootTurnLimit === 0 ? UNBOUNDED_TURN_RESERVATION : rootTurnLimit) + 2), maxTokens: 1_000_000_000, } const started = Date.now() @@ -1109,7 +1171,7 @@ export async function runAgentic runId: `agentic:${strategy.name}:${opts.task.id}`, journal: new InMemorySpawnJournal(), blobs: new InMemoryResultBlobStore(), - executors: agenticRegistry(opts.surface, opts), + executors: agenticRegistry(opts.surface, exactOpts), maxDepth: 3, ...(opts.hooks ? { hooks: opts.hooks } : {}), }) @@ -1122,11 +1184,16 @@ export async function runAgentic } // Drivers deliver the strategy outcome; the cost vector is stamped here from `result.spentTotal` // (the journal aggregate: settled child work + metered driver inference) + wall clock. - const core = result.out.deliverable as Omit + const core = result.out.deliverable as Omit< + AgenticRunResult & Result, + 'usd' | 'usdKnown' | 'ms' | 'tokens' | 'tokensKnown' + > return { ...core, usd: result.spentTotal.usd, + usdKnown: result.spentTotal.usdKnown !== false, tokens: result.spentTotal.tokens, + tokensKnown: result.spentTotal.tokensKnown !== false, ms: Date.now() - started, } as AgenticRunResult & Result } diff --git a/src/runtime/stream-agent-turn.test.ts b/src/runtime/stream-agent-turn.test.ts index 5952dccf..e89a27ec 100644 --- a/src/runtime/stream-agent-turn.test.ts +++ b/src/runtime/stream-agent-turn.test.ts @@ -11,9 +11,15 @@ import type { SandboxEvent } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { inProcessSandboxClient } from './in-process-sandbox-client' -import { collectAgentTurn, streamAgentTurn } from './stream-agent-turn' +import { collectAgentTurn, streamAgentTurn, streamObservedAgentTurn } from './stream-agent-turn' +import { attestRuntimeOwnedExecutor } from './supervise/materialization' import type { Executor, ExecutorFactory, ExecutorResult } from './supervise/types' +const TEST_PROFILE = { + name: 'stream-agent-turn-test', + model: { default: 'offline-test-model' }, +} as const + function finalOf(events: RuntimeStreamEvent[]): RuntimeStreamEvent & { type: 'final' } { const final = events.at(-1) if (final?.type !== 'final') throw new Error('no terminal final event') @@ -35,7 +41,7 @@ describe('streamAgentTurn: box backend', () => { ] as SandboxEvent[]) const seen: RuntimeStreamEvent[] = [] - for await (const event of streamAgentTurn({ kind: 'box', box }, 'say hello')) { + for await (const event of streamObservedAgentTurn({ kind: 'box', box }, 'say hello')) { seen.push(event) } // Incremental events surface in order, before the terminal event. @@ -51,7 +57,7 @@ describe('streamAgentTurn: box backend', () => { expect(final.text).toBe('Hello world') expect(final.metadata).toMatchObject({ tokenUsage: { input: 100, output: 40 }, - costUsd: 0.02, + usdKnown: false, model: 'kimi-k2', }) }) @@ -62,9 +68,9 @@ describe('streamAgentTurn: box backend', () => { { type: 'done', data: { tokenUsage: { inputTokens: 7, outputTokens: 3 } } }, ] as SandboxEvent[]) - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'answer')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'answer')) expect(turn.finalText).toBe('42') - expect(turn.usage).toEqual({ input: 7, output: 3 }) + expect(turn.usage).toEqual({ input: 7, output: 3, usdKnown: false }) expect(turn.status).toBe('completed') expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -82,7 +88,7 @@ describe('streamAgentTurn: box backend', () => { }, }) const box = await client.create() - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'boom')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'boom')) expect(turn.status).toBe('failed') expect(turn.error).toMatchObject({ kind: 'backend', message: 'box exploded' }) const types = turn.events.map((e) => e.type) @@ -113,8 +119,12 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn( - { kind: 'box', box, options: { sessionId: 'sess-1', model: 'kimi-k2' } }, + streamObservedAgentTurn( + { + kind: 'box', + box, + options: { sessionId: 'sess-1', model: 'kimi-k2' }, + }, 'do the task', ), ) @@ -125,7 +135,7 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { if (start?.type !== 'backend_start') throw new Error('expected backend_start') expect(start.backend).toBe('box') expect(turn.finalText).toBe('task output') - expect(turn.usage).toEqual({ input: 9, output: 4, costUsd: 0.01, model: 'kimi-k2' }) + expect(turn.usage).toEqual({ input: 9, output: 4, usdKnown: false, model: 'kimi-k2' }) expect(turn.status).toBe('completed') }) @@ -141,7 +151,7 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'hang', { timeoutMs: 25 }), + streamObservedAgentTurn({ kind: 'box', box }, 'hang', { timeoutMs: 25 }), ) expect(turn.status).toBe('failed') expect(turn.error?.message).toContain('timed out after 25ms') @@ -196,7 +206,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { it('preserveToolParts: true surfaces deduped tool_call/tool_result in-stream', async () => { const box = await makeBox(toolFrames) const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'list files', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'list files', { preserveToolParts: true }), ) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -214,12 +224,12 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { expect(result).toMatchObject({ toolName: 'bash', toolCallId: 'call-1', result: 'file.txt' }) // The projection is additive: text/usage folding is unchanged. expect(turn.finalText).toBe('listed') - expect(turn.usage).toEqual({ input: 5, output: 2 }) + expect(turn.usage).toEqual({ input: 5, output: 2, usdKnown: false }) }) it('default (off) leaves the stream vocabulary unchanged — no tool events', async () => { const box = await makeBox(toolFrames) - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'list files')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'list files')) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', 'text_delta', @@ -244,7 +254,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, ] as SandboxEvent[]) const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'fetch', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'fetch', { preserveToolParts: true }), ) const types = turn.events.map((e) => e.type) expect(types).toEqual(['backend_start', 'tool_call', 'tool_result', 'llm_call', 'final']) @@ -264,7 +274,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'search', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'search', { preserveToolParts: true }), ) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -291,7 +301,7 @@ describe('streamAgentTurn: raw-event tap (onRawEvent)', () => { ] as SandboxEvent[], }) const box = await client.create() - const stream = streamAgentTurn({ kind: 'box', box }, 'go', { + const stream = streamObservedAgentTurn({ kind: 'box', box }, 'go', { onRawEvent: async (event) => { // Async on purpose: the drive must AWAIT the tap before projecting. await Promise.resolve() @@ -334,7 +344,7 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () }, }) const box = await client.create() - for await (const event of streamAgentTurn({ kind: 'box', box }, 'go')) { + for await (const event of streamObservedAgentTurn({ kind: 'box', box }, 'go')) { log.push(`consumed:${event.type}`) // The mid-stream escape: arbitrary awaited work (a vault sync, a retry // decision) runs here while the producer is suspended. @@ -380,11 +390,14 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () let synced = false async function* withLifecycle(): AsyncGenerator { - const first = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'attempt')) + const first = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'attempt')) const noop = first.finalText === '' && first.status === 'completed' if (noop) { // Retry with a steering prompt — the first `final` is never forwarded. - for await (const event of streamAgentTurn({ kind: 'box', box }, 'attempt (retry)')) { + for await (const event of streamObservedAgentTurn( + { kind: 'box', box }, + 'attempt (retry)', + )) { if (event.type === 'final') { synced = true // pre-done lifecycle work completes before forwarding } @@ -408,46 +421,83 @@ describe('streamAgentTurn: executor backend', () => { onTeardown?: () => void hangUntilAbort?: boolean }): ExecutorFactory { - return (_spec, ctx): Executor => ({ - runtime: 'inline', - async execute(task, signal): Promise> { - if (opts?.hangUntilAbort) { - await new Promise((_resolve, reject) => { - const onAbort = () => reject(signal.reason ?? new Error('aborted')) - if (signal.aborted) onAbort() - else signal.addEventListener('abort', onAbort, { once: true }) - // ctx.signal must be the same channel — assert linkage indirectly. - if (ctx.signal.aborted) onAbort() - }) - } - return { - outRef: 'stub-1', - out: { content: `echo: ${String(task)}` }, - spent: { iterations: 1, tokens: { input: 11, output: 6 }, usd: 0.005, ms: 1 }, - } - }, - async teardown() { - opts?.onTeardown?.() - return { destroyed: true } - }, - resultArtifact(): ExecutorResult { - throw new Error('one-shot executor: resultArtifact unused') - }, - }) + return (spec, ctx): Executor => { + const attemptId = ctx.node?.attemptId ?? 'stub-attempt' + const executor: Executor = { + runtime: 'inline', + async execute(task, signal): Promise> { + if (opts?.hangUntilAbort) { + await new Promise((_resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new Error('aborted')) + if (signal.aborted) onAbort() + else signal.addEventListener('abort', onAbort, { once: true }) + // ctx.signal must be the same channel — assert linkage indirectly. + if (ctx.signal.aborted) onAbort() + }) + } + return { + outRef: 'stub-1', + out: { content: `echo: ${String(task)}`, transportAttempts: 2 }, + spent: { iterations: 1, tokens: { input: 11, output: 6 }, usd: 0.005, ms: 1 }, + } + }, + async teardown() { + opts?.onTeardown?.() + return { destroyed: true } + }, + resultArtifact(): ExecutorResult { + throw new Error('one-shot executor: resultArtifact unused') + }, + } + return attestRuntimeOwnedExecutor( + executor, + { + effectiveProfile: spec.profile, + backend: 'inline-test', + model: { status: 'known', id: 'offline-test-model' }, + execution: { kind: 'request', id: attemptId }, + materializer: 'offline-test-executor', + plan: { kind: 'offline-test' }, + }, + { + attemptId, + binding: { kind: 'offline-test', attemptId }, + descriptor: { kind: 'offline-test', transport: 'in-process' }, + }, + ) + } } it('runs the factory once and terminates with the executor usage', async () => { let toreDown = 0 const stream = streamAgentTurn( - { kind: 'executor', factory: stubFactory({ onTeardown: () => toreDown++ }) }, + { + kind: 'executor', + factory: stubFactory({ onTeardown: () => toreDown++ }), + profile: TEST_PROFILE, + }, 'ping', ) const turn = await collectAgentTurn(stream) expect(turn.finalText).toBe('echo: ping') - expect(turn.usage).toEqual({ input: 11, output: 6, costUsd: 0.005 }) + expect(turn.usage).toEqual({ + input: 11, + output: 6, + costUsd: 0.005, + }) expect(turn.status).toBe('completed') + expect(turn.transportAttempts).toBe(2) + const final = turn.events.at(-1) + expect(final?.type).toBe('final') + if (final?.type !== 'final') throw new Error('expected final turn event') + expect(final.metadata).toMatchObject({ transportAttempts: 2 }) // Incremental metering surfaces before the terminal event. - expect(turn.events.map((e) => e.type)).toEqual(['backend_start', 'llm_call', 'final']) + expect(turn.events.map((e) => e.type)).toEqual([ + 'backend_start', + 'llm_call', + 'artifact', + 'final', + ]) expect(toreDown).toBe(1) }) @@ -458,6 +508,7 @@ describe('streamAgentTurn: executor backend', () => { { kind: 'executor', factory: stubFactory({ hangUntilAbort: true, onTeardown: () => toreDown++ }), + profile: TEST_PROFILE, }, 'hang', { signal: controller.signal }, @@ -492,7 +543,10 @@ describe('streamAgentTurn: chat backend', () => { it('streams normalized events and terminates with usage + model', async () => { const seen: RuntimeStreamEvent[] = [] - for await (const event of streamAgentTurn({ kind: 'chat', backend: stubChatBackend() }, 'hi')) { + for await (const event of streamObservedAgentTurn( + { kind: 'chat', backend: stubChatBackend() }, + 'hi', + )) { seen.push(event) } expect(seen.map((e) => e.type)).toEqual([ @@ -512,13 +566,14 @@ describe('streamAgentTurn: chat backend', () => { expect(final.metadata).toMatchObject({ tokenUsage: { input: 21, output: 9 }, model: 'glm-4.6', + usdKnown: false, }) expect(final.metadata).not.toHaveProperty('costUsd') }) it('abort mid-stream terminates with status aborted after partial deltas', async () => { const controller = new AbortController() - const stream = streamAgentTurn( + const stream = streamObservedAgentTurn( { kind: 'chat', backend: stubChatBackend({ hangUntilAbort: true }) }, 'hang', { signal: controller.signal }, @@ -539,8 +594,11 @@ describe('streamAgentTurn: chat backend', () => { it('timeoutMs expiry terminates with status failed (not aborted)', async () => { const turn = await collectAgentTurn( - streamAgentTurn( - { kind: 'chat', backend: stubChatBackend({ hangUntilAbort: true }) }, + streamObservedAgentTurn( + { + kind: 'chat', + backend: stubChatBackend({ hangUntilAbort: true }), + }, 'slow', { timeoutMs: 25, diff --git a/src/runtime/stream-agent-turn.ts b/src/runtime/stream-agent-turn.ts index 79f3bb05..547fa01c 100644 --- a/src/runtime/stream-agent-turn.ts +++ b/src/runtime/stream-agent-turn.ts @@ -1,8 +1,8 @@ /** * `streamAgentTurn` — the ONE run-a-turn event-stream contract over every * execution substrate: a sandbox box (`SandboxInstance.streamPrompt`), a - * one-shot `Executor` (cli-bridge / router / BYO, via `ExecutorFactory`), and - * an in-process `AgentExecutionBackend` (the `resolveAgentBackend` output). + * one-shot Runtime-owned `Executor` (cli-bridge / router / sandbox, via + * `ExecutorFactory`), and an in-process `AgentExecutionBackend`. * * One function, one vocabulary: every backend kind yields the existing * `RuntimeStreamEvent` union incrementally and ALWAYS terminates with a @@ -14,13 +14,14 @@ * This is a UNIFICATION seam, not a new stream parser — each kind is a thin * adapter over code that already exists and is already hardened: * - `box` — `mapSandboxEvent` + `extractLlmCallEvent` (sandbox-events.ts) - * project the sandbox event stream; nothing is re-mapped here. - * - `executor` — `inlineSandboxClient` (the ONE executor→box adapter) turns - * the factory into a box, then the box path drives it. The - * executor's settle/teardown lifecycle stays in that adapter. + * project the sandbox event stream; its requested profile is + * explicitly recorded as unverified because the box already exists. + * - `executor` — Runtime materializes the exact `AgentProfile`, records its + * identity receipts, drives the executor once, and tears it + * down after capturing the terminal artifact. * - `chat` — the backend's own `stream()` surface, normalized by - * `normalizeBackendStreamEvent` (the same projection - * `runAgentTaskStream` applies). + * `normalizeBackendStreamEvent`; its requested profile is likewise + * unverified because an arbitrary backend cannot attest its setup. * * Distinct from `openSandboxRun` (box-only, session resume over one persistent * artifact, raw `SandboxEvent` deliverables) and from `runAgentTaskStream` @@ -47,11 +48,17 @@ */ import { scoreKnowledgeReadiness } from '@tangle-network/agent-eval' +import { + type AgentProfile, + agentProfileSchema, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' import type { PromptOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { normalizeBackendStreamEvent } from '../backends' -import { BackendTransportError } from '../errors' +import { BackendTransportError, ValidationError } from '../errors' import { newRuntimeSession, nowIso } from '../sessions' import type { + AgentBackendInput, AgentExecutionBackend, AgentTaskSpec, AgentTaskStatus, @@ -59,9 +66,31 @@ import type { RuntimeSession, RuntimeStreamEvent, } from '../types' -import { inlineSandboxClient } from './inline-sandbox-client' import { createSandboxToolPartState, mapSandboxEvent, mapSandboxToolEvent } from './sandbox-events' -import type { ExecutorFactory } from './supervise/types' +import { + authoredProfileDigest, + knownExecutionBindingReceipt, + knownMaterializationReceipt, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, + runtimeOwnedPendingExecutorMaterialization, + unknownExecutionBindingReceipt, + unknownMaterializationReceipt, +} from './supervise/materialization' +import { + concreteModelId, + profileBridgeWireModel, + profileProviderModel, +} from './supervise/model-policy' +import type { + ExecutionBindingReceipt, + Executor, + ExecutorFactory, + ExecutorResult, + NodeExecutionIdentity, + ProfileMaterializationReceipt, + UsageEvent, +} from './supervise/types' /** * The execution substrate one turn runs on — a closed discriminated union over @@ -69,7 +98,19 @@ import type { ExecutorFactory } from './supervise/types' * * @experimental */ -export type AgentTurnBackend = +export type AgentTurnBackend = { + /** A Runtime-owned executor factory materialized from this exact canonical profile. */ + kind: 'executor' + factory: ExecutorFactory + /** Exact canonical identity materialized by the executor. */ + profile: AgentProfile + /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ + agentRunName?: string +} + +/** Lower-level observation adapters. They normalize an already-created execution surface but do + * not bind an AgentProfile to it, so they are deliberately absent from the public kernel export. */ +type ObservedAgentTurnBackend = | { /** A live sandbox box: the turn is one `box.streamPrompt(prompt)` call. */ kind: 'box' @@ -85,18 +126,6 @@ export type AgentTurnBackend = /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ agentRunName?: string } - | { - /** - * A one-shot `Executor` (cli-bridge / router / BYO): the factory is - * instantiated fresh for the turn via `inlineSandboxClient`, run once on - * the prompt, and torn down — the same per-spawn lifecycle the supervise - * runtime gives it. - */ - kind: 'executor' - factory: ExecutorFactory - /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ - agentRunName?: string - } | { /** * An in-process `AgentExecutionBackend` (`resolveAgentBackend` output or @@ -106,6 +135,11 @@ export type AgentTurnBackend = backend: AgentExecutionBackend } +/** One prompt or an exact OpenAI-compatible conversation carried as the turn input. */ +export type AgentTurnInput = + | string + | { readonly messages: ReadonlyArray>> } + /** @experimental */ export interface StreamAgentTurnOptions { /** Caller-initiated cancellation. Terminates the stream with `final.status: 'aborted'`. */ @@ -116,6 +150,10 @@ export interface StreamAgentTurnOptions { * (a blown deadline is a turn failure, not a caller cancellation). */ timeoutMs?: number + /** Stable logical paid-call id, forwarded as the provider idempotency key and retained in evidence. */ + callId?: string + /** Caller trace tag retained in evidence and forwarded when the transport supports it. */ + correlationId?: string /** * Opt-in tool-part projection for box and executor backends: sandbox tool * parts additionally surface in-stream as @@ -137,18 +175,205 @@ export interface StreamAgentTurnOptions { onRawEvent?: (event: SandboxEvent) => void | Promise } +function turnIntent(input: AgentTurnInput): string { + if (typeof input === 'string') return input + for (let index = input.messages.length - 1; index >= 0; index -= 1) { + const message = input.messages[index] + if (message?.role === 'user' && typeof message.content === 'string') return message.content + } + return 'structured agent turn' +} + +function turnBackendInput(task: AgentTaskSpec, input: AgentTurnInput): AgentBackendInput { + if (typeof input === 'string') return { task, message: input } + return { + task, + messages: input.messages.map((message) => ({ ...message })) as AgentBackendInput['messages'], + } +} + +function executorEvidence( + executor: Executor, + profile: AgentProfile, + attemptId: string, +): { + materialization: ProfileMaterializationReceipt + executionBinding: ExecutionBindingReceipt +} { + const profileDigest = authoredProfileDigest(profile) + const declaration = runtimeOwnedExecutorMaterialization(executor) + if (!profileDigest || !declaration) { + const materialization = unknownMaterializationReceipt({ + ...(profileDigest ? { authoredProfileDigest: profileDigest } : {}), + runtime: executor.runtime, + reason: declaration ? 'invalid-executor-report' : 'executor-did-not-report', + }) + return { + materialization, + executionBinding: unknownExecutionBindingReceipt( + materialization, + attemptId, + declaration ? 'invalid-executor-report' : 'executor-did-not-report', + ), + } + } + try { + const materialization = knownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + declaration, + }) + const binding = runtimeOwnedExecutorExecutionBinding(executor) + if (!binding || binding.attemptId !== attemptId) throw new Error('executor attempt id mismatch') + return { + materialization, + executionBinding: knownExecutionBindingReceipt(materialization, binding), + } + } catch { + const materialization = unknownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + reason: 'invalid-executor-report', + }) + return { + materialization, + executionBinding: unknownExecutionBindingReceipt( + materialization, + attemptId, + 'invalid-executor-report', + ), + } + } +} + +/** Exact turns require the model-bearing profile to be the sole behavioral authority. Runtime + * checks the trusted declaration before execution so a seam fallback cannot hide behind the same + * profile digest. */ +function assertExactExecutorDeclaration(executor: Executor, profile: AgentProfile): void { + const profileModel = profileProviderModel(profile) + if (!profileModel) { + throw new ValidationError( + 'streamAgentTurn: exact AgentProfile.model.default must name the concrete model', + ) + } + const declaration = + runtimeOwnedExecutorMaterialization(executor) ?? + runtimeOwnedPendingExecutorMaterialization(executor)?.declaration + if (!declaration) return + if (declaration.model.status !== 'known') { + throw new ValidationError( + 'streamAgentTurn: exact executor did not materialize a known model identity', + ) + } + if (declaration.backend === 'router' || declaration.backend === 'router-tools') { + const plan = declaration.plan as { + configuredModel?: unknown + configuredReasoningEffort?: unknown + reasoningEffort?: unknown + } + if (declaration.model.id !== profileModel) { + throw new ValidationError( + 'streamAgentTurn: Router executor model differs from AgentProfile.model.default', + ) + } + if ( + plan.configuredModel !== null && + plan.configuredModel !== undefined && + concreteModelId(String(plan.configuredModel)) !== profileModel + ) { + throw new ValidationError( + 'streamAgentTurn: configured Router model conflicts with AgentProfile.model.default', + ) + } + const expectedEffort = profile.model?.reasoningEffort ?? null + if ( + plan.configuredReasoningEffort !== null && + plan.configuredReasoningEffort !== undefined && + plan.configuredReasoningEffort !== expectedEffort + ) { + throw new ValidationError( + 'streamAgentTurn: configured Router reasoning conflicts with AgentProfile.model.reasoningEffort', + ) + } + if ((plan.reasoningEffort ?? null) !== expectedEffort) { + throw new ValidationError( + 'streamAgentTurn: Router reasoning effort must come from AgentProfile.model.reasoningEffort', + ) + } + } + if (declaration.backend === 'bridge' || declaration.backend === 'bridge-worktree') { + const expectedWireModel = profileBridgeWireModel(profile) + if (!expectedWireModel || declaration.model.id !== expectedWireModel) { + throw new ValidationError( + 'streamAgentTurn: bridge executor model differs from the AgentProfile harness/provider/model wire id', + ) + } + const plan = declaration.plan as { configuredModel?: unknown } + if ( + plan.configuredModel !== null && + plan.configuredModel !== undefined && + concreteModelId(String(plan.configuredModel)) !== expectedWireModel + ) { + throw new ValidationError( + 'streamAgentTurn: configured bridge model conflicts with the AgentProfile harness/provider/model wire id', + ) + } + } +} + +function turnProvenance( + startedAt: number, + timeoutMs: number | undefined, + profileDigest: string | undefined, + taskDigest: string, + materialization: ProfileMaterializationReceipt | undefined, + executionBinding: ExecutionBindingReceipt | undefined, + correlation: Readonly>, +): Record { + const endedAt = Date.now() + return { + ...(profileDigest + ? { + identity: { + profileDigest, + taskDigest, + ...(Object.keys(correlation).length > 0 ? { correlation } : {}), + }, + } + : { taskDigest }), + ...(materialization ? { materialization } : {}), + ...(executionBinding ? { executionBindings: [executionBinding] } : {}), + budget: { timeoutMs: timeoutMs ?? null }, + timing: { + startedAt: new Date(startedAt).toISOString(), + endedAt: new Date(endedAt).toISOString(), + durationMs: endedAt - startedAt, + }, + } +} + /** * Metered usage of one turn, summed over every cost-bearing event the backend - * emitted. `input`/`output` are token counts (0 when the backend reported - * none — the honest sum, never a fabricated estimate). `costUsd`/`model` are - * present only when the backend actually reported them. + * emitted. `input`/`output` are token counts and are accompanied by + * `tokensKnown: false` when the backend did not report them. `costUsd`/`model` + * are present only when the backend actually reported them. * * @experimental */ export interface AgentTurnUsage { input: number output: number + /** Present when a real turn ran but the provider did not report token usage. */ + tokensKnown?: false costUsd?: number + /** Present when Runtime could not prove the full dollar amount. */ + usdKnown?: false + /** Separately-labelled local/catalog estimate; never billed spend. */ + estimatedCostUsd?: number + /** Provider-reported prompt-cache fields; absent fields remain unknown. */ + promptCache?: Readonly> + /** Provider-reported reasoning-token subset of output, when available. */ + reasoningTokens?: number model?: string } @@ -161,7 +386,12 @@ export interface AgentTurnUsage { */ export interface CollectedAgentTurn { finalText: string + /** Exact terminal artifact output from a Runtime-owned executor. */ + output?: unknown usage: AgentTurnUsage + /** Exact underlying transport calls when the Runtime-owned executor reports them. */ + transportAttempts?: number + toolCalls: Array<{ id?: string; name: string; arguments: string }> events: RuntimeStreamEvent[] status: AgentTaskStatus error?: BackendErrorDetail @@ -178,7 +408,17 @@ interface TurnAccumulator { input: number output: number costUsd: number + estimatedCostUsd: number + sawEstimatedCost: boolean + promptCache: Record + reasoningTokens?: number + tokensKnown: boolean + usdKnown: boolean + sawLlmCall: boolean model?: string + stopReason?: string + transportAttempts?: number + result?: ExecutorResult } /** @@ -193,44 +433,226 @@ interface TurnAccumulator { */ export async function* streamAgentTurn( backend: AgentTurnBackend, - prompt: string, + input: AgentTurnInput, opts: StreamAgentTurnOptions = {}, +): AsyncGenerator { + assertTurnTimeout(opts.timeoutMs) + if ((backend as { kind?: unknown }).kind !== 'executor') { + throw new ValidationError( + "streamAgentTurn: exact execution accepts only kind 'executor'; use Runtime-owned creation", + ) + } + yield* streamAgentTurnInternal(backend, input, opts) +} + +/** @internal Normalize a pre-created box/chat stream without asserting an AgentProfile identity. */ +export async function* streamObservedAgentTurn( + backend: ObservedAgentTurnBackend, + input: AgentTurnInput, + opts: StreamAgentTurnOptions = {}, +): AsyncGenerator { + assertTurnTimeout(opts.timeoutMs) + yield* streamAgentTurnInternal(backend, input, opts) +} + +function assertTurnTimeout(timeoutMs: number | undefined): void { + if (timeoutMs === undefined) return + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647) { + throw new ValidationError('streamAgentTurn: timeoutMs must be an integer from 1 to 2147483647') + } +} + +function assertTurnIdentity(value: string | undefined, field: string): void { + if (value !== undefined && (typeof value !== 'string' || value.trim().length === 0)) { + throw new ValidationError(`streamAgentTurn: ${field} must be a non-empty string`) + } +} + +async function* streamAgentTurnInternal( + backend: AgentTurnBackend | ObservedAgentTurnBackend, + input: AgentTurnInput, + opts: StreamAgentTurnOptions, ): AsyncGenerator { const label = backend.kind === 'chat' ? backend.backend.kind : backend.kind - const task: AgentTaskSpec = { id: `turn-${crypto.randomUUID()}`, intent: prompt } - const acc: TurnAccumulator = { deltaText: '', input: 0, output: 0, costUsd: 0 } + const profile = + backend.kind === 'executor' ? agentProfileSchema.parse(backend.profile) : undefined + const profileDigest = profile ? authoredProfileDigest(profile) : undefined + assertTurnIdentity(opts.callId, 'callId') + assertTurnIdentity(opts.correlationId, 'correlationId') + const correlation = { + ...(opts.callId ? { callId: opts.callId } : {}), + ...(opts.correlationId ? { correlationId: opts.correlationId } : {}), + } + const taskInput = input + const taskDigest = canonicalCandidateDigest(taskInput) + const task: AgentTaskSpec = { + id: `turn-${crypto.randomUUID()}`, + intent: turnIntent(input), + ...(profileDigest + ? { + metadata: { + identity: { + profileDigest, + taskDigest, + ...(Object.keys(correlation).length > 0 ? { correlation } : {}), + } satisfies NodeExecutionIdentity, + }, + } + : {}), + } + const acc: TurnAccumulator = { + deltaText: '', + input: 0, + output: 0, + costUsd: 0, + estimatedCostUsd: 0, + sawEstimatedCost: false, + promptCache: {}, + tokensKnown: false, + usdKnown: false, + sawLlmCall: false, + } const deadline = deriveTurnSignal(opts.signal, opts.timeoutMs ?? 0) + const startedAt = Date.now() let session: RuntimeSession | undefined + let executor: Executor | undefined + let materialization: ProfileMaterializationReceipt | undefined + let executionBinding: ExecutionBindingReceipt | undefined try { - session = await startTurnSession(backend, task, prompt, deadline.signal, label) - yield { type: 'backend_start', task, session, backend: label, timestamp: nowIso() } + const nodeId = task.id + const attemptId = `${nodeId}:attempt:${crypto.randomUUID()}` + if (backend.kind === 'executor') { + executor = backend.factory( + { profile: profile!, harness: null }, + { + signal: deadline.signal, + seams: {}, + node: { + rootId: nodeId, + parentId: nodeId, + nodeId, + attemptId, + identity: { + profileDigest: profileDigest!, + taskDigest, + ...(Object.keys(correlation).length > 0 ? { correlation } : {}), + }, + }, + }, + ) + assertExactExecutorDeclaration(executor, profile!) + const pending = runtimeOwnedPendingExecutorMaterialization(executor) + if (pending !== undefined) { + if (pending.runtime !== executor.runtime || pending.binding.attemptId !== attemptId) { + throw new ValidationError( + 'streamAgentTurn: pending executor did not bind the kernel-minted attempt', + ) + } + if (authoredProfileDigest(pending.declaration.effectiveProfile) !== profileDigest) { + throw new ValidationError( + 'streamAgentTurn: pending executor changed the authored AgentProfile before execution', + ) + } + materialization = unknownMaterializationReceipt({ + authoredProfileDigest: profileDigest!, + runtime: executor.runtime, + reason: 'executor-receipt-pending', + }) + executionBinding = unknownExecutionBindingReceipt( + materialization, + attemptId, + 'executor-receipt-pending', + ) + } else { + ;({ materialization, executionBinding } = executorEvidence(executor, profile!, attemptId)) + assertExactExecutorEvidence(materialization, executionBinding) + } + } else { + materialization = unknownMaterializationReceipt({ + runtime: backend.kind === 'box' ? 'sandbox' : label, + reason: 'executor-did-not-report', + }) + executionBinding = unknownExecutionBindingReceipt( + materialization, + attemptId, + 'executor-did-not-report', + ) + } + session = await startTurnSession(backend, task, input, deadline.signal, label) + yield { + type: 'backend_start', + task, + session, + backend: executor?.runtime ?? label, + metadata: { + ...(profileDigest + ? { + identity: { + profileDigest, + taskDigest, + ...(Object.keys(correlation).length > 0 ? { correlation } : {}), + }, + } + : { taskDigest }), + ...(materialization ? { materialization } : {}), + ...(executionBinding ? { executionBindings: [executionBinding] } : {}), + budget: { timeoutMs: opts.timeoutMs ?? null }, + timing: { startedAt: new Date(startedAt).toISOString() }, + }, + timestamp: nowIso(), + } const inner = backend.kind === 'chat' - ? driveChatTurn(backend.backend, task, session, prompt, deadline.signal, acc) - : driveBoxTurn( - backend.kind === 'executor' - ? await inlineSandboxClient(backend.factory).create() - : backend.box, - prompt, - deadline.signal, - backend.agentRunName ?? 'agent', - acc, - { - ...(backend.kind !== 'executor' && backend.options - ? { options: backend.options } - : {}), - preserveToolParts: opts.preserveToolParts === true, - ...(opts.onRawEvent ? { onRawEvent: opts.onRawEvent } : {}), - }, - ) + ? driveChatTurn(backend.backend, task, session, input, deadline.signal, acc) + : backend.kind === 'executor' + ? driveExecutorTurn( + executor!, + task, + session, + input, + deadline.signal, + acc, + materializedModel(materialization, profile!), + ) + : driveBoxTurn( + backend.box, + turnIntent(input), + deadline.signal, + backend.agentRunName ?? 'agent', + acc, + { + ...(backend.options ? { options: backend.options } : {}), + preserveToolParts: opts.preserveToolParts === true, + ...(opts.onRawEvent ? { onRawEvent: opts.onRawEvent } : {}), + }, + ) for await (const event of inner) { yield event throwIfAborted(deadline.signal) } - yield buildFinalEvent(task, session, acc, { status: 'completed', reason: 'turn completed' }) + if (backend.kind === 'executor') { + ;({ materialization, executionBinding } = executorEvidence(executor!, profile!, attemptId)) + assertExactExecutorEvidence(materialization, executionBinding) + } + + yield buildFinalEvent( + task, + session, + acc, + { status: 'completed', reason: 'turn completed' }, + turnProvenance( + startedAt, + opts.timeoutMs, + profileDigest, + taskDigest, + materialization, + executionBinding, + correlation, + ), + ) } catch (err) { const callerAborted = opts.signal?.aborted === true const status: AgentTaskStatus = callerAborted ? 'aborted' : 'failed' @@ -249,12 +671,43 @@ export async function* streamAgentTurn( error, timestamp: nowIso(), } - yield buildFinalEvent(task, session, acc, { status, reason: message, error }) + yield buildFinalEvent( + task, + session, + acc, + { status, reason: message, error }, + turnProvenance( + startedAt, + opts.timeoutMs, + profileDigest, + taskDigest, + materialization, + executionBinding, + correlation, + ), + ) } finally { + await executor?.teardown('brutalKill').catch(() => undefined) deadline.dispose() } } +function assertExactExecutorEvidence( + materialization: ProfileMaterializationReceipt, + executionBinding: ExecutionBindingReceipt, +): void { + if (materialization.status !== 'known' || executionBinding.status !== 'known') { + throw new ValidationError( + 'streamAgentTurn: exact profile execution requires terminally validated Runtime materialization and execution binding evidence', + ) + } + if (materialization.effectiveProfileDigest !== materialization.authoredProfileDigest) { + throw new ValidationError( + 'streamAgentTurn: executor changed the authored AgentProfile; exact turn execution refuses profile overlays', + ) + } +} + /** * Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that * honors its terminal contract) into the turn summary plus the full event @@ -275,6 +728,10 @@ export async function collectAgentTurn( ) } const metadata = final.metadata ?? {} + const resultMetadata = + metadata.result && typeof metadata.result === 'object' + ? (metadata.result as Record) + : undefined const tokenUsage = metadata.tokenUsage && typeof metadata.tokenUsage === 'object' ? (metadata.tokenUsage as Record) @@ -282,15 +739,35 @@ export async function collectAgentTurn( const usage: AgentTurnUsage = { input: finiteNumber(tokenUsage.input) ?? 0, output: finiteNumber(tokenUsage.output) ?? 0, + ...(metadata.tokensKnown === false ? { tokensKnown: false as const } : {}), } const costUsd = finiteNumber(metadata.costUsd) if (costUsd !== undefined) usage.costUsd = costUsd + if (metadata.usdKnown === false) usage.usdKnown = false + const estimatedCostUsd = finiteNumber(metadata.estimatedCostUsd) + if (estimatedCostUsd !== undefined) usage.estimatedCostUsd = estimatedCostUsd + if (metadata.promptCache && typeof metadata.promptCache === 'object') { + usage.promptCache = metadata.promptCache as Record + } + const reasoningTokens = finiteNumber(metadata.reasoningTokens) + if (reasoningTokens !== undefined) usage.reasoningTokens = reasoningTokens if (typeof metadata.model === 'string' && metadata.model.length > 0) { usage.model = metadata.model } + const transportAttempts = finiteNumber(metadata.transportAttempts) + const toolCalls = events + .filter((event) => event.type === 'tool_call') + .map((event) => ({ + ...(event.toolCallId ? { id: event.toolCallId } : {}), + name: event.toolName, + arguments: typeof event.args === 'string' ? event.args : JSON.stringify(event.args ?? {}), + })) return { finalText: final.text ?? '', + ...(resultMetadata && 'output' in resultMetadata ? { output: resultMetadata.output } : {}), usage, + ...(transportAttempts !== undefined ? { transportAttempts } : {}), + toolCalls, events, status: final.status, ...(final.error ? { error: final.error } : {}), @@ -301,17 +778,18 @@ export async function collectAgentTurn( * correlation session otherwise. Box/executor turns carry no server session * here — resume lives in `openSandboxRun`/`SandboxLineage`, not this primitive. */ async function startTurnSession( - backend: AgentTurnBackend, + backend: AgentTurnBackend | ObservedAgentTurnBackend, task: AgentTaskSpec, - prompt: string, + input: AgentTurnInput, signal: AbortSignal, label: string, ): Promise { if (backend.kind === 'chat' && backend.backend.start) { - return backend.backend.start( - { task, message: prompt }, - { task, knowledge: emptyReadiness(task), signal }, - ) + return backend.backend.start(turnBackendInput(task, input), { + task, + knowledge: emptyReadiness(task), + signal, + }) } return newRuntimeSession(label) } @@ -370,19 +848,180 @@ async function* driveChatTurn( backend: AgentExecutionBackend, task: AgentTaskSpec, session: RuntimeSession, - prompt: string, + input: AgentTurnInput, signal: AbortSignal, acc: TurnAccumulator, ): AsyncGenerator { - const input = { task, message: prompt } + const backendInput = turnBackendInput(task, input) const context = { task, knowledge: emptyReadiness(task), session, signal } - for await (const raw of backend.stream(input, context)) { + for await (const raw of backend.stream(backendInput, context)) { const event = normalizeBackendStreamEvent(raw, task, session) foldEvent(event, acc) yield event } } +async function* driveExecutorTurn( + executor: Executor, + task: AgentTaskSpec, + session: RuntimeSession, + input: AgentTurnInput, + signal: AbortSignal, + acc: TurnAccumulator, + declaredModel: string | undefined, +): AsyncGenerator { + const taskValue = typeof input === 'string' ? input : { messages: input.messages } + const run = executor.execute(taskValue, signal) + let result: ExecutorResult + if (isAsyncIterable(run)) { + for await (const _usage of run) { + throwIfAborted(signal) + } + result = executor.resultArtifact() + } else { + result = await run + } + acc.result = result + acc.terminalText = executorResultText(result.out) + acc.input = result.spent.tokens.input + acc.output = result.spent.tokens.output + acc.costUsd = result.spent.usd + const estimatedCostUsd = executorResultEstimatedCost(result.out) + if (estimatedCostUsd !== undefined) { + acc.estimatedCostUsd = estimatedCostUsd + acc.sawEstimatedCost = true + } + Object.assign(acc.promptCache, executorResultPromptCache(result.out)) + acc.reasoningTokens = executorResultReasoningTokens(result.out) + acc.tokensKnown = result.spent.tokensKnown !== false + acc.usdKnown = result.spent.usdKnown !== false + acc.sawLlmCall = true + + // Usage model identity is observational: the executor must report what answered. The profile's + // declared model remains available in materialization evidence, but it is not fabricated into + // `usage.model` when the provider/transport omitted an actual response model. + const model = executorResultModel(result.out) + if (model) acc.model = model + acc.stopReason = executorResultStopReason(result.out) + acc.transportAttempts = executorResultTransportAttempts(result.out) + const latencyMs = result.spent.ms + yield { + type: 'llm_call', + task, + session, + model: model ?? declaredModel ?? executor.runtime, + ...(acc.tokensKnown ? { tokensIn: acc.input, tokensOut: acc.output } : {}), + ...(acc.usdKnown ? { costUsd: acc.costUsd } : {}), + ...(acc.tokensKnown ? {} : { tokensKnown: false }), + ...(acc.usdKnown ? {} : { usdKnown: false }), + ...(acc.sawEstimatedCost ? { estimatedCostUsd: acc.estimatedCostUsd } : {}), + ...(Object.keys(acc.promptCache).length > 0 ? { promptCache: acc.promptCache } : {}), + latencyMs, + timestamp: nowIso(), + } + for (const call of executorResultToolCalls(result.out)) { + yield { + type: 'tool_call', + task, + session, + toolName: call.name, + ...(call.id ? { toolCallId: call.id } : {}), + args: call.arguments, + timestamp: nowIso(), + } + } + yield { + type: 'artifact', + task, + session, + artifactId: result.outRef, + name: 'agent-turn-result', + metadata: { + spend: result.spent, + ...(result.verdict ? { verdict: result.verdict } : {}), + }, + timestamp: nowIso(), + } +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return typeof value === 'object' && value !== null && Symbol.asyncIterator in value +} + +function executorResultText(value: unknown): string { + if (typeof value === 'string') return value + if (!value || typeof value !== 'object') return '' + const content = (value as Record).content + return typeof content === 'string' ? content : '' +} + +function executorResultModel(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const model = (value as Record).model + return typeof model === 'string' && model.length > 0 ? model : undefined +} + +function executorResultStopReason(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const reason = (value as Record).finishReason + return typeof reason === 'string' && reason.length > 0 ? reason : undefined +} + +function executorResultTransportAttempts(value: unknown): number | undefined { + if (!value || typeof value !== 'object') return undefined + const attempts = (value as Record).transportAttempts + return typeof attempts === 'number' && Number.isSafeInteger(attempts) && attempts > 0 + ? attempts + : undefined +} + +function executorResultEstimatedCost(value: unknown): number | undefined { + if (!value || typeof value !== 'object') return undefined + return finiteNumber((value as Record).estimatedCostUsd) +} + +function executorResultPromptCache(value: unknown): Record { + if (!value || typeof value !== 'object') return {} + const raw = (value as Record).promptCache + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {} + const cache: Record = {} + for (const [key, entry] of Object.entries(raw)) { + if ((typeof entry === 'number' && Number.isFinite(entry)) || typeof entry === 'string') { + cache[key] = entry + } + } + return cache +} + +function executorResultReasoningTokens(value: unknown): number | undefined { + if (!value || typeof value !== 'object') return undefined + const count = (value as Record).reasoningTokens + return Number.isSafeInteger(count) && (count as number) >= 0 ? (count as number) : undefined +} + +function executorResultToolCalls( + value: unknown, +): Array<{ id?: string; name: string; arguments: string }> { + if (!value || typeof value !== 'object') return [] + const raw = (value as Record).toolCalls + if (!Array.isArray(raw)) return [] + return raw.flatMap((entry) => { + if (!entry || typeof entry !== 'object') return [] + const call = entry as Record + if (typeof call.name !== 'string') return [] + return [ + { + ...(typeof call.id === 'string' ? { id: call.id } : {}), + name: call.name, + arguments: + typeof call.arguments === 'string' + ? call.arguments + : JSON.stringify(call.arguments ?? {}), + }, + ] + }) +} + /** Fold one normalized event into the turn accumulator (text + usage). * `fallbackModelLabel` — a mapper-stamped run label to exclude from * `usage.model` (it is not a backend-reported model). */ @@ -396,9 +1035,25 @@ function foldEvent( return } if (event.type === 'llm_call') { + const tokensReported = + event.tokensKnown !== false && event.tokensIn !== undefined && event.tokensOut !== undefined + const usdReported = event.usdKnown !== false && event.costUsd !== undefined + if (!acc.sawLlmCall) { + acc.tokensKnown = tokensReported + acc.usdKnown = usdReported + acc.sawLlmCall = true + } else { + acc.tokensKnown &&= tokensReported + acc.usdKnown &&= usdReported + } acc.input += event.tokensIn ?? 0 acc.output += event.tokensOut ?? 0 acc.costUsd += event.costUsd ?? 0 + if (event.estimatedCostUsd !== undefined) { + acc.estimatedCostUsd += event.estimatedCostUsd + acc.sawEstimatedCost = true + } + if (event.promptCache) Object.assign(acc.promptCache, event.promptCache) if (event.model && event.model !== fallbackModelLabel) acc.model = event.model } } @@ -424,6 +1079,7 @@ function buildFinalEvent( session: RuntimeSession | undefined, acc: TurnAccumulator, outcome: { status: AgentTaskStatus; reason: string; error?: BackendErrorDetail }, + provenance: Record, ): RuntimeStreamEvent { const finalText = acc.terminalText ?? acc.deltaText return { @@ -431,18 +1087,44 @@ function buildFinalEvent( task, ...(session ? { session } : {}), status: outcome.status, - reason: outcome.reason, + reason: outcome.status === 'completed' ? (acc.stopReason ?? outcome.reason) : outcome.reason, ...(finalText ? { text: finalText } : {}), metadata: { tokenUsage: { input: acc.input, output: acc.output }, - ...(acc.costUsd > 0 ? { costUsd: acc.costUsd } : {}), + ...(acc.tokensKnown ? {} : { tokensKnown: false }), + ...(acc.usdKnown ? { costUsd: acc.costUsd } : { usdKnown: false }), + ...(acc.sawEstimatedCost ? { estimatedCostUsd: acc.estimatedCostUsd } : {}), + ...(Object.keys(acc.promptCache).length > 0 ? { promptCache: acc.promptCache } : {}), + ...(acc.reasoningTokens !== undefined ? { reasoningTokens: acc.reasoningTokens } : {}), ...(acc.model ? { model: acc.model } : {}), + ...(acc.stopReason ? { stopReason: acc.stopReason } : {}), + ...(acc.transportAttempts !== undefined ? { transportAttempts: acc.transportAttempts } : {}), + ...(acc.result + ? { + result: { + outRef: acc.result.outRef, + output: acc.result.out, + ...(acc.result.verdict ? { verdict: acc.result.verdict } : {}), + spent: acc.result.spent, + }, + } + : {}), + ...provenance, }, ...(outcome.error ? { error: outcome.error } : {}), timestamp: nowIso(), } } +function materializedModel( + receipt: ProfileMaterializationReceipt | undefined, + profile: AgentProfile, +): string | undefined { + if (receipt?.status === 'known' && receipt.model.status === 'known') return receipt.model.id + const fallback = profile.model?.default + return typeof fallback === 'string' && fallback.length > 0 ? fallback : undefined +} + /** Minimal ready-by-construction readiness report for a requirement-free turn. */ function emptyReadiness(task: AgentTaskSpec) { return scoreKnowledgeReadiness({ taskId: task.id, requirements: [] }) diff --git a/src/runtime/structural-rollout.test.ts b/src/runtime/structural-rollout.test.ts index 3e784cf2..8629cf78 100644 --- a/src/runtime/structural-rollout.test.ts +++ b/src/runtime/structural-rollout.test.ts @@ -85,7 +85,7 @@ describe('repair keep-best guard', () => { }) describe('modelAuthoredChecks — the assert filter (visible info only, frozen per task)', () => { - const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'Write add.' } + const task: AgenticTask = { id: 't', userPrompt: 'Write add.' } it('keeps only single-line paren-balanced asserts mentioning the entry symbol, capped', async () => { const reply = [ @@ -141,11 +141,10 @@ describe('officialChecksFromMeta', () => { const source = officialChecksFromMeta() const withMeta: AgenticTask = { id: 't', - systemPrompt: 's', userPrompt: 'u', meta: { visibleChecks: ['assert f() == 1', 42, ' '] }, } - const bare: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const bare: AgenticTask = { id: 't', userPrompt: 'u' } const ctx = { count: 0, consult: async () => null } expect(await source.generate(withMeta, ctx)).toEqual([ { code: 'assert f() == 1', kind: 'official' }, @@ -155,7 +154,7 @@ describe('officialChecksFromMeta', () => { }) describe('sandboxCheckRunner', () => { - const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const task: AgenticTask = { id: 't', userPrompt: 'u' } const checks: VisibleCheck[] = [ { code: 'assert f() == 1', kind: 'official' }, { code: 'assert f() != 2', kind: 'authored' }, @@ -277,10 +276,15 @@ describe('structuralRollout — the strategy, end to end (offline transport, fak const result = await runAgentic({ surface, - task: { id: 'starved', systemPrompt: 'Solve it.', userPrompt: 'Solve it.' }, + task: { id: 'starved', userPrompt: 'Solve it.' }, routerBaseUrl: 'http://offline.test/v1', routerKey: 'k', - model: 'stub-model', + workerProfile: { + name: 'starved-worker', + model: { default: 'stub-model', metadata: { maxTokens: 64 } }, + prompt: { systemPrompt: 'Solve it.' }, + tools: {}, + }, complete: async () => { throw new Error('candidate should not run') }, @@ -394,13 +398,21 @@ describe('structuralRollout — the strategy, end to end (offline transport, fak const result = await runAgentic({ surface, - task: { id: 't1', systemPrompt: 'Solve it.', userPrompt: 'Write f.\n\ndef f():\n ...' }, + task: { id: 't1', userPrompt: 'Write f.\n\ndef f():\n ...' }, routerBaseUrl: 'http://offline.test/v1', routerKey: 'k', - model: 'stub-model', + workerProfile: { + name: 'structural-worker', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'stub-model', + metadata: { maxTurns: 2 }, + }, + prompt: { systemPrompt: 'Solve it.' }, + tools: { submit_answer: true }, + }, complete, - innerTurns: 2, - maxTokens: 64, strategy: structuralRollout({ policy: { k: 2, repairRounds: 2, testgen: 0 }, checkSource: { generate }, diff --git a/src/runtime/structural-rollout.ts b/src/runtime/structural-rollout.ts index 85d4989a..3a9b4744 100644 --- a/src/runtime/structural-rollout.ts +++ b/src/runtime/structural-rollout.ts @@ -53,8 +53,6 @@ export interface StructuralRolloutPolicy { /** Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). * Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. */ diverse?: boolean - /** Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. */ - temperature?: number } /** The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. */ @@ -684,12 +682,5 @@ export function structuralRollout( }, ) - if (policy.temperature === undefined) return inner - // The shot temperature is an AgenticOptions concern; the policy override threads in at - // the driver seam so the strategy stays a plain defineStrategy member. - return { - name: inner.name, - driver: (surface, task, opts, budget) => - inner.driver(surface, task, { ...opts, temperature: policy.temperature }, budget), - } + return inner } diff --git a/src/runtime/supervise-surface.ts b/src/runtime/supervise-surface.ts index 0ffc9523..f221b4b2 100644 --- a/src/runtime/supervise-surface.ts +++ b/src/runtime/supervise-surface.ts @@ -16,7 +16,7 @@ import { OUTPUT_VALUE, type TraceAnalysisStore } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import type { AnalystRegistry, MakeWorkerAgent } from '../mcp/tools/coordination' -import type { RouterConfig } from './router-client' +import type { RouterTransportConfig } from './router-client' import { type AgenticSurface, type AgenticTask, @@ -185,8 +185,9 @@ function missingRunTestsEvidence(): { summary: string } { export interface SurfaceWorkerConfig { readonly routerBaseUrl: string readonly routerKey: string - readonly model: string - readonly maxTokens?: number + /** Exact worker behavior, tools, and model. */ + readonly profile: AgentProfile + readonly analystProfile?: AgentProfile readonly innerTurns?: number /** Refine-shot budget for ONE worker attempt (max steered shots). Default 1. */ readonly budget?: number @@ -210,7 +211,7 @@ function surfaceWorkerExecutor( const attemptTask: AgenticTask = guidance ? { ...task, - systemPrompt: `${task.systemPrompt ?? ''}\n\n— Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) —\n${guidance}`, + userPrompt: `${task.userPrompt}\n\n— Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) —\n${guidance}`, } : task const r = await runAgentic({ @@ -220,8 +221,8 @@ function surfaceWorkerExecutor( budget: worker.budget ?? 1, routerBaseUrl: worker.routerBaseUrl, routerKey: worker.routerKey, - model: worker.model, - ...(worker.maxTokens !== undefined ? { maxTokens: worker.maxTokens } : {}), + workerProfile: worker.profile, + ...(worker.analystProfile ? { analystProfile: worker.analystProfile } : {}), ...(worker.innerTurns !== undefined ? { innerTurns: worker.innerTurns } : {}), }) const out: SurfaceWorkerOut = { @@ -231,7 +232,14 @@ function surfaceWorkerExecutor( summary: `${strategy.name} ${r.shots} shot(s) → ${(100 * r.score).toFixed(0)}% (${r.resolved ? 'resolved' : 'unresolved'})`, failing: r.resolved ? [] : traced.failing(), } - const spent: Spend = { iterations: r.completions, tokens: r.tokens, usd: r.usd, ms: r.ms } + const spent: Spend = { + iterations: r.completions, + tokens: r.tokens, + ...(r.tokensKnown ? {} : { tokensKnown: false }), + usd: r.usd, + ...(r.usdKnown ? {} : { usdKnown: false }), + ms: r.ms, + } artifact = { outRef: `surface-worker:${task.id}:${r.shots}:${r.resolved ? 'ok' : 'no'}`, out, @@ -257,9 +265,8 @@ export interface SuperviseSurfaceOptions { /** The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop * bounds for a handful of worker spawns — raise it to let the driver try more. */ readonly budget?: Budget - /** The driver brain's router substrate (its own inference). Default: the worker's router + model — the - * driver and workers share one router unless you separate them (e.g. a stronger driver model). */ - readonly router?: RouterConfig + /** The driver brain's Router endpoint/auth. Model and behavior remain owned by `profile`. */ + readonly router?: RouterTransportConfig /** The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()` * (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the * within-run self-improvement OFF (the driver sees raw settled outputs). */ @@ -296,15 +303,16 @@ export async function superviseSurface( // Default the driver to the worker's router (one router unless separated) and the pool to a handful of // worker spawns sized off the worker bounds — so the minimal call is // `superviseSurface(profile, task, { surface, worker })`. - const router = opts.router ?? { + const router: RouterTransportConfig = opts.router ?? { routerBaseUrl: opts.worker.routerBaseUrl, routerKey: opts.worker.routerKey, - model: opts.worker.model, } const budget = opts.budget ?? { maxIterations: (innerTurns + 2) * 5 + 16, - maxTokens: (opts.worker.maxTokens ?? 4000) * 8, + maxTokens: 1_000_000_000, } + const workerMaxTokens = + profileMaxTokens(opts.worker.profile) ?? Math.max(1, Math.floor(budget.maxTokens / 8)) // Every spawned worker is a BYO executor that runs the surface task; the deliverable is the completion // oracle (delivered ⟺ the surface check passed). @@ -341,7 +349,7 @@ export async function superviseSurface( maxLiveWorkers: opts.maxLiveWorkers ?? 1, // A SMALL per-worker reservation so MULTIPLE workers fit the pool (the default reserves the whole pool // per worker → only one ever spawns, defeating the spawn-a-targeted-worker steering). - perWorker: { maxIterations: innerTurns + 2, maxTokens: opts.worker.maxTokens ?? 4000 }, + perWorker: { maxIterations: innerTurns + 2, maxTokens: workerMaxTokens }, router, ...(analysts ? { analysts, analyzeOnSettle: analysts.kinds.map((k) => k.id) } : {}), }) @@ -358,3 +366,14 @@ export async function superviseSurface( completions: sp.iterations, } } + +function profileMaxTokens(profile: AgentProfile): number | undefined { + const value = profile.model?.metadata?.maxTokens + if (value === undefined) return undefined + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new Error( + 'superviseSurface: AgentProfile.model.metadata.maxTokens must be a positive safe integer', + ) + } + return value as number +} diff --git a/src/runtime/supervise/authoring.ts b/src/runtime/supervise/authoring.ts index 43f910cd..74802240 100644 --- a/src/runtime/supervise/authoring.ts +++ b/src/runtime/supervise/authoring.ts @@ -5,8 +5,7 @@ * Every agent here is three things: instructions (system prompt), tools, and a model — its * `AgentProfile`. The supervisor's job is to WRITE those profiles: read the task, decompose it, * and for each sub-task author a tailored worker recipe. `supervisorInstructions` is the how-to the - * supervisor reads (its system prompt); `authoredWorker` builds a worker AGENT from a profile the - * supervisor authored — the authored systemPrompt + model shape the worker's call. + * supervisor reads; canonical Runtime executors materialize the resulting profile. * * The skill is the single OPTIMIZABLE surface: edit it → the supervisor designs better agents. * That is the self-improvement lever (the prompt/skill lever), not the execution plumbing. @@ -20,14 +19,7 @@ import { type AgentProfilePrompt, agentProfileSchema, } from '@tangle-network/agent-interface' -import { contentAddress } from '../../durable/spawn-journal' -import { ValidationError } from '../../errors' -import { type RouterConfig, routerChatWithUsage } from '../router-client' -import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' -import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' -import { concreteProfileModel } from './model-policy' import { supervisorPolicyPrompt } from './prompt-registry' -import type { Agent, AgentSpec, Executor, ExecutorResult } from './types' /** What the supervisor AUTHORS per sub-task: one complete canonical profile whose name and * task-specific system prompt are present. Every other `AgentProfile` axis is preserved exactly. */ @@ -53,52 +45,6 @@ export function asAuthoredProfile(raw: unknown): AuthoredProfile | null { } } -/** - * Lift a profile the supervisor AUTHORED into the canonical shape every executor reads. - * - * The skill asks for `systemPrompt` and `model` as flat fields — the vocabulary a model writes - * well — while `AgentProfile` carries them as `prompt.systemPrompt` and `model.default`. Nothing - * downstream reads the flat form: the router and cli-bridge leaves read `profile.prompt - * .systemPrompt`, and the sandbox leaf hands the profile to a strict schema that REJECTS the flat - * key outright (`Unrecognized key: "systemPrompt"`), which fails the worker's every round. Lift - * both here, once, so what the supervisor writes is what the worker runs. - * - * Purely additive: a profile already canonical is returned untouched, and a flat field is dropped - * only after its canonical slot is filled. Both spellings of the same standing instruction, set to - * DIFFERENT text, is a contradiction with no safe reading — it fails loud, matching - * `resolveSupervisorProfile`'s rule for the supervisor's own profile. - */ -export function canonicalizeAuthoredProfile(raw: unknown): AgentProfile { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return (raw ?? {}) as AgentProfile - const authored = { ...(raw as Record) } - - const flatPrompt = authored.systemPrompt - if (typeof flatPrompt === 'string' && flatPrompt.trim().length > 0) { - const prompt = - authored.prompt && typeof authored.prompt === 'object' && !Array.isArray(authored.prompt) - ? { ...(authored.prompt as Record) } - : {} - const canonicalPrompt = prompt.systemPrompt - if (typeof canonicalPrompt === 'string' && canonicalPrompt !== flatPrompt) { - throw new ValidationError( - 'canonicalizeAuthoredProfile: prompt.systemPrompt and systemPrompt are both set and differ ' + - '— they are the same standing instruction, so author exactly one ' + - `(prompt.systemPrompt: ${JSON.stringify(canonicalPrompt.slice(0, 80))}; ` + - `systemPrompt: ${JSON.stringify(flatPrompt.slice(0, 80))})`, - ) - } - if (typeof canonicalPrompt !== 'string') prompt.systemPrompt = flatPrompt - authored.prompt = prompt - delete authored.systemPrompt - } - - if (typeof authored.model === 'string' && authored.model.trim().length > 0) { - authored.model = { default: authored.model } - } - - return authored as AgentProfile -} - /** The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable * surface: editing this changes how the supervisor designs every agent it spawns. * @@ -128,86 +74,6 @@ export function supervisorInstructions(opts?: { goal?: string }): string { ].join('\n') } -/** Build a router-only worker from an authored profile. This helper executes the prompt/model axes; - * use `workerFromBackend` for full materialization of tools, MCP, resources, hooks, and subagents. */ -export function authoredWorker( - profile: AuthoredProfile, - opts: { - cfg: RouterConfig - taskPrompt: string - deliverable: DeliverableSpec - temperature?: number - }, -): Agent { - const model = concreteProfileModel(profile) ?? opts.cfg.model - const executorFactory: NonNullable = (spec, ctx) => { - let artifact: ExecutorResult | undefined - const executionId = ctx.node?.nodeId ?? `authored-router-${profile.name}` - const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) - const inner: Executor = attestRuntimeOwnedExecutor( - { - runtime: 'router', - async execute(_t, signal) { - const res = await routerChatWithUsage( - { ...opts.cfg, model }, - [ - { role: 'system', content: profile.prompt.systemPrompt }, - { role: 'user', content: opts.taskPrompt }, - ], - { temperature: opts.temperature ?? 0.4, ...(signal ? { signal } : {}) }, - ) - artifact = { - outRef: contentAddress(res.content), - out: res.content, - spent: { - iterations: 1, - tokens: res.usage ?? { input: 0, output: 0 }, - usd: res.costUsd ?? 0, - ms: 0, - }, - } - return artifact - }, - teardown: () => Promise.resolve({ destroyed: true }), - resultArtifact: () => { - if (!artifact) throw new Error('authoredWorker: resultArtifact read before execute') - return artifact - }, - }, - { - effectiveProfile: spec.profile, - backend: 'router', - model: { status: 'known', id: model }, - execution: { kind: 'request', id: executionId }, - materializer: 'authored-router-prompt', - plan: { - kind: 'authored-router-completion', - model, - temperature: opts.temperature ?? 0.4, - taskPrompt: opts.taskPrompt, - }, - }, - { - attemptId, - binding: { endpoint: opts.cfg.routerBaseUrl, executionId, model }, - descriptor: { kind: 'router-request', transport: 'http', backend: 'router' }, - }, - ) - return gateOnDeliverable(inner, opts.deliverable) - } - const spec: AgentSpec = { - profile, - harness: null, - executorFactory, - } - return { name: profile.name, act: async () => '', executorSpec: spec } as Agent< - unknown, - unknown - > & { - executorSpec: AgentSpec - } -} - // ── Profile-richness gate ──────────────────────────────────────────────────── // // The supervisor's product is the worker PROFILE it authors. The failure mode the existing diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index dbabfd00..a3a72e32 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -1,12 +1,105 @@ -import { createServer, type Server } from 'node:http' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + type AgentProfile, + canonicalAgentProfileDigest, + type ReasoningEffort, +} from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { spendFromUsageEvents } from './budget' import { bridgeExecutor } from './runtime' import type { UsageEvent } from './types' const TEST_RUN_DIGEST = `sha256:${'b'.repeat(64)}` +const TEST_WORKSPACE_DIGEST = `sha256:${'a'.repeat(64)}` + +function respondBridgeCapabilities(req: IncomingMessage, res: ServerResponse): boolean { + if (req.method !== 'GET' || req.url !== '/') return false + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + return true +} + +function appliedReasoning(harness: string, requested: ReasoningEffort | null): string | null { + if (requested === null) return null + if (harness === 'pi') { + if (requested === 'none') return 'off' + return requested === 'ultracode' ? 'xhigh' : requested + } + if (harness === 'claude-code') { + if (requested === 'none' || requested === 'minimal') return 'low' + return requested === 'ultracode' ? 'max' : requested + } + if (harness === 'codex') { + if (requested === 'none') return 'minimal' + return requested === 'xhigh' || requested === 'ultracode' ? 'high' : requested + } + if (harness === 'kimi-code') { + if (requested === 'medium') return null + return requested === 'none' || requested === 'minimal' || requested === 'low' + ? '--no-thinking' + : '--thinking' + } + if (harness === 'gemini') return null + return requested +} + +function bridgeProfileReceipt(body: Record): Record { + const profile = body.agent_profile as AgentProfile + const model = String(body.model) + const harness = model.split('/')[0] ?? model + const parts = model.split('/') + const requested = profile.model?.reasoningEffort ?? null + return { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: canonicalAgentProfileDigest(profile), + harness, + provider: parts.length >= 3 ? (parts[1] ?? null) : (profile.model?.provider ?? null), + model, + reasoningEffort: { requested, applied: appliedReasoning(harness, requested) }, + workspacePlanDigest: TEST_WORKSPACE_DIGEST, + files: [], + unsupported: [], + } +} + +/** Upgrade a fixture to the exact v2 wire: explicit cost provenance on every usage frame and one + * terminal profile acknowledgement before `[DONE]`. */ +function bridgeProtocolSse(body: string, requestBody: Record): string { + const usageBound = body + .split('\n') + .map((line) => { + if (!line.startsWith('data: ') || line === 'data: [DONE]') return line + try { + const payload = JSON.parse(line.slice('data: '.length)) as Record + if (payload.usage && typeof payload.usage === 'object') { + const usage = payload.usage as Record + payload.usage = { + ...usage, + ...(typeof usage.cost === 'number' + ? { cost_known: true, cost_provenance: 'provider-receipt' } + : { cost_known: false }), + } + } + return `data: ${JSON.stringify(payload)}` + } catch { + return line + } + }) + .join('\n') + if (!usageBound.includes('data: [DONE]')) return usageBound + return usageBound.replace( + 'data: [DONE]', + `data: ${JSON.stringify({ profile_materialization: bridgeProfileReceipt(requestBody) })}\n\ndata: [DONE]`, + ) +} function numberSseDataFrames(body: string): string { let seq = 0 @@ -52,10 +145,12 @@ async function startBridgeStub( opts: { status?: number contentType?: string + protocol?: boolean onRequest?: (body: Record) => void } = {}, ): Promise<{ url: string; server: Server }> { const server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const chunks: Buffer[] = [] for await (const chunk of req) chunks.push(Buffer.from(chunk)) const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') as Record< @@ -72,7 +167,7 @@ async function startBridgeStub( }) res.end( (opts.contentType ?? 'text/event-stream') === 'text/event-stream' - ? numberSseDataFrames(body) + ? numberSseDataFrames(opts.protocol === false ? body : bridgeProtocolSse(body, requestBody)) : body, ) }) @@ -82,12 +177,16 @@ async function startBridgeStub( } function makeExecutor(bridgeUrl: string) { - const profile: AgentProfile = { name: 'bridge-test-worker' } + const profile: AgentProfile = { + name: 'bridge-test-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'glm-5.2' }, + } return bridgeExecutor( { profile, harness: null }, { signal: new AbortController().signal, - seams: { bridge: { bridgeUrl, bridgeBearer: 'test-bearer', model: 'kimi-k2' } }, + seams: { bridge: { bridgeUrl, bridgeBearer: 'test-bearer' } }, }, ) } @@ -145,6 +244,75 @@ describe('bridgeExecutor upstream-error propagation', () => { ) }) + it('refuses an old bridge before any model POST', async () => { + let posts = 0 + server = createServer((req, res) => { + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ capabilities: {} })) + return + } + posts += 1 + res.writeHead(500) + res.end() + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + + await expect( + drain( + executor.execute( + 'must not dispatch', + new AbortController().signal, + ) as AsyncIterable, + ), + ).rejects.toThrow(/does not advertise cli-bridge\.profile-materialization\.v2/u) + expect(posts).toBe(0) + }) + + it('rejects a v2 bridge that completes without its terminal profile acknowledgement', async () => { + const stub = await startBridgeStub( + `data: ${JSON.stringify({ choices: [{ delta: { content: 'untrusted' } }] })}\n\ndata: [DONE]\n\n`, + { protocol: false }, + ) + server = stub.server + const executor = makeExecutor(stub.url) + + await expect( + drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ), + ).rejects.toThrow(/completed without cli-bridge\.profile-materialization\.v2/u) + expect(() => executor.resultArtifact()).toThrow(/before stream drained/u) + }) + + it('rejects a terminal profile acknowledgement with the wrong effective profile digest', async () => { + const badReceipt = { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: `sha256:${'f'.repeat(64)}`, + harness: 'kimi-k2', + provider: null, + model: 'kimi-k2', + reasoningEffort: { requested: null, applied: null }, + workspacePlanDigest: TEST_WORKSPACE_DIGEST, + files: [], + unsupported: [], + } + const stub = await startBridgeStub( + `data: ${JSON.stringify({ profile_materialization: badReceipt })}\n\ndata: [DONE]\n\n`, + { protocol: false }, + ) + server = stub.server + const executor = makeExecutor(stub.url) + + await expect( + drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ), + ).rejects.toThrow(/bridge materialized profile .* expected/u) + }) + it('drains a healthy stream unchanged and settles the artifact (tail parse is inert)', async () => { const chunks = [ `data: ${JSON.stringify({ choices: [{ delta: { content: 'final answer' } }] })}`, @@ -201,7 +369,7 @@ describe('bridgeExecutor upstream-error propagation', () => { systemPrompt: 'Lead the pursuit.', instructions: ['Prefer falsifiable hypotheses.'], }, - model: { default: 'gpt-5.6', reasoningEffort: 'high' }, + model: { provider: 'openai', default: 'gpt-5.6', reasoningEffort: 'high' }, permissions: { shell: 'ask' }, tools: { web: true }, mcp: { @@ -223,7 +391,7 @@ describe('bridgeExecutor upstream-error propagation', () => { { signal: new AbortController().signal, seams: { - bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer', model: 'kimi-code/k2' }, + bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer' }, }, }, ) @@ -234,7 +402,7 @@ describe('bridgeExecutor upstream-error propagation', () => { ) as AsyncIterable, ) - expect(requestBody?.model).toBe('codex/gpt-5.6') + expect(requestBody?.model).toBe('codex/openai/gpt-5.6') expect(requestBody?.agent_profile).toEqual(profile) expect(requestBody?.messages).toEqual([{ role: 'user', content: 'design the experiment' }]) }) @@ -255,10 +423,68 @@ describe('bridgeExecutor upstream-error propagation', () => { }) }) + it('lets a trusted terminal total supersede an earlier incomplete cost chunk', async () => { + const body = [ + `data: ${JSON.stringify({ + usage: { + prompt_tokens: 3, + completion_tokens: 2, + cost_known: false, + cost_scope: 'incremental', + }, + })}`, + `data: ${JSON.stringify({ + usage: { + cost: 0.012, + cost_known: true, + cost_provenance: 'billing-receipt', + cost_scope: 'total', + }, + })}`, + 'data: [DONE]', + ].join('\n\n') + const stub = await startBridgeStub(`${body}\n\n`) + server = stub.server + const executor = makeExecutor(stub.url) + + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(events).toContainEqual({ kind: 'cost', usd: 0.012 }) + expect(executor.resultArtifact().spent).toMatchObject({ usd: 0.012 }) + expect(executor.resultArtifact().spent.usdKnown).not.toBe(false) + }) + + it('preserves absent prompt-cache fields instead of inventing zeroes', async () => { + const body = `data: ${JSON.stringify({ + usage: { + prompt_tokens: 7, + completion_tokens: 2, + fresh_input_tokens: 7, + }, + })}\n\ndata: [DONE]\n\n` + const stub = await startBridgeStub(body) + server = stub.server + const executor = makeExecutor(stub.url) + + await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + const out = executor.resultArtifact().out as { + promptCache?: Record + } + expect(out.promptCache).toEqual({ freshInput: 7 }) + expect(out.promptCache).not.toHaveProperty('readInput') + expect(out.promptCache).not.toHaveProperty('writeInput') + }) + it('keeps dollar cost unknown when a later completed turn omits price', async () => { let requests = 0 let deliver: (message: unknown) => void = () => {} server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const chunks: Buffer[] = [] for await (const chunk of req) chunks.push(Buffer.from(chunk)) const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< @@ -278,7 +504,10 @@ describe('bridgeExecutor upstream-error propagation', () => { : { prompt_tokens: 4, completion_tokens: 1 } res.end( numberSseDataFrames( - `data: ${JSON.stringify({ choices: [{ delta: { content: `turn-${requests}` } }], usage })}\n\ndata: [DONE]\n\n`, + bridgeProtocolSse( + `data: ${JSON.stringify({ choices: [{ delta: { content: `turn-${requests}` } }], usage })}\n\ndata: [DONE]\n\n`, + requestBody, + ), ), ) }) @@ -310,6 +539,7 @@ describe('bridgeExecutor upstream-error propagation', () => { lastEventId: string | undefined }> = [] server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const chunks: Buffer[] = [] for await (const chunk of req) chunks.push(Buffer.from(chunk)) const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record @@ -327,12 +557,14 @@ describe('bridgeExecutor upstream-error propagation', () => { 'x-run-request-digest': responseDigest, }) if (requests.length === 1) { - res.end(`id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 1 } })}\n\n`) + res.end( + `id: 1\n${bridgeProtocolSse(`data: ${JSON.stringify({ usage: { prompt_tokens: 1 } })}\n\n`, body)}`, + ) return } const eventId = defect === 'skipped replay event' ? 3 : 2 res.end( - `id: ${eventId}\ndata: ${JSON.stringify({ usage: { completion_tokens: 1 } })}\n\ndata: [DONE]\n\n`, + `id: ${eventId}\n${bridgeProtocolSse(`data: ${JSON.stringify({ usage: { completion_tokens: 1 } })}\n\ndata: [DONE]\n\n`, body)}`, ) }) await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) @@ -371,6 +603,7 @@ describe('bridgeExecutor upstream-error propagation', () => { }) server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const cancelledId = cancelledRunId(req.url) if (cancelledId) { cancelledRuns.push(cancelledId) @@ -400,7 +633,7 @@ describe('bridgeExecutor upstream-error propagation', () => { }) if (chatRequests.length === 1) { res.write( - `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, + `id: 1\n${bridgeProtocolSse(`data: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, body)}`, ) setTimeout(() => res.destroy(), 5) return @@ -451,6 +684,7 @@ describe('bridgeExecutor upstream-error propagation', () => { const liveRuns = new Set() const cancelledRuns: string[] = [] server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const cancelledId = cancelledRunId(req.url) if (cancelledId) { cancelledRuns.push(cancelledId) @@ -477,16 +711,19 @@ describe('bridgeExecutor upstream-error propagation', () => { }) if (requestBodies.length === 1) { res.write( - `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, + `id: 1\n${bridgeProtocolSse(`data: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, requestBody)}`, ) return } res.end( numberSseDataFrames( - `data: ${JSON.stringify({ - choices: [{ delta: { content: 'corrected answer' } }], - usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, - })}\n\ndata: [DONE]\n\n`, + bridgeProtocolSse( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'corrected answer' } }], + usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, + })}\n\ndata: [DONE]\n\n`, + requestBody, + ), ), ) liveRuns.delete(runId) @@ -534,6 +771,7 @@ describe('bridgeExecutor upstream-error propagation', () => { firstRequestSeen = resolve }) server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return const cancelledId = cancelledRunId(req.url) if (cancelledId) { cancelledRuns.push(cancelledId) @@ -562,10 +800,13 @@ describe('bridgeExecutor upstream-error propagation', () => { }) res.end( numberSseDataFrames( - `data: ${JSON.stringify({ - choices: [{ delta: { content: 'resumed answer' } }], - usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, - })}\n\ndata: [DONE]\n\n`, + bridgeProtocolSse( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'resumed answer' } }], + usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, + })}\n\ndata: [DONE]\n\n`, + requestBody, + ), ), ) }) @@ -627,6 +868,8 @@ describe('bridgeExecutor harness control rides the profile, not argv', () => { // state between arms does not need its own executor. const profile: AgentProfile = { name: 'isolated-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'glm-5.2' }, extensions: { pi: { load: [] } }, } const executor = bridgeExecutor( @@ -634,7 +877,7 @@ describe('bridgeExecutor harness control rides the profile, not argv', () => { { signal: new AbortController().signal, seams: { - bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer', model: 'pi/glm-5.2' }, + bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer' }, }, }, ) @@ -650,33 +893,4 @@ describe('bridgeExecutor harness control rides the profile, not argv', () => { // And no argv channel was invented alongside it. expect(Object.keys(bodies[0] ?? {})).not.toContain('args') }) - - it('forwards the seam overlay’s extension controls after the profile merge', async () => { - const bodies: Record[] = [] - const stub = await startBridgeStub(okFrame, { onRequest: (body) => bodies.push(body) }) - server = stub.server - const executor = bridgeExecutor( - { profile: { name: 'worker', prompt: { systemPrompt: 'be exact' } }, harness: null }, - { - signal: new AbortController().signal, - seams: { - bridge: { - bridgeUrl: stub.url, - bridgeBearer: 'test-bearer', - model: 'pi/glm-5.2', - agentProfile: { name: 'worker', extensions: { pi: { load: ['pi-zai-glm'] } } }, - }, - }, - }, - ) - await drain( - executor.execute('do the task', new AbortController().signal) as AsyncIterable, - ) - - // The merge must not drop either side: the spawn profile's prompt AND the overlay's controls. - expect(bodies[0]?.agent_profile).toMatchObject({ - prompt: { systemPrompt: 'be exact' }, - extensions: { pi: { load: ['pi-zai-glm'] } }, - }) - }) }) diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index 7bf79267..6ac5f9e8 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -183,12 +183,14 @@ export interface BudgetPool { * pool does not read wall-clock). */ export function spendFromUsageEvents(events: UsageEvent[]): Spend { const tokens = zeroTokenUsage() + let tokensKnown = true let usd = 0 let usdKnown = true let iterations = 0 for (const ev of events) { if (ev.kind === 'tokens') { addTokenUsage(tokens, { input: ev.input, output: ev.output }) + if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd if (ev.usdKnown === false) usdKnown = false @@ -199,6 +201,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { return { iterations, tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), ms: 0, @@ -208,12 +211,14 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { async function foldUsage(events: AsyncIterable | UsageEvent[]): Promise { if (Array.isArray(events)) return spendFromUsageEvents(events) const tokens = zeroTokenUsage() + let tokensKnown = true let usd = 0 let usdKnown = true let iterations = 0 for await (const ev of events) { if (ev.kind === 'tokens') { addTokenUsage(tokens, { input: ev.input, output: ev.output }) + if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd if (ev.usdKnown === false) usdKnown = false @@ -224,6 +229,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom return { iterations, tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), ms: 0, diff --git a/src/runtime/supervise/chat-transport-executor.ts b/src/runtime/supervise/chat-transport-executor.ts index 1bbfa029..4454d1bd 100644 --- a/src/runtime/supervise/chat-transport-executor.ts +++ b/src/runtime/supervise/chat-transport-executor.ts @@ -1,34 +1,13 @@ /** - * The chat-transport leaf executor: a worker whose runtime is a plain OpenAI-compatible - * `/v1/chat/completions` transport — the worker IS a model conversation, not a sandboxed process - * (#721). Tool calls are optional (none, or a caller-provided tool table executed on this host). - * A chat worker gets everything real workers get through the open `Executor` port: node pinning, - * conserved spend, settle/verdict, journal + edge ledger. + * A session-owning composition over Runtime's canonical Router tool-loop executor. * - * Module home: a standalone leaf-executor module beside `worktree-cli-executor.ts` — a direct - * `(options) → Executor` constructor, NOT a `createExecutor` backend variant. The reason is - * continuity: `workerFromBackend` (the backend-as-data path every `ExecutorConfig` rides) creates - * a fresh executor per spawn with no session re-attachment and deliberately FAILS LOUD on a - * `continuity: 'resume'` spawn; the documented resume consumer is a session-owning - * `makeWorkerAgent` seam. {@link chatWorkerSeam} is that seam, and this module ships both halves - * together so no caller re-derives the resume wiring. - * - * Transport shape: NON-streaming, one buffered POST per turn — the simplest honest choice. - * A streaming executor cannot mark an unmetered turn today (`UsageEvent`'s `tokens` variant has - * no `tokensKnown: false` twin — see the documented limitation in `./types`), while the one-shot - * path returns a whole `Spend` that carries both markers. Honesty wins over liveness here. - * - * Metering: tokens come from the transport's `usage` fields; a turn without usage marks - * `tokensKnown: false`. Dollars come ONLY from the response's own cost fields (`usage.cost` / - * `usage.cost_usd`, the cli-bridge and OpenRouter conventions); a turn without one marks - * `usdKnown: false`. NEVER estimated from a local price table — this executor speaks to arbitrary - * OpenAI-compatible endpoints whose models a local table cannot price, and a silent estimate is a - * fabricated measurement. + * This module adds conversation persistence for graph-edge `resume` continuity. It does not own + * model selection, prompts, generation controls, retries, tool policy, or provider accounting: + * those are lowered from one exact `AgentProfile` by `createExecutor({ backend: 'router-tools' })`. * * @experimental */ -import { randomUUID } from 'node:crypto' import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import { contentAddress } from '../../durable/content-address' import { ValidationError } from '../../errors' @@ -38,482 +17,191 @@ import type { WorkerSpawnContext, } from '../../mcp/tools/coordination' import type { ToolSpec } from '../router-client' -import { zeroTokenUsage } from '../util' -import { canonicalizeAuthoredProfile } from './authoring' -import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' -import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' -import { concreteModelId, concreteProfileModel } from './model-policy' -import { mergeAbortSignals, taskToPrompt } from './runtime' -import type { Agent, AgentSpec, Executor, ExecutorResult, Runtime, Spend } from './types' +import { type DeliverableSpec, gateOnDeliverable, mapExecutorResult } from './completion-gate' +import { assertExecutableAgentProfile } from './model-policy' +import { createExecutor, type RouterToolsSeam } from './runtime' +import type { Agent, AgentSpec, Executor, ExecutorContext, ExecutorResult } from './types' -// ── The transport ────────────────────────────────────────────────────────────── +/** Buffered OpenAI-compatible completion port used only for offline execution. */ +export type ChatCompletionsTransport = NonNullable -/** One buffered chat-completions call: the OpenAI-shape request body in, the parsed completion - * JSON out. The ONE wire function of this module — the executor's default transport is built - * from it, and a harness that must prove two arms share a substrate (P1 parity) drives BOTH - * through the same instance. */ -export type ChatCompletionsTransport = ( - body: Record, - signal?: AbortSignal, -) => Promise - -/** The default transport: POST `${url}/chat/completions` with an optional bearer. Fail-loud on - * any non-2xx — the status and body head become the settle reason. */ -export function chatCompletionsTransport(opts: { - url: string - bearer?: string -}): ChatCompletionsTransport { - if (typeof opts.url !== 'string' || opts.url.length === 0) { - throw new ValidationError('chatCompletionsTransport: url required') - } - const endpoint = `${opts.url.replace(/\/$/, '')}/chat/completions` - return async (body, signal) => { - const res = await fetch(endpoint, { - method: 'POST', - headers: { - 'content-type': 'application/json', - ...(opts.bearer ? { authorization: `Bearer ${opts.bearer}` } : {}), - }, - body: JSON.stringify(body), - ...(signal ? { signal } : {}), - }) - if (!res.ok) { - throw new ValidationError(`chat transport ${res.status}: ${(await res.text()).slice(0, 200)}`) - } - return res.json() - } -} - -// ── The session store (continuity substrate) ─────────────────────────────────── - -/** Conversation history keyed by the settled worker id — the resume substrate. The kernel owns - * identity, ordering, ledger truth, and spend continuity; this store owns only the message - * lists a `'resume'` spawn continues (`WorkerSpawnContext.resume.ofWorker` is the load key). - * PROCESS-LOCAL by the same boundary the kernel documents for resume itself: a prior process's - * workers are not resume targets. */ +/** Conversation history keyed by the settled Runtime worker id. */ export interface ChatSessionStore { - load(workerId: string): ReadonlyArray> | undefined - save(workerId: string, messages: ReadonlyArray>): void + load(workerId: string): ReadonlyArray>> | undefined + save(workerId: string, messages: ReadonlyArray>>): void } -/** In-memory `ChatSessionStore`. Entries are detached copies — a caller mutating a saved array - * cannot corrupt a recorded session. */ +/** In-memory, process-local conversation store with detached reads and writes. */ export function createChatSessionStore(): ChatSessionStore { - const sessions = new Map>>() + const sessions = new Map>>>() return { - load: (workerId) => sessions.get(workerId), - save: (workerId, messages) => { - sessions.set(workerId, structuredClone(messages) as ReadonlyArray>) + load(workerId) { + const messages = sessions.get(workerId) + return messages === undefined + ? undefined + : (structuredClone(messages) as ReadonlyArray>>) + }, + save(workerId, messages) { + sessions.set( + workerId, + structuredClone(messages) as ReadonlyArray>>, + ) }, } } -// ── The executor ─────────────────────────────────────────────────────────────── - -/** One entry of the caller-provided tool table: the OpenAI function spec the model sees, and the - * host-side implementation run when the model calls it. */ +/** One profile-authorized function tool and its host implementation. */ export interface ChatTransportTool { readonly spec: ToolSpec - /** Runs ON THIS HOST; the returned string folds back as the `tool` message. A throw is fed - * back as an error message for the model to correct — a bad tool call is a real outcome, not - * an infra fault. */ readonly execute: (args: Record, task: unknown) => Promise } +/** + * Transport and session data for one exact profile-driven conversation. + * Behavioral controls belong only in `profile.model.metadata`. + */ export interface ChatTransportExecutorOptions { - /** OpenAI-compatible base URL (with or without `/v1`); the executor POSTs to - * `${url}/chat/completions`. Ignored when `complete` is injected. */ - url: string - /** Bearer token for the default transport. Omit for an unauthenticated endpoint. */ - bearer?: string - /** The wire model id sent on every completion. */ - model: string - /** System prompt seeding a FRESH conversation. A resumed conversation keeps the system message - * it was recorded with — a session continues; it is not re-primed. */ - system?: string - /** Tool table. Omitted = a pure conversation (no `tools` field on the wire). */ - tools?: ReadonlyArray - temperature?: number - /** Output-token ceiling for ONE completion, sent as `max_tokens` on every request when set. - * Omitted = no field on the wire, so the endpoint's own default governs. A harness pairing - * this executor against another sampling path (P1 parity) pins BOTH arms to one value. */ - maxTokens?: number - /** Inference-turn cap for ONE shot (one `execute`). Default 200 — a runaway backstop, not a - * workflow limit (mirrors `routerToolsInlineExecutor.maxTurns`). */ - maxTurnsPerShot?: number - /** Injected buffered transport — the offline seam (mirrors `RouterConfig.complete`). When set, - * `url`/`bearer` are unused and NO network is touched. */ - complete?: ChatCompletionsTransport - /** Session store backing continuity. Required to record this conversation (with `sessionKey`) - * or to continue a prior one (with `resume`). */ - sessions?: ChatSessionStore - /** The id this worker's conversation is recorded under at settle — the kernel node id when - * spawned through a scope, so a later `'resume'` spawn's `resume.ofWorker` finds it. */ - sessionKey?: string - /** The resume lineage from `WorkerSpawnContext.resume`: this shot continues `ofWorker`'s - * recorded message list. Requires `sessions` holding that conversation — fails loud before - * any spend when it does not. */ - resume?: WorkerResumeContext - /** Profile this executor materializes, for the kernel's materialization receipt. Omitted = - * the node's receipt reads `executor-did-not-report` (a direct, unsupervised use). */ - profile?: AgentProfile - /** Kernel-minted attempt id (`ExecutorNodeContext.attemptId`) binding the receipt to this - * exact spawn. */ - attemptId?: string + readonly profile: AgentProfile + readonly url?: string + readonly bearer?: string + readonly tools?: ReadonlyArray + readonly complete?: ChatCompletionsTransport + readonly sessions?: ChatSessionStore + readonly sessionKey?: string + readonly resume?: WorkerResumeContext } -interface ChatCompletionMessage { - content?: string | null - tool_calls?: Array<{ id?: string; function?: { name?: string; arguments?: string } }> +function exactProfile(profile: AgentProfile, context: string): AgentProfile { + const parsed = agentProfileSchema.safeParse(profile) + if (!parsed.success) + throw new ValidationError(`${context}: invalid AgentProfile: ${parsed.error.message}`) + assertExecutableAgentProfile(parsed.data, context) + return parsed.data } -interface ChatCompletionResponse { - choices?: Array<{ message?: ChatCompletionMessage }> - usage?: { - prompt_tokens?: number - completion_tokens?: number - cost?: number - cost_usd?: number +function initialMessages( + opts: ChatTransportExecutorOptions, +): ReadonlyArray>> | undefined { + if (!opts.resume) return undefined + if (!opts.sessions) { + throw new ValidationError( + "chat transport: a 'resume' spawn needs the session store holding the prior conversation", + ) } + const prior = opts.sessions.load(opts.resume.ofWorker) + if (prior === undefined) { + throw new ValidationError( + `chat transport: no recorded conversation for worker '${opts.resume.ofWorker}'`, + ) + } + return prior } -const CHAT_TRANSPORT_RUNTIME: Runtime = 'chat-transport' - -/** - * Build the chat-transport `Executor`: one `execute` = one conversation SHOT — seed (fresh system - * prompt, or the resumed session's recorded history) + the task as the next user message, then - * loop completion → host tool calls → tool messages until the model answers without a tool call - * (or the turn cap). Settles with the final assistant text as `out`. - * - * Fail-loud contract: transport failures (non-2xx, network faults, malformed completions) throw - * `ValidationError`, which the scope settles as an INFRA failure (`Settled.down.infra`) — never a - * fake success. The accumulated conversation is still recorded before the throw when a store is - * configured, because the inference HAPPENED and a resume may continue a failed session (the - * kernel deliberately allows resume-after-failure; the seam decides). - */ -export function chatTransportExecutor(opts: ChatTransportExecutorOptions): Executor { - const model = concreteModelId(opts.model) - if (!model) throw new ValidationError('chatTransportExecutor: model required') - if (!opts.complete && (typeof opts.url !== 'string' || opts.url.length === 0)) { - throw new ValidationError('chatTransportExecutor: url required (or inject `complete`)') - } - for (const tool of opts.tools ?? []) { - if (typeof tool.spec?.function?.name !== 'string' || typeof tool.execute !== 'function') { - throw new ValidationError( - 'chatTransportExecutor: every tools entry needs spec.function.name + execute', - ) - } - } - const maxTurns = opts.maxTurnsPerShot ?? 200 - if (!Number.isInteger(maxTurns) || maxTurns < 1) { - throw new ValidationError('chatTransportExecutor: maxTurnsPerShot must be a positive integer') - } - if (opts.maxTokens !== undefined && (!Number.isInteger(opts.maxTokens) || opts.maxTokens < 1)) { - throw new ValidationError('chatTransportExecutor: maxTokens must be a positive integer') +function executorConfig(opts: ChatTransportExecutorOptions): { + config: { backend: 'router-tools' } & RouterToolsSeam + profile: AgentProfile +} { + const profile = exactProfile(opts.profile, 'chat transport') + if (!opts.complete && !opts.url) { + throw new ValidationError('chat transport: url required unless complete is injected') } - // Resolve the seed BEFORE any spend: a resume that cannot re-attach is a configuration fault. - let seed: Array> - if (opts.resume) { - if (!opts.sessions) { - throw new ValidationError( - "chatTransportExecutor: a 'resume' spawn needs `sessions` — the store holding the " + - 'conversation this shot continues', - ) - } - const prior = opts.sessions.load(opts.resume.ofWorker) - if (prior === undefined) { - throw new ValidationError( - `chatTransportExecutor: no recorded conversation for worker '${opts.resume.ofWorker}' — ` + - 'the session store holds only conversations recorded by this process (the kernel’s ' + - 'process-local resume boundary)', - ) + const tools = opts.tools ?? [] + for (const tool of tools) { + if (!tool.spec.function.name || typeof tool.execute !== 'function') { + throw new ValidationError('chat transport: every tool needs spec.function.name and execute') } - seed = structuredClone(prior) as Array> - } else { - seed = - opts.system !== undefined && opts.system.length > 0 - ? [{ role: 'system', content: opts.system }] - : [] } - const transport = - opts.complete ?? - chatCompletionsTransport({ url: opts.url, ...(opts.bearer ? { bearer: opts.bearer } : {}) }) - const toolSpecs = (opts.tools ?? []).map((tool) => tool.spec) - const toolByName = new Map((opts.tools ?? []).map((tool) => [tool.spec.function.name, tool])) - - const controller = new AbortController() - let artifact: ExecutorResult | undefined - let executed = false - const executionId = opts.sessionKey ?? `chat-session-${randomUUID()}` - const attemptId = opts.attemptId ?? newExecutionAttemptId(executionId) - - const executor: Executor = { - runtime: CHAT_TRANSPORT_RUNTIME, - async execute(task, signal): Promise> { - // The seed is consumed once: a second execute would replay a stale conversation and - // double-record the session. One executor instance = one shot, per the spawn contract. - if (executed) { - throw new ValidationError('chatTransportExecutor: execute() called twice on one instance') - } - executed = true - const started = Date.now() - const messages = seed - messages.push({ role: 'user', content: taskToPrompt(task) }) - const linked = mergeAbortSignals(signal, controller.signal) - const tokens = zeroTokenUsage() - let tokensKnown = true - let usd = 0 - let usdKnown = true - let turns = 0 - let lastText = '' - try { - for (let t = 0; t < maxTurns; t += 1) { - const body: Record = { - model, - messages, - ...(toolSpecs.length > 0 ? { tools: toolSpecs, tool_choice: 'auto' } : {}), - ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts.maxTokens !== undefined ? { max_tokens: opts.maxTokens } : {}), - } - let raw: unknown - try { - raw = await transport(body, linked) - } catch (cause) { - // An abort is the caller's own teardown/deadline — propagate untouched so the scope - // classifies it as the abort it is, not as a transport fault of this executor. - if (cause instanceof Error && cause.name === 'AbortError') throw cause - if (cause instanceof ValidationError) throw cause - throw new ValidationError( - `chatTransportExecutor: transport failed: ${cause instanceof Error ? cause.message : String(cause)}`, - ) - } - turns += 1 - const data = raw as ChatCompletionResponse - const usage = data?.usage - if ( - usage && - typeof usage.prompt_tokens === 'number' && - typeof usage.completion_tokens === 'number' - ) { - tokens.input += usage.prompt_tokens - tokens.output += usage.completion_tokens - } else { - tokensKnown = false - } - const turnCost = - typeof usage?.cost === 'number' - ? usage.cost - : typeof usage?.cost_usd === 'number' - ? usage.cost_usd - : undefined - if (turnCost !== undefined) usd += turnCost - else usdKnown = false - const msg = data?.choices?.[0]?.message - if (msg === undefined) { - throw new ValidationError( - 'chatTransportExecutor: transport returned no choices[0].message', - ) - } - if (typeof msg.content === 'string' && msg.content.length > 0) lastText = msg.content - const toolCalls = msg.tool_calls ?? [] - if (toolCalls.length === 0 || toolSpecs.length === 0) { - // Record the terminal assistant turn so a resumed session continues from it. - messages.push({ role: 'assistant', content: msg.content ?? '' }) - break - } - // Record the assistant turn verbatim, then run each call on the host and fold the - // result back as a `tool` message for the next turn (the routerToolsInline shape). - messages.push({ - role: 'assistant', - content: msg.content ?? '', - tool_calls: toolCalls.map((tc, i) => ({ - id: tc.id ?? `call_${i}`, - type: 'function', - function: { - name: tc.function?.name ?? '', - arguments: tc.function?.arguments ?? '{}', - }, - })), - }) - for (let i = 0; i < toolCalls.length; i += 1) { - const tc = toolCalls[i] - const id = tc?.id ?? `call_${i}` - const name = tc?.function?.name ?? '' - const tool = toolByName.get(name) - if (!tool) { - messages.push({ - role: 'tool', - tool_call_id: id, - content: `error: unknown tool '${name}'`, - }) - continue - } - let args: Record - try { - args = JSON.parse(tc?.function?.arguments ?? '{}') as Record - } catch { - messages.push({ - role: 'tool', - tool_call_id: id, - content: 'error: tool arguments were not valid JSON', - }) - continue - } - let result: string - try { - result = await tool.execute(args, task) - } catch (cause) { - result = `error: ${cause instanceof Error ? cause.message : String(cause)}` - } - messages.push({ role: 'tool', tool_call_id: id, content: result }) + const resumed = initialMessages(opts) + return { + profile, + config: { + backend: 'router-tools', + routerBaseUrl: opts.url ?? 'http://injected.invalid', + routerKey: opts.bearer ?? (opts.complete ? 'injected-transport' : ''), + tools: tools.map((tool) => tool.spec), + executeToolCall: async (name, args, task) => { + const tool = tools.find((candidate) => candidate.spec.function.name === name) + if (!tool) throw new ValidationError(`chat transport: unknown tool ${JSON.stringify(name)}`) + return tool.execute(args, task) + }, + ...(opts.complete ? { complete: opts.complete } : {}), + ...(resumed ? { initialMessages: resumed } : {}), + ...(opts.sessions && opts.sessionKey + ? { + onMessages: (messages: ReadonlyArray>>) => { + opts.sessions?.save(opts.sessionKey as string, messages) + }, } - } - } finally { - // The turns that RAN are recorded even when the shot failed: the kernel deliberately - // allows resuming a failed prior worker, and the seam decides. Recording only successes - // would silently amputate a resumed session's real history. - if (opts.sessions && opts.sessionKey !== undefined) { - opts.sessions.save(opts.sessionKey, messages) - } - } - const spent: Spend = { - iterations: turns, - tokens, - ...(tokensKnown ? {} : { tokensKnown: false }), - usd, - ...(usdKnown ? {} : { usdKnown: false }), - ms: Date.now() - started, - } - artifact = { - outRef: contentAddress({ kind: 'chat-transport', model, content: lastText, turns }), - out: lastText, - spent, - } - return artifact - }, - teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) - }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('chatTransportExecutor: resultArtifact() read before execute()') - } - return { ...artifact, spent: artifact.spent } + : {}), }, } - if (opts.profile === undefined) return executor - return attestRuntimeOwnedExecutor( - executor, - { - effectiveProfile: opts.profile, - backend: 'chat-transport', - model: { status: 'known', id: model }, - execution: { kind: 'session', id: executionId }, - materializer: 'chat-transport-conversation', - plan: { - kind: 'openai-chat-conversation', - model, - maxTurnsPerShot: maxTurns, - tools: toolSpecs, - resumeOf: opts.resume?.ofWorker ?? null, - }, - }, - { - attemptId, - binding: { - endpoint: opts.complete ? 'injected-transport' : opts.url, - model, - sessionKey: opts.sessionKey ?? null, - }, - descriptor: { - kind: 'chat-transport-session', - transport: opts.complete ? 'injected' : 'http', - backend: 'chat-transport', - }, - }, - ) } -// ── The worker seam (the resume consumer) ────────────────────────────────────── - -export interface ChatWorkerSeamOptions { - /** OpenAI-compatible base URL every spawned worker speaks. Unused when `complete` is set. */ - url: string - bearer?: string - /** Fallback wire model when a spawned profile carries none (`profile.model.default` wins). */ - model?: string - tools?: ReadonlyArray - temperature?: number - /** Per-completion `max_tokens` for every spawned worker (see - * {@link ChatTransportExecutorOptions.maxTokens}). */ - maxTokens?: number - maxTurnsPerShot?: number - /** Injected buffered transport — the offline seam; no network is touched when set. */ - complete?: ChatCompletionsTransport - /** Session store backing continuity. Default: one fresh in-memory store PER SEAM, matching the - * kernel's process-local resume boundary (one seam = one run's sessions). */ - sessions?: ChatSessionStore - /** The completion oracle: each worker settles `valid` ⟺ this check passes on its final - * assistant text (`gateOnDeliverable` — settled ⟺ DELIVERED, exactly how `workerFromBackend` - * composes it). Pass the graph's deliverable so a keep-best driver can pick a winner; omitted, - * workers settle unverdicted and only a driver `submit_result` can win. */ - deliverable?: DeliverableSpec +function buildChatTransportExecutor( + opts: ChatTransportExecutorOptions, + context: ExecutorContext, +): Executor { + const { config, profile } = executorConfig(opts) + const spec: AgentSpec = { profile, harness: null } + const inner = createExecutor(config)(spec, context) as Executor + return mapExecutorResult(inner, (result: ExecutorResult) => { + const raw = result.out as { content?: unknown } + const content = typeof raw?.content === 'string' ? raw.content : '' + return { + outRef: contentAddress({ kind: 'chat-transport', profile, content }), + out: content, + ...(result.verdict ? { verdict: result.verdict } : {}), + } + }) } /** - * The `makeWorkerAgent` seam over {@link chatTransportExecutor} — the continuity consumer - * `workerFromBackend` refuses to be. Every spawn becomes one conversation shot: the spawned - * profile's system prompt + instructions (which is where a graph's delegates directive lands) - * seed a fresh session, and a `'resume'` spawn re-attaches by loading `resume.ofWorker`'s - * recorded message list from the seam's session store. Conversations are recorded under the - * kernel node id, which is exactly what a later `resume.ofWorker` names. + * Build one exact profile-driven chat executor through `createExecutor`. + * Prefer `chatWorkerSeam` for supervised work because it supplies trusted node identity. */ +export function chatTransportExecutor(opts: ChatTransportExecutorOptions): Executor { + return buildChatTransportExecutor(opts, { + signal: new AbortController().signal, + seams: {}, + }) +} + +/** Transport/session configuration shared by every spawned exact profile. */ +export interface ChatWorkerSeamOptions { + readonly url?: string + readonly bearer?: string + readonly tools?: ReadonlyArray + readonly complete?: ChatCompletionsTransport + readonly sessions?: ChatSessionStore + readonly deliverable?: DeliverableSpec +} + +/** Session-owning worker factory for graph continuity. */ export function chatWorkerSeam(opts: ChatWorkerSeamOptions): MakeWorkerAgent { - if (!opts.complete && (typeof opts.url !== 'string' || opts.url.length === 0)) { - throw new ValidationError('chatWorkerSeam: url required (or inject `complete`)') + if (!opts.complete && !opts.url) { + throw new ValidationError('chatWorkerSeam: url required unless complete is injected') } const sessions = opts.sessions ?? createChatSessionStore() return (rawProfile, spawnContext?: WorkerSpawnContext) => { - // The supervisor authors in the skill's flat vocabulary; lift + validate here exactly like - // `workerFromBackend` — the one other place a profile becomes a spawnable worker. - const parsed = agentProfileSchema.safeParse(canonicalizeAuthoredProfile(rawProfile)) - if (!parsed.success) { - throw new ValidationError(`chatWorkerSeam: invalid AgentProfile: ${parsed.error.message}`) - } - const profile = parsed.data - const model = concreteProfileModel(profile) ?? concreteModelId(opts.model) - if (!model) { - throw new ValidationError( - 'chatWorkerSeam: no model — set ChatWorkerSeamOptions.model or AgentProfile.model.default', - ) - } - const system = [profile.prompt?.systemPrompt, ...(profile.prompt?.instructions ?? [])] - .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) - .join('\n') + const profile = exactProfile(rawProfile, 'chatWorkerSeam') const name = profile.name ?? 'chat-worker' const spec: AgentSpec = { profile, harness: null, - // Per-spawn factory: built only after admission, with the kernel node context — the node id - // is the session-record key a later resume names, and the attempt id binds the receipt. - executorFactory: (executorSpec, ctx) => { - const executor = chatTransportExecutor({ - url: opts.url, - ...(opts.bearer !== undefined ? { bearer: opts.bearer } : {}), - model, - ...(system.length > 0 ? { system } : {}), - ...(opts.tools !== undefined ? { tools: opts.tools } : {}), - ...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts.maxTokens !== undefined ? { maxTokens: opts.maxTokens } : {}), - ...(opts.maxTurnsPerShot !== undefined ? { maxTurnsPerShot: opts.maxTurnsPerShot } : {}), - ...(opts.complete !== undefined ? { complete: opts.complete } : {}), - sessions, - ...(ctx.node?.nodeId !== undefined ? { sessionKey: ctx.node.nodeId } : {}), - ...(spawnContext?.resume !== undefined ? { resume: spawnContext.resume } : {}), - profile: executorSpec.profile, - ...(ctx.node?.attemptId !== undefined ? { attemptId: ctx.node.attemptId } : {}), - }) + executorFactory: (executorSpec, context) => { + const executor = buildChatTransportExecutor( + { + profile: executorSpec.profile, + ...(opts.url ? { url: opts.url } : {}), + ...(opts.bearer ? { bearer: opts.bearer } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + ...(opts.complete ? { complete: opts.complete } : {}), + sessions, + ...(context.node?.nodeId ? { sessionKey: context.node.nodeId } : {}), + ...(spawnContext?.resume ? { resume: spawnContext.resume } : {}), + }, + context, + ) return opts.deliverable ? gateOnDeliverable(executor, opts.deliverable) : executor }, } diff --git a/src/runtime/supervise/completion-gate.ts b/src/runtime/supervise/completion-gate.ts index 133f12e1..78ef5906 100644 --- a/src/runtime/supervise/completion-gate.ts +++ b/src/runtime/supervise/completion-gate.ts @@ -141,6 +141,68 @@ export function gateOnDeliverable( return inheritRuntimeOwnedExecutorAttestation(inner, wrapped) } +export interface ExecutorResultMapping { + outRef: string + out: Out + verdict?: DefaultVerdict +} + +/** + * Transform a Runtime executor's terminal artifact without losing its private + * profile-materialization attestation or altering its measured spend. This is + * the composition point for deterministic post-processing and grading; callers + * must not rebuild an Executor around a model transport merely to change `out`. + */ +export function mapExecutorResult( + inner: Executor, + map: ( + result: ExecutorResult, + task: unknown, + ) => ExecutorResultMapping | Promise>, +): Executor { + let mapped: ExecutorResult | undefined + + const settle = async ( + result: ExecutorResult, + task: unknown, + ): Promise> => { + const transformed = await map(result, task) + mapped = { + outRef: transformed.outRef, + out: transformed.out, + ...(transformed.verdict ? { verdict: transformed.verdict } : {}), + spent: result.spent, + } + return mapped + } + + const wrapped: Executor = { + runtime: inner.runtime, + ...(inner.budgetExempt !== undefined ? { budgetExempt: inner.budgetExempt } : {}), + ...(inner.deliver ? { deliver: (message: unknown) => inner.deliver?.(message) } : {}), + ...(inner.progress ? { progress: () => inner.progress?.() } : {}), + ...(inner.traceSource ? { traceSource: () => inner.traceSource?.() } : {}), + ...(inner.accounting ? { accounting: () => inner.accounting?.() } : {}), + ...(inner.metered ? { metered: () => inner.metered?.() } : {}), + execute(task, signal) { + const execution = inner.execute(task, signal) + if (isAsyncIterable(execution)) { + return (async function* () { + for await (const event of execution) yield event + await settle(inner.resultArtifact(), task) + })() + } + return (async () => settle(await execution, task))() + }, + teardown: (grace) => inner.teardown(grace), + resultArtifact() { + if (!mapped) throw new Error('mapExecutorResult: resultArtifact() read before execute()') + return mapped + }, + } + return inheritRuntimeOwnedExecutorAttestation(inner, wrapped) +} + function isAsyncIterable(v: unknown): v is AsyncIterable { return ( v != null && diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index 80e7f992..0e973c77 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -222,13 +222,6 @@ function formatRosterNode(node: NodeSnapshot, settled?: SettledWorker): string { return `- ${node.id}: ${node.status}, label=${node.label}, runtime=${node.runtime}${result}` } -/** maxTurns=0 anti-runaway tripwire: a finite ceiling for the ONE case the conserved pool can't - * bound — a driver whose chat seam reports NO usage (so `scope.meter`/`pool.observe` is never - * called and its turns don't drain the pool). With a usage-reporting seam, driver inference now - * meters into the pool and `poolStarved` halts it; the pool + deadline + abort are the real bounds - * and no healthy run approaches this. */ -const runawayTripwireTurns = 2000 - /** Spawn-progress is impossible: the pool can't afford another worker AND nothing is in flight to * await. A long-horizon driver bounded by the conserved pool stops here instead of spinning (the * in-loop budget guard the turn cap alone never provided). Checks BOTH conserved channels: tokens @@ -338,11 +331,9 @@ export function driverAgent(opts: DriverAgentOptions): Agent { 'driverAgent: maxTurns must be >= 0 (0 lifts the turn cap; bounds become the conserved pool + deadline + abort)', ) } - // maxTurns=0 lifts the turn-COUNT cap: a long-horizon decomposition must not die on an - // arbitrary number of turns. It is bounded instead by the conserved budget pool, an absolute - // deadline, the driver's own stop, and abort — all checked in-loop below. The tripwire is a - // pure anti-runaway guard, NOT the intended limit. - const maxTurns = opts.maxTurns === 0 ? runawayTripwireTurns : (opts.maxTurns ?? 16) + // maxTurns=0 lifts the turn-count cap exactly. The conserved pool, deadline, abort, and explicit + // stop remain the caller-visible bounds; Runtime does not substitute a hidden sentinel. + const maxTurns = opts.maxTurns ?? 16 const now = opts.now ?? Date.now const inbox = opts.inbox ?? createInbox() @@ -411,8 +402,8 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // Meter the driver's OWN inference on EVERY turn into the conserved pool — the largest single // token consumer in the loop, and what makes maxTurns=0 genuinely bounded (a thinking driver - // drains the pool → poolStarved). iterations:0 — the conserved iteration channel budgets - // CHILD rounds, not driver turns. + // drains the pool → poolStarved). Driver turns never consume the child-iteration budget; + // failed inference stays visible through unknown token/USD channels and journal detail. // // There is deliberately no branch that skips `scope.meter`. A turn whose brain reported no // usage is metered as an UNKNOWN turn (`tokensKnown: false`), not omitted: omitting it let the @@ -427,33 +418,31 @@ export function driverAgent(opts: DriverAgentOptions): Agent { tools: ReadonlyArray, detail: Record, ) => { - // KNOWN LIMITATION (pre-existing, predates the streamed transport): a turn that THROWS is - // never metered at all. The throw propagates out of `meteredBrain` before `scope.meter` - // below, so a brain call that burned prefill and then failed (a 5xx after generation, an - // abort mid-completion) leaves NO record — not even an unknown-marked one — and the pool's - // `tokensKnown` stays true while real tokens were spent. Deferred deliberately: metering - // the throw path means deciding what a partial turn debits and re-throwing after the meter - // in every driver exit, which changes error-handling semantics across this file and belongs - // in its own change. Filed separately; do not fix inline. - const res = await opts.brain(messages, tools) - // Nothing at all came back about what the turn cost: neither tokens nor dollars. Both - // channels are marked unknown, mirroring how the pool already represents unknown dollars. - // - // KNOWN LIMITATION (pre-existing): "a turn that reported tokens but no price is a - // known-ZERO dollar turn" is true for a genuinely free model and WRONG for an UNPRICED one. - // `meterTurn` in `router-client.ts` returns `costUsd: undefined` both when a model costs - // nothing and when it is simply missing from the price table, and the two are - // indistinguishable here — so an unpriced model's real dollars are recorded as a measured - // $0 with `usdKnown` left true, and a dollar-denominated cap under-counts. Fixing it means - // the router result must distinguish "free" from "unpriced", which is an upstream contract - // change. Filed separately; do not fix inline. - const nothingReported = res.usage === undefined && res.costUsd === undefined + let res: Awaited> + try { + res = await opts.brain(messages, tools) + } catch (error) { + await scope.meter( + { + iterations: 0, + tokens: { input: 0, output: 0 }, + tokensKnown: false, + usd: 0, + usdKnown: false, + ms: 0, + }, + { driver: opts.name, inferenceFailed: true, ...detail }, + ) + throw error + } + const trustedCost = + res.costProvenance === 'provider-receipt' || res.costProvenance === 'billing-receipt' const turnSpend: Spend = { iterations: 0, tokens: { input: res.usage?.input ?? 0, output: res.usage?.output ?? 0 }, ...(res.usage === undefined ? { tokensKnown: false } : {}), - usd: res.costUsd ?? 0, - ...(nothingReported ? { usdKnown: false } : {}), + usd: trustedCost ? (res.costUsd ?? 0) : 0, + ...(trustedCost && res.costUsd !== undefined ? {} : { usdKnown: false }), ms: 0, } await scope.meter(turnSpend, { @@ -463,6 +452,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // `include_usage` contract), which is a different fact from a brain that never reports // usage at all. Recorded on the turn so the journal distinguishes them. ...(res.usageUnknown === true ? { streamUsageMissing: true } : {}), + ...(res.costProvenance === 'catalog-estimate' ? { estimatedCostUsd: res.costUsd } : {}), ...detail, }) return res @@ -488,14 +478,6 @@ export function driverAgent(opts: DriverAgentOptions): Agent { opts.compaction.distill ?? (async (msgs) => { const roster = summarizeRoster(scope.view, coord.settled()) - // KNOWN LIMITATION (pre-existing, predates the streamed transport): this `catch` - // degrades a failed distill to a roster-only summary, which is the right product - // behavior — compaction must not kill the run — but it also SWALLOWS the turn's - // cost. `meteredBrain` throws before reaching `scope.meter`, so a distill that - // burned a full O(history) prefill and then failed debits nothing and leaves the - // pool's `tokensKnown` true. It is the same unmetered-throw gap as the main loop - // (see `meteredBrain` above), reached through a different exit; both are deferred - // to one change that decides what a thrown turn debits. Filed separately. try { const res = await meteredBrain( [...msgs, { role: 'user', content: distillInstruction }], diff --git a/src/runtime/supervise/delegate.ts b/src/runtime/supervise/delegate.ts index e9932149..208bb7cb 100644 --- a/src/runtime/supervise/delegate.ts +++ b/src/runtime/supervise/delegate.ts @@ -20,9 +20,7 @@ */ import { ConfigError } from '../../errors' -import type { RouterConfig } from '../router-client' -import type { ToolLoopChat } from '../tool-loop' -import { supervisorInstructions } from './authoring' +import type { RouterTransportConfig } from '../router-client' import type { DeliverableSpec } from './completion-gate' import type { ExecutorConfig } from './runtime' import { supervise } from './supervise' @@ -35,7 +33,7 @@ import type { Budget, SupervisedResult } from './types' export const defaultDelegateBudget: Budget = { maxIterations: 50, maxTokens: 200_000 } /** Inputs to {@link delegate}. The intent is the first positional arg; everything here is optional - * with sensible defaults, so the common call is `delegate(intent, { backend, router })`. */ + * with explicit execution identity, so the common call names one exact supervisor profile. */ export interface DelegateOptions { /** The completion oracle (settled ⟺ delivered) the authored workers settle against. Strongly * recommended — without it the supervisor trusts a worker's self-report. For a code intent, @@ -47,41 +45,15 @@ export interface DelegateOptions { readonly backend?: ExecutorConfig /** The conserved compute pool for the whole delegation. Defaults to {@link defaultDelegateBudget}. */ readonly budget?: Budget - /** The model the supervisor BRAIN runs on (the router model). The brain must tool-call - * (`spawn_agent` / `await_event`), so a delegator model, not a hidden-reasoning model. */ - readonly model?: string - /** The supervisor brain's router substrate. REQUIRED for the default router-brained supervisor - * (the brain is resolved from this), unless a test injects `brain` directly. `model` overrides - * `router.model`. (Design delta vs the bare `supervise()` profile: the brain needs a router.) */ - readonly router?: RouterConfig - /** Inject the supervisor brain directly (tests / advanced) instead of resolving it from `router`. */ - readonly brain?: ToolLoopChat - /** Override the default authoring-supervisor profile (name / extra system-prompt stance). The - * default already carries the authoring skill; override only to add a goal or rename. */ - readonly supervisor?: { - readonly name?: string - readonly systemPrompt?: string - } + /** Exact executable authoring supervisor. Model, prompt, harness, and provider live here. */ + readonly supervisorProfile: SupervisorProfile + /** Router endpoint/auth for a `cli-base` supervisor; contains no behavioral settings. */ + readonly router: RouterTransportConfig /** Restrict the run to this subset of models (forwarded to `supervise()`). */ readonly allowedModels?: readonly string[] readonly runId?: string } -/** Build the DEFAULT authoring supervisor profile: a router-brained supervisor (`harness: cli-base`) - * whose standing instruction IS the authoring-agent-profiles skill, so it decomposes the intent and - * AUTHORS a worker profile per sub-task. No worker profile is baked in here. */ -function authoringSupervisorProfile( - model: string | undefined, - override?: { readonly name?: string; readonly systemPrompt?: string }, -): SupervisorProfile { - return { - name: override?.name ?? 'delegate-supervisor', - harness: 'cli-base', - ...(model ? { model: { default: model } } : {}), - prompt: { systemPrompt: override?.systemPrompt ?? supervisorInstructions() }, - } -} - /** * Delegate an INTENT to a default authoring supervisor and return its `SupervisedResult` unchanged. * @@ -91,25 +63,16 @@ function authoringSupervisorProfile( */ export async function delegate( intent: string, - opts: DelegateOptions = {}, + opts: DelegateOptions, ): Promise> { if (typeof intent !== 'string' || intent.trim().length === 0) { throw new ConfigError('delegate: `intent` must be a non-empty string') } - if (!opts.brain && !opts.router) { - throw new ConfigError( - 'delegate: provide opts.router (the supervisor brain substrate) or opts.brain (tests)', - ) - } - - const profile = authoringSupervisorProfile(opts.model, opts.supervisor) - - return supervise(profile, intent, { + return supervise(opts.supervisorProfile, intent, { budget: opts.budget ?? defaultDelegateBudget, ...(opts.backend ? { backend: opts.backend } : {}), ...(opts.deliverable ? { deliverable: opts.deliverable as DeliverableSpec } : {}), - ...(opts.router ? { router: opts.router } : {}), - ...(opts.brain ? { brain: opts.brain } : {}), + router: opts.router, ...(opts.allowedModels ? { allowedModels: opts.allowedModels } : {}), ...(opts.runId ? { runId: opts.runId } : {}), }) as Promise> diff --git a/src/runtime/supervise/graph.ts b/src/runtime/supervise/graph.ts index 8b5b78c0..42db3feb 100644 --- a/src/runtime/supervise/graph.ts +++ b/src/runtime/supervise/graph.ts @@ -61,7 +61,7 @@ import type { WorkerWatchOptions, } from '../../mcp/tools/coordination' import { composeRuntimeHooks, type RuntimeHooks } from '../../runtime-hooks' -import type { RouterConfig } from '../router-client' +import type { RouterTransportConfig } from '../router-client' import type { ToolLoopChat } from '../tool-loop' import type { DeliverableSpec } from './completion-gate' import { @@ -202,7 +202,7 @@ export interface RunGraphOptions { * directive delivery, and the edge ledger AROUND this seam — only the leaf `act` is yours. */ readonly makeWorkerAgent?: MakeWorkerAgent /** The driver brain's router substrate (`profile.harness` omitted or `cli-base`). */ - readonly router?: RouterConfig + readonly router?: RouterTransportConfig /** Caller-side runtime hooks (telemetry, policy, product extensions). Composed AFTER the * graph's own spawn-binding hook on the SAME event stream — the graph never swallows the * seam supervise() exposes. */ diff --git a/src/runtime/supervise/materialization.ts b/src/runtime/supervise/materialization.ts index 508f6a3f..520a0d43 100644 --- a/src/runtime/supervise/materialization.ts +++ b/src/runtime/supervise/materialization.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto' import { agentProfileSchema, + canonicalAgentProfileDigest, canonicalCandidateDigest, type Sha256Digest, } from '@tangle-network/agent-interface' @@ -22,11 +23,25 @@ type RuntimeOwnedExecutorMaterialization = readonly declaration: ExecutorMaterialization readonly binding?: ExecutorExecutionBinding } + | { + /** A Runtime-owned external executor whose exact launch is known, but whose remote + * materializer has not yet returned its terminal acknowledgement. This private state lets + * Runtime validate the planned authority before execution without presenting it as proof + * that the remote process actually used that plan. */ + readonly kind: 'pending' + readonly runtime: Runtime + readonly declaration: ExecutorMaterialization + readonly binding: ExecutorExecutionBinding + } | { readonly kind: 'deferred'; readonly runtime: Runtime } +interface RuntimeOwnedExecutorMaterializationRef { + current: RuntimeOwnedExecutorMaterialization +} + const runtimeOwnedExecutorMaterializations = new WeakMap< object, - RuntimeOwnedExecutorMaterialization + RuntimeOwnedExecutorMaterializationRef >() const runtimeOwnedScopeOwners = new WeakMap() @@ -62,9 +77,8 @@ export function attestRuntimeOwnedExecutor( declaration: ExecutorMaterialization, binding?: ExecutorExecutionBinding, ): Executor { - runtimeOwnedExecutorMaterializations.set( - executor as object, - Object.freeze({ + runtimeOwnedExecutorMaterializations.set(executor as object, { + current: Object.freeze({ kind: 'known', declaration: detachedSnapshot(declaration, 'runtime-owned executor materialization'), ...(binding === undefined @@ -73,7 +87,48 @@ export function attestRuntimeOwnedExecutor( binding: detachedSnapshot(binding, 'runtime-owned executor execution binding'), }), }), - ) + }) + return executor +} + +/** Brand an external Runtime executor whose exact remote materialization is provable only after + * its terminal protocol acknowledgement. The declaration is a planned authority check, never a + * known receipt; callers must finalize this same object after validating the acknowledgement. */ +export function attestRuntimeOwnedPendingExecutor( + executor: Executor, + runtime: Runtime, + declaration: ExecutorMaterialization, + binding: ExecutorExecutionBinding, +): Executor { + runtimeOwnedExecutorMaterializations.set(executor as object, { + current: Object.freeze({ + kind: 'pending', + runtime, + declaration: detachedSnapshot(declaration, 'pending executor materialization'), + binding: detachedSnapshot(binding, 'pending executor execution binding'), + }), + }) + return executor +} + +/** Replace a private pending brand with terminally acknowledged evidence. A caller-owned executor + * cannot enter this path because only this module can create the pending WeakMap entry. */ +export function finalizeRuntimeOwnedPendingExecutor( + executor: Executor, + declaration: ExecutorMaterialization, + binding: ExecutorExecutionBinding, +): Executor { + const ref = runtimeOwnedExecutorMaterializations.get(executor as object) + if (ref?.current.kind !== 'pending' || ref.current.runtime !== executor.runtime) { + throw new ValidationError( + 'executor materialization: terminal acknowledgement has no matching Runtime-owned pending executor', + ) + } + ref.current = Object.freeze({ + kind: 'known', + declaration: detachedSnapshot(declaration, 'runtime-owned executor materialization'), + binding: detachedSnapshot(binding, 'runtime-owned executor execution binding'), + }) return executor } @@ -83,21 +138,20 @@ export function attestRuntimeOwnedDeferredExecutor( executor: Executor, runtime: Runtime, ): Executor { - runtimeOwnedExecutorMaterializations.set( - executor as object, - Object.freeze({ kind: 'deferred', runtime }), - ) + runtimeOwnedExecutorMaterializations.set(executor as object, { + current: Object.freeze({ kind: 'deferred', runtime }), + }) return executor } /** Preserve the runtime-owned attestation when a trusted wrapper changes result semantics only. */ -export function inheritRuntimeOwnedExecutorAttestation( - source: Executor, +export function inheritRuntimeOwnedExecutorAttestation( + source: Executor, wrapper: Executor, ): Executor { - const attestation = runtimeOwnedExecutorMaterializations.get(source as object) - if (attestation !== undefined) { - runtimeOwnedExecutorMaterializations.set(wrapper as object, attestation) + const ref = runtimeOwnedExecutorMaterializations.get(source as object) + if (ref !== undefined) { + runtimeOwnedExecutorMaterializations.set(wrapper as object, ref) } return wrapper } @@ -106,7 +160,7 @@ export function inheritRuntimeOwnedExecutorAttestation( export function runtimeOwnedExecutorMaterialization( executor: Executor, ): ExecutorMaterialization | undefined { - const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object)?.current return attestation?.kind === 'known' ? attestation.declaration : undefined } @@ -114,15 +168,34 @@ export function runtimeOwnedExecutorMaterialization( export function runtimeOwnedExecutorExecutionBinding( executor: Executor, ): ExecutorExecutionBinding | undefined { - const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object)?.current return attestation?.kind === 'known' ? attestation.binding : undefined } +/** Read a planned external declaration only for Runtime's private pending brand. This is useful for + * pre-execution authority checks, but must never be turned into a known durable receipt. */ +export function runtimeOwnedPendingExecutorMaterialization(executor: Executor): + | { + readonly runtime: Runtime + readonly declaration: ExecutorMaterialization + readonly binding: ExecutorExecutionBinding + } + | undefined { + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object)?.current + return attestation?.kind === 'pending' + ? { + runtime: attestation.runtime, + declaration: attestation.declaration, + binding: attestation.binding, + } + : undefined +} + /** Read the expected leaf runtime only for a privately branded deferred orchestration executor. */ export function runtimeOwnedDeferredExecutorRuntime( executor: Executor, ): Runtime | undefined { - const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object)?.current return attestation?.kind === 'deferred' ? attestation.runtime : undefined } @@ -158,9 +231,7 @@ export function knownMaterializationReceipt( let materializationPlanDigest: Sha256Digest let platformAttachmentsDigest: Sha256Digest | undefined try { - // AgentProfile parsing represents omitted optional fields as `undefined`, while the backend's - // JSON request omits them. Commit the exact JSON document that crosses that boundary. - effectiveProfileDigest = canonicalCandidateDigest(jsonWireSnapshot(effectiveProfile.data)) + effectiveProfileDigest = canonicalAgentProfileDigest(effectiveProfile.data) materializationPlanDigest = canonicalCandidateDigest(jsonWireSnapshot(declaration.plan)) platformAttachmentsDigest = declaration.platformAttachments === undefined @@ -252,7 +323,7 @@ export function unknownExecutionBindingReceipt( /** Derive an authored-profile digest without turning non-canonical input into a fake identity. */ export function authoredProfileDigest(profile: unknown): Sha256Digest | undefined { try { - return canonicalCandidateDigest(profile) + return canonicalAgentProfileDigest(agentProfileSchema.parse(profile)) } catch { return undefined } diff --git a/src/runtime/supervise/model-policy.ts b/src/runtime/supervise/model-policy.ts index 370376f0..58d42d2b 100644 --- a/src/runtime/supervise/model-policy.ts +++ b/src/runtime/supervise/model-policy.ts @@ -7,6 +7,7 @@ import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { ConfigError } from '../../errors' +import { agentHarness } from '../harness-role' /** * Return the model id an executor may send to a provider. @@ -31,19 +32,164 @@ export function concreteProfileModel(profile: Pick): stri return concreteModelId(profile.model?.default) } +/** The provider-facing model id declared by a profile. Direct Router execution uses this form: + * the endpoint already selects the provider, so the model id itself is not harness-prefixed. */ +export function profileProviderModel(profile: Pick): string | undefined { + return concreteProfileModel(profile) +} + +/** The full cli-bridge wire id declared by a profile: harness/provider/model. */ +export function profileBridgeWireModel( + profile: Pick, +): string | undefined { + const model = concreteProfileModel(profile) + const provider = profile.model?.provider + const providerModel = model + ? provider && !model.includes('/') + ? `${provider}/${model}` + : model + : undefined + const harness = agentHarness(profile.harness) + if (!harness) return providerModel + if (!providerModel) return harness + return providerModel.startsWith(`${harness}/`) ? providerModel : `${harness}/${providerModel}` +} + /** - * Remove only Eval's runtime-selected model marker before a profile crosses an execution boundary. - * Every other model hint remains intact, including provider, reasoning effort, and small-model - * preferences. The input profile is never mutated. + * Refuse an incomplete execution identity before any backend may fill it from ambient config. + * `AgentProfile` is the sole behavioral authority: harness, provider, and concrete model all + * participate in its digest. Eval's runtime-selected marker is a matrix-planning value, never an + * executable model. */ -export function profileForExecution(profile: AgentProfile): AgentProfile { - const model = profile.model - if (!isHarnessNativeModel(model?.default) || model === undefined) return profile - const { default: _runtimeSelected, ...remainingModel } = model - const { model: _model, ...remainingProfile } = profile - return Object.keys(remainingModel).length > 0 - ? { ...remainingProfile, model: remainingModel } - : remainingProfile +export function assertExecutableAgentProfile(profile: AgentProfile, context: string): void { + if (profile.harness === undefined) { + throw new ConfigError(`${context}: AgentProfile.harness must be explicit before execution`) + } + const declared = profile.model?.default + const model = concreteModelId(declared) + if (!model) { + const reason = isHarnessNativeModel(declared) ? 'runtime-selected' : 'missing' + throw new ConfigError( + `${context}: AgentProfile.model.default is ${reason}; execution requires a concrete model`, + ) + } + if (!profile.model?.provider?.trim()) { + throw new ConfigError( + `${context}: AgentProfile.model.provider must be explicit before execution`, + ) + } +} + +/** Generation and loop controls that may affect one model execution. */ +export interface ProfileModelExecutionSettings { + readonly temperature?: number + readonly maxTokens?: number + /** Total transport attempts including the first request. */ + readonly maxAttempts?: number + readonly seed?: number + readonly toolChoice?: 'auto' | 'required' | 'none' + readonly extraBody?: Readonly> + /** Zero means no turn-count cap; conserved budgets and deadlines still apply. */ + readonly maxTurns?: number + readonly stream?: boolean +} + +const PROFILE_MODEL_METADATA_KEYS = new Set([ + 'extraBody', + 'maxRetries', + 'maxTokens', + 'maxTurns', + 'seed', + 'stream', + 'temperature', + 'toolChoice', +]) + +/** + * Read every Router-affecting control from the exact profile and reject unknown controls. + * Backends receive endpoint/auth and executable ports only; they cannot silently alter behavior. + */ +export function profileModelExecutionSettings( + profile: Pick, + context: string, +): ProfileModelExecutionSettings { + const metadata = profile.model?.metadata ?? {} + const unknown = Object.keys(metadata).filter((key) => !PROFILE_MODEL_METADATA_KEYS.has(key)) + if (unknown.length > 0) { + throw new ConfigError( + `${context}: unsupported AgentProfile.model.metadata fields: ${unknown.join(', ')}`, + ) + } + const temperature = finiteNumber(metadata.temperature, `${context}: temperature`) + const maxTokens = positiveInteger(metadata.maxTokens, `${context}: maxTokens`) + const maxRetries = nonnegativeInteger(metadata.maxRetries, `${context}: maxRetries`) + const seed = safeInteger(metadata.seed, `${context}: seed`) + const maxTurns = nonnegativeInteger(metadata.maxTurns, `${context}: maxTurns`) + const stream = optionalBoolean(metadata.stream, `${context}: stream`) + const toolChoice = metadata.toolChoice + if ( + toolChoice !== undefined && + toolChoice !== 'auto' && + toolChoice !== 'required' && + toolChoice !== 'none' + ) { + throw new ConfigError(`${context}: toolChoice must be auto, required, or none`) + } + const extraBody = metadata.extraBody + if ( + extraBody !== undefined && + (typeof extraBody !== 'object' || extraBody === null || Array.isArray(extraBody)) + ) { + throw new ConfigError(`${context}: extraBody must be an object`) + } + return { + ...(temperature !== undefined ? { temperature } : {}), + ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(maxRetries !== undefined ? { maxAttempts: maxRetries + 1 } : {}), + ...(seed !== undefined ? { seed } : {}), + ...(toolChoice !== undefined ? { toolChoice } : {}), + ...(extraBody !== undefined + ? { extraBody: Object.freeze({ ...(extraBody as Record) }) } + : {}), + ...(maxTurns !== undefined ? { maxTurns } : {}), + ...(stream !== undefined ? { stream } : {}), + } +} + +function finiteNumber(value: unknown, context: string): number | undefined { + if (value === undefined) return undefined + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new ConfigError(`${context} must be a finite number`) + } + return value +} + +function safeInteger(value: unknown, context: string): number | undefined { + if (value === undefined) return undefined + if (!Number.isSafeInteger(value)) throw new ConfigError(`${context} must be a safe integer`) + return value as number +} + +function positiveInteger(value: unknown, context: string): number | undefined { + const parsed = safeInteger(value, context) + if (parsed !== undefined && parsed < 1) { + throw new ConfigError(`${context} must be positive`) + } + return parsed +} + +function nonnegativeInteger(value: unknown, context: string): number | undefined { + const parsed = safeInteger(value, context) + if (parsed !== undefined && parsed < 0) { + throw new ConfigError(`${context} must be nonnegative`) + } + return parsed +} + +function optionalBoolean(value: unknown, context: string): boolean | undefined { + if (value === undefined) return undefined + if (typeof value !== 'boolean') throw new ConfigError(`${context} must be boolean`) + return value } /** diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index dc6d6478..fbc28b02 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -32,10 +32,18 @@ import { Readable } from 'node:stream' import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import { type AgentProfile, + type AgentProfileResourceRef, agentProfileSchema, - mergeAgentProfiles, + canonicalAgentProfileDigest, + profileMaterializationAxes, + REASONING_EFFORTS, + type ReasoningEffort, } from '@tangle-network/agent-interface' import type { BackendType, SandboxEvent } from '@tangle-network/sandbox' +import { + assertProfileMaterialization, + defineProfileMaterializationContract, +} from '../../agent/profile-materialization' import { ValidationError } from '../../errors' import type { LocalHarness } from '../../mcp/local-harness' import { mergeTraceEnv } from '../../mcp/trace-propagation' @@ -60,7 +68,16 @@ import { resolveAgentEnvironmentProvider, } from '../environment-provider' import { agentHarness } from '../harness-role' -import { routerChatWithUsage, type ToolSpec } from '../router-client' +import { + type PromptCacheUsage, + type RouterChatResult, + type RouterChatToolsResult, + type RouterConfig, + routerChatWithTools, + routerChatWithUsage, + streamRouterChatWithTools, + type ToolSpec, +} from '../router-client' import type { RunAgentRoundsOptions } from '../run-loop' import { runAgentRounds } from '../run-loop' import type { @@ -73,12 +90,19 @@ import type { } from '../types' import { zeroTokenUsage } from '../util' import { createInbox, type Inbox } from './inbox' -import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import { - concreteModelId, + attestRuntimeOwnedExecutor, + attestRuntimeOwnedPendingExecutor, + finalizeRuntimeOwnedPendingExecutor, + newExecutionAttemptId, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, +} from './materialization' +import { + assertExecutableAgentProfile, concreteProfileModel, - isHarnessNativeModel, - profileForExecution, + profileBridgeWireModel, + profileModelExecutionSettings, } from './model-policy' import { type ActivityLog, @@ -112,15 +136,41 @@ import { createWorktreeCliExecutor } from './worktree-cli-executor' // ── Seam contracts (read off ExecutorContext.seams, narrowed per built-in) ───── /** - * Router/inline connection seam. A direct OpenAI-compatible Router endpoint — - * the cheapest leaf, no box, no tools. `model` overrides the profile's model - * hint when present; otherwise the profile's `model.default` is required. + * Router/inline transport seam. The profile owns model, prompt, and generation behavior. */ export interface RouterSeam { routerBaseUrl: string routerKey: string - model?: string -} + /** Injectable transport for offline/local execution; still passes through Runtime metering. */ + complete?: RouterConfig['complete'] + /** When present, return one turn's requested tool calls without executing them. */ + tools?: ReadonlyArray +} + +const routerTurnProfileMaterialization = defineProfileMaterializationContract({ + name: 'router-profile-turn', + axes: [ + 'name', + 'description', + 'version', + 'tags', + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelProvider', + 'modelReasoningEffort', + 'modelMetadata', + 'harness', + 'tools', + 'files', + 'resourceTools', + 'skills', + 'resourceAgents', + 'commands', + 'resourceInstructions', + 'metadata', + ], +}) /** * Sandbox executor seam. The `sandboxClient` the composed `runAgentRounds` creates @@ -182,7 +232,7 @@ export interface CliSeam { /** * cli-worktree seam. A supervisor-authored `AgentProfile` driving a local coding-harness CLI * (claude / codex / opencode) on its own git worktree — the leaf `createWorktreeCliExecutor` - * named as data. `harness` + `repoRoot` are required; the task comes from `Executor.execute`. + * named as data. `repoRoot` is transport data; `AgentProfile.harness` selects the CLI. * `taskPrompt` remains an optional direct-call fallback for callers that execute with `undefined`. * The authored * `profile.prompt.systemPrompt` + `profile.model.default` reach the harness via the §1.5 @@ -190,8 +240,6 @@ export interface CliSeam { */ export interface CliWorktreeSeam { repoRoot: string - /** Local CLI harness transport. Omit when `bridge` is set. */ - harness?: LocalHarness taskPrompt?: string runId?: string baseRef?: string @@ -217,25 +265,20 @@ export interface CliWorktreeSeam { export interface CliWorktreeBridgeSeam { bridgeUrl: string bridgeBearer: string - /** Bridge model/harness id. Defaults to the profile's model hint when omitted. */ - model?: string - /** Canonical profile overlay merged over the spawned profile. */ - agentProfile?: AgentProfile /** Caller-owned deadline for each bridge turn. Runtime enforces it locally and sends the * same value in `execution.timeoutMs` so cli-bridge cannot substitute its own cutoff. */ timeoutMs?: number /** Stable cli-bridge session id. Defaults to `bridge-worktree-${runId}`. */ sessionId?: string - maxTurns?: number + /** Transport reconnects allowed after the first POST. Default 3; set 0 to disable. */ + maxReconnects?: number } /** * cli-bridge seam. A local OpenAI-compatible bridge that fronts harness CLIs - * (claude-code / opencode / kimi / pi) behind one HTTP surface; `model` doubles - * as the harness selector (e.g. `claude-code/sonnet`, `opencode//`). - * `agentProfile` is the bridge-dialect profile (metadata.disallowedTools, mcp) - * forwarded verbatim per request — how an arm disables native tools or injects - * a provider search MCP. + * (claude-code / opencode / kimi / pi) behind one HTTP surface. The spawned + * `AgentProfile` is the sole harness/provider/model and behavioral authority and + * is forwarded verbatim per request; this seam carries transport data only. * * The executor opens a resumable cli-bridge session. `sessionId` identifies the * harness conversation across turns; each turn also receives its own durable run id. @@ -249,7 +292,7 @@ export interface CliWorktreeBridgeSeam { * loses its pairing: an installed extension that persists memory across runs carries arm A's state * into arm B, and nothing reports it. * - * That is what the `AgentProfile` on this seam (and on the spawn spec) is FOR. `agent_profile` + * That is what the spawned `AgentProfile` is FOR. `agent_profile` * rides every request verbatim, and cli-bridge maps it onto each harness's own native controls: * * - Materializing any profile at all already starts the harness isolated from ambient @@ -279,21 +322,16 @@ export interface CliWorktreeBridgeSeam { export interface BridgeSeam { bridgeUrl: string bridgeBearer: string - /** Fallback bridge wire id. A spawned profile may select its own harness and model. */ - model?: string /** Optional working directory forwarded to cli-bridge and persisted with the session. */ cwd?: string - /** Canonical profile overlay merged over the spawned profile. */ - agentProfile?: AgentProfile /** Caller-owned deadline for each bridge turn. Runtime enforces it locally and sends the * same value in `execution.timeoutMs` so the bridge-owned process follows the same policy. */ timeoutMs?: number /** Stable, caller-owned cli-bridge session id for harness-side resume. Defaults * to a freshly minted per-spawn id so each worker is its own resumable session. */ sessionId?: string - /** Per-resume-turn inference cap before the worker settles on its last output. - * Mirrors `routerToolsInlineExecutor.maxTurns`; default 200 (runaway backstop). */ - maxTurns?: number + /** Transport reconnects allowed after the first POST. Default 3; set 0 to disable. */ + maxReconnects?: number /** Newest-last activity window `progress()` reports. Default 12 (matches `PiSeam`). */ activityWindow?: number } @@ -318,6 +356,8 @@ const sandboxSeamKey = 'sandbox' const cliSeamKey = 'cli' const bridgeSeamKey = 'bridge' const maxBridgeTimeoutMs = 2_147_483_647 +const bridgeProfileMaterializationSchema = 'cli-bridge.profile-materialization.v2' +const bridgeUsageCostSchema = 'cli-bridge.usage-cost.v1' const cliWorktreeSeamKey = 'cli-worktree' const providerSeamKey = 'provider' @@ -357,18 +397,19 @@ function contentRef(prefix: string, value: unknown): string { * as a ceiling rather than a measurement. Use this — never a bare zero — whenever a runtime cannot * see what its worker spent. * - * DELIBERATELY NOT `usdKnown: false`, and this is not an oversight. On the dollar channel that flag - * is not a marker but a REFUSAL: `budget.ts` treats unknown dollars under a dollar-capped root as a - * reconcile violation and fails the child. Applying it here would contradict `budgetExempt`, whose - * whole documented contract is that such a worker settles OUT of the conserved pool rather than - * against it (`scope.ts`) — a worker the kernel already agreed not to budget would start failing - * after its work had burned, which is a policy change about which configurations are allowed, not a - * fix to how honestly spend is reported. The token marker taints only token accounting. Under the - * current `budgetExempt` policy the surviving `usd: 0` remains dollar-known; callers that require - * dollar accounting must use a backend that returns priced usage. + * The same rule applies to dollars. A dollar-capped run must refuse an executor whose billed spend + * is unknowable; `budgetExempt` does not authorize Runtime to relabel unknown spend as a measured + * zero. Callers that require dollar accounting must use a backend with a trusted billing receipt. */ function unmeteredSpend(ms: number): Spend { - return { iterations: 0, tokens: zeroTokenUsage(), tokensKnown: false, usd: 0, ms } + return { + iterations: 0, + tokens: zeroTokenUsage(), + tokensKnown: false, + usd: 0, + usdKnown: false, + ms, + } } // ── router/inline executor (harness === null) ────────────────────────────────── @@ -386,15 +427,12 @@ function unmeteredSpend(ms: number): Spend { */ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerSeamKey, 'router/inline') - const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) - if (!model) { - throw new ValidationError( - 'routerInlineExecutor: no model — set RouterSeam.model or AgentProfile.model.default', - ) - } + const model = exactRouterModel(spec.profile, 'routerInlineExecutor') if (!seam.routerBaseUrl || !seam.routerKey) { throw new ValidationError('routerInlineExecutor: RouterSeam.routerBaseUrl + routerKey required') } + const profileExecution = routerProfileExecution(spec.profile, seam, { multiTurn: false }) + const requestIdentity = routerRequestIdentity(ctx) const controller = new AbortController() const abortIfSignalled = () => { @@ -411,24 +449,104 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { { runtime: 'router' as Runtime, async execute(task, signal): Promise> { - const messages = taskToMessages(task, spec) + const messages = taskToMessages(task, spec, profileExecution.systemPrompt) const started = Date.now() const linked = linkSignals(signal, controller.signal) - const r = await routerChatWithUsage( - { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, - messages, - linked ? { signal: linked } : {}, + const extraBody = { + ...(profileExecution.extraBody ?? {}), + ...(profileExecution.reasoningEffort + ? { reasoning_effort: profileExecution.reasoningEffort } + : {}), + } + const r = await runRouterTransport( + 'routerInlineExecutor', + () => + seam.tools + ? (profileExecution.stream === true + ? streamRouterChatWithTools + : routerChatWithTools)( + { + routerBaseUrl: seam.routerBaseUrl, + routerKey: seam.routerKey, + model, + ...(profileExecution.maxAttempts !== undefined + ? { maxAttempts: profileExecution.maxAttempts } + : {}), + ...(seam.complete ? { complete: seam.complete } : {}), + }, + messages, + seam.tools, + { + ...(profileExecution.temperature !== undefined + ? { temperature: profileExecution.temperature } + : {}), + ...(linked ? { signal: linked } : {}), + ...(profileExecution.toolChoice + ? { toolChoice: profileExecution.toolChoice } + : {}), + ...(profileExecution.maxTokens !== undefined + ? { maxTokens: profileExecution.maxTokens } + : {}), + ...(profileExecution.seed !== undefined ? { seed: profileExecution.seed } : {}), + ...(Object.keys(extraBody).length > 0 ? { extraBody } : {}), + ...requestIdentity, + }, + ) + : routerChatWithUsage( + { + routerBaseUrl: seam.routerBaseUrl, + routerKey: seam.routerKey, + model, + ...(profileExecution.maxAttempts !== undefined + ? { maxAttempts: profileExecution.maxAttempts } + : {}), + ...(seam.complete ? { complete: seam.complete } : {}), + }, + messages, + { + ...(profileExecution.temperature !== undefined + ? { temperature: profileExecution.temperature } + : {}), + ...(linked ? { signal: linked } : {}), + ...(profileExecution.maxTokens !== undefined + ? { maxTokens: profileExecution.maxTokens } + : {}), + ...(profileExecution.seed !== undefined ? { seed: profileExecution.seed } : {}), + ...(profileExecution.reasoningEffort + ? { reasoningEffort: profileExecution.reasoningEffort } + : {}), + ...(profileExecution.extraBody + ? { extraBody: profileExecution.extraBody } + : {}), + ...requestIdentity, + }, + ), ) const spent: Spend = { iterations: 1, tokens: r.usage ? { input: r.usage.input, output: r.usage.output } : zeroTokenUsage(), - usd: r.costUsd ?? 0, + usd: r.billedCostUsd ?? 0, ...(r.usage ? {} : { tokensKnown: false }), - ...(r.costUsd === undefined ? { usdKnown: false } : {}), + ...(r.billedCostUsd === undefined ? { usdKnown: false } : {}), ms: Date.now() - started, } - const out = { content: r.content } as unknown - artifact = { outRef: contentRef('router', { model, content: r.content }), out, spent } + assertObservedRouterModel(r.model, model, 'routerInlineExecutor') + const out = { + content: r.content ?? '', + ...(r.model !== undefined ? { model: r.model } : {}), + transportAttempts: r.transportAttempts, + ...(r.costUsd !== undefined ? { estimatedCostUsd: r.costUsd } : {}), + ...(r.cache ? { promptCache: r.cache } : {}), + ...(r.usage?.reasoning !== undefined ? { reasoningTokens: r.usage.reasoning } : {}), + ...('toolCalls' in r ? { toolCalls: r.toolCalls } : {}), + ...(r.reasoning ? { reasoning: r.reasoning } : {}), + ...(r.finishReason + ? { finishReason: r.finishReason } + : 'toolCalls' in r && r.toolCalls.length > 0 + ? { finishReason: 'tool_calls' } + : {}), + } as unknown + artifact = { outRef: contentRef('router', { model, out }), out, spent } return artifact }, teardown(_grace): Promise<{ destroyed: boolean }> { @@ -451,7 +569,20 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { id: executionId, }, materializer: 'router-prompt-model', - plan: { kind: 'openai-chat-completion', model }, + plan: { + kind: 'openai-chat-completion', + model, + provider: spec.profile.model?.provider ?? null, + temperature: profileExecution.temperature ?? null, + maxTokens: profileExecution.maxTokens ?? null, + maxAttempts: profileExecution.maxAttempts ?? null, + seed: profileExecution.seed ?? null, + reasoningEffort: profileExecution.reasoningEffort ?? null, + extraBody: profileExecution.extraBody ?? null, + tools: seam.tools ?? null, + toolChoice: profileExecution.toolChoice ?? null, + systemPrompt: profileExecution.systemPrompt || null, + }, }, { attemptId, @@ -480,9 +611,13 @@ export type { ToolSpec } export interface RouterToolsSeam { routerBaseUrl: string routerKey: string - model?: string + complete?: RouterConfig['complete'] tools: ReadonlyArray executeToolCall: (name: string, args: Record, task: unknown) => Promise + /** Exact conversation to continue. Runtime validates its system message against the profile. */ + initialMessages?: ReadonlyArray>> + /** Observe the detached final conversation for session persistence. */ + onMessages?: (messages: ReadonlyArray>>) => void | Promise /** Online observer of each tool step — the seam a `DetectorMonitor` taps to watch the live pipe * (raise a `finding` when the worker loops/errors). Called after every tool call resolves, with * real per-call wall-clock (`startedAt`/`endedAt`/`durationMs`) so a push `TraceSource` can carry @@ -498,21 +633,38 @@ export interface RouterToolsSeam { endedAt?: number durationMs?: number }) => void - /** Max inference turns. Default 200 (runaway backstop — set far above any - * legitimate workflow). For tighter per-workflow limits use a cost budget - * or wall-clock deadline at the call site. */ - maxTurns?: number } const routerToolsSeamKey = 'router-tools' -interface RouterToolsResponse { - choices?: Array<{ - message?: { - content?: string | null - tool_calls?: Array<{ id?: string; function?: { name?: string; arguments?: string } }> - } - }> - usage?: { prompt_tokens?: number; completion_tokens?: number } +function mergePromptCache( + target: Record, + cache: PromptCacheUsage | undefined, +): void { + if (!cache) return + for (const key of ['readTokens', 'writeTokens', 'missTokens', 'readSavingsUsd'] as const) { + const value = cache[key] + if (value !== undefined) target[key] = (Number(target[key]) || 0) + value + } + if (cache.status !== undefined) target.status = cache.status +} + +async function runRouterTransport(context: string, call: () => Promise): Promise { + try { + return await call() + } catch (error) { + throwRouterTransportFailure(context, error) + } +} + +function throwRouterTransportFailure(context: string, error: unknown): never { + if ( + error instanceof ValidationError || + (error && typeof error === 'object' && 'name' in error && error.name === 'AbortError') + ) { + throw error + } + const message = error instanceof Error ? error.message : String(error) + throw new ValidationError(`${context}: transport failed: ${message}`) } /** @@ -521,18 +673,24 @@ interface RouterToolsResponse { */ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerToolsSeamKey, 'router-tools') - const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) - if (!model) { - throw new ValidationError( - 'routerToolsInlineExecutor: no model — set RouterToolsSeam.model or AgentProfile.model.default', - ) - } + const model = exactRouterModel(spec.profile, 'routerToolsInlineExecutor') if (!seam.routerBaseUrl || !seam.routerKey) { throw new ValidationError( 'routerToolsInlineExecutor: RouterToolsSeam.routerBaseUrl + routerKey required', ) } - const maxTurns = seam.maxTurns ?? 200 + const profileExecution = routerProfileExecution( + spec.profile, + { + routerBaseUrl: seam.routerBaseUrl, + routerKey: seam.routerKey, + tools: seam.tools, + }, + { multiTurn: true }, + ) + const enabledToolNames = new Set(seam.tools.map((tool) => tool.function.name)) + const maxTurns = profileExecution.maxTurns ?? 0 + const requestIdentity = routerRequestIdentity(ctx) const controller = new AbortController() const abortIfSignalled = () => { @@ -554,12 +712,25 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = deliver: (m) => inbox.deliver(m), async execute(task, signal): Promise> { const started = Date.now() - const messages: Array> = [ - ...(taskToMessages(task, spec) as Array>), - ] + const messages: Array> = seam.initialMessages + ? taskToMessages( + { + messages: [...seam.initialMessages, { role: 'user', content: taskToPrompt(task) }], + }, + spec, + profileExecution.systemPrompt, + ) + : taskToMessages(task, spec, profileExecution.systemPrompt) const tokens = zeroTokenUsage() let tokensKnown = true + let billedUsd = 0 + let usdKnown = true let turns = 0 + let transportAttempts = 0 + let observedModel: string | undefined + let reasoningTokens = 0 + let reasoningKnown = true + const promptCache: Record = {} let lastText = '' // Fold any queued down-messages into the conversation as one operator turn (the boundary flush). const flush = () => { @@ -572,148 +743,197 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = // re-register listeners on these long-lived signals every turn. const external = mergeAbortSignals(signal, controller.signal) - for (let t = 0; t < maxTurns; t += 1) { - // QUEUED messages flush at the step boundary, before this turn's inference. - flush() - // A forceful (interrupt) message aborts THIS turn so the worker re-plans immediately. The - // per-turn controller fires on `external` OR a fresh interrupt; its listener on `external` is - // removed after the turn (`cleanup`) so nothing accumulates across turns. - const interruptSig = inbox.freshInterrupt() - const turnController = new AbortController() - const abortTurn = () => turnController.abort() - if (external.aborted) turnController.abort() - else external.addEventListener('abort', abortTurn) - interruptSig.addEventListener('abort', abortTurn, { once: true }) - const cleanup = () => external.removeEventListener('abort', abortTurn) - let res: Response - try { - res = await fetch(`${seam.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${seam.routerKey}`, - }, - body: JSON.stringify({ - model, + try { + for (let t = 0; maxTurns === 0 || t < maxTurns; t += 1) { + // QUEUED messages flush at the step boundary, before this turn's inference. + flush() + // A forceful (interrupt) message aborts THIS turn so the worker re-plans immediately. The + // per-turn controller fires on `external` OR a fresh interrupt; its listener on `external` is + // removed after the turn (`cleanup`) so nothing accumulates across turns. + const interruptSig = inbox.freshInterrupt() + const turnController = new AbortController() + const abortTurn = () => turnController.abort() + if (external.aborted) turnController.abort() + else external.addEventListener('abort', abortTurn) + interruptSig.addEventListener('abort', abortTurn, { once: true }) + const cleanup = () => external.removeEventListener('abort', abortTurn) + let res: Awaited> + try { + res = await (profileExecution.stream === true + ? streamRouterChatWithTools + : routerChatWithTools)( + { + routerBaseUrl: seam.routerBaseUrl, + routerKey: seam.routerKey, + model, + ...(profileExecution.maxAttempts !== undefined + ? { maxAttempts: profileExecution.maxAttempts } + : {}), + ...(seam.complete ? { complete: seam.complete } : {}), + }, messages, - tools: seam.tools, - tool_choice: 'auto', - temperature: 0.2, - }), - signal: turnController.signal, - }) - } catch (e) { + seam.tools, + { + ...(profileExecution.temperature !== undefined + ? { temperature: profileExecution.temperature } + : {}), + signal: turnController.signal, + ...(profileExecution.toolChoice + ? { toolChoice: profileExecution.toolChoice } + : {}), + ...(profileExecution.maxTokens !== undefined + ? { maxTokens: profileExecution.maxTokens } + : {}), + ...(profileExecution.seed !== undefined ? { seed: profileExecution.seed } : {}), + ...(profileExecution.reasoningEffort + ? { reasoningEffort: profileExecution.reasoningEffort } + : {}), + ...(profileExecution.extraBody ? { extraBody: profileExecution.extraBody } : {}), + ...routerTurnRequestIdentity(requestIdentity, t), + }, + ) + } catch (e) { + cleanup() + // Re-plan ONLY when a forceful inbox message aborted this turn (a real AbortError, with the + // interrupt — not the external teardown/budget signal). The re-planned turn still consumes a + // loop slot when the caller configured a finite maxTurns, but does not bill a turn. + // Any other error — incl. a network fault coincident with an interrupt — is fatal: rethrow. + const interruptAbort = + e instanceof DOMException && + e.name === 'AbortError' && + interruptSig.aborted && + !signal.aborted && + !controller.signal.aborted + if (interruptAbort) continue + throwRouterTransportFailure('routerToolsInlineExecutor', e) + } cleanup() - // Re-plan ONLY when a forceful inbox message aborted this turn (a real AbortError, with the - // interrupt — not the external teardown/budget signal). The re-planned turn still consumes a - // loop slot (so interrupt spam is bounded by maxTurns, not a hang) but does not bill a turn. - // Any other error — incl. a network fault coincident with an interrupt — is fatal: rethrow. - const interruptAbort = - e instanceof DOMException && - e.name === 'AbortError' && - interruptSig.aborted && - !signal.aborted && - !controller.signal.aborted - if (interruptAbort) continue - throw e - } - cleanup() - // The inference completed — count the turn now (an interrupted, re-planned turn doesn't bill). - turns += 1 - if (!res.ok) { - throw new ValidationError( - `routerToolsInlineExecutor: router ${res.status}: ${(await res.text()).slice(0, 200)}`, - ) - } - const data = (await res.json()) as RouterToolsResponse - const u = data.usage - if (u && typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number') { - tokens.input += u.prompt_tokens - tokens.output += u.completion_tokens - } else { - tokensKnown = false - } - const msg = data.choices?.[0]?.message - if (msg?.content) lastText = msg.content - const toolCalls = msg?.tool_calls ?? [] - if (toolCalls.length === 0) { - // Before settling, flush once more — a worker may not finish while a steer/answer it never - // read is still pending. If anything flushed, keep going; otherwise it is truly done. - if (flush()) continue - break - } - - // Record the assistant turn verbatim, then run each call on the host and - // fold the result back as a `tool` message for the next turn. - messages.push({ - role: 'assistant', - content: msg?.content ?? '', - tool_calls: toolCalls.map((tc, i) => ({ - id: tc.id ?? `call_${i}`, - type: 'function', - function: { - name: tc.function?.name ?? '', - arguments: tc.function?.arguments ?? '{}', - }, - })), - }) - for (let i = 0; i < toolCalls.length; i += 1) { - const tc = toolCalls[i] - const id = tc?.id ?? `call_${i}` - let args: Record = {} - try { - args = JSON.parse(tc?.function?.arguments ?? '{}') as Record - } catch { - // Malformed args are a real outcome, not an infra fault — feed the error - // back so the model can correct, rather than aborting the whole loop. - messages.push({ - role: 'tool', - tool_call_id: id, - content: 'error: tool arguments were not valid JSON', - }) - continue + // The inference completed — count the turn now (an interrupted, re-planned turn doesn't bill). + turns += 1 + transportAttempts += res.transportAttempts + assertObservedRouterModel(res.model, model, 'routerToolsInlineExecutor') + if (res.model !== undefined) observedModel = res.model + mergePromptCache(promptCache, res.cache) + if (res.usage) { + tokens.input += res.usage.input + tokens.output += res.usage.output + if (res.usage.reasoning !== undefined) reasoningTokens += res.usage.reasoning + else reasoningKnown = false + } else { + tokensKnown = false + reasoningKnown = false } - const toolName = tc?.function?.name ?? '' - let result: string - let status: 'ok' | 'error' = 'ok' - const toolStartedAt = Date.now() - try { - result = await seam.executeToolCall(toolName, args, task) - } catch (e) { - status = 'error' - result = `error: ${e instanceof Error ? e.message : String(e)}` + if (res.billedCostUsd !== undefined) billedUsd += res.billedCostUsd + else usdKnown = false + if (res.content) lastText = res.content + const toolCalls = res.toolCalls + if (toolCalls.length === 0) { + // Before settling, flush once more — a worker may not finish while a steer/answer it never + // read is still pending. If anything flushed, keep going; otherwise it is truly done. + if (flush()) continue + messages.push({ role: 'assistant', content: res.content ?? '' }) + break } - const toolEndedAt = Date.now() - messages.push({ role: 'tool', tool_call_id: id, content: result }) - // Feed the online detector pipe (stuck-loop / error-streak) — a worker repeating the same - // call or hammering errors is caught mid-run, not only at settle. This is an observability - // side-channel: a throwing monitor must never crash the production inference loop. - try { - seam.onToolStep?.({ - toolName, - args, - status, - startedAt: toolStartedAt, - endedAt: toolEndedAt, - durationMs: toolEndedAt - toolStartedAt, - }) - } catch { - // ignore — monitoring must not break the worker + + // Record the assistant turn verbatim, then run each call on the host and + // fold the result back as a `tool` message for the next turn. + messages.push({ + role: 'assistant', + content: res.content ?? '', + tool_calls: toolCalls.map((tc) => ({ + id: tc.id, + type: 'function', + function: { + name: tc.name, + arguments: tc.arguments, + }, + })), + }) + for (let i = 0; i < toolCalls.length; i += 1) { + const tc = toolCalls[i] + const id = tc?.id ?? `call_${i}` + const toolName = tc?.name ?? '' + if (!enabledToolNames.has(toolName)) { + messages.push({ + role: 'tool', + tool_call_id: id, + content: `error: tool ${JSON.stringify(toolName)} is not enabled by AgentProfile.tools`, + }) + try { + seam.onToolStep?.({ toolName, args: {}, status: 'error' }) + } catch { + // Monitoring cannot authorize or execute a refused tool call. + } + continue + } + let args: Record = {} + try { + args = JSON.parse(tc?.arguments ?? '{}') as Record + } catch { + // Malformed args are a real outcome, not an infra fault — feed the error + // back so the model can correct, rather than aborting the whole loop. + messages.push({ + role: 'tool', + tool_call_id: id, + content: 'error: tool arguments were not valid JSON', + }) + continue + } + let result: string + let status: 'ok' | 'error' = 'ok' + const toolStartedAt = Date.now() + try { + result = await seam.executeToolCall(toolName, args, task) + } catch (e) { + status = 'error' + result = `error: ${e instanceof Error ? e.message : String(e)}` + } + const toolEndedAt = Date.now() + messages.push({ role: 'tool', tool_call_id: id, content: result }) + // Feed the online detector pipe (stuck-loop / error-streak) — a worker repeating the same + // call or hammering errors is caught mid-run, not only at settle. This is an observability + // side-channel: a throwing monitor must never crash the production inference loop. + try { + seam.onToolStep?.({ + toolName, + args, + status, + startedAt: toolStartedAt, + endedAt: toolEndedAt, + durationMs: toolEndedAt - toolStartedAt, + }) + } catch { + // ignore — monitoring must not break the worker + } } } + } finally { + await seam.onMessages?.( + structuredClone(messages) as ReadonlyArray>>, + ) } const priced = isModelPriced(model) - const usd = priced ? estimateCost(tokens.input, tokens.output, model) : 0 + const estimatedUsd = priced ? estimateCost(tokens.input, tokens.output, model) : undefined const spent: Spend = { iterations: turns, tokens, ...(tokensKnown ? {} : { tokensKnown: false }), - usd, - ...(!priced || !tokensKnown ? { usdKnown: false } : {}), + usd: billedUsd, + ...(usdKnown ? {} : { usdKnown: false }), ms: Date.now() - started, } - const out = { content: lastText } as unknown + const out = { + content: lastText, + ...(observedModel !== undefined ? { model: observedModel } : {}), + messages, + turns, + toolCalls: messages.filter((message) => message.role === 'tool').length, + transportAttempts, + ...(estimatedUsd !== undefined ? { estimatedCostUsd: estimatedUsd } : {}), + ...(Object.keys(promptCache).length > 0 ? { promptCache } : {}), + ...(reasoningKnown && turns > 0 ? { reasoningTokens } : {}), + } as unknown artifact = { outRef: contentRef('router-tools', { model, content: lastText }), out, spent } return artifact }, @@ -739,7 +959,20 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = id: executionId, }, materializer: 'router-tools-prompt-model', - plan: { kind: 'openai-tool-loop', model, maxTurns, tools: seam.tools }, + plan: { + kind: 'openai-tool-loop', + model, + provider: spec.profile.model?.provider ?? null, + maxTurns, + tools: seam.tools, + temperature: profileExecution.temperature ?? null, + maxTokens: profileExecution.maxTokens ?? null, + maxAttempts: profileExecution.maxAttempts ?? null, + toolChoice: profileExecution.toolChoice ?? null, + extraBody: profileExecution.extraBody ?? null, + reasoningEffort: profileExecution.reasoningEffort ?? null, + systemPrompt: profileExecution.systemPrompt || null, + }, }, { attemptId, @@ -753,6 +986,39 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = ) } +function assertObservedRouterModel( + observed: string | undefined, + expected: string, + context: string, +): void { + if (observed !== undefined && observed !== expected) { + throw new ValidationError( + `${context}: provider reported model ${JSON.stringify(observed)} but AgentProfile requires ${JSON.stringify(expected)}`, + ) + } +} + +function routerRequestIdentity(ctx: ExecutorContext): { + readonly callId?: string + readonly correlationId?: string +} { + const correlation = ctx.node?.identity?.correlation + return { + ...(correlation?.callId ? { callId: correlation.callId } : {}), + ...(correlation?.correlationId ? { correlationId: correlation.correlationId } : {}), + } +} + +function routerTurnRequestIdentity( + identity: { readonly callId?: string; readonly correlationId?: string }, + turnIndex: number, +): { readonly callId?: string; readonly correlationId?: string } { + return { + ...(identity.callId ? { callId: `${identity.callId}:turn:${turnIndex + 1}` } : {}), + ...(identity.correlationId ? { correlationId: identity.correlationId } : {}), + } +} + // ── sandbox executor (harness is a BackendType) ──────────────────────────────── /** @@ -986,23 +1252,41 @@ async function* streamSandboxLeaf(args: StreamSandboxArgs): AsyncIterable 0 || result.tokenUsage.input || result.tokenUsage.output) { + yield { + kind: 'tokens', + input: result.tokenUsage.input, + output: result.tokenUsage.output, + ...(tokensKnown ? {} : { tokensKnown: false }), + } + } + if (result.iterations.length > 0 || result.costUsd) { + yield { kind: 'cost', usd: result.costUsd, ...(usdKnown ? {} : { usdKnown: false }) } } - if (result.costUsd) yield { kind: 'cost', usd: result.costUsd } } finally { args.signal.removeEventListener('abort', cascade) args.controller.signal.removeEventListener('abort', cascade) @@ -1246,80 +1530,31 @@ function killWithGrace( * * Reports REAL usage when the bridge surfaces it, never a fabricated cost. */ -/** Resolve the bridge wire model for this spawn. Per-create matrix settings win, then the - * canonical profile's harness/model preferences, then the bridge's configured fallback. */ -/** - * A profile's selected model with its provider attached, the way a harness addresses one. - * - * Returns undefined when the profile selects no model, and the id unchanged when it is already - * qualified or when the profile names no provider — there, the harness's own provider resolution - * IS the caller's declared intent rather than a gap to fill. - */ -function qualifyProviderModel(model: AgentProfile['model']): string | undefined { - const id = concreteModelId(model?.default) - if (!id) return undefined - const provider = model?.provider - if (!provider || id.includes('/')) return id - return `${provider}/${id}` +interface ResolvedBridgeSeam extends BridgeSeam { + /** Derived once from the exact AgentProfile; never accepted as backend configuration. */ + readonly model: string } -function bridgeCellModel( - seamModel: string | undefined, - ctx: ExecutorContext, - profile: AgentProfile, -): string | undefined { - const create = ctx.seams.createOptions as - | { backend?: { type?: string; model?: { model?: string } } } - | undefined - const backend = create?.backend - const profileHarness = agentHarness(profile.harness) - const harness = backend?.type ?? profileHarness - // The PROVIDER rides with the model. A harness addresses a model as `provider/model`, so a wire - // id built from `model.default` alone loses it: `{provider:'tangle-router', default:'glm-5.2'}` - // becomes `pi/glm-5.2`, which routes to the right BACKEND and then hands pi a bare id it cannot - // place. pi falls back to its own default provider and dies with "No API key found for - // " — a credential error naming a provider the caller never chose. Measured live; - // the same request with `pi/tangle-router/glm-5.2` returns 200. - // - // A concrete per-cell `backend.model.model` override is left exactly as supplied: it is a - // caller-authored wire id, not a profile hint, and qualifying it would rewrite what the caller - // asked for. Eval's runtime-selected marker is the one exception: it selects the harness's - // configured model and must never cross the wire as a provider model id. - const backendModel = backend?.model?.model - const hasBackendModel = backendModel !== undefined - // cli-bridge already treats a bare harness id (`pi`, `codex`, …) as "use that - // harness's configured model". Translate Eval's marker into that existing - // wire form rather than leaking `default` or substituting an unrelated fallback. - if (hasBackendModel && isHarnessNativeModel(backendModel)) { - return harness ?? concreteModelId(seamModel) - } - const model = hasBackendModel - ? concreteModelId(backendModel) - : qualifyProviderModel(profile.model) - const fallback = concreteModelId(seamModel) - if (!harness && !model) return fallback - if (!harness) return model - if (model) return model.startsWith(`${harness}/`) ? model : `${harness}/${model}` - if (!fallback) return undefined - return fallback.startsWith(`${harness}/`) ? fallback : `${harness}/${fallback}` +/** Resolve the exact bridge wire id from the profile and nowhere else. */ +function bridgeProfileModel(profile: AgentProfile, context: string): string { + assertExecutableAgentProfile(profile, context) + if (!agentHarness(profile.harness)) { + throw new ValidationError(`${context}: AgentProfile.harness must select a coding-agent harness`) + } + const model = profileBridgeWireModel(profile) + if (!model) { + throw new ValidationError(`${context}: AgentProfile did not resolve a bridge wire model`) + } + return model } export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { const base = readSeam(ctx, bridgeSeamKey, 'bridge') - // A per-create `backend` override (threaded by `inlineSandboxClient` as - // `seams.createOptions`) targets the bridge model per cell without a second - // client: `backend.type` is the harness, `backend.model.model` the model, and - // the wire id is `${harness}/${model}` (an already-`${harness}/`-prefixed model - // passes through). This is how ONE bridge `SandboxClient` drives every - // harness×model cell of a matrix — the seam `model` is the fixed default. - const effectiveProfile = agentProfileSchema.parse( - mergeAgentProfiles(spec.profile, base.agentProfile) ?? spec.profile, - ) - const seam = { ...base, model: bridgeCellModel(base.model, ctx, effectiveProfile) } - if (!seam.bridgeUrl || !seam.bridgeBearer || !seam.model) { - throw new ValidationError( - 'bridgeExecutor: bridgeUrl + bridgeBearer and a profile or bridge model are required', - ) + const effectiveProfile = agentProfileSchema.parse(spec.profile) + const model = bridgeProfileModel(effectiveProfile, 'bridgeExecutor') + const seam: ResolvedBridgeSeam = { ...base, model } + if (!seam.bridgeUrl || !seam.bridgeBearer) { + throw new ValidationError('bridgeExecutor: bridgeUrl + bridgeBearer are required') } if ( seam.timeoutMs !== undefined && @@ -1331,7 +1566,14 @@ export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { `bridgeExecutor: timeoutMs must be an integer from 1 to ${maxBridgeTimeoutMs}`, ) } - const maxTurns = seam.maxTurns ?? 200 + if ( + seam.maxReconnects !== undefined && + (!Number.isSafeInteger(seam.maxReconnects) || seam.maxReconnects < 0) + ) { + throw new ValidationError('bridgeExecutor: maxReconnects must be a nonnegative safe integer') + } + const maxTurns = profileModelExecutionSettings(effectiveProfile, 'bridgeExecutor').maxTurns ?? 0 + const maxReconnects = seam.maxReconnects ?? 3 // A stable per-spawn session id (caller can pin one) — cli-bridge keys harness // resume off this exactly as a box id keys a sandbox session. const sessionId = seam.sessionId ?? `bridge-${spec.profile.name ?? 'worker'}-${randomUUID()}` @@ -1361,122 +1603,138 @@ export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { refreshing: false, closed: false, } - // What this executor changed about what the caller declared. Recorded up front, never evicted, - // so a run that fails on turn 40 still answers "what was this worker actually given?". - if (base.agentProfile !== undefined) { - observation.derived.push( - "profile: merged the bridge seam's agentProfile over the spawn profile before materialization", - ) - } - if (seam.model !== base.model) { - observation.derived.push( - `model: resolved the bridge wire model to ${seam.model} (seam default ${base.model ?? 'none'})`, - ) - } // One push source per SPAWN, not per turn: a steered worker's turn 4 tool calls belong to the // same trace as its turn 0 calls, and `collect()` must still answer after the stream drains. const trace = createPushTraceSource({ runId: sessionId }) - return attestRuntimeOwnedExecutor( - { - runtime: 'cli' as Runtime, - deliver: (m) => inbox.deliver(m), - /** - * The LIVE read of this worker, answered entirely from local mirrors so it is synchronous - * and cannot block or fail the run. The bridge's own run state is refreshed OUT OF BAND — - * a read schedules a fetch whose answer lands for the NEXT read — because the executor - * cannot know from its own stream whether a silent run is thinking, detached mid-reconnect, - * or already cancelled server-side, and that is exactly the distinction a supervisor's - * respawn decision turns on. - */ - progress: (): ExecutorProgress | undefined => { - try { - scheduleBridgeRunStateRefresh(seam, activeRuns, observation) - return { - turns: observation.turns, - pendingMessages: inbox.pending(), - recentActivity: observation.activity.read(), - ...(observation.derived.length > 0 ? { derived: [...observation.derived] } : {}), - note: bridgeProgressNote(observation, liveBridgeRunId(activeRuns)), - } - } catch { - // An observability read must degrade to "no progress available", never take a live - // run down. There is no partial answer worth risking the worker for. - return undefined - } - }, - traceSource: (): TraceSource => trace.source, - execute(task, signal): AsyncIterable { - return streamBridgeSession({ - task, - signal, - profile: effectiveProfile, - seam, - sessionId, - maxTurns, - inbox, - controller, - activeRuns, - observation, - record: (step: ToolStepInput) => { - trace.record(step) - }, - onArtifact: (a) => { - artifact = a - }, - }) - }, - async teardown(grace): Promise<{ destroyed: boolean }> { - controller.abort() - const remaining = [...activeRuns.values()].filter((run) => !run.terminal) - if (remaining.length === 0) return { destroyed: true } - const terminal = await Promise.all( - remaining.map((run) => cancelBridgeRunToTerminal(seam, run, grace)), - ) - return { destroyed: terminal.every(Boolean) } - }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('bridgeExecutor: resultArtifact() read before stream drained') - } - return { ...artifact, spent: artifact.spent } - }, + // Interface's AgentExecutionPreparationReceipt cannot be reused here yet: it is a pre-compute + // contract requiring an execution-bound workspace lease, source/prepared workspace digests, + // profile-activation evidence, and full per-axis ownership. cli-bridge currently has only its + // terminal applied WorkspacePlan acknowledgement. Keep that acknowledgement explicitly terminal, + // bind it into Runtime's existing declaration, and never present this planned declaration as + // evidence that the remote process actually used it. + const plannedDeclaration = { + effectiveProfile, + backend: 'bridge', + model: { status: 'known' as const, id: seam.model }, + execution: { kind: 'session', id: sessionId }, + materializer: 'cli-bridge-agent-profile', + plan: { + kind: 'cli-bridge-session', + cwd: seam.cwd ?? null, + maxTurns, + maxReconnects, + timeoutMs: seam.timeoutMs ?? null, + streaming: true, + terminalAcknowledgement: null, }, - { + } + const plannedBinding = { + attemptId, + binding: { + bridgeUrl: seam.bridgeUrl, + cwd: seam.cwd ?? null, effectiveProfile, - backend: 'bridge', - model: { status: 'known', id: seam.model }, - execution: { kind: 'session', id: sessionId }, - materializer: 'cli-bridge-agent-profile', - plan: { - kind: 'cli-bridge-session', - cwd: seam.cwd ?? null, - maxTurns, - timeoutMs: seam.timeoutMs ?? null, - streaming: true, - }, + model: seam.model, + sessionId, }, - { - attemptId, - binding: { - bridgeUrl: seam.bridgeUrl, - cwd: seam.cwd ?? null, - effectiveProfile, - model: seam.model, + descriptor: { kind: 'bridge-session', transport: 'http', backend: 'bridge' }, + } + let acknowledged: BridgeProfileMaterializationReceipt | undefined + let executor!: Executor + executor = { + runtime: 'cli' as Runtime, + deliver: (m) => inbox.deliver(m), + /** + * The LIVE read of this worker, answered entirely from local mirrors so it is synchronous + * and cannot block or fail the run. The bridge's own run state is refreshed OUT OF BAND — + * a read schedules a fetch whose answer lands for the NEXT read — because the executor + * cannot know from its own stream whether a silent run is thinking, detached mid-reconnect, + * or already cancelled server-side, and that is exactly the distinction a supervisor's + * respawn decision turns on. + */ + progress: (): ExecutorProgress | undefined => { + try { + scheduleBridgeRunStateRefresh(seam, activeRuns, observation) + return { + turns: observation.turns, + pendingMessages: inbox.pending(), + recentActivity: observation.activity.read(), + ...(observation.derived.length > 0 ? { derived: [...observation.derived] } : {}), + note: bridgeProgressNote(observation, liveBridgeRunId(activeRuns)), + } + } catch { + return undefined + } + }, + traceSource: (): TraceSource => trace.source, + execute(task, signal): AsyncIterable { + return streamBridgeSession({ + task, + signal, + profile: effectiveProfile, + seam, sessionId, - }, - descriptor: { kind: 'bridge-session', transport: 'http', backend: 'bridge' }, + maxTurns, + maxReconnects, + inbox, + controller, + activeRuns, + observation, + record: (step: ToolStepInput) => { + trace.record(step) + }, + onArtifact: (a) => { + artifact = a + }, + onProfileMaterialization: (receipt) => { + if (acknowledged !== undefined) { + if (JSON.stringify(acknowledged) !== JSON.stringify(receipt)) { + throw new ValidationError( + 'bridgeExecutor: profile materialization changed across session turns', + ) + } + return + } + acknowledged = receipt + finalizeRuntimeOwnedPendingExecutor( + executor, + { + ...plannedDeclaration, + plan: { ...plannedDeclaration.plan, terminalAcknowledgement: receipt }, + }, + plannedBinding, + ) + }, + }) }, - ) + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + const remaining = [...activeRuns.values()].filter((run) => !run.terminal) + if (remaining.length === 0) return { destroyed: true } + const terminal = await Promise.all( + remaining.map((run) => cancelBridgeRunToTerminal(seam, run, grace)), + ) + return { destroyed: terminal.every(Boolean) } + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError('bridgeExecutor: resultArtifact() read before stream drained') + } + return { ...artifact, spent: artifact.spent } + }, + } + return attestRuntimeOwnedPendingExecutor(executor, 'cli', plannedDeclaration, plannedBinding) } interface StreamBridgeArgs { task: unknown signal: AbortSignal profile: AgentProfile - seam: BridgeSeam + seam: ResolvedBridgeSeam sessionId: string maxTurns: number + maxReconnects: number inbox: Inbox controller: AbortController activeRuns: Map @@ -1484,6 +1742,7 @@ interface StreamBridgeArgs { observation: BridgeObservation record: (step: ToolStepInput) => void onArtifact: (a: ExecutorResult) => void + onProfileMaterialization: (receipt: BridgeProfileMaterializationReceipt) => void } /** Everything `bridgeExecutor.progress()` answers from. Every field is written by the turn loop as @@ -1651,11 +1910,30 @@ function readBridgeRunState(runId: string, body: string): BridgeRunStateRead | u interface ActiveBridgeRun { readonly id: string requestDigest?: string + profileMaterialization?: BridgeProfileMaterializationReceipt + transportAttempts: number lastEventId: number terminal: boolean cancelInFlight?: Promise } +interface BridgeProfileMaterializationReceipt { + readonly schema: typeof bridgeProfileMaterializationSchema + readonly effectiveProfileDigest: string + readonly harness: string + readonly provider: string | null + /** Exact full bridge wire id, for example `pi/tangle-router/deepseek-v4-flash`. */ + readonly model: string + readonly reasoningEffort: { + readonly requested: ReasoningEffort | null + /** Exact native argv/config value after the bridge's backend mapping. */ + readonly applied: string | null + } + readonly workspacePlanDigest: string + readonly files: ReadonlyArray<{ path: string; mode: number }> + readonly unsupported: ReadonlyArray<{ dimension: string; reason: string }> +} + /** * One resumable cli-bridge session, run as a streamed turn loop. Turn 0 sends the * task; each subsequent turn fires ONLY when the inbox has a steer/answer to fold — @@ -1671,14 +1949,18 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0 they ARE the prompt (resume content). const pending = inbox.drain() if (pending.length) { @@ -1725,6 +2007,7 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0) { - usd += chunk.cost - yield { kind: 'cost', usd: chunk.cost } + const increment = + chunk.costScope === 'total' ? chunk.cost - turnKnownCostSubtotal : chunk.cost + if (increment < 0) { + throw new ValidationError('bridgeExecutor: total billed cost decreased within a turn') + } + turnKnownCostSubtotal += increment + if (increment > 0) { + usd += increment + yield { kind: 'cost', usd: increment } + } + } + if (typeof chunk.estimatedCost === 'number') { + const increment = + chunk.costScope === 'total' + ? chunk.estimatedCost - turnEstimatedCostSubtotal + : chunk.estimatedCost + if (increment < 0) { + throw new ValidationError( + 'bridgeExecutor: total estimated cost decreased within a turn', + ) } + turnEstimatedCostSubtotal += increment + estimatedUsd += increment + sawEstimatedUsd = true } } + args.onProfileMaterialization(activeRun.profileMaterialization!) } catch (error) { // A forceful steer first detaches this HTTP reader, then explicitly cancels // the durable run and waits for terminal proof. Starting the resume turn @@ -1809,6 +2145,7 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0 ? { promptCache } : {}), + ...(sawEstimatedUsd ? { estimatedCostUsd: estimatedUsd } : {}), + } as unknown args.onArtifact({ outRef: contentRef('bridge', { model: seam.model, session: args.sessionId, content: lastText }), out, @@ -1852,16 +2196,17 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { + await assertBridgeExecutionCapabilities(args.seam, args.signal) let reconnects = 0 let pendingUpstreamError: ValidationError | undefined for (;;) { let res: BridgeResponse try { + args.run.transportAttempts += 1 res = await bridgeStreamPost(args.seam.bridgeUrl, { bearer: args.seam.bridgeBearer, sessionId: args.sessionId, @@ -1889,7 +2236,7 @@ async function* streamDurableBridgeRun( }) } catch (error) { if (args.signal.aborted) throw error - if (reconnects >= BRIDGE_MAX_RECONNECTS) { + if (reconnects >= args.maxReconnects) { throw new ValidationError( `bridgeExecutor: run ${args.run.id} disconnected before terminal acknowledgement after ${reconnects + 1} attempts: ${errorMessage(error)}`, ) @@ -1923,12 +2270,28 @@ async function* streamDurableBridgeRun( } args.run.lastEventId = event.id if (event.error) pendingUpstreamError = event.error + if (event.chunk?.profileMaterialization) { + const receipt = assertBridgeProfileMaterialization( + event.chunk.profileMaterialization, + args.profile, + args.seam.model, + ) + if ( + args.run.profileMaterialization !== undefined && + JSON.stringify(args.run.profileMaterialization) !== JSON.stringify(receipt) + ) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} profile materialization changed across replay`, + ) + } + args.run.profileMaterialization = receipt + } if (event.chunk) yield event.chunk } } catch (error) { if (args.signal.aborted) throw error if (error instanceof ValidationError) throw error - if (reconnects >= BRIDGE_MAX_RECONNECTS) { + if (reconnects >= args.maxReconnects) { throw new ValidationError( `bridgeExecutor: run ${args.run.id} stream disconnected before terminal acknowledgement after ${reconnects + 1} attempts: ${errorMessage(error)}`, ) @@ -1938,6 +2301,11 @@ async function* streamDurableBridgeRun( } if (sawDone) { + if (args.run.profileMaterialization === undefined) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} completed without ${bridgeProfileMaterializationSchema}`, + ) + } args.run.terminal = true if (pendingUpstreamError) throw pendingUpstreamError return @@ -1949,7 +2317,7 @@ async function* streamDurableBridgeRun( if (args.signal.aborted) { throw new DOMException('bridgeExecutor: turn aborted', 'AbortError') } - if (reconnects >= BRIDGE_MAX_RECONNECTS) { + if (reconnects >= args.maxReconnects) { throw new ValidationError( `bridgeExecutor: run ${args.run.id} ended without terminal acknowledgement after ${reconnects + 1} attempts`, ) @@ -1958,6 +2326,68 @@ async function* streamDurableBridgeRun( } } +/** Refuse an old bridge before it can start a paid harness turn. This is intentionally uncached: + * a process may restart behind the same URL, and a remembered capability from the prior process + * is not evidence about the process that will receive the next POST. The terminal receipt remains + * mandatory because the bridge can still restart between this GET and the run request. */ +async function assertBridgeExecutionCapabilities( + seam: BridgeSeam, + signal: AbortSignal, +): Promise { + const target = new URL(`${seam.bridgeUrl.replace(/\/$/, '')}/`) + const requestFn = target.protocol === 'https:' ? httpsRequest : httpRequest + const response = await new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new DOMException('bridgeExecutor: aborted before capability preflight', 'AbortError')) + return + } + const req = requestFn( + target, + { + method: 'GET', + headers: { authorization: `Bearer ${seam.bridgeBearer}` }, + timeout: 0, + }, + (res) => { + void (async () => { + const chunks: Buffer[] = [] + for await (const chunk of res) chunks.push(Buffer.from(chunk)) + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + text: Buffer.concat(chunks).toString('utf8'), + }) + })().catch(reject) + }, + ) + const abort = () => + req.destroy(new DOMException('bridgeExecutor: preflight aborted', 'AbortError')) + signal.addEventListener('abort', abort, { once: true }) + req.on('error', reject) + req.on('close', () => signal.removeEventListener('abort', abort)) + req.end() + }) + if (response.status < 200 || response.status >= 300) { + throw new ValidationError( + `bridgeExecutor: capability preflight returned ${response.status}: ${response.text.slice(0, 200)}`, + ) + } + let body: { capabilities?: Record } + try { + body = JSON.parse(response.text) as typeof body + } catch { + throw new ValidationError('bridgeExecutor: capability preflight returned invalid JSON') + } + if (body.capabilities?.profileMaterialization !== bridgeProfileMaterializationSchema) { + throw new ValidationError( + `bridgeExecutor: bridge does not advertise ${bridgeProfileMaterializationSchema}`, + ) + } + if (body.capabilities?.usageCostProvenance !== bridgeUsageCostSchema) { + throw new ValidationError(`bridgeExecutor: bridge does not advertise ${bridgeUsageCostSchema}`) + } +} + /** The subset of `Response` `streamBridgeSession` consumes: status gate, an error * body reader, and a web `ReadableStream` the SSE parser drains. */ interface BridgeResponse { @@ -2231,33 +2661,247 @@ interface BridgeStreamChunk { content?: string /** Every tool call the delta carried, decoded into the shared tool-step currency. */ toolCalls?: ReadonlyArray - usage?: { input: number; output: number } + usage?: { + input: number + output: number + known: boolean + promptCache?: { freshInput?: number; readInput?: number; writeInput?: number } + } cost?: number + costKnown?: boolean + estimatedCost?: number + costScope?: 'incremental' | 'total' + profileMaterialization?: BridgeProfileMaterializationReceipt } -/** - * Decode the OpenAI-shaped `tool_calls` of one delta into the shared `ToolStepInput` currency, - * through the SAME `decodeOpenAiPart` adapter the sandbox/parts trace source uses — the wire shape - * cli-bridge emits (`{index, id, type:'function', function:{name, arguments}}`) is exactly the one - * that decoder owns, so there is no second mapping to drift. - * - * FIDELITY, stated once: this wire carries the model's DECISION to call a tool. cli-bridge's - * backends surface the call the moment the harness announces it and NEVER report the call - * finishing, so no outcome, result, or duration exists to read. Each step is therefore marked - * `statusCaptured: false` and carries no `startedAt`/`endedAt` — its span is an instant with no - * status. Synthesising an end time would inject a fabricated 0ms latency, and defaulting to 'ok' - * would count an unobserved call as a success; both would silently corrupt any downstream latency - * or error-rate analysis. An honest lower-fidelity span beats a fabricated one. A harness whose - * native protocol reports tool completion could carry true durations; this wire does not. - * - * cli-bridge emits each call complete in ONE delta (`{id, name, arguments}` together), so no - * cross-delta argument-fragment assembly is needed; a frame that carries argument bytes without a - * name decodes to nothing rather than to a nameless call. - */ -function decodeBridgeToolCalls(raw: unknown): ToolStepInput[] { - if (!Array.isArray(raw)) return [] - const steps: ToolStepInput[] = [] - for (const call of raw) { +function assertBridgeProfileMaterialization( + value: unknown, + profile: AgentProfile, + wireModel: string | undefined, +): BridgeProfileMaterializationReceipt { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new ValidationError('bridgeExecutor: profile materialization receipt must be an object') + } + const raw = value as Record + const exactKeys = [ + 'effectiveProfileDigest', + 'files', + 'harness', + 'model', + 'provider', + 'reasoningEffort', + 'schema', + 'unsupported', + 'workspacePlanDigest', + ] + if (Object.keys(raw).sort().join(',') !== exactKeys.sort().join(',')) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has missing or unknown fields', + ) + } + if (raw.schema !== bridgeProfileMaterializationSchema) { + throw new ValidationError( + `bridgeExecutor: profile materialization receipt is not ${bridgeProfileMaterializationSchema}`, + ) + } + const effectiveProfileDigest = raw.effectiveProfileDigest + if ( + typeof effectiveProfileDigest !== 'string' || + !/^sha256:[a-f0-9]{64}$/u.test(effectiveProfileDigest) + ) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has an invalid effectiveProfileDigest', + ) + } + const expectedDigest = canonicalAgentProfileDigest(profile) + if (effectiveProfileDigest !== expectedDigest) { + throw new ValidationError( + `bridgeExecutor: bridge materialized profile ${effectiveProfileDigest}, expected ${expectedDigest}`, + ) + } + if (typeof raw.harness !== 'string' || raw.harness.length === 0) { + throw new ValidationError('bridgeExecutor: profile materialization receipt has no harness') + } + if (raw.provider !== null && (typeof raw.provider !== 'string' || raw.provider.length === 0)) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid provider', + ) + } + if (typeof raw.model !== 'string' || raw.model.length === 0 || raw.model !== wireModel) { + throw new ValidationError( + `bridgeExecutor: bridge materialized model ${JSON.stringify(raw.model)}, expected ${JSON.stringify(wireModel)}`, + ) + } + const expectedHarness = agentHarness(profile.harness) ?? wireModel?.split('/')[0] + if (!expectedHarness || raw.harness !== expectedHarness) { + throw new ValidationError( + `bridgeExecutor: bridge materialized harness ${JSON.stringify(raw.harness)}, expected ${JSON.stringify(expectedHarness)}`, + ) + } + const expectedProvider = profile.model?.provider ?? null + if (expectedProvider !== null && raw.provider !== expectedProvider) { + throw new ValidationError( + `bridgeExecutor: bridge materialized provider ${JSON.stringify(raw.provider)}, expected ${JSON.stringify(expectedProvider)}`, + ) + } + if ( + !raw.reasoningEffort || + typeof raw.reasoningEffort !== 'object' || + Array.isArray(raw.reasoningEffort) + ) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid reasoningEffort', + ) + } + const reasoningEffort = raw.reasoningEffort as Record + if (Object.keys(reasoningEffort).sort().join(',') !== 'applied,requested') { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt reasoningEffort has missing or unknown fields', + ) + } + const requested = reasoningEffort.requested + const applied = reasoningEffort.applied + if ( + (requested !== null && + (typeof requested !== 'string' || + !REASONING_EFFORTS.includes(requested as ReasoningEffort))) || + (applied !== null && (typeof applied !== 'string' || applied.length === 0)) + ) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid reasoning effort values', + ) + } + const expectedRequested = profile.model?.reasoningEffort ?? null + const expectedApplied = expectedBridgeAppliedReasoning(raw.harness, expectedRequested) + if (requested !== expectedRequested || applied !== expectedApplied) { + throw new ValidationError( + `bridgeExecutor: bridge materialized reasoning effort ${JSON.stringify({ requested, applied })}, expected ${JSON.stringify({ requested: expectedRequested, applied: expectedApplied })}`, + ) + } + if ( + typeof raw.workspacePlanDigest !== 'string' || + !/^sha256:[a-f0-9]{64}$/u.test(raw.workspacePlanDigest) + ) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid workspacePlanDigest', + ) + } + if (!Array.isArray(raw.files) || !Array.isArray(raw.unsupported)) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt files/unsupported must be arrays', + ) + } + const files = raw.files.map((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new ValidationError('bridgeExecutor: profile materialization receipt has invalid file') + } + const file = entry as Record + if ( + Object.keys(file).sort().join(',') !== 'mode,path' || + typeof file.path !== 'string' || + file.path.length === 0 || + !Number.isSafeInteger(file.mode) || + (file.mode as number) < 0 + ) { + throw new ValidationError('bridgeExecutor: profile materialization receipt has invalid file') + } + return { path: file.path, mode: file.mode as number } + }) + const unsupported = raw.unsupported.map((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid unsupported entry', + ) + } + const item = entry as Record + if ( + Object.keys(item).sort().join(',') !== 'dimension,reason' || + typeof item.dimension !== 'string' || + item.dimension.length === 0 || + typeof item.reason !== 'string' || + item.reason.length === 0 + ) { + throw new ValidationError( + 'bridgeExecutor: profile materialization receipt has invalid unsupported entry', + ) + } + return { dimension: item.dimension, reason: item.reason } + }) + if (unsupported.length > 0) { + throw new ValidationError( + `bridgeExecutor: bridge did not materialize profile dimensions: ${unsupported.map((item) => item.dimension).join(', ')}`, + ) + } + return Object.freeze({ + schema: bridgeProfileMaterializationSchema, + effectiveProfileDigest, + harness: raw.harness, + provider: raw.provider as string | null, + model: raw.model, + reasoningEffort: { + requested: requested as ReasoningEffort | null, + applied: applied as string | null, + }, + workspacePlanDigest: raw.workspacePlanDigest, + files: Object.freeze(files), + unsupported: Object.freeze(unsupported), + }) +} + +/** Expected native control for the bridge backends that can emit the v2 acknowledgement. These + * mappings mirror the actual cli-bridge argv functions, so the acknowledgement is checked against + * what the process must have received rather than merely echoing the canonical request. */ +function expectedBridgeAppliedReasoning( + harness: string, + requested: ReasoningEffort | null, +): string | null { + if (requested === null) return null + switch (harness) { + case 'pi': + if (requested === 'none') return 'off' + return requested === 'ultracode' ? 'xhigh' : requested + case 'claude-code': + if (requested === 'none' || requested === 'minimal') return 'low' + return requested === 'ultracode' ? 'max' : requested + case 'codex': + if (requested === 'none') return 'minimal' + return requested === 'xhigh' || requested === 'ultracode' ? 'high' : requested + case 'kimi-code': + if (requested === 'medium') return null + return requested === 'none' || requested === 'minimal' || requested === 'low' + ? '--no-thinking' + : '--thinking' + case 'gemini': + return null + default: + // OpenCode and bridge backends with direct reasoning variants preserve the canonical label. + return requested + } +} + +/** + * Decode the OpenAI-shaped `tool_calls` of one delta into the shared `ToolStepInput` currency, + * through the SAME `decodeOpenAiPart` adapter the sandbox/parts trace source uses — the wire shape + * cli-bridge emits (`{index, id, type:'function', function:{name, arguments}}`) is exactly the one + * that decoder owns, so there is no second mapping to drift. + * + * FIDELITY, stated once: this wire carries the model's DECISION to call a tool. cli-bridge's + * backends surface the call the moment the harness announces it and NEVER report the call + * finishing, so no outcome, result, or duration exists to read. Each step is therefore marked + * `statusCaptured: false` and carries no `startedAt`/`endedAt` — its span is an instant with no + * status. Synthesising an end time would inject a fabricated 0ms latency, and defaulting to 'ok' + * would count an unobserved call as a success; both would silently corrupt any downstream latency + * or error-rate analysis. An honest lower-fidelity span beats a fabricated one. A harness whose + * native protocol reports tool completion could carry true durations; this wire does not. + * + * cli-bridge emits each call complete in ONE delta (`{id, name, arguments}` together), so no + * cross-delta argument-fragment assembly is needed; a frame that carries argument bytes without a + * name decodes to nothing rather than to a nameless call. + */ +function decodeBridgeToolCalls(raw: unknown): ToolStepInput[] { + if (!Array.isArray(raw)) return [] + const steps: ToolStepInput[] = [] + for (const call of raw) { if (!call || typeof call !== 'object') continue const record = call as Record const fn = record.function as Record | undefined @@ -2384,7 +3028,20 @@ function parseSseFrame(frame: string): BridgeSseEvent | undefined { message?: { content?: string | null } }> error?: { message?: string; type?: string } - usage?: { prompt_tokens?: number; completion_tokens?: number; cost?: number } + usage?: { + prompt_tokens?: unknown + completion_tokens?: unknown + fresh_input_tokens?: unknown + cache_read_input_tokens?: unknown + cache_write_input_tokens?: unknown + cost?: unknown + estimated_cost?: unknown + cost_known?: unknown + cost_provenance?: unknown + cost_scope?: unknown + estimated?: unknown + } + profile_materialization?: unknown } try { parsed = JSON.parse(data) @@ -2412,10 +3069,50 @@ function parseSseFrame(frame: string): BridgeSseEvent | undefined { const toolCalls = decodeBridgeToolCalls(choice?.delta?.tool_calls) if (toolCalls.length > 0) out.toolCalls = toolCalls const u = parsed.usage - if (u && (typeof u.prompt_tokens === 'number' || typeof u.completion_tokens === 'number')) { - out.usage = { input: u.prompt_tokens ?? 0, output: u.completion_tokens ?? 0 } + if (u) { + const input = optionalBridgeTokenCount(u.prompt_tokens, 'prompt_tokens') + const output = optionalBridgeTokenCount(u.completion_tokens, 'completion_tokens') + const freshInput = optionalBridgeTokenCount(u.fresh_input_tokens, 'fresh_input_tokens') + const readInput = optionalBridgeTokenCount(u.cache_read_input_tokens, 'cache_read_input_tokens') + const writeInput = optionalBridgeTokenCount( + u.cache_write_input_tokens, + 'cache_write_input_tokens', + ) + if ( + input !== undefined || + output !== undefined || + freshInput !== undefined || + readInput !== undefined || + writeInput !== undefined + ) { + out.usage = { + input: input ?? 0, + output: output ?? 0, + known: input !== undefined && output !== undefined && u.estimated !== true, + ...(freshInput !== undefined || readInput !== undefined || writeInput !== undefined + ? { + promptCache: { + ...(freshInput !== undefined ? { freshInput } : {}), + ...(readInput !== undefined ? { readInput } : {}), + ...(writeInput !== undefined ? { writeInput } : {}), + }, + } + : {}), + } + } + if (u.estimated !== undefined && typeof u.estimated !== 'boolean') { + throw new ValidationError('bridgeExecutor: usage.estimated must be boolean') + } + if (u.cost_scope !== undefined && u.cost_scope !== 'incremental' && u.cost_scope !== 'total') { + throw new ValidationError("bridgeExecutor: usage.cost_scope must be 'incremental' or 'total'") + } + out.costScope = u.cost_scope === 'total' ? 'total' : 'incremental' + applyBridgeCostReceipt(out, u) + } + if (parsed.profile_materialization !== undefined) { + out.profileMaterialization = + parsed.profile_materialization as BridgeProfileMaterializationReceipt } - if (typeof u?.cost === 'number') out.cost = u.cost return { kind: 'event', id, @@ -2423,6 +3120,70 @@ function parseSseFrame(frame: string): BridgeSseEvent | undefined { } } +function optionalBridgeTokenCount(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new ValidationError(`bridgeExecutor: usage.${field} must be a nonnegative safe integer`) + } + return value as number +} + +function optionalBridgeMoney(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new ValidationError(`bridgeExecutor: usage.${field} must be a finite nonnegative number`) + } + return value +} + +/** Admit billed spend only with explicit trusted provenance. A catalog estimate remains visible on + * the result but can never debit a dollar budget or turn unknown dollars into a known zero. */ +function applyBridgeCostReceipt( + out: BridgeStreamChunk, + usage: { + cost?: unknown + estimated_cost?: unknown + cost_known?: unknown + cost_provenance?: unknown + }, +): void { + const cost = optionalBridgeMoney(usage.cost, 'cost') + const estimatedCost = optionalBridgeMoney(usage.estimated_cost, 'estimated_cost') + const provenance = usage.cost_provenance + if (usage.cost_known !== true && usage.cost_known !== false) { + throw new ValidationError('bridgeExecutor: usage.cost_known must be an explicit boolean') + } + if (usage.cost_known) { + if ( + cost === undefined || + (provenance !== 'provider-receipt' && provenance !== 'billing-receipt') || + estimatedCost !== undefined + ) { + throw new ValidationError( + 'bridgeExecutor: known cost requires cost plus provider-receipt or billing-receipt provenance', + ) + } + out.costKnown = true + out.cost = cost + return + } + if (cost !== undefined) { + throw new ValidationError('bridgeExecutor: unknown cost cannot carry billed cost') + } + if (estimatedCost !== undefined && provenance !== 'catalog-estimate') { + throw new ValidationError('bridgeExecutor: estimated cost requires catalog-estimate provenance') + } + if ( + estimatedCost === undefined && + provenance !== undefined && + provenance !== 'catalog-estimate' + ) { + throw new ValidationError('bridgeExecutor: unknown cost has invalid provenance') + } + out.costKnown = false + if (estimatedCost !== undefined) out.estimatedCost = estimatedCost +} + function bridgeWorktreeExecutor( spec: AgentSpec, ctx: ExecutorContext, @@ -2441,10 +3202,8 @@ function bridgeWorktreeExecutor( const runId = seam.runId ?? randomUUID() const sessionId = bridge.sessionId ?? `bridge-worktree-${runId}` const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(runId) - const effectiveProfile = agentProfileSchema.parse( - mergeAgentProfiles(spec.profile, bridge.agentProfile) ?? spec.profile, - ) - const model = bridgeCellModel(bridge.model, ctx, effectiveProfile) + const effectiveProfile = agentProfileSchema.parse(spec.profile) + const model = bridgeProfileModel(effectiveProfile, 'cliWorktreeExecutor bridge') const controller = new AbortController() const pending: unknown[] = [] let inner: Executor | undefined @@ -2472,158 +3231,181 @@ function bridgeWorktreeExecutor( pending.push(msg) } - return attestRuntimeOwnedExecutor( - { - runtime: 'cli' as Runtime, - budgetExempt: seam.budgetExempt ?? false, - deliver, - execute(task, signal): AsyncIterable { - return (async function* bridgeWorktreeStream() { - const started = Date.now() - const linked = mergeAbortSignals(signal, controller.signal) - let bridgeArtifact: ExecutorResult | undefined - - try { - worktree = await createWorktree({ - repoRoot: seam.repoRoot, - runId, - ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), - ...(seam.runGit ? { runGit: seam.runGit } : {}), - }) - removed = false - - const bridgeSeam: BridgeSeam = { - bridgeUrl: bridge.bridgeUrl, - bridgeBearer: bridge.bridgeBearer, - cwd: worktree.path, - sessionId, - ...(bridge.model ? { model: bridge.model } : {}), - ...(bridge.agentProfile ? { agentProfile: bridge.agentProfile } : {}), - ...(bridge.timeoutMs !== undefined ? { timeoutMs: bridge.timeoutMs } : {}), - ...(bridge.maxTurns !== undefined ? { maxTurns: bridge.maxTurns } : {}), - } - const bridgeCtx: ExecutorContext = { - ...ctx, - signal: linked, - seams: { ...ctx.seams, [bridgeSeamKey]: bridgeSeam }, - } - inner = bridgeExecutor(spec, bridgeCtx) - for (const msg of pending.splice(0)) inner.deliver?.(msg) + const plannedDeclaration = { + effectiveProfile, + backend: 'bridge-worktree', + model: { status: 'known' as const, id: model }, + execution: { kind: 'worktree-session', id: `${runId}:${sessionId}` }, + materializer: 'bridge-worktree-agent-profile', + plan: { + kind: 'bridge-worktree-session', + runId, + sessionId, + baseRef: seam.baseRef ?? 'HEAD', + model, + testCmd: seam.testCmd ?? null, + typecheckCmd: seam.typecheckCmd ?? null, + checkTimeoutMs: + seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, + checkOutputCap: seam.checkOutputCap ?? 16_000, + bridgeAcknowledgement: null, + }, + } + const plannedBinding = { + attemptId, + binding: { + bridgeUrl: bridge.bridgeUrl, + effectiveProfile, + model, + repoRoot: seam.repoRoot, + runId, + sessionId, + }, + descriptor: { + kind: 'bridge-worktree-session', + transport: 'http', + backend: 'bridge-worktree', + }, + } + let executor!: Executor + executor = { + runtime: 'cli' as Runtime, + budgetExempt: seam.budgetExempt ?? false, + deliver, + execute(task, signal): AsyncIterable { + return (async function* bridgeWorktreeStream() { + const started = Date.now() + const linked = mergeAbortSignals(signal, controller.signal) + let bridgeArtifact: ExecutorResult | undefined - const run = inner.execute(task, linked) - if (isAsyncIterable(run)) { - for await (const event of run) yield event - bridgeArtifact = inner.resultArtifact() - } else { - bridgeArtifact = await run - } + try { + worktree = await createWorktree({ + repoRoot: seam.repoRoot, + runId, + ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), + ...(seam.runGit ? { runGit: seam.runGit } : {}), + }) + removed = false + + const bridgeSeam: BridgeSeam = { + bridgeUrl: bridge.bridgeUrl, + bridgeBearer: bridge.bridgeBearer, + cwd: worktree.path, + sessionId, + ...(bridge.timeoutMs !== undefined ? { timeoutMs: bridge.timeoutMs } : {}), + ...(bridge.maxReconnects !== undefined ? { maxReconnects: bridge.maxReconnects } : {}), + } + const bridgeCtx: ExecutorContext = { + ...ctx, + signal: linked, + seams: { ...ctx.seams, [bridgeSeamKey]: bridgeSeam }, + } + inner = bridgeExecutor(spec, bridgeCtx) + for (const msg of pending.splice(0)) inner.deliver?.(msg) - const diff = await captureWorktreeDiff({ - worktree, - ...(seam.runGit ? { runGit: seam.runGit } : {}), - }) - const checks = await runWorktreeChecks({ - worktreePath: worktree.path, - ...(seam.testCmd !== undefined ? { testCmd: seam.testCmd } : {}), - ...(seam.typecheckCmd !== undefined ? { typecheckCmd: seam.typecheckCmd } : {}), - timeoutMs: - seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, - cap: seam.checkOutputCap ?? 16_000, - ...(seam.runCommand ? { runCommand: seam.runCommand } : {}), - signal: linked, - }) + const run = inner.execute(task, linked) + if (isAsyncIterable(run)) { + for await (const event of run) yield event + bridgeArtifact = inner.resultArtifact() + } else { + bridgeArtifact = await run + } - const result: WorktreeHarnessResult = { - branch: worktree.branch, - patch: diff.patch, - stats: diff.stats, - harness: { - name: 'bridge', - exitCode: null, - timedOut: false, - killedBySignal: null, - durationMs: bridgeArtifact.spent.ms || Date.now() - started, - stdout: bridgeOutputText(bridgeArtifact.out), - stderr: '', + const bridgeDeclaration = runtimeOwnedExecutorMaterialization(inner) + const bridgeBinding = runtimeOwnedExecutorExecutionBinding(inner) + if (bridgeDeclaration === undefined || bridgeBinding === undefined) { + throw new ValidationError( + 'cliWorktreeExecutor: bridge completed without a terminal materialization acknowledgement', + ) + } + finalizeRuntimeOwnedPendingExecutor( + executor, + { + ...plannedDeclaration, + plan: { + ...plannedDeclaration.plan, + bridgeAcknowledgement: bridgeDeclaration.plan, }, - ...(checks ? { checks } : {}), - } - const spent: Spend = { - ...bridgeArtifact.spent, - ms: bridgeArtifact.spent.ms || Date.now() - started, - } - artifact = { - outRef: contentRef('bridge-worktree', { sessionId, result }), - out: result, - spent, - } - } catch (err) { - controller.abort() - await inner?.teardown('brutalKill').catch(() => undefined) - await cleanupWorktree() - throw err + }, + { + ...plannedBinding, + binding: { + ...plannedBinding.binding, + worktreePath: worktree.path, + bridgeBinding: bridgeBinding.binding, + }, + }, + ) + + const diff = await captureWorktreeDiff({ + worktree, + ...(seam.runGit ? { runGit: seam.runGit } : {}), + }) + const checks = await runWorktreeChecks({ + worktreePath: worktree.path, + ...(seam.testCmd !== undefined ? { testCmd: seam.testCmd } : {}), + ...(seam.typecheckCmd !== undefined ? { typecheckCmd: seam.typecheckCmd } : {}), + timeoutMs: + seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, + cap: seam.checkOutputCap ?? 16_000, + ...(seam.runCommand ? { runCommand: seam.runCommand } : {}), + signal: linked, + }) + + const result: WorktreeHarnessResult = { + branch: worktree.branch, + patch: diff.patch, + stats: diff.stats, + harness: { + name: 'bridge', + exitCode: null, + timedOut: false, + killedBySignal: null, + durationMs: bridgeArtifact.spent.ms || Date.now() - started, + stdout: bridgeOutputText(bridgeArtifact.out), + stderr: '', + }, + ...(checks ? { checks } : {}), } - })() - }, - async teardown(grace): Promise<{ destroyed: boolean }> { - controller.abort() - let destroyed = true - try { - if (inner) { - destroyed = (await inner.teardown(grace)).destroyed + const spent: Spend = { + ...bridgeArtifact.spent, + ms: bridgeArtifact.spent.ms || Date.now() - started, } - } finally { + artifact = { + outRef: contentRef('bridge-worktree', { sessionId, result }), + out: result, + spent, + } + } catch (err) { + controller.abort() + await inner?.teardown('brutalKill').catch(() => undefined) await cleanupWorktree() + throw err } - return { destroyed } - }, - resultArtifact() { - if (!artifact) { - throw new ValidationError( - 'cliWorktreeExecutor: bridge resultArtifact() read before stream drained', - ) - } - return artifact - }, + })() }, - { - effectiveProfile, - backend: 'bridge-worktree', - model: model - ? { status: 'known', id: model } - : { status: 'unknown', reason: 'bridge worktree profile did not select a model' }, - execution: { kind: 'worktree-session', id: `${runId}:${sessionId}` }, - materializer: 'bridge-worktree-agent-profile', - plan: { - kind: 'bridge-worktree-session', - runId, - sessionId, - baseRef: seam.baseRef ?? 'HEAD', - model: model ?? null, - testCmd: seam.testCmd ?? null, - typecheckCmd: seam.typecheckCmd ?? null, - checkTimeoutMs: - seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, - checkOutputCap: seam.checkOutputCap ?? 16_000, - }, + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + let destroyed = true + try { + if (inner) { + destroyed = (await inner.teardown(grace)).destroyed + } + } finally { + await cleanupWorktree() + } + return { destroyed } }, - { - attemptId, - binding: { - bridgeUrl: bridge.bridgeUrl, - effectiveProfile, - model: model ?? null, - repoRoot: seam.repoRoot, - runId, - sessionId, - }, - descriptor: { - kind: 'bridge-worktree-session', - transport: 'http', - backend: 'bridge-worktree', - }, + resultArtifact() { + if (!artifact) { + throw new ValidationError( + 'cliWorktreeExecutor: bridge resultArtifact() read before stream drained', + ) + } + return artifact }, - ) + } + return attestRuntimeOwnedPendingExecutor(executor, 'cli', plannedDeclaration, plannedBinding) } function bridgeOutputText(out: unknown): string { @@ -2660,15 +3442,18 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { throw new ValidationError('cliWorktreeExecutor: CliWorktreeSeam.repoRoot required') } if (seam.bridge) return bridgeWorktreeExecutor(spec, ctx, seam) - if (!seam.harness) { + const effectiveProfile = agentProfileSchema.parse(spec.profile) + assertExecutableAgentProfile(effectiveProfile, 'cliWorktreeExecutor') + const harness = localWorktreeHarness(agentHarness(effectiveProfile.harness)) + if (!harness) { throw new ValidationError( - 'cliWorktreeExecutor: CliWorktreeSeam.harness required when bridge is not set', + 'cliWorktreeExecutor: AgentProfile.harness must select claude-code, codex, or opencode when bridge is not set', ) } return createWorktreeCliExecutor({ repoRoot: seam.repoRoot, - profile: spec.profile, - harness: seam.harness, + profile: effectiveProfile, + harness, ...(seam.taskPrompt !== undefined ? { taskPrompt: seam.taskPrompt } : {}), ...(seam.runId ? { runId: seam.runId } : {}), ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), @@ -2686,6 +3471,12 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { }) as Executor } +function localWorktreeHarness(harness: string | undefined): LocalHarness | undefined { + return harness === 'claude-code' || harness === 'codex' || harness === 'opencode' + ? harness + : undefined +} + // ── createExecutor: the ONE built-in factory (backend as data) ────────────────── /** @@ -2702,22 +3493,52 @@ export type ExecutorConfig = | ({ backend: 'provider' } & ProviderSeam) | ({ backend: 'sandbox'; harness?: BackendType } & SandboxSeam) +function assertExactConfigKeys( + value: Readonly>, + allowed: ReadonlySet, + context: string, +): void { + const unknown = Object.keys(value).filter((key) => !allowed.has(key)) + if (unknown.length > 0) { + throw new ValidationError( + `${context}: unknown fields ${unknown.sort().join(', ')}; execution behavior belongs in AgentProfile`, + ) + } +} + /** Capture one public executor configuration at its call boundary. All data that selects policy, - * model, process, limits, profile overlays, or backend behavior is detached and deeply frozen. + * model, process, limits, or backend behavior is detached and deeply frozen. * Explicit service/function fields remain live by reference because they are executable ports, * not portable configuration. */ export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { switch (config.backend) { case 'router-tools': { - const { executeToolCall, onToolStep, ...decisionData } = config + const { complete, executeToolCall, onMessages, onToolStep, ...decisionData } = config const snapshot = detachedSnapshot(decisionData, 'createExecutor router-tools config') return Object.freeze({ ...snapshot, + ...(complete === undefined ? {} : { complete }), executeToolCall, + ...(onMessages === undefined ? {} : { onMessages }), ...(onToolStep === undefined ? {} : { onToolStep }), }) } + case 'router': { + const { complete, ...decisionData } = config + const snapshot = detachedSnapshot(decisionData, 'createExecutor router config') + return Object.freeze({ + ...snapshot, + ...(complete === undefined ? {} : { complete }), + }) + } case 'cli-worktree': { + if (config.bridge) { + assertExactConfigKeys( + config.bridge as unknown as Readonly>, + new Set(['bridgeBearer', 'bridgeUrl', 'maxReconnects', 'sessionId', 'timeoutMs']), + 'createExecutor cli-worktree bridge config', + ) + } const { runGit, runCommand, ...decisionData } = config const snapshot = detachedSnapshot(decisionData, 'createExecutor cli-worktree config') return Object.freeze({ @@ -2763,32 +3584,33 @@ export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { }), }) } - case 'router': case 'bridge': + assertExactConfigKeys( + config as unknown as Readonly>, + new Set([ + 'activityWindow', + 'backend', + 'bridgeBearer', + 'bridgeUrl', + 'cwd', + 'maxReconnects', + 'sessionId', + 'timeoutMs', + ]), + 'createExecutor bridge config', + ) + return detachedSnapshot(config, 'createExecutor bridge config') case 'cli': return detachedSnapshot(config, `createExecutor ${config.backend} config`) } } -/** A backend config reused for multiple workers/managers cannot pin execution identity or carry a - * profile overlay applied after Scope hashed the authored profile. Direct single-execution - * `createExecutor` calls may still use those fields. */ +/** A backend config reused for multiple workers/managers cannot pin execution identity. */ export function captureReusableExecutorConfig( config: ExecutorConfig, context: string, ): ExecutorConfig { const captured = snapshotExecutorConfig(config) - const profileOverlay = - captured.backend === 'bridge' - ? captured.agentProfile - : captured.backend === 'cli-worktree' - ? captured.bridge?.agentProfile - : undefined - if (profileOverlay !== undefined) { - throw new ValidationError( - `${context}: backend agentProfile overlays are not allowed because they change the effective profile after spawn identity is fixed`, - ) - } const fixedIdentity = captured.backend === 'bridge' && captured.sessionId !== undefined ? 'sessionId' @@ -2845,6 +3667,7 @@ export function bindReusableExecutorExecutionId( export function createExecutor(config: ExecutorConfig): ExecutorFactory { const captured = snapshotExecutorConfig(config) return (spec, ctx) => { + assertExecutableAgentProfile(spec.profile, `createExecutor(${captured.backend})`) const { backend, ...seamData } = captured as ExecutorConfig & Record const seam = Object.freeze(seamData) const seamed: ExecutorContext = { ...ctx, seams: { ...ctx.seams, [backend]: seam } } @@ -2904,8 +3727,16 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory const profileForCreate = providerSeam.profileForCreate return providerAsExecutor(provider, { ...providerSeam, - profileForCreate: (profile) => - profileForExecution(profileForCreate?.(profile) ?? profile), + profileForCreate: (profile) => { + const prepared = profileForCreate?.(profile) ?? profile + assertExecutableAgentProfile(prepared, 'createExecutor(provider)') + if (canonicalAgentProfileDigest(prepared) !== canonicalAgentProfileDigest(profile)) { + throw new ValidationError( + 'createExecutor(provider): profileForCreate changed the exact AgentProfile; execution overlays are not allowed', + ) + } + return prepared + }, })(spec, seamed) } case 'sandbox': { @@ -3024,17 +3855,188 @@ export function taskToPrompt(task: unknown): string { return JSON.stringify(task) } -/** Router messages from the opaque task + every portable profile prompt instruction. */ -function taskToMessages(task: unknown, spec: AgentSpec): Array<{ role: string; content: string }> { - const messages: Array<{ role: string; content: string }> = [] - const system = [spec.profile.prompt?.systemPrompt, ...(spec.profile.prompt?.instructions ?? [])] - .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) - .join('\n') - if (system.length > 0) { - messages.push({ role: 'system', content: system }) +interface RouterProfileExecution { + systemPrompt: string + reasoningEffort?: ReasoningEffort + temperature?: number + maxTokens?: number + maxAttempts?: number + seed?: number + toolChoice?: 'auto' | 'required' | 'none' + extraBody?: Readonly> + maxTurns?: number + stream?: boolean +} + +/** Validate and render every AgentProfile axis the direct Router path claims to carry. + * Unsupported behavioral axes fail before the HTTP request; inline resources become + * named system-prompt attachments because this executor has no workspace to mount. */ +function routerProfileExecution( + profile: AgentProfile, + seam: RouterSeam, + mode: { multiTurn: boolean }, +): RouterProfileExecution { + assertProfileMaterialization({ + contract: routerTurnProfileMaterialization, + changedAxes: profileMaterializationAxes(profile), + context: 'routerInlineExecutor', + }) + + if (agentHarness(profile.harness) !== undefined) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile.harness ${JSON.stringify(profile.harness)} requires a harness executor; the direct Router executor cannot materialize it`, + ) + } + + const profileEffort = profile.model?.reasoningEffort + const settings = profileModelExecutionSettings(profile, 'routerInlineExecutor') + + if (!mode.multiTurn && settings.maxTurns !== undefined) { + throw new ValidationError( + 'routerInlineExecutor: AgentProfile.model.metadata.maxTurns requires the router-tools backend', + ) + } + if (settings.stream === true && seam.tools === undefined) { + throw new ValidationError( + 'routerInlineExecutor: streamed chat without tool schemas is not supported; omit stream or use a harness executor', + ) + } + + const declaredTools = profile.tools ?? {} + const suppliedTools = seam.tools ?? [] + if (settings.toolChoice !== undefined && suppliedTools.length === 0) { + throw new ValidationError( + 'routerInlineExecutor: AgentProfile.model.metadata.toolChoice requires at least one enabled tool', + ) } - messages.push({ role: 'user', content: taskToPrompt(task) }) - return messages + const suppliedNames = new Set() + for (const tool of suppliedTools) { + const name = tool.function.name + if (!name || suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: caller tool names must be non-empty and unique (${JSON.stringify(name)})`, + ) + } + suppliedNames.add(name) + if (declaredTools[name] !== true) { + throw new ValidationError( + `routerInlineExecutor: caller tool ${JSON.stringify(name)} is not enabled by AgentProfile.tools`, + ) + } + } + for (const [name, enabled] of Object.entries(declaredTools)) { + if (enabled && !suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile enables tool ${JSON.stringify(name)} but the caller supplied no matching schema`, + ) + } + if (!enabled && suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile disables tool ${JSON.stringify(name)}`, + ) + } + } + + return { + systemPrompt: renderRouterProfilePrompt(profile), + ...(profileEffort ? { reasoningEffort: profileEffort } : {}), + ...settings, + } +} + +/** Resolve the one model id that will cross the Router boundary from the exact profile only. */ +function exactRouterModel(profile: AgentProfile, context: string): string { + assertExecutableAgentProfile(profile, context) + if (agentHarness(profile.harness) !== undefined) { + throw new ValidationError( + `${context}: AgentProfile.harness ${JSON.stringify(profile.harness)} conflicts with direct Router execution; use "cli-base"`, + ) + } + return concreteProfileModel(profile)! +} + +function renderRouterProfilePrompt(profile: AgentProfile): string { + const sections: string[] = [ + profile.prompt?.systemPrompt, + ...(profile.prompt?.instructions ?? []), + ].filter((value): value is string => typeof value === 'string' && value.trim().length > 0) + const resources = profile.resources + if (!resources) return sections.join('\n') + + if (typeof resources.instructions === 'string') { + if (resources.instructions.trim()) sections.push(resources.instructions) + } else if (resources.instructions) { + sections.push(renderRouterResource('instructions', resources.instructions)) + } + for (const file of resources.files ?? []) { + if (file.executable === true) { + throw new ValidationError( + `routerInlineExecutor: executable resource ${JSON.stringify(file.path)} requires a workspace backend`, + ) + } + sections.push(renderRouterResource(`file ${file.path}`, file.resource)) + } + for (const [kind, refs] of [ + ['tool', resources.tools], + ['skill', resources.skills], + ['agent', resources.agents], + ['command', resources.commands], + ] as const) { + for (const ref of refs ?? []) sections.push(renderRouterResource(kind, ref)) + } + return sections.join('\n\n') +} + +function renderRouterResource(kind: string, resource: AgentProfileResourceRef): string { + if (resource.kind !== 'inline') { + throw new ValidationError( + `routerInlineExecutor: ${kind} resource ${JSON.stringify(resource.name ?? resource.path)} is not inline and cannot be resolved by the direct Router executor`, + ) + } + return `## Attached ${kind}: ${resource.name}\n${resource.content}` +} + +/** Router messages from the opaque task + every portable profile prompt instruction. + * The profile prompt is always the immutable first message. Later system-role messages are + * preserved as per-call task context; they cannot replace or precede the profile policy. */ +function taskToMessages( + task: unknown, + spec: AgentSpec, + resolvedSystem?: string, +): Array<{ role: string; content: unknown } & Record> { + const system = + resolvedSystem ?? + [spec.profile.prompt?.systemPrompt, ...(spec.profile.prompt?.instructions ?? [])] + .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) + .join('\n') + + if ( + task && + typeof task === 'object' && + Array.isArray((task as { messages?: unknown }).messages) + ) { + const supplied = (task as { messages: unknown[] }).messages.map((value, index) => { + if (!value || typeof value !== 'object') { + throw new ValidationError(`routerInlineExecutor: messages[${index}] must be an object`) + } + const message = { ...(value as Record) } + if (typeof message.role !== 'string' || !('content' in message)) { + throw new ValidationError( + `routerInlineExecutor: messages[${index}] requires role and content`, + ) + } + return message as { role: string; content: unknown } & Record + }) + if (system.length > 0 && !(supplied[0]?.role === 'system' && supplied[0].content === system)) { + return [{ role: 'system', content: system }, ...supplied] + } + return supplied + } + + return [ + ...(system.length > 0 ? [{ role: 'system', content: system }] : []), + { role: 'user', content: taskToPrompt(task) }, + ] } /** A driver that refines a single task up to `maxIterations` times then stops — diff --git a/src/runtime/supervise/sandbox-session.ts b/src/runtime/supervise/sandbox-session.ts index 9823fa31..db82e72c 100644 --- a/src/runtime/supervise/sandbox-session.ts +++ b/src/runtime/supervise/sandbox-session.ts @@ -95,10 +95,12 @@ export interface SteerableSandboxArgs { /** One steerable sandbox worker. The returned session is inert until `stream()` is drained. */ export function createSteerableSandboxSession(args: SteerableSandboxArgs): SteerableSandboxSession { const now = args.now ?? Date.now - const maxTurns = Math.max( - 1, - Math.floor(args.options?.maxTurns ?? DEFAULT_SANDBOX_STEERING_MAX_TURNS), - ) + const maxTurns = args.options?.maxTurns ?? DEFAULT_SANDBOX_STEERING_MAX_TURNS + if (!Number.isSafeInteger(maxTurns) || maxTurns < 0) { + throw new ValidationError( + 'steerable sandbox worker: maxTurns must be a nonnegative safe integer (0 means unbounded)', + ) + } const activity: ActivityLog = createActivityLog(args.options?.activityWindow ?? 12) const trace = createPushTraceSource({ runId: `sandbox-${args.harness}-${now()}`, now }) @@ -157,12 +159,13 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer const started = now() const tokens = zeroTokenUsage() + let tokensKnown = true let usd = 0 let handle: SandboxLineageHandle | undefined let nextPrompt: string | undefined = args.taskToPrompt(task) try { - for (let turn = 0; turn < maxTurns; turn += 1) { + for (let turn = 0; maxTurns === 0 || turn < maxTurns; turn += 1) { // Drain the down-leg FIRST. On turn 0 a steer that landed during box acquisition is // appended to the task; on later turns the steer IS the whole prompt, because the // worker's prior context lives server-side in the session. @@ -202,6 +205,7 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer } let events: AsyncIterable + let sawLlmCall = false try { if (!handle) { const opened = await lineage.start(spec, prompt, turnController.signal, promptOptions) @@ -214,16 +218,28 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer recordEvent(event) const call = extractLlmCallEvent(event, spec.name ?? String(args.harness)) if (!call) continue + sawLlmCall = true + const callTokensKnown = + call.tokensKnown !== false && + typeof call.tokensIn === 'number' && + typeof call.tokensOut === 'number' + if (!callTokensKnown) tokensKnown = false const input = call.tokensIn ?? 0 const output = call.tokensOut ?? 0 - if (input || output) { + if (input || output || !callTokensKnown) { tokens.input += input tokens.output += output - yield { kind: 'tokens', input, output } + yield { + kind: 'tokens', + input, + output, + ...(callTokensKnown ? {} : { tokensKnown: false }), + } } if (typeof call.costUsd === 'number' && call.costUsd > 0) { usd += call.costUsd - yield { kind: 'cost', usd: call.costUsd } + // Numeric sandbox cost has no billing-provenance/completeness receipt. + yield { kind: 'cost', usd: call.costUsd, usdKnown: false } } } } catch (e) { @@ -238,6 +254,11 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer } cleanup() + if (!sawLlmCall) { + tokensKnown = false + yield { kind: 'tokens', input: 0, output: 0, tokensKnown: false } + } + state.turns += 1 activity.push({ at: now(), kind: 'turn', label: `turn ${turn}` }) yield { kind: 'iteration' } @@ -252,7 +273,17 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer state.teardown = undefined } - const spent: Spend = { iterations: state.turns, tokens, usd, ms: now() - started } + // Mark the dollar channel unknown even when no event carried a numeric subtotal: a completed + // sandbox turn is not proof that the provider billed exactly zero. + if (state.turns > 0) yield { kind: 'cost', usd: 0, usdKnown: false } + const spent: Spend = { + iterations: state.turns, + tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), + usd, + ...(state.turns > 0 ? { usdKnown: false } : {}), + ms: now() - started, + } const out = { content: state.lastText, turns: state.turns, diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 1e1fa49b..de0aae65 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -26,6 +26,7 @@ */ import { + canonicalAgentProfileDigest, canonicalCandidateDigest, type Sha256Digest, sha256DigestSchema, @@ -51,6 +52,7 @@ import { runtimeOwnedDeferredExecutorRuntime, runtimeOwnedExecutorExecutionBinding, runtimeOwnedExecutorMaterialization, + runtimeOwnedPendingExecutorMaterialization, unknownExecutionBindingReceipt, unknownMaterializationReceipt, } from './materialization' @@ -751,11 +753,13 @@ export function createScope(args: ScopeArgs): Scope { at: new Date(now()).toISOString(), }) }) + let pendingEvidence: { complete: () => Promise; fail: () => Promise } | undefined const materializationCommitted = spawnCommitted.then(async () => { const profileDigest = identity?.profileDigest ?? authoredProfileDigest(spec.profile) let receipt: ProfileMaterializationReceipt let binding: ExecutionBindingReceipt const declaration = runtimeOwnedExecutorMaterialization(executor) + const pending = runtimeOwnedPendingExecutorMaterialization(executor) const deferredRuntime = runtimeOwnedDeferredExecutorRuntime(executor) if (deferredRuntime !== undefined) { deferredOwner.ownerMaterialization = { @@ -773,6 +777,88 @@ export function createScope(args: ScopeArgs): Scope { } return } + if (pending !== undefined) { + if (profileDigest === undefined) { + throw new ValidationError( + 'scope.spawn: a pending materialization requires a canonical authored profile', + ) + } + if (pending.runtime !== executor.runtime || pending.binding.attemptId !== attemptId) { + throw new ValidationError( + 'scope.spawn: pending executor did not bind the kernel-minted attempt id', + ) + } + // Validate the planned shape before spend, but do not persist it as proof that the + // external bridge actually used it. Only the terminal acknowledgement can finalize it. + const plannedReceipt = knownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + declaration: pending.declaration, + }) + if ( + plannedReceipt.status !== 'known' || + plannedReceipt.effectiveProfileDigest !== plannedReceipt.authoredProfileDigest + ) { + throw new ValidationError( + 'scope.spawn: pending executor changed the authored AgentProfile before execution', + ) + } + let recorded = false + pendingEvidence = { + complete: async () => { + if (recorded) return + const acknowledged = runtimeOwnedExecutorMaterialization(executor) + const acknowledgedBinding = runtimeOwnedExecutorExecutionBinding(executor) + if (acknowledged === undefined || acknowledgedBinding?.attemptId !== attemptId) { + throw new ValidationError( + 'scope.spawn: external executor completed without a terminal materialization acknowledgement', + ) + } + const finalReceipt = knownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + declaration: acknowledged, + }) + if (finalReceipt.status !== 'known') { + throw new ValidationError('scope.spawn: terminal materialization remained unknown') + } + if (finalReceipt.effectiveProfileDigest !== finalReceipt.authoredProfileDigest) { + throw new ValidationError( + 'scope.spawn: external executor changed the authored AgentProfile', + ) + } + const finalBinding = knownExecutionBindingReceipt(finalReceipt, acknowledgedBinding) + await appendNodeMaterialization(args, id, ordinal, finalReceipt, finalBinding, now) + live.materialization = finalReceipt + live.executionBindings.push(finalBinding) + recorded = true + }, + fail: async () => { + if (recorded) return + // The remote turn can be acknowledged before a later local persistence/check step + // fails. Preserve that known execution truth even though the child settles down. + if (runtimeOwnedExecutorMaterialization(executor) !== undefined) { + await pendingEvidence?.complete() + return + } + const unknown = unknownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + reason: 'executor-receipt-pending', + }) + const unknownBinding = unknownExecutionBindingReceipt( + unknown, + attemptId, + 'executor-receipt-pending', + ) + await appendNodeMaterialization(args, id, ordinal, unknown, unknownBinding, now) + live.materialization = unknown + live.executionBindings.push(unknownBinding) + recorded = true + }, + } + return + } if (declaration === undefined) { receipt = unknownMaterializationReceipt({ ...(profileDigest === undefined ? {} : { authoredProfileDigest: profileDigest }), @@ -857,6 +943,10 @@ export function createScope(args: ScopeArgs): Scope { args.blobs, now, materializationCommitted, + { + complete: async () => pendingEvidence?.complete(), + fail: async () => pendingEvidence?.fail(), + }, childDeadlineAtMs, ) .then((s) => { @@ -1324,7 +1414,7 @@ export async function recordScopeOwnerMaterialization( 'scope owner execution binding does not use the kernel-minted attempt id', ) } - if (canonicalCandidateDigest(declaration.effectiveProfile) !== state.authoredProfileDigest) { + if (canonicalAgentProfileDigest(declaration.effectiveProfile) !== state.authoredProfileDigest) { throw new ValidationError( 'scope owner stable effective profile conflicts with its admitted authored profile', ) @@ -1743,6 +1833,10 @@ async function runChild( blobs: ResultBlobStore, now: () => number, executionReady: Promise, + executionEvidence: { + complete: () => Promise + fail: () => Promise + }, deadlineAtMs: number | undefined, ): Promise { let reconciled = false @@ -1800,6 +1894,7 @@ async function runChild( childAbort.signal, ) live.spent = spend + await executionEvidence.complete() artifact = executor.resultArtifact() as ExecutorResult const accounting = executor.accounting?.() const terminalSpend = preserveUnknownTelemetry(spend, artifact.spent) @@ -1810,6 +1905,7 @@ async function runChild( if (reconcileError !== undefined) throw reconcileError } else { const terminal = await awaitAbortable(Promise.resolve(ran), childAbort.signal) + await executionEvidence.complete() const accounting = executor.accounting?.() live.spent = accounting?.reported ?? terminal.spent artifact = terminal @@ -1854,6 +1950,12 @@ async function runChild( // A thrown executor has also finished its own work — only the down-record persistence // remains, so the non-blocking drain may await this child too. live.executorDone = true + let evidenceError: unknown + try { + await executionEvidence.fail() + } catch (error) { + evidenceError = error + } // A recursive executor can still report the nested work committed before it threw. // Reconcile that whole partial subtree while journaling its child-work component separately. // A box-backed trace must be collected before teardown destroys the session that owns it. @@ -1884,7 +1986,11 @@ async function runChild( // settlement as infrastructure-related, while the unknown spend flags retain the accounting // failure itself in the durable record. errMessage(err), - teardownError !== undefined || reconcileError !== undefined || aborted || isInfraError(err), + evidenceError !== undefined || + teardownError !== undefined || + reconcileError !== undefined || + aborted || + isInfraError(err), trace, executor.metered?.(), ) @@ -2011,7 +2117,7 @@ export function deriveNodeExecutionIdentity( return undefined } } - const profileDigest = digest(spec.profile) + const profileDigest = authoredProfileDigest(spec.profile) const taskDigest = digest(task) const candidateDigest = spec.execution?.candidateDigest if (candidateDigest !== undefined && !sha256DigestSchema.safeParse(candidateDigest).success) { @@ -2074,6 +2180,7 @@ async function foldStream( signal?: AbortSignal, ): Promise { const tokens = { input: 0, output: 0 } + let tokensKnown = true let usd = 0 let usdKnown = true let iterations = 0 @@ -2088,6 +2195,7 @@ async function foldStream( if (ev.kind === 'tokens') { tokens.input += ev.input tokens.output += ev.output + if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd if (ev.usdKnown === false) usdKnown = false @@ -2097,6 +2205,7 @@ async function foldStream( onProgress?.({ iterations, tokens: { ...tokens }, + ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), ms: 0, @@ -2111,6 +2220,7 @@ async function foldStream( return { iterations, tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), ms: 0, diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index fc43d23f..9bbd6d6a 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -13,6 +13,7 @@ import { type AgentProfile, type AgentProfileSecurityPolicy, agentProfileSchema, + canonicalAgentProfileDigest, canonicalCandidateDigest, type Sha256Digest, validateAgentProfileSecurity, @@ -44,9 +45,8 @@ import type { } from '../../mcp/tools/coordination' import { composeRuntimeHooks, type RuntimeHooks } from '../../runtime-hooks' import { harnessRunsAgent } from '../harness-role' -import type { RouterConfig } from '../router-client' +import type { RouterTransportConfig } from '../router-client' import type { ToolLoopChat, ToolLoopCompactionOptions } from '../tool-loop' -import { canonicalizeAuthoredProfile } from './authoring' import { assertValidBudget, spendFromUsageEvents } from './budget' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' import { DEFAULT_SUCCESSFUL_SHUTDOWN_MS, teardownExecutor } from './deadline' @@ -57,9 +57,14 @@ import { attestRuntimeOwnedScopeOwner, runtimeOwnedExecutorExecutionBinding, runtimeOwnedExecutorMaterialization, + runtimeOwnedPendingExecutorMaterialization, runtimeOwnedScopeOwnerRuntime, } from './materialization' -import { assertModelAllowed, assertProfileModelsAllowed } from './model-policy' +import { + assertExecutableAgentProfile, + assertModelAllowed, + assertProfileModelsAllowed, +} from './model-policy' import { createSupervisorSpanRecorder, type SupervisorSpanOptions, @@ -130,10 +135,7 @@ export function workerFromBackend( const unscopedNamespace = randomUUID() let unscopedOrdinal = 0 return (rawProfile, spawnContext) => { - // The supervisor authors in the skill's flat vocabulary; every leaf reads the canonical - // profile. Lift it HERE — the one place a backend becomes a spawnable worker — so no leaf - // has to guess which shape it was handed, then hold the LIFTED form to the canonical schema. - const parsed = agentProfileSchema.safeParse(canonicalizeAuthoredProfile(rawProfile)) + const parsed = agentProfileSchema.safeParse(rawProfile) if (!parsed.success) { throw new ValidationError(`workerFromBackend: invalid AgentProfile: ${parsed.error.message}`) } @@ -253,12 +255,10 @@ function assertBackendProfileMaterialization( /** * The ROOT router-brained supervisor's materialization claim. The router arm consumes the - * identity fields, the resolved system prompt (`systemPrompt` + `prompt.instructions` + + * identity fields, the resolved system prompt (`prompt.systemPrompt` + `prompt.instructions` + * `resources.instructions`), and the resolved model id (`model.default`); the remaining model - * HINTS (`small`, `provider`, `reasoningEffort`, `metadata`) are accepted as documented-unhonored - * router-arm material (`supervisorAgent`'s contract table states each one), so a canonical - * profile carrying ordinary hints is not refused. Every behavioral axis — tools, permissions, - * MCP, hooks, modes, subagents, file resources — still fails loud before any compute. + * fields are either applied by the profile-bound Router adapter or refused. Every behavioral axis + * the Router brain cannot materialize fails before compute. */ const routerSupervisorProfileMaterialization = defineProfileMaterializationContract({ name: 'router-supervisor-execution', @@ -271,7 +271,6 @@ const routerSupervisorProfileMaterialization = defineProfileMaterializationContr 'instructions', 'resourceInstructions', 'modelDefault', - 'modelSmall', 'modelProvider', 'modelReasoningEffort', 'modelMetadata', @@ -303,15 +302,6 @@ function automaticDriverBackendSupported(backend: ExecutorConfig): boolean { return backend.backend === 'bridge' } -function backendProfileOverlays(backend: ExecutorConfig | undefined): AgentProfile[] { - if (!backend) return [] - if (backend.backend === 'bridge' && backend.agentProfile) return [backend.agentProfile] - if (backend.backend === 'cli-worktree' && backend.bridge?.agentProfile) { - return [backend.bridge.agentProfile] - } - return [] -} - /** Run a harness-brained manager through the same executor factory as its children. The manager's * full profile is preserved, the live coordination server is added under one reserved alias, and * every streamed turn is charged to the manager's scope before it may continue. */ @@ -410,44 +400,81 @@ function driveHarnessFromBackend( // unmetered runtime reaches the single bounded teardown path below. const declaration = runtimeOwnedExecutorMaterialization(executor) const executionBinding = runtimeOwnedExecutorExecutionBinding(executor) - if (declaration === undefined || executionBinding === undefined) { - throw new ValidationError( - `driveHarnessFromBackend: built-in runtime ${JSON.stringify(executor.runtime)} has no trusted materialization declaration or execution binding`, + const pending = runtimeOwnedPendingExecutorMaterialization(executor) + const authoredProfileFromDriverExecution = (profile: AgentProfile): AgentProfile => { + const mcp = profile.mcp ?? {} + const { [coordinationMcpAlias]: _runtimeAttachment, ...authoredMcp } = mcp + const { mcp: _mcp, ...withoutMcp } = profile + return agentProfileSchema.parse( + Object.keys(authoredMcp).length > 0 ? { ...withoutMcp, mcp: authoredMcp } : withoutMcp, ) } - await recordScopeOwnerMaterialization( - scope, - executor.runtime, - { - ...declaration, - // The endpoint is one attempt's transport binding, not AgentProfile identity. Keep the - // admitted profile stable and commit the logical coordination capability separately. - effectiveProfile: canonicalDriverProfile, - platformAttachments: { - [coordinationMcpAlias]: { - kind: 'coordination-mcp', - transport: 'http', - tools: stableCoordinationTools, - }, + const ownerDeclaration = ( + exactDeclaration: NonNullable, + ): NonNullable => ({ + ...exactDeclaration, + // The coordination MCP is a Runtime-owned platform attachment, not authored behavior. + effectiveProfile: canonicalDriverProfile, + platformAttachments: { + [coordinationMcpAlias]: { + kind: 'coordination-mcp', + transport: 'http', + tools: stableCoordinationTools, }, }, - { - ...executionBinding, - binding: { - stableBinding: executionBinding.binding, - platformAttachments: { - [coordinationMcpAlias]: { - transport: 'http', - url: coordinationMcpUrl, + }) + const publishMaterialization = async ( + exactDeclaration: NonNullable, + exactBinding: NonNullable, + ) => { + await recordScopeOwnerMaterialization( + scope, + executor.runtime, + ownerDeclaration(exactDeclaration), + { + ...exactBinding, + binding: { + stableBinding: exactBinding.binding, + platformAttachments: { + [coordinationMcpAlias]: { + transport: 'http', + url: coordinationMcpUrl, + }, }, }, + descriptor: { + ...exactBinding.descriptor, + coordination: true, + }, }, - descriptor: { - ...executionBinding.descriptor, - coordination: true, - }, - }, - ) + ) + } + if (pending === undefined && (declaration === undefined || executionBinding === undefined)) { + throw new ValidationError( + `driveHarnessFromBackend: built-in runtime ${JSON.stringify(executor.runtime)} has no trusted materialization declaration or execution binding`, + ) + } + if (pending !== undefined) { + if ( + pending.runtime !== executor.runtime || + pending.binding.attemptId !== scopeOwnerExecutorNodeContext(scope).attemptId + ) { + throw new ValidationError( + 'driveHarnessFromBackend: pending executor did not bind the kernel-minted attempt', + ) + } + if ( + canonicalAgentProfileDigest( + authoredProfileFromDriverExecution(pending.declaration.effectiveProfile), + ) !== canonicalAgentProfileDigest(canonicalDriverProfile) + ) { + throw new ValidationError( + 'driveHarnessFromBackend: pending executor changed the authored AgentProfile before execution', + ) + } + } else { + await publishMaterialization(declaration!, executionBinding!) + } if (executor.budgetExempt) { throw new ValidationError( `driveHarnessFromBackend: runtime ${JSON.stringify(executor.runtime)} does not report usage and cannot drive a budgeted supervisor`, @@ -490,6 +517,16 @@ function driveHarnessFromBackend( { role: 'driver', runtime: executor.runtime }, ) } + if (pending !== undefined) { + const acknowledged = runtimeOwnedExecutorMaterialization(executor) + const acknowledgedBinding = runtimeOwnedExecutorExecutionBinding(executor) + if (acknowledged === undefined || acknowledgedBinding === undefined) { + throw new ValidationError( + 'driveHarnessFromBackend: external executor completed without a terminal materialization acknowledgement', + ) + } + await publishMaterialization(acknowledged, acknowledgedBinding) + } completed = true } catch (error) { failed = true @@ -695,7 +732,7 @@ export interface SuperviseOptions { readonly isDriverProfile?: (input: AuthorizedSpawnContext) => boolean /** The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's * model wins. */ - readonly router?: RouterConfig + readonly router?: RouterTransportConfig /** Inject the supervisor brain directly (tests / advanced). */ readonly brain?: ToolLoopChat /** Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a @@ -1041,39 +1078,6 @@ function freezeDetachedProfile(value: unknown): AgentProfile { return freezeDetached(agentProfileSchema.parse(value)) } -/** - * Map the two loose `SupervisorProfile` spellings onto their canonical `AgentProfile` form before - * the strict schema parse, so both documented spellings run the SAME canonical pipeline and share - * one identity digest: - * - a string `model` IS `model.default`; - * - a top-level `systemPrompt` IS `prompt.systemPrompt` (two disagreeing values are a fault); - * - `harness: null` selects the router brain, which canonically is an ABSENT harness. - * A canonical profile passes through byte-identical; every other field is left for the schema to - * accept or refuse. - */ -function canonicalSupervisorProfileInput(profile: SupervisorProfile): unknown { - if (typeof profile !== 'object' || profile === null) return profile - const { harness, model, systemPrompt, prompt, ...rest } = profile as SupervisorProfile & - Record - const promptSystem = prompt?.systemPrompt - if (systemPrompt !== undefined && promptSystem !== undefined && systemPrompt !== promptSystem) { - throw new ValidationError( - 'supervise: profile.prompt.systemPrompt and profile.systemPrompt are both set and differ — ' + - 'they are the same standing instruction, so keep exactly one', - ) - } - const canonicalPrompt = - systemPrompt !== undefined ? { ...prompt, systemPrompt } : (prompt as unknown) - return { - ...rest, - ...(harness === null || harness === undefined ? {} : { harness }), - ...(model === undefined - ? {} - : { model: typeof model === 'string' ? { default: model } : model }), - ...(canonicalPrompt === undefined ? {} : { prompt: canonicalPrompt }), - } -} - function canonicalExecution( profile: AgentProfile, task: unknown, @@ -1179,11 +1183,12 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super assertValidBudget(options.budget, 'supervise budget') // Fail loud before any compute: every configured model must be in the allowed subset (no-op // when allowedModels is unset). The backend seam carries its own model on most backends. - const parsedProfile = agentProfileSchema.safeParse(canonicalSupervisorProfileInput(profile)) + const parsedProfile = agentProfileSchema.safeParse(profile) if (!parsedProfile.success) { throw new ValidationError(`supervise: invalid AgentProfile: ${parsedProfile.error.message}`) } const canonicalProfile = freezeDetachedProfile(parsedProfile.data) + assertExecutableAgentProfile(canonicalProfile, 'supervise root') const canonicalTask = freezeDetached(task) if (options.makeWorkerAgent && options.authorizeSpawn) { throw new ValidationError( @@ -1225,16 +1230,6 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super ) const backendModel = (options.backend as { model?: unknown } | undefined)?.model const driverBackendModel = (options.driverBackend as { model?: unknown } | undefined)?.model - const overlays = [ - ...backendProfileOverlays(options.backend), - ...backendProfileOverlays(options.driverBackend), - ] - if (overlays.length > 0) { - throw new ValidationError( - 'supervise: backend agentProfile overlays are not allowed because they run after spawn authorization; merge the overlay into the exact profile before calling supervise', - ) - } - assertModelAllowed(options.router?.model, options.allowedModels) assertProfileModelsAllowed(canonicalProfile, options.allowedModels) assertModelAllowed( typeof backendModel === 'string' ? backendModel : undefined, diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 33401e3d..f7493fc6 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -15,10 +15,10 @@ * Both arms spawn children through the SAME `makeWorkerAgent` seam and apply the SAME independent * deliverable check to direct submissions. Raw driver prose is never eligible. */ -import type { - AgentProfileModelHints, - AgentProfilePrompt, - AgentProfileResources, +import { + type AgentProfile, + type AgentProfileResources, + agentProfileSchema, } from '@tangle-network/agent-interface' import { ConfigError, ValidationError } from '../../errors' import type { McpToolDescriptor } from '../../mcp/server' @@ -33,7 +33,7 @@ import type { } from '../../mcp/tools/coordination' import { coordinationVerbNames } from '../../mcp/tools/coordination' import { agentHarness } from '../harness-role' -import { type RouterConfig, routerBrain } from '../router-client' +import { type RouterTransportConfig, routerBrain } from '../router-client' import type { ToolLoopChat, ToolLoopCompactionOptions } from '../tool-loop' import type { DeliverableSpec } from './completion-gate' import { driverAgent } from './coordination-driver' @@ -43,7 +43,11 @@ import type { BusRecord } from './event-bus' import { bestDelivered, runFinalizer, runTree, type SupervisorFinalizer } from './finalizer' import { createInbox } from './inbox' import { attestRuntimeOwnedScopeOwner, runtimeOwnedScopeOwnerRuntime } from './materialization' -import { concreteModelId } from './model-policy' +import { + assertExecutableAgentProfile, + concreteProfileModel, + profileModelExecutionSettings, +} from './model-policy' import { supervisorPolicyPrompt } from './prompt-registry' import { detachedSnapshot } from './snapshot' import type { StopRule } from './stop-rules' @@ -60,80 +64,17 @@ import type { Agent, Budget, NodeExecutionIdentity, ResultBlobStore, Scope } fro * ship two contradictory defaults selected by entry point. */ export const defaultSupervisorPrompt = supervisorPolicyPrompt.text -/** - * The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. - * `harness` is the backend-as-data discriminant; `systemPrompt` is the standing instruction. - * - * A canonical `AgentProfile` from `@tangle-network/agent-interface` satisfies this interface - * structurally: its `model` is a hints OBJECT and its system prompt lives at `prompt.systemPrompt`, - * so both spellings are accepted here and reduced by {@link resolveSupervisorProfile}. Before that, - * a canonical profile's model object reached `RouterConfig.model` (a string) as an object and its - * `prompt.systemPrompt` was dropped — a request the provider rejects, and a supervisor running the - * default strategy while its profile named another. - * - * WHAT EACH ARM HONORS — the two brains read different amounts of a profile, so state it rather - * than let a caller infer that a field took effect: - * - * - ROUTER arm (`harness` null): only `name`, the resolved model id (`model`, or - * `model.default`), and the resolved system prompt (`prompt.systemPrompt`/`systemPrompt` plus - * `prompt.instructions` and `resources.instructions`) reach the brain. A full `AgentProfile`'s - * `tools`, `mcp`, `permissions`, `resources.skills`/`files`, `hooks`, `modes`, `subagents`, - * `model.provider`, `model.small` and `model.reasoningEffort` are NOT honored here: the router - * brain is one `ToolLoopChat` over the coordination verbs, and neither of its two tool-calling - * transports (`routerChatWithTools` buffered, `streamRouterChatWithTools` when - * `RouterConfig.stream` is set) has a parameter for any of them. - * - HARNESS arm (`harness` set): the WHOLE profile object is handed to `deps.driveHarness` - * untouched, plus the resolved system prompt as a separate argument. Everything the profile - * declares is the harness's to materialize; this module changes none of it. - */ -export interface SupervisorProfile { - readonly name?: string - /** null/undefined/`cli-base` → router brain (in-process tool-loop); a coding-CLI harness → an - * external harness brain. */ - readonly harness?: string | null - /** The router model when the brain is router-driven: a model id, or a canonical profile's model - * hints whose `default` IS the id. Absent (including a hints object with no `default`) → the - * deps router config's model applies. Other hints (`small`, `provider`, `reasoningEffort`) are - * harness-arm material only. */ - readonly model?: string | AgentProfileModelHints - /** Canonical `AgentProfile` prompt shaping. `prompt.systemPrompt` and the top-level `systemPrompt` - * are the same standing instruction in two spellings; disagreeing values are a fault, not a pick. - * `prompt.instructions` lines are appended to the resolved prompt, one per line. */ - readonly prompt?: AgentProfilePrompt - /** Canonical `AgentProfile` resources. Only `instructions` shapes the brain here (appended to the - * resolved system prompt); every other resource is the harness's to materialize. */ - readonly resources?: AgentProfileResources - /** The standing instructions ("you delegate, you do not solve"). */ - readonly systemPrompt?: string -} +/** A supervisor is an exact canonical AgentProfile; no looser model/prompt shape exists. */ +export type SupervisorProfile = AgentProfile -/** A `SupervisorProfile` reduced to the scalars the two brain arms consume. `modelId`/`systemPrompt` - * stay `undefined` when the profile named none — the caller's fallback (`deps.router.model`, - * the built-in default supervisor prompt) then applies, and this type cannot hide which happened. - * - * There is deliberately no `reasoningEffort` here: the router brain runs on `chatWithTools` (the - * buffered/streamed switch in the router client), and neither transport has a `reasoning_effort` - * parameter — only the chat-only `routerChatWithUsage` does — so a field carrying it would be a - * public promise nothing keeps. `model.reasoningEffort` still reaches the harness arm inside the - * profile. */ +/** The exact profile fields consumed by supervisor materialization. */ export interface ResolvedSupervisorProfile { readonly name: string readonly harness: string | null - readonly modelId?: string + readonly modelId: string readonly systemPrompt?: string } -/** Longest prompt excerpt an error message may carry. A supervisor system prompt is routinely - * thousands of characters; two of them interpolated whole turn a configuration fault into an - * unreadable wall, so a fault reports each prompt's LENGTH plus a leading excerpt instead. */ -const PROMPT_EXCERPT_CHARS = 60 - -/** ` chars starting "…"` — enough to tell two prompts apart without printing either. */ -function describePrompt(value: string): string { - const head = value.slice(0, PROMPT_EXCERPT_CHARS) - return `${value.length} chars starting ${JSON.stringify(head)}${value.length > PROMPT_EXCERPT_CHARS ? '…' : ''}` -} - /** * The instruction lines a canonical `resources.instructions` contributes. A plain string and an * `inline` resource are their own text; a `github` reference names bytes that live elsewhere and @@ -157,31 +98,19 @@ function resourceInstructionLines( } /** - * The standing instruction both arms run under, assembled from every canonical spelling that - * carries one: the system prompt (`prompt.systemPrompt` or the top-level `systemPrompt`), then the - * `prompt.instructions` lines, then `resources.instructions` — each on its own line, in that order. + * The standing instruction both arms run under: `prompt.systemPrompt`, then canonical prompt and + * resource instruction lines. * `undefined` only when the profile names none at all. * - * Two disagreeing system prompts throw: they are the same standing instruction in two spellings, so - * picking one silently changes what the supervisor runs and there is no defensible winner. */ function resolveSupervisorSystemPrompt( profile: SupervisorProfile, activePrompt?: string, ): string | undefined { const promptSystem = profile.prompt?.systemPrompt - const topSystem = profile.systemPrompt - if (promptSystem !== undefined && topSystem !== undefined && promptSystem !== topSystem) { - throw new ValidationError( - 'supervisorAgent: profile.prompt.systemPrompt and profile.systemPrompt are both set and ' + - 'differ — they are the same standing instruction, so keep exactly one ' + - `(prompt.systemPrompt: ${describePrompt(promptSystem)}; ` + - `systemPrompt: ${describePrompt(topSystem)})`, - ) - } // Instruction lines are APPENDED to the active prompt, so a profile that names only // instructions keeps whatever prompt the arm would otherwise run — never replaces it. - const base = promptSystem ?? topSystem ?? activePrompt + const base = promptSystem ?? activePrompt const lines = [ ...(profile.prompt?.instructions ?? []), ...resourceInstructionLines(profile.resources?.instructions), @@ -190,37 +119,24 @@ function resolveSupervisorSystemPrompt( return (base !== undefined ? [base, ...lines] : lines).join('\n') } -/** - * The router model id, or `undefined` when the profile names none. A string `model` IS the id; an - * object `model` is canonical model hints and `default` is the id. `AgentProfileModelHints.default` - * is OPTIONAL upstream (`{ provider: 'anthropic' }` is a valid canonical profile), so hints without - * a resolvable id are the documented "profile names no model" case: the router config's own model - * applies, exactly as when `model` is absent. - */ -export function resolveSupervisorModelId(profile: SupervisorProfile): string | undefined { - if (typeof profile.model === 'string') return concreteModelId(profile.model) - return concreteModelId(profile.model?.default) +/** Resolve the model after refusing any incomplete execution identity. */ +export function resolveSupervisorModelId(profile: SupervisorProfile): string { + assertExecutableAgentProfile(profile, 'supervisorAgent') + return concreteProfileModel(profile)! } /** - * Reduce either profile spelling — a hand-written `SupervisorProfile` or a canonical `AgentProfile` - * — to the scalars the brain arms consume: - * - * - `modelId`: a string `model` verbatim, else `model.default`. Absent or unresolvable → the - * router config's own model applies unchanged. - * - `systemPrompt`: the system prompt plus the `prompt.instructions` and `resources.instructions` - * lines, one per line. - * - * `supervisorAgent` resolves each piece only where it is consumed (the model id on the router arm - * only); this whole-profile reduction is the caller-facing view of the same rules. + * Reduce one canonical executable profile to the scalars the two brain arms consume. */ export function resolveSupervisorProfile(profile: SupervisorProfile): ResolvedSupervisorProfile { - const systemPrompt = resolveSupervisorSystemPrompt(profile) - const modelId = resolveSupervisorModelId(profile) + const exact = agentProfileSchema.parse(profile) + assertExecutableAgentProfile(exact, 'resolveSupervisorProfile') + const systemPrompt = resolveSupervisorSystemPrompt(exact) + const modelId = resolveSupervisorModelId(exact) return { - name: profile.name ?? 'supervisor', - harness: profile.harness ?? null, - ...(modelId !== undefined ? { modelId } : {}), + name: exact.name ?? 'supervisor', + harness: agentHarness(exact.harness) ?? null, + modelId, ...(systemPrompt !== undefined ? { systemPrompt } : {}), } } @@ -352,7 +268,7 @@ export interface SupervisorAgentDeps { readonly maxLiveWorkers?: number /** Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The * profile's model wins. */ - readonly router?: RouterConfig + readonly router?: RouterTransportConfig /** Inject the brain directly (tests / advanced) instead of resolving `routerBrain` from the profile. */ readonly brain?: ToolLoopChat /** Required to run an external-harness supervisor: runs the harness as the driver. */ @@ -432,7 +348,9 @@ export function supervisorAgent( profile: SupervisorProfile, deps: SupervisorAgentDeps, ): Agent { - const stableProfile = detachedSnapshot(profile, 'supervisorAgent profile') + const exactProfile = agentProfileSchema.parse(profile) + assertExecutableAgentProfile(exactProfile, 'supervisorAgent') + const stableProfile = detachedSnapshot(exactProfile, 'supervisorAgent profile') const resolveTools = deps.resolveSupervisorTools const observeNodeEvent = deps.observeNodeEvent const nodeContextSeed = @@ -738,14 +656,19 @@ function routerBrainFromProfile( 'supervisorAgent: a router-brained supervisor (harness omitted or cli-base) needs deps.router (or deps.brain)', ) } - // The model id is resolved HERE, the one place it is consumed. `model.reasoningEffort` is not - // carried with it: `routerBrain` runs on `chatWithTools` — `routerChatWithTools` buffered, or - // `streamRouterChatWithTools` when the spread `deps.router` sets `stream` — and neither transport - // has a `reasoning_effort` parameter (only the chat-only `routerChatWithUsage` does), so - // forwarding it would need a router-client change, not a local workaround. const modelId = resolveSupervisorModelId(profile) - return routerBrain({ - ...deps.router, - ...(modelId !== undefined ? { model: modelId } : {}), - }) + const settings = profileModelExecutionSettings(profile, 'supervisorAgent') + return routerBrain( + { + ...deps.router, + model: modelId, + ...(settings.maxAttempts !== undefined ? { maxAttempts: settings.maxAttempts } : {}), + ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), + ...(settings.stream !== undefined ? { stream: settings.stream } : {}), + }, + { + ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), + ...(profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}), + }, + ) } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index d8dc0d78..60734123 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -219,18 +219,18 @@ export interface ExecutorResult { * conserved pool meters all runtimes identically. `tokens` carries `LoopTokenUsage`'s * `{ input, output }`; `usd` is a SEPARATE channel (never folded into tokens). * - * KNOWN LIMITATION (pre-existing): the `cost` variant can say its dollars are a subtotal - * (`usdKnown: false`), and the `tokens` variant has NO twin — there is no way to report "this turn - * happened and its token count is unknown". `Spend.tokensKnown` exists downstream, but nothing - * upstream of `foldStream` (`scope.ts`) can ever set it, so a STREAMING executor whose provider - * omitted usage reports the turn as costing zero tokens rather than as unmeasured. Only the - * non-streaming path, which returns a whole `Spend`, can carry the marker today. Closing it means - * widening this union (a `tokensKnown: false` field on `tokens`, or an `unknown` variant) and - * threading it through `foldStream` — a change to the metering contract every executor implements, - * which is why it is not folded into a streaming-transport fix. Filed separately. + * Either channel can explicitly say its numeric subtotal is incomplete. A missing provider receipt + * therefore remains unknown through live metering and terminal reconciliation instead of becoming + * a fabricated zero. */ export type UsageEvent = - | { kind: 'tokens'; input: number; output: number } + | { + kind: 'tokens' + /** Known token subtotal. When false, these counts are only the observed/estimated floor. */ + tokensKnown?: false + input: number + output: number + } | { kind: 'cost' /** Known dollar subtotal. When false, `usd` must not be treated as total cost. */ @@ -327,6 +327,7 @@ export interface ExecutorExecutionBinding { /** Why exact materialization evidence is unavailable for a node. */ export type UnknownMaterializationReason = | 'executor-did-not-report' + | 'executor-receipt-pending' | 'invalid-executor-report' | 'root-agent-did-not-report' diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index 8bd08f27..135482e6 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -8,6 +8,8 @@ * steerable concerns the call sites add (a driver's conserved-pool + deadline bound; an inline * executor's inbox flush + abort) attach via optional `hooks`; the skeleton stays one copy. */ + +import { ValidationError } from '../errors' import type { RouterToolCall, ToolSpec } from './router-client' /** Provider-neutral conversation record accepted by a tool-loop brain. */ @@ -22,9 +24,9 @@ export type ToolLoopChat = ( content?: string | null toolCalls: RouterToolCall[] usage?: { input: number; output: number } - /** The turn's inference cost (usd) when the provider priced it — for callers that meter usd - * into a conserved pool (the supervisor brain). `runBrainLoop` itself ignores it. */ + /** Dollar value reported for the turn. It is not billed spend unless provenance says so. */ costUsd?: number + costProvenance?: 'provider-receipt' | 'billing-receipt' | 'catalog-estimate' /** The turn ran but its usage was not reported when the transport EXPECTED one (the streamed * router transport asks for usage and this says it never arrived). A metering caller records an * unknown turn on it; `runBrainLoop` itself ignores it. */ @@ -132,10 +134,28 @@ export interface ToolLoopResult { /** The behavior trace: each call + its result, in order — what a trace-analyst steerer reads. */ toolTrace: Array<{ name: string; args: string; result: string }> usage: { input: number; output: number } + /** False when any completed inference turn omitted token usage. */ + tokensKnown?: false /** The full conversation after the loop — lets a caller CARRY the messages into the next shot. */ messages: ToolLoopMessageRecord[] } +/** Resolve one tool-loop turn limit. Zero removes only the turn-count cap; abort, time, and + * resource budgets remain available to callers. */ +export function resolveToolLoopMaxTurns( + value: number | undefined, + fallback: number, + context = 'tool loop', +): number { + const resolved = value ?? fallback + if (!Number.isSafeInteger(resolved) || resolved < 0) { + throw new ValidationError( + `${context}: maxTurns must be a nonnegative safe integer (0 means unbounded)`, + ) + } + return resolved +} + export async function runBrainLoop(opts: { chat: ToolLoopChat tools: ReadonlyArray @@ -148,14 +168,16 @@ export async function runBrainLoop(opts: { * free). Off by default — when unset the conversation accumulates exactly as before. */ compaction?: ToolLoopCompaction }): Promise { - const maxTurns = opts.maxTurns ?? 4 + const maxTurns = resolveToolLoopMaxTurns(opts.maxTurns, 4, 'runBrainLoop') const messages: ToolLoopMessageRecord[] = [...opts.initialMessages] let toolCalls = 0 let lastText = '' const usage = { input: 0, output: 0 } + let tokensKnown = true + let completedTurns = 0 const toolTrace: Array<{ name: string; args: string; result: string }> = [] - for (let turn = 1; turn <= maxTurns; turn += 1) { + for (let turn = 1; maxTurns === 0 || turn <= maxTurns; turn += 1) { if (opts.hooks?.stopBefore?.(turn)) break await opts.hooks?.beforeTurn?.(turn, messages) // Close the chapter BEFORE the inference turn that would otherwise re-bill the whole transcript: @@ -163,11 +185,12 @@ export async function runBrainLoop(opts: { // not O(total-history). A clean boundary — the prior turn's tool replies are already folded in. if (opts.compaction) await maybeCompact(messages, opts.compaction, turn) const r = await opts.chat(messages, opts.tools) + completedTurns = turn if (r.usage) { usage.input += r.usage.input usage.output += r.usage.output opts.hooks?.onUsage?.(r.usage) - } + } else tokensKnown = false if (r.content) lastText = r.content if (r.toolCalls.length === 0) { // The stopping reply is part of the conversation (the contract on `messages`: seed + @@ -176,7 +199,15 @@ export async function runBrainLoop(opts: { // solution the model can't see, and candidate extraction (structural-rollout's // fenced-code fallback) reads an empty conversation on non-tool-calling models. if (r.content) messages.push({ role: 'assistant', content: r.content }) - return { final: lastText, turns: turn, toolCalls, toolTrace, usage, messages } + return { + final: lastText, + turns: turn, + toolCalls, + toolTrace, + usage, + ...(tokensKnown ? {} : { tokensKnown: false }), + messages, + } } // Record the assistant turn verbatim (content + the tool_calls it requested), then run each @@ -210,5 +241,13 @@ export async function runBrainLoop(opts: { toolTrace.push({ name: tc.name, args: tc.arguments, result: out }) } } - return { final: lastText, turns: maxTurns, toolCalls, toolTrace, usage, messages } + return { + final: lastText, + turns: completedTurns, + toolCalls, + toolTrace, + usage, + ...(tokensKnown ? {} : { tokensKnown: false }), + messages, + } } diff --git a/src/runtime/types.ts b/src/runtime/types.ts index ac82590b..bbeee3db 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -116,6 +116,8 @@ export interface OutputAdapter { export interface LoopTokenUsage { input: number output: number + /** False when the subtotal is incomplete. */ + tokensKnown?: false } /** @@ -206,6 +208,12 @@ export interface Iteration { startedAt: number endedAt: number costUsd: number + /** False when `costUsd` is only the observed subtotal, not a complete bill. */ + costUsdKnown?: false + /** Local/catalog estimates remain separate from billed spend. */ + estimatedCostUsd?: number + /** Provider-reported prompt-cache fields; absent fields remain unknown. */ + promptCache?: Record /** Summed LLM token usage across every `llm_call` event in this iteration. */ tokenUsage: LoopTokenUsage } @@ -284,6 +292,12 @@ export interface LoopResult { durationMs: number /** Sum of every iteration's `costUsd`. */ costUsd: number + /** False when `costUsd` is only the observed subtotal, not a complete bill. */ + costUsdKnown?: false + /** Sum of separately-labelled local/catalog estimates. */ + estimatedCostUsd?: number + /** Aggregated provider-reported prompt-cache fields. */ + promptCache?: Record /** Sum of every iteration's token usage. `loopDispatch` commits it through * the campaign's paid-call receipt. */ tokenUsage: LoopTokenUsage @@ -510,6 +524,8 @@ export interface LoopIterationEndedPayload { verdict?: DefaultVerdict error?: string costUsd: number + costUsdKnown?: false + estimatedCostUsd?: number durationMs: number /** Summed LLM token usage for this iteration — maps to gen_ai.usage.* on the * branch span. Omitted when no `llm_call` events carried token counts. */ @@ -533,6 +549,8 @@ export interface LoopDecisionPayload { export interface LoopEndedPayload { winnerIterationIndex?: number totalCostUsd: number + costUsdKnown?: false + estimatedCostUsd?: number durationMs: number iterations: number } diff --git a/src/runtime/util.ts b/src/runtime/util.ts index 08c0ac29..50e5855a 100644 --- a/src/runtime/util.ts +++ b/src/runtime/util.ts @@ -137,10 +137,11 @@ export function zeroTokenUsage(): LoopTokenUsage { return { input: 0, output: 0 } } -/** Add `delta` into `acc` in place. Missing fields count as zero. */ +/** Add the observed subtotal into `acc`; incompleteness is sticky. */ export function addTokenUsage(acc: LoopTokenUsage, delta: Partial): void { acc.input += delta.input ?? 0 acc.output += delta.output ?? 0 + if (delta.tokensKnown === false) acc.tokensKnown = false } /** diff --git a/src/types.ts b/src/types.ts index 7d767cea..340bd0c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -320,6 +320,10 @@ export type RuntimeStreamEvent = task: AgentTaskSpec session: RuntimeSession backend: string + /** Canonical execution identity and materialization evidence for this turn, when Runtime + * owns the selected executor. Generic metadata keeps the event vocabulary open while the + * values use Runtime's existing identity/materialization receipt shapes. */ + metadata?: Record timestamp: string } | { @@ -361,7 +365,15 @@ export type RuntimeStreamEvent = model: string tokensIn?: number tokensOut?: number + /** False when the numeric token subtotal is incomplete or absent. */ + tokensKnown?: false costUsd?: number + /** False when `costUsd` is only an observed floor, estimate, or absent. */ + usdKnown?: false + /** Separately-labelled local/catalog estimate; never billed spend. */ + estimatedCostUsd?: number + /** Provider-reported prompt-cache fields; absent fields remain unknown. */ + promptCache?: Readonly> latencyMs?: number finishReason?: string timestamp?: string diff --git a/tests/backends-anthropic-usage.test.ts b/tests/backends-anthropic-usage.test.ts deleted file mode 100644 index d3d9001a..00000000 --- a/tests/backends-anthropic-usage.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - createOpenAICompatibleBackend, - type KnowledgeRequirement, - type RuntimeStreamEvent, - runAgentTaskStream, -} from '../src/index' - -const readyReq: KnowledgeRequirement = { - id: 'build-command', - description: 'Build command', - requiredFor: ['test'], - category: 'codebase_specific', - acquisitionMode: 'inspect_repo', - importance: 'blocking', - freshness: 'weekly', - sensitivity: 'public', - confidenceNeeded: 0.8, - currentConfidence: 0.9, -} - -async function collect(iter: AsyncIterable): Promise { - const out: RuntimeStreamEvent[] = [] - for await (const e of iter) out.push(e) - return out -} - -describe('createOpenAICompatibleBackend — Anthropic usage emission', () => { - it('synthesizes llm_call from Anthropic message_start + message_delta usage', async () => { - // The router proxies Anthropic models (e.g. claude-sonnet-4-6) through the - // same chat-completions endpoint. Anthropic SSE carries: - // - message_start.message.usage.input_tokens - // - content_block_delta with delta.text - // - message_delta.usage.output_tokens + delta.stop_reason - // - message_stop - // None of the OpenAI fields (`choices`, `usage` at the top) are present. - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-sonnet-4-6', - fetchImpl: async () => - new Response( - 'data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-6","content":[],"stop_reason":null,"usage":{"input_tokens":128,"output_tokens":1}}}\n\n' + - 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hel"}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"lo"}}\n\n' + - 'data: {"type":"content_block_stop","index":0}\n\n' + - 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":24}}\n\n' + - 'data: {"type":"message_stop"}\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'anthropic-task', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - - const llmCalls = events.filter((e) => e.type === 'llm_call') - expect(llmCalls).toHaveLength(1) - const llmCall = llmCalls[0] - if (llmCall.type !== 'llm_call') throw new Error('expected llm_call') - expect(llmCall.tokensIn).toBe(128) - // message_start carries output_tokens:1 (the streamed acknowledgement - // partial), message_delta adds 24 — total 25. - expect(llmCall.tokensOut).toBe(25) - expect(llmCall.model).toBe('claude-sonnet-4-6') - expect(llmCall.finishReason).toBe('end_turn') - - // Anthropic-shape text deltas should also flow through. - expect( - events - .filter((e) => e.type === 'text_delta') - .map((e) => e.text) - .join(''), - ).toBe('hello') - }) - - it('captures usage even when only message_delta carries output_tokens (no message_start usage)', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-haiku-4-6', - fetchImpl: async () => - new Response( - 'data: {"type":"message_start","message":{"id":"msg_2","model":"claude-haiku-4-6","usage":{"input_tokens":11}}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"text":"ok"}}\n\n' + - 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":3}}\n\n' + - 'data: {"type":"message_stop"}\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'anthropic-min', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const llmCall = events.find((e) => e.type === 'llm_call') - expect(llmCall).toBeDefined() - if (llmCall?.type !== 'llm_call') throw new Error('expected llm_call') - expect(llmCall.tokensIn).toBe(11) - expect(llmCall.tokensOut).toBe(3) - expect(llmCall.model).toBe('claude-haiku-4-6') - expect(llmCall.finishReason).toBe('end_turn') - }) -}) diff --git a/tests/backends-fail-loud.test.ts b/tests/backends-fail-loud.test.ts deleted file mode 100644 index c79dd282..00000000 --- a/tests/backends-fail-loud.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - type AgentBackendInput, - type AgentExecutionBackend, - BackendTransportError, - createOpenAICompatibleBackend, - type KnowledgeRequirement, - type RuntimeStreamEvent, - runAgentTaskStream, -} from '../src/index' - -const readyReq: KnowledgeRequirement = { - id: 'build-command', - description: 'Build command', - requiredFor: ['test'], - category: 'codebase_specific', - acquisitionMode: 'inspect_repo', - importance: 'blocking', - freshness: 'weekly', - sensitivity: 'public', - confidenceNeeded: 0.8, - currentConfidence: 0.9, -} - -async function collect(iter: AsyncIterable): Promise { - const out: RuntimeStreamEvent[] = [] - for await (const e of iter) out.push(e) - return out -} - -describe('createOpenAICompatibleBackend — typed transport errors surface to backend_error + final', () => { - it('surfaces a 402 free-tier denial with kind=transport, status=402, and the upstream body', async () => { - // This is the literal failure from /tmp/live-smoke-evidence.md: the - // router returns 402 with `free_tier_limit` JSON and the runtime used to - // emit `backend_error` with only the synthesized message string, so - // consumers couldn't distinguish "model rejected free tier" from - // "DNS failure". The error detail must now carry status + body. - const body = JSON.stringify({ - error: { - message: 'Model "claude-sonnet-4-6" requires credits.', - type: 'insufficient_funds', - code: 'free_tier_limit', - }, - }) - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-free-tier', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-sonnet-4-6', - retry: { maxAttempts: 1 }, - fetchImpl: async () => new Response(body, { status: 402 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: '402-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const backendError = events.find((e) => e.type === 'backend_error') - expect(backendError).toBeDefined() - if (backendError?.type !== 'backend_error') { - throw new Error('expected backend_error') - } - expect(backendError.error).toEqual({ - kind: 'transport', - message: expect.stringContaining('402'), - status: 402, - body, - }) - // final carries the same typed detail. - const final = events.at(-1) - expect(final?.type).toBe('final') - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('failed') - expect(final.error).toEqual({ - kind: 'transport', - message: expect.stringContaining('402'), - status: 402, - body, - }) - }) - - it.each([ - [401, 'invalid api key'], - [403, 'forbidden'], - [404, 'model not found'], - ])('surfaces hard 4xx (%i) with typed transport detail and a failed final', async (status) => { - const body = JSON.stringify({ error: { code: 'auth_failed', message: 'rejected' } }) - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-bad', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - retry: { maxAttempts: 1 }, - fetchImpl: async () => new Response(body, { status }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: `${status}-task`, intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - expect(final?.type).toBe('final') - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('failed') - expect(final.error?.kind).toBe('transport') - expect(final.error?.status).toBe(status) - expect(final.error?.body).toBe(body) - }) - - it('surfaces a 5xx after retry exhaustion with the last status and body', async () => { - const body = 'Bad Gateway' - let attempts = 0 - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - retry: { maxAttempts: 3, initialBackoffMs: 1, maxBackoffMs: 2 }, - fetchImpl: async () => { - attempts += 1 - return new Response(body, { status: 502 }) - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: '502-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(attempts).toBe(3) - const final = events.at(-1) - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('failed') - expect(final.error?.kind).toBe('transport') - expect(final.error?.status).toBe(502) - expect(final.error?.body).toBe(body) - }) - - it('surfaces a connection failure (every attempt throws) with kind=transport, status=0', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - retry: { maxAttempts: 2, initialBackoffMs: 1, maxBackoffMs: 2 }, - fetchImpl: async () => { - throw new TypeError('fetch failed: ENETUNREACH') - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'net-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('failed') - expect(final.error?.kind).toBe('transport') - expect(final.error?.status).toBe(0) - expect(final.error?.message).toContain('unreachable') - }) - - it('truncates oversized error bodies to ~2 KiB so persisted events stay bounded', async () => { - const huge = 'A'.repeat(8192) - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - retry: { maxAttempts: 1 }, - fetchImpl: async () => new Response(huge, { status: 500 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'huge-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - if (final?.type !== 'final') throw new Error('expected final') - // 2 KiB cap + 1 ellipsis char. - expect(final.error?.body?.length).toBeLessThanOrEqual(2049) - expect(final.error?.body?.endsWith('…')).toBe(true) - }) - - it('classifies non-transport backend stream errors as kind=backend', async () => { - // A custom backend whose stream() throws a plain Error (sandbox crash, - // adapter bug). The runtime must still surface a typed error — just with - // kind=backend and no status. - const backend: AgentExecutionBackend = { - kind: 'custom-sandbox', - async *stream() { - yield { type: 'text_delta', text: 'partial' } - throw new Error('sandbox kernel died') - }, - } - const events = await collect( - runAgentTaskStream({ - task: { id: 'custom-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('failed') - expect(final.error).toEqual({ - kind: 'backend', - message: 'sandbox kernel died', - }) - // Partial text is still surfaced so consumers can show whatever the model - // produced before the crash. - expect(final.text).toBe('partial') - }) - - it('leaves final.error undefined on a successful run', async () => { - // The fail-loud change must not regress the success path — an absent - // `error` field is the contract for "stream completed cleanly". - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - fetchImpl: async () => - new Response( - 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n' + - 'data: {"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":1}}\n\n' + - 'data: [DONE]\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'ok-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - if (final?.type !== 'final') throw new Error('expected final') - expect(final.status).toBe('completed') - expect(final.error).toBeUndefined() - const backendError = events.find((e) => e.type === 'backend_error') - expect(backendError).toBeUndefined() - }) - - it('BackendTransportError exposes status + body for direct catchers', async () => { - // Some consumers wrap the backend instead of running it through - // runAgentTaskStream — they need to see the typed error directly. - const body = 'rate-limited' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - retry: { maxAttempts: 1 }, - fetchImpl: async () => new Response(body, { status: 429 }), - }) - let caught: unknown - try { - for await (const _ of backend.stream( - { task: { id: 't', intent: 'go' }, message: 'hi' }, - { - task: { id: 't', intent: 'go' }, - knowledge: { - taskId: 't', - blockingMissingRequirements: [], - nonBlockingGaps: [], - satisfiedRequirements: [], - readinessScore: 1, - severity: 'none', - recommendedAction: 'proceed', - reason: 'ready', - }, - session: { - id: 's', - backend: 'tcloud', - status: 'active', - createdAt: '2026-05-24T00:00:00Z', - updatedAt: '2026-05-24T00:00:00Z', - }, - }, - )) { - // drain - } - } catch (err) { - caught = err - } - expect(caught).toBeInstanceOf(BackendTransportError) - const transportErr = caught as BackendTransportError - expect(transportErr.status).toBe(429) - expect(transportErr.body).toBe(body) - expect(transportErr.backend).toBe('tcloud') - }) -}) diff --git a/tests/backends-openai-tools.test.ts b/tests/backends-openai-tools.test.ts deleted file mode 100644 index c915cdc4..00000000 --- a/tests/backends-openai-tools.test.ts +++ /dev/null @@ -1,551 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - createOpenAICompatibleBackend, - type KnowledgeRequirement, - type OpenAIChatTool, - type RuntimeStreamEvent, - runAgentTaskStream, -} from '../src/index' - -const readyReq: KnowledgeRequirement = { - id: 'build-command', - description: 'Build command', - requiredFor: ['test'], - category: 'codebase_specific', - acquisitionMode: 'inspect_repo', - importance: 'blocking', - freshness: 'weekly', - sensitivity: 'public', - confidenceNeeded: 0.8, - currentConfidence: 0.9, -} - -async function collect(iter: AsyncIterable): Promise { - const out: RuntimeStreamEvent[] = [] - for await (const e of iter) out.push(e) - return out -} - -const delegateResearchTool: OpenAIChatTool = { - type: 'function', - function: { - name: 'delegate_research', - description: 'Spin up a long-running researcher loop and return a taskId for polling.', - parameters: { - type: 'object', - properties: { - question: { type: 'string' }, - namespace: { type: 'string' }, - }, - required: ['question'], - }, - }, -} - -const delegateCodeTool: OpenAIChatTool = { - type: 'function', - function: { - name: 'delegate_code', - description: 'Spin up a coder iteration and return a taskId.', - parameters: { - type: 'object', - properties: { goal: { type: 'string' }, repoRoot: { type: 'string' } }, - required: ['goal'], - }, - }, -} - -describe('createOpenAICompatibleBackend — tools[] request shape', () => { - it('omits tools/tool_choice from the body when no tools option is set', async () => { - let captured: Record | undefined - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - fetchImpl: async (_url, init) => { - captured = JSON.parse((init?.body as string) ?? '{}') as Record - return new Response('data: [DONE]\n\n', { status: 200 }) - }, - }) - await collect( - runAgentTaskStream({ - task: { id: 'no-tools', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(captured).toBeDefined() - expect(captured?.tools).toBeUndefined() - expect(captured?.tool_choice).toBeUndefined() - }) - - it('includes the tools[] array verbatim when configured', async () => { - let captured: Record | undefined - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-sonnet-4-6', - tools: [delegateResearchTool, delegateCodeTool], - fetchImpl: async (_url, init) => { - captured = JSON.parse((init?.body as string) ?? '{}') as Record - return new Response('data: [DONE]\n\n', { status: 200 }) - }, - }) - await collect( - runAgentTaskStream({ - task: { id: 'with-tools', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(captured?.tools).toEqual([delegateResearchTool, delegateCodeTool]) - // tool_choice defaults to omitted — provider falls back to its own default - // (typically 'auto') so callers can keep the wire shape minimal. - expect(captured?.tool_choice).toBeUndefined() - }) - - it('includes response_format verbatim when configured', async () => { - let captured: Record | undefined - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - responseFormat: { type: 'json_object' }, - fetchImpl: async (_url, init) => { - captured = JSON.parse((init?.body as string) ?? '{}') as Record - return new Response('data: [DONE]\n\n', { status: 200 }) - }, - }) - await collect( - runAgentTaskStream({ - task: { id: 'json-response', intent: 'return JSON', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(captured).toMatchObject({ - stream: true, - stream_options: { include_usage: true }, - response_format: { type: 'json_object' }, - }) - }) - - it('includes generation options when configured', async () => { - let captured: Record | undefined - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'kimi-k2.7-code-highspeed', - temperature: 1, - maxTokens: 8192, - fetchImpl: async (_url, init) => { - captured = JSON.parse((init?.body as string) ?? '{}') as Record - return new Response('data: [DONE]\n\n', { status: 200 }) - }, - }) - await collect( - runAgentTaskStream({ - task: { id: 'generation-options', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(captured).toMatchObject({ - temperature: 1, - max_tokens: 8192, - }) - }) - - it('honors an explicit tool_choice value (auto / none / required / pin)', async () => { - const captures: Record[] = [] - const make = (toolChoice: Parameters[0]['toolChoice']) => - createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool], - toolChoice, - fetchImpl: async (_url, init) => { - captures.push(JSON.parse((init?.body as string) ?? '{}') as Record) - return new Response('data: [DONE]\n\n', { status: 200 }) - }, - }) - - for (const tc of [ - 'auto' as const, - 'none' as const, - 'required' as const, - { type: 'function' as const, function: { name: 'delegate_research' } }, - ]) { - await collect( - runAgentTaskStream({ - task: { id: `choice-${JSON.stringify(tc)}`, intent: 'hi', requiredKnowledge: [readyReq] }, - backend: make(tc), - input: { message: 'hi' }, - }), - ) - } - expect(captures.map((c) => c.tool_choice)).toEqual([ - 'auto', - 'none', - 'required', - { type: 'function', function: { name: 'delegate_research' } }, - ]) - }) -}) - -describe('createOpenAICompatibleBackend — OpenAI-shape tool_call streaming', () => { - it('emits a single tool_call event after a finish_reason: tool_calls chunk', async () => { - // OpenAI streams tool calls as fragmented deltas — the model emits the - // call id once, the function name once, and the arguments JSON in - // arbitrary-sized chunks. We must accumulate until finish_reason fires. - const sse = - 'data: {"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_42","type":"function","function":{"name":"delegate_research","arguments":""}}]}}],"model":"gpt-4.1-mini"}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"que"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"stion\\":\\""}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"competitor scan\\"}"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"model":"gpt-4.1-mini"}\n\n' + - 'data: {"choices":[],"model":"gpt-4.1-mini","usage":{"prompt_tokens":50,"completion_tokens":12}}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-stream', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'find competitor pricing' }, - }), - ) - const toolCalls = events.filter((e) => e.type === 'tool_call') - expect(toolCalls).toHaveLength(1) - const call = toolCalls[0] - if (call.type !== 'tool_call') throw new Error('expected tool_call') - expect(call.toolName).toBe('delegate_research') - expect(call.toolCallId).toBe('call_42') - expect(call.args).toEqual({ question: 'competitor scan' }) - // Usage still flows. - const usage = events.find((e) => e.type === 'llm_call') - expect(usage).toBeDefined() - if (usage?.type !== 'llm_call') throw new Error('expected llm_call') - expect(usage.tokensIn).toBe(50) - expect(usage.tokensOut).toBe(12) - }) - - it('emits multiple tool_call events when the model fires parallel tool calls', async () => { - // Parallel-tool-call SSE: deltas for index 0 and index 1 interleave; the - // single finish_reason:tool_calls flushes both. - const sse = - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"delegate_research","arguments":"{\\"question\\":\\"a\\"}"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"delegate_code","arguments":"{\\"goal\\":\\"b\\"}"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool, delegateCodeTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-parallel', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'do both' }, - }), - ) - const toolCalls = events.filter((e) => e.type === 'tool_call') - expect(toolCalls).toHaveLength(2) - const names = toolCalls.map((e) => (e.type === 'tool_call' ? e.toolName : '')) - expect(names.sort()).toEqual(['delegate_code', 'delegate_research']) - const byName = Object.fromEntries( - toolCalls.map((e) => (e.type === 'tool_call' ? [e.toolName, e] : ['', e])), - ) - expect(byName.delegate_research?.type === 'tool_call' && byName.delegate_research.args).toEqual( - { question: 'a' }, - ) - expect(byName.delegate_code?.type === 'tool_call' && byName.delegate_code.args).toEqual({ - goal: 'b', - }) - }) - - it('falls back to raw string args when the streamed JSON fragments are malformed', async () => { - // Reality: provider proxies occasionally truncate the last arguments - // chunk. We surface the raw concatenation instead of dropping the call so - // the dispatcher can attempt repair. - const sse = - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_x","type":"function","function":{"name":"delegate_research","arguments":"{\\"question\\":\\"unfini"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-trunc', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(typeof calls[0].args).toBe('string') - expect(calls[0].args).toBe('{"question":"unfini') - }) - - it('flushes a tool_call that never received an explicit finish_reason', async () => { - // Some routers drop the terminal choice chunk and go straight to [DONE]. - // The pending call must still surface — dropping it silently is the same - // class of bug as the 402 swallow. - const sse = - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_z","type":"function","function":{"name":"delegate_code","arguments":"{\\"goal\\":\\"ship\\"}"}}]}}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateCodeTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-flush', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(calls[0].toolName).toBe('delegate_code') - expect(calls[0].args).toEqual({ goal: 'ship' }) - }) - - it('emits tool_call events alongside text deltas in mixed-content streams', async () => { - const sse = - 'data: {"choices":[{"index":0,"delta":{"content":"Sure, "}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"content":"delegating now."}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_m","type":"function","function":{"name":"delegate_research","arguments":"{\\"question\\":\\"q\\"}"}}]}}]}\n\n' + - 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-mixed', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const text = events - .filter((e) => e.type === 'text_delta') - .map((e) => (e.type === 'text_delta' ? e.text : '')) - .join('') - expect(text).toBe('Sure, delegating now.') - expect(events.filter((e) => e.type === 'tool_call')).toHaveLength(1) - }) - - it('handles non-streamed message.tool_calls (single-chunk completion)', async () => { - // Some providers (and the router fallback path for non-stream upstreams) - // collapse the whole assistant turn into one chunk with `message.tool_calls`. - const sse = - 'data: {"choices":[{"index":0,"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_q","type":"function","function":{"name":"delegate_research","arguments":"{\\"question\\":\\"x\\"}"}}]},"finish_reason":"tool_calls"}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-single', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(calls[0].toolName).toBe('delegate_research') - expect(calls[0].toolCallId).toBe('call_q') - expect(calls[0].args).toEqual({ question: 'x' }) - }) - - /** - * The Tangle router's Gemini lane is OpenAI-COMPATIBLE but not OpenAI-exact: - * it emits every parallel tool call COMPLETE inside a single delta and puts - * NO `index` on the entries (the one `"index":0` in the frame is the CHOICE - * index). Keying those on `index ?? 0` collapsed all of them into one - * accumulator and concatenated their `arguments` JSON, which then failed to - * parse and surfaced as a raw string — a six-deliverable turn produced none, - * silently. Frames below are the captured shape from - * router.tangle.tools/v1 + gemini-2.5-flash-lite. - */ - it('keeps parallel calls distinct when a compat gateway omits per-call index', async () => { - const sse = - 'data: {"choices":[{"delta":{"role":"assistant","tool_calls":[' + - '{"function":{"arguments":"{\\"question\\":\\"first\\"}","name":"delegate_research"},"id":"function-call-1","type":"function"},' + - '{"function":{"arguments":"{\\"goal\\":\\"second\\"}","name":"delegate_code"},"id":"function-call-2","type":"function"},' + - '{"function":{"arguments":"{\\"question\\":\\"third\\"}","name":"delegate_research"},"id":"function-call-3","type":"function"}' + - ']},"finish_reason":"tool_calls","index":0}],"model":"gemini-2.5-flash-lite"}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gemini-2.5-flash-lite', - tools: [delegateResearchTool, delegateCodeTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-noindex', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'do three things' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(3) - expect(calls.map((c) => (c.type === 'tool_call' ? c.toolCallId : undefined))).toEqual([ - 'function-call-1', - 'function-call-2', - 'function-call-3', - ]) - expect(calls.map((c) => (c.type === 'tool_call' ? c.toolName : undefined))).toEqual([ - 'delegate_research', - 'delegate_code', - 'delegate_research', - ]) - // Every args payload is a parsed object, never the concatenated raw string. - expect(calls.map((c) => (c.type === 'tool_call' ? c.args : undefined))).toEqual([ - { question: 'first' }, - { goal: 'second' }, - { question: 'third' }, - ]) - }) - - /** - * An index-less gateway may still fragment: the first frame carries the id + - * name, later frames carry arguments only. Those continuations must append to - * the call they belong to, not open a new one. - */ - it('appends index-less argument-only fragments to the open call', async () => { - const sse = - 'data: {"choices":[{"delta":{"tool_calls":[{"id":"fc-1","type":"function","function":{"name":"delegate_research","arguments":"{\\"question\\":"}}]}}]}\n\n' + - 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\\"split arg\\"}"}}]}}]}\n\n' + - 'data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}\n\n' + - 'data: [DONE]\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gemini-2.5-flash-lite', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'tc-noindex-frag', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(calls[0].toolCallId).toBe('fc-1') - expect(calls[0].args).toEqual({ question: 'split arg' }) - }) -}) - -describe('createOpenAICompatibleBackend — Anthropic-shape tool_use streaming', () => { - it('assembles a tool_call event from content_block_start/delta/stop sequence', async () => { - // Router proxies Anthropic models with their native SSE shape. tool_use - // blocks come through as: content_block_start (with id+name), - // content_block_delta (input_json_delta chunks of partial_json), then - // content_block_stop. - const sse = - 'data: {"type":"message_start","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":120}}}\n\n' + - 'data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_a","name":"delegate_research","input":{}}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\\"question\\":\\""}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"q1\\"}"}}\n\n' + - 'data: {"type":"content_block_stop","index":0}\n\n' + - 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":15}}\n\n' + - 'data: {"type":"message_stop"}\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-sonnet-4-6', - tools: [delegateResearchTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'anthropic-tc', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(calls[0].toolName).toBe('delegate_research') - expect(calls[0].toolCallId).toBe('toolu_a') - expect(calls[0].args).toEqual({ question: 'q1' }) - // Anthropic stop_reason 'tool_use' must propagate. - const usage = events.find((e) => e.type === 'llm_call') - expect(usage?.type === 'llm_call' && usage.finishReason).toBe('tool_use') - }) - - it('interleaves text deltas and tool_use blocks correctly', async () => { - const sse = - 'data: {"type":"message_start","message":{"id":"msg_2","model":"claude-sonnet-4-6","usage":{"input_tokens":50}}}\n\n' + - 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I will "}}\n\n' + - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"delegate."}}\n\n' + - 'data: {"type":"content_block_stop","index":0}\n\n' + - 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_b","name":"delegate_code","input":{}}}\n\n' + - 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"goal\\":\\"ship\\"}"}}\n\n' + - 'data: {"type":"content_block_stop","index":1}\n\n' + - 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":8}}\n\n' + - 'data: {"type":"message_stop"}\n\n' - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'claude-sonnet-4-6', - tools: [delegateCodeTool], - fetchImpl: async () => new Response(sse, { status: 200 }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'anthropic-mixed', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const text = events - .filter((e) => e.type === 'text_delta') - .map((e) => (e.type === 'text_delta' ? e.text : '')) - .join('') - expect(text).toBe('I will delegate.') - const calls = events.filter((e) => e.type === 'tool_call') - expect(calls).toHaveLength(1) - if (calls[0].type !== 'tool_call') throw new Error('expected tool_call') - expect(calls[0].toolName).toBe('delegate_code') - expect(calls[0].args).toEqual({ goal: 'ship' }) - }) -}) diff --git a/tests/backends-openai-usage.test.ts b/tests/backends-openai-usage.test.ts deleted file mode 100644 index 5ae2fa11..00000000 --- a/tests/backends-openai-usage.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - createOpenAICompatibleBackend, - type KnowledgeRequirement, - type RuntimeStreamEvent, - runAgentTaskStream, -} from '../src/index' - -const readyReq: KnowledgeRequirement = { - id: 'build-command', - description: 'Build command', - requiredFor: ['test'], - category: 'codebase_specific', - acquisitionMode: 'inspect_repo', - importance: 'blocking', - freshness: 'weekly', - sensitivity: 'public', - confidenceNeeded: 0.8, - currentConfidence: 0.9, -} - -async function collect(iter: AsyncIterable): Promise { - const out: RuntimeStreamEvent[] = [] - for await (const e of iter) out.push(e) - return out -} - -describe('createOpenAICompatibleBackend — OpenAI usage emission', () => { - it('emits a synthesized llm_call event from the final OpenAI usage chunk', async () => { - // Regression: prior to this fix the backend never read `parsed.usage` and - // never emitted `llm_call`, so RunRecord.tokenUsage was always zero. - let capturedBody: string | undefined - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - fetchImpl: async (_url, init) => { - capturedBody = init?.body as string | undefined - return new Response( - 'data: {"choices":[{"index":0,"delta":{"content":"hel"}}],"model":"gpt-4.1-mini"}\n\n' + - 'data: {"choices":[{"index":0,"delta":{"content":"lo"}}],"model":"gpt-4.1-mini"}\n\n' + - 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"model":"gpt-4.1-mini"}\n\n' + - 'data: {"choices":[],"model":"gpt-4.1-mini","usage":{"prompt_tokens":42,"completion_tokens":7,"total_tokens":49}}\n\n' + - 'data: [DONE]\n\n', - { status: 200 }, - ) - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'usage-task', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - - // The request body must include `stream_options.include_usage` so the - // router forwards usage upstream and back. - expect(capturedBody).toBeDefined() - expect(JSON.parse(capturedBody as string)).toMatchObject({ - stream: true, - stream_options: { include_usage: true }, - }) - - const llmCalls = events.filter((e) => e.type === 'llm_call') - expect(llmCalls).toHaveLength(1) - const llmCall = llmCalls[0] - if (llmCall.type !== 'llm_call') throw new Error('expected llm_call') - expect(llmCall.tokensIn).toBe(42) - expect(llmCall.tokensOut).toBe(7) - expect(llmCall.model).toBe('gpt-4.1-mini') - expect(llmCall.finishReason).toBe('stop') - // Latency is wall-clock, so just check it's a finite non-negative number. - expect(typeof llmCall.latencyMs).toBe('number') - expect(llmCall.latencyMs).toBeGreaterThanOrEqual(0) - // Text deltas still flow. - expect( - events - .filter((e) => e.type === 'text_delta') - .map((e) => e.text) - .join(''), - ).toBe('hello') - }) - - it('does NOT synthesize an llm_call event when the stream never reports usage', async () => { - // Truthful absence: better to emit nothing than to emit zero tokens and - // poison every downstream ledger with silent zeros. - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.tangle.tools/v1', - model: 'gpt-4.1-mini', - fetchImpl: async () => - new Response('data: {"choices":[{"delta":{"content":"ok"}}]}\n\n' + 'data: [DONE]\n\n', { - status: 200, - }), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'no-usage', intent: 'hi', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const llmCalls = events.filter((e) => e.type === 'llm_call') - expect(llmCalls).toHaveLength(0) - }) -}) diff --git a/tests/kernel/chat-transport-executor.test.ts b/tests/kernel/chat-transport-executor.test.ts index 9e3bd395..010fa1e0 100644 --- a/tests/kernel/chat-transport-executor.test.ts +++ b/tests/kernel/chat-transport-executor.test.ts @@ -13,15 +13,15 @@ * field marks `usdKnown: false` — never a silent estimate, never a fabricated zero. * 5. The tool table: tool_calls run on the host and fold back as `tool` messages; unknown tools * and malformed arguments are fed back to the model, never thrown. - * 6. Transport failures throw `ValidationError` (the scope's INFRA class) — and the turns that - * DID run are still recorded, because resume-after-failure is a kernel-supported path. + * 6. Transport failures remain visible to the scope as INFRA — and the turns that DID run are + * still recorded, because resume-after-failure is a kernel-supported path. * 7. `chatWorkerSeam`: profile model/prompt win, the graph's appended directive instructions * reach the system message, the node id keys the recorded session, and a missing model * fails loud. */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' -import { ValidationError } from '../../src/errors' import type { WorkerSpawnContext } from '../../src/mcp/tools/coordination' import { type ChatCompletionsTransport, @@ -51,6 +51,16 @@ function buildExecutor(spec: AgentSpec, ctx: ExecutorContext): Executor const never = new AbortController().signal +function chatProfile( + overrides: Partial & Pick = { name: 'chat-worker' }, +): AgentProfile { + return { + harness: 'cli-base', + model: { provider: 'scripted', default: 'test/model' }, + ...overrides, + } +} + /** A scripted transport: replies in order (last repeats), captures every request body. */ function scriptedTransport(replies: Array>): { transport: ChatCompletionsTransport @@ -78,8 +88,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) const ex = chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', - system: 'Be terse.', + profile: chatProfile({ name: 'terse', prompt: { systemPrompt: 'Be terse.' } }), complete: transport, }) const result = await (ex.execute('what is 2+2?', never) as Promise>) @@ -123,10 +132,16 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile({ + name: 'tool-user', + model: { + provider: 'scripted', + default: 'test/model', + metadata: { temperature: 0.7, maxTokens: 2500 }, + }, + tools: { step: true }, + }), complete: transport, - temperature: 0.7, - maxTokens: 2500, tools: [ { spec: { type: 'function', function: { name: 'step', parameters: {} } }, @@ -149,11 +164,17 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', expect(() => chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile({ + name: 'invalid-token-limit', + model: { + provider: 'scripted', + default: 'test/model', + metadata: { maxTokens }, + }, + }), complete: transport, - maxTokens, }), - ).toThrow(/maxTokens must be a positive integer/) + ).toThrow(/maxTokens must (?:be positive|be a safe integer)/) } expect(requests).toHaveLength(0) }) @@ -165,8 +186,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', - system: 'Persist.', + profile: chatProfile({ name: 'persistent', prompt: { systemPrompt: 'Persist.' } }), complete: shot1.transport, sessions, sessionKey: 'run:s0', @@ -177,9 +197,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) const result = await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', - // A resumed session keeps its recorded system message; this one must NOT re-seed. - system: 'IGNORED ON RESUME', + profile: chatProfile({ name: 'persistent', prompt: { systemPrompt: 'Persist.' } }), complete: shot2.transport, sessions, sessionKey: 'run:s1', @@ -204,7 +222,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', expect(() => chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile(), complete: transport, sessions: createChatSessionStore(), resume: { ofWorker: 'ghost:s9', sequence: 2 }, @@ -213,11 +231,11 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', expect(() => chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile(), complete: transport, resume: { ofWorker: 'ghost:s9', sequence: 2 }, }), - ).toThrow(/needs `sessions`/) + ).toThrow(/needs the session store/) expect(requests).toHaveLength(0) }) @@ -225,7 +243,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', const noUsage = scriptedTransport([{ choices: [{ message: { content: 'blind turn' } }] }]) const r1 = await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile(), complete: noUsage.transport, }).execute('t', never) as Promise>) expect(r1.spent.tokensKnown).toBe(false) @@ -240,7 +258,10 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) const r2 = await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'openai/gpt-4o-mini', + profile: chatProfile({ + name: 'priced-model', + model: { provider: 'openai', default: 'openai/gpt-4o-mini' }, + }), complete: tokensOnly.transport, }).execute('t', never) as Promise>) expect(r2.spent.tokens).toEqual({ input: 3, output: 4 }) @@ -271,7 +292,7 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', ]) const result = await (chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile({ name: 'lookup-user', tools: { lookup: true } }), complete: transport, tools: [ { @@ -290,20 +311,27 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', const turn2 = requests[1]?.messages as Array> expect(turn2.slice(-3)).toEqual([ { role: 'tool', tool_call_id: 'c1', content: 'result:42' }, - { role: 'tool', tool_call_id: 'c2', content: "error: unknown tool 'nope'" }, + { + role: 'tool', + tool_call_id: 'c2', + content: 'error: tool "nope" is not enabled by AgentProfile.tools', + }, { role: 'tool', tool_call_id: 'c3', content: 'error: tool arguments were not valid JSON' }, ]) expect(requests[0]?.tools).toBeDefined() expect(requests[0]?.tool_choice).toBe('auto') }) - it('throws ValidationError on transport failure — and still records the turns that ran', async () => { + it('surfaces a transport failure — and still records the turns that ran', async () => { const sessions = createChatSessionStore() let turn = 0 const ex = chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', - system: 'S.', + profile: chatProfile({ + name: 'fallible-tool-user', + prompt: { systemPrompt: 'S.' }, + tools: { step: true }, + }), sessions, sessionKey: 'run:s0', tools: [ @@ -330,36 +358,21 @@ describe('chatTransportExecutor — one conversation shot on a bare transport', throw new Error('socket hang up') }, }) - await expect(ex.execute('go', never) as Promise).rejects.toThrow( - new ValidationError('chatTransportExecutor: transport failed: socket hang up'), - ) + await expect(ex.execute('go', never) as Promise).rejects.toThrow('socket hang up') // The failed shot's REAL first turn is recorded, so a resume can continue the session. const recorded = sessions.load('run:s0') expect(recorded?.map((m) => m.role)).toEqual(['system', 'user', 'assistant', 'tool']) }) - it('fails loud on a malformed completion and on a second execute of one instance', async () => { + it('fails loud on a malformed completion', async () => { const empty = scriptedTransport([{ choices: [] }]) await expect( chatTransportExecutor({ url: 'http://unused.invalid', - model: 'test/model', + profile: chatProfile(), complete: empty.transport, }).execute('t', never) as Promise, ).rejects.toThrow(/no choices\[0\]\.message/) - - const ok = scriptedTransport([ - reply('once', { prompt_tokens: 1, completion_tokens: 1, cost: 0 }), - ]) - const ex = chatTransportExecutor({ - url: 'http://unused.invalid', - model: 'test/model', - complete: ok.transport, - }) - await (ex.execute('t', never) as Promise) - await expect(ex.execute('t', never) as Promise).rejects.toThrow( - /execute\(\) called twice/, - ) }) }) @@ -386,16 +399,18 @@ describe('chatWorkerSeam — the continuity-honoring makeWorkerAgent over the ex const sessions = createChatSessionStore() const seam = chatWorkerSeam({ url: 'http://unused.invalid', - model: 'seam/fallback', complete: transport, sessions, - temperature: 0.7, - maxTokens: 2500, }) const agent = seam( { name: 'product-agent', - model: { default: 'profile/model' }, + harness: 'cli-base', + model: { + provider: 'scripted', + default: 'profile/model', + metadata: { temperature: 0.7, maxTokens: 2500 }, + }, // The graph pins the delegates directive by APPENDING to instructions — it must reach // the system message. prompt: { systemPrompt: 'Sell.', instructions: ['directive: be helpful'] }, @@ -427,7 +442,11 @@ describe('chatWorkerSeam — the continuity-honoring makeWorkerAgent over the ex url: 'http://unused.invalid', complete: transport, }) - const profile = { name: 'agent', model: { default: 'm' }, prompt: { systemPrompt: 'S.' } } + const profile = chatProfile({ + name: 'agent', + model: { provider: 'scripted', default: 'm' }, + prompt: { systemPrompt: 'S.' }, + }) const first = seam(profile, spawnContext({})) await (buildExecutor(specOf(first), nodeCtx('g:s0')).execute( @@ -452,10 +471,10 @@ describe('chatWorkerSeam — the continuity-honoring makeWorkerAgent over the ex ]) }) - it('fails loud when neither the profile nor the seam names a model', () => { + it('fails loud when the exact profile omits execution identity', () => { const seam = chatWorkerSeam({ url: 'http://unused.invalid', complete: async () => reply('x') }) expect(() => seam({ name: 'agent', prompt: { systemPrompt: 'S.' } }, undefined)).toThrow( - /no model/, + /harness must be explicit/, ) }) }) diff --git a/tests/kernel/coordination-driver.test.ts b/tests/kernel/coordination-driver.test.ts index d0ad0d81..ec0aed5d 100644 --- a/tests/kernel/coordination-driver.test.ts +++ b/tests/kernel/coordination-driver.test.ts @@ -512,12 +512,11 @@ describe('driverAgent — maxTurns=0 lifts the turn cap; the conserved pool + de expect(result.kind).toBe('no-winner') }) - it('runs the driver PAST the default 16-turn cap until it stops on its own', async () => { + it('runs past the former 2000-turn sentinel until the driver stops on its own', async () => { SHARED_BLOBS = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() const seen: SeenMessages = [] - // 20 benign turns then a no-tool-call stop: a run the old default (16) would have force-finalized. - const turns: ScriptedTurn[] = Array.from({ length: 20 }, () => benignTurn) + const turns: ScriptedTurn[] = Array.from({ length: 2001 }, () => benignTurn) turns.push({ content: 'nothing left to do' }) const chat = scriptedBrain(turns, seen) @@ -532,9 +531,8 @@ describe('driverAgent — maxTurns=0 lifts the turn cap; the conserved pool + de now: () => 0, }) - // 20 benign turns + 1 stop = 21 driver turns — proof maxTurns=0 blew past the old 16 cap - // instead of force-finalizing at it. - expect(seen.length).toBe(21) + // 2,001 benign turns + 1 stop proves Runtime did not remap 0 to the old 2,000 sentinel. + expect(seen.length).toBe(2002) }) it('breaks the unlimited loop the moment the scope signal aborts (mid-loop)', async () => { diff --git a/tests/kernel/driver-inference-metering.test.ts b/tests/kernel/driver-inference-metering.test.ts index 1103cf6c..d584bdc6 100644 --- a/tests/kernel/driver-inference-metering.test.ts +++ b/tests/kernel/driver-inference-metering.test.ts @@ -523,6 +523,7 @@ describe("driver inference metering — the driver's own tokens count against th toolCalls: [{ id: `call-${n}`, name: 'list_questions', arguments: '{}' }], usage: { input: 5, output: 5 }, costUsd: 0.04, + costProvenance: 'provider-receipt', } } const opts: DriverAgentOptions = { diff --git a/tests/kernel/router-brain.test.ts b/tests/kernel/router-brain.test.ts index 0ab5d4d4..4f3f12e7 100644 --- a/tests/kernel/router-brain.test.ts +++ b/tests/kernel/router-brain.test.ts @@ -156,7 +156,7 @@ describe('routerBrain — the production ToolLoopChat seam over the router tool- const init = fetchMock.mock.calls[0]![1] as { body: string } const sent = JSON.parse(init.body) expect(sent.temperature).toBe(0.1) - expect(sent.tool_choice).toBe('auto') + expect(sent.tool_choice).toBeUndefined() }) it('forwards the router usage + cost so the driver can meter its inference', async () => { @@ -220,7 +220,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { expect(result.reasoning).toBe('The user wants me to calculate.') expect(result.finishReason).toBe('tool_calls') // The whole point: the streamed turn meters exactly what the buffered turn would have. - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) expect(result.costUsd).toBeGreaterThan(0) // Usage arrived, so the turn is NOT marked unknown. expect(result.usageUnknown).toBeUndefined() @@ -248,7 +248,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { ]) // Frame 4 — the terminal usage chunk, the one whose loss makes a real turn look free. expect(result.finishReason).toBe('tool_calls') - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) expect(result.usageUnknown).toBeUndefined() } }) @@ -266,7 +266,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { const result = await streamRouterChatWithTools(cfg, [], []) expect(result.reasoning).toBe('The user wants me to calculate.') expect(result.toolCalls).toHaveLength(1) - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) }) it('asks for the stream and for usage inside it, and keeps every other request field identical', async () => { @@ -427,7 +427,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { { id: 'call_83f2da0d', name: 'calc', arguments: '{"expr":"21*2"}' }, ]) expect(result.finishReason).toBe('tool_calls') - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) expect(result.usageUnknown).toBeUndefined() }) @@ -441,7 +441,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { const result = await streamRouterChatWithTools(cfg, [], []) expect(result.reasoning).toBe('The user wants me to calculate.') expect(result.toolCalls).toHaveLength(1) - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) }) // A streamed turn stops reading its body EARLY on both of its exits: `[DONE]` arrives before the @@ -452,7 +452,7 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { const cancelled = stubUnclosedStream(fixture) const result = await streamRouterChatWithTools(cfg, [], []) // The turn still parsed completely — cancelling is cleanup, not truncation. - expect(result.usage).toEqual({ input: 162, output: 28 }) + expect(result.usage).toEqual({ input: 162, output: 28, reasoning: 15 }) expect(cancelled).toHaveBeenCalledTimes(1) }) @@ -498,7 +498,7 @@ describe('routerBrain transport selection', () => { const sent = JSON.parse(fetchMock.mock.calls[0]![1].body) expect(sent.stream).toBe(true) expect(sent.temperature).toBe(0.4) - expect(sent.tool_choice).toBe('auto') + expect(sent.tool_choice).toBeUndefined() expect(result.content).toBe('ok') // Same numbers the buffered brain reports for the same usage — the conserved pool is unaffected // by the transport choice. diff --git a/tests/kernel/scripted-brain.ts b/tests/kernel/scripted-brain.ts index 0a8bcd5f..bc2bc1f3 100644 --- a/tests/kernel/scripted-brain.ts +++ b/tests/kernel/scripted-brain.ts @@ -5,7 +5,9 @@ export interface ScriptedTurn { content?: string toolCalls?: Array<{ id?: string; name: string; arguments: Record }> usage?: { input: number; output: number } + /** Scripted costs model an exact provider receipt unless explicitly overridden. */ costUsd?: number + costProvenance?: 'provider-receipt' | 'billing-receipt' | 'catalog-estimate' } /** Build a scripted `ToolLoopChat` brain from a fixed turn sequence: converts parsed tool args to @@ -28,7 +30,12 @@ export function scriptedBrain( arguments: JSON.stringify(tc.arguments), })), ...(turn.usage ? { usage: turn.usage } : {}), - ...(turn.costUsd !== undefined ? { costUsd: turn.costUsd } : {}), + ...(turn.costUsd !== undefined + ? { + costUsd: turn.costUsd, + costProvenance: turn.costProvenance ?? ('provider-receipt' as const), + } + : {}), } } } diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index 9c69df78..dbb91d92 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -441,15 +441,8 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(executor?.runtime).toBe('router') }) - it('workerFromBackend rejects post-identity profile overlays and shared execution ids', () => { + it('workerFromBackend rejects shared execution ids', () => { const invalid: ExecutorConfig[] = [ - { - backend: 'bridge', - bridgeUrl: 'http://bridge.test', - bridgeBearer: 'secret', - model: 'model', - agentProfile: { name: 'late-overlay' }, - }, { backend: 'bridge', bridgeUrl: 'http://bridge.test', @@ -734,22 +727,6 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ ).toThrow(new RegExp(`${rejected}.*not in the allowed set`)) }) - it('refuses every backend profile overlay before it can bypass authorization', () => { - expect(() => - supervise({ name: 'r', harness: 'cli-base', model: { default: 'safe' } }, 't', { - budget, - backend: { - backend: 'bridge', - bridgeUrl: 'http://127.0.0.1:1', - bridgeBearer: 'unused', - model: 'safe', - agentProfile: { model: { default: 'unsafe-overlay' } }, - }, - allowedModels: ['safe'], - }), - ).toThrow(/backend agentProfile overlays are not allowed/) - }) - it('refuses a fixed session id on the reusable driver backend', () => { expect(() => supervise({ name: 'r', harness: 'codex' }, 't', { diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index 04d573c7..abb9fbf4 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -3,7 +3,11 @@ import { createServer, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { + type AgentProfile, + canonicalAgentProfileDigest, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' @@ -20,6 +24,42 @@ type BridgeRequest = { const TEST_RUN_DIGEST = `sha256:${'c'.repeat(64)}` +function codexTestProfile(name: string, systemPrompt?: string): AgentProfile { + return { + name, + harness: 'codex', + model: { provider: 'openai', default: 'test' }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + } +} + +function routerTestProfile(name: string, systemPrompt?: string): AgentProfile { + return { + name, + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'test' }, + ...(systemPrompt ? { prompt: { systemPrompt } } : {}), + } +} + +function createBridgeServer(handler: Parameters[0]): Server { + return createServer((req, res) => { + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + return + } + return handler(req, res) + }) +} + function numberSseDataFrames(body: string): string { let seq = 0 return body.replace(/^data: (?!\[DONE\])/gmu, () => `id: ${++seq}\ndata: `) @@ -35,7 +75,43 @@ function respondWithBridgeStream( 'x-run-id': request.run_id, 'x-run-request-digest': TEST_RUN_DIGEST, }) - res.end(numberSseDataFrames(stream)) + const normalized = stream + .split('\n') + .map((line) => { + if (!line.startsWith('data: ') || line === 'data: [DONE]') return line + const payload = JSON.parse(line.slice('data: '.length)) as { usage?: Record } + if (!payload.usage) return line + payload.usage = { + ...payload.usage, + ...(typeof payload.usage.cost === 'number' + ? { cost_known: true, cost_provenance: 'provider-receipt' } + : { cost_known: false }), + } + return `data: ${JSON.stringify(payload)}` + }) + .join('\n') + const parts = request.model.split('/') + const requested = request.agent_profile.model?.reasoningEffort ?? null + const applied = + requested === 'none' + ? 'minimal' + : requested === 'xhigh' || requested === 'ultracode' + ? 'high' + : requested + const receipt = `data: ${JSON.stringify({ + profile_materialization: { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: canonicalAgentProfileDigest(request.agent_profile), + harness: parts[0], + provider: parts[1] ?? null, + model: request.model, + reasoningEffort: { requested, applied }, + workspacePlanDigest: `sha256:${'d'.repeat(64)}`, + files: [], + unsupported: [], + }, + })}` + res.end(numberSseDataFrames(normalized.replace('data: [DONE]', `${receipt}\n\ndata: [DONE]`))) } function cancelledRunId(url: string | undefined): string | undefined { @@ -123,7 +199,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( const firstRequest = new Promise((resolve) => { markFirstRequest = resolve }) - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const cancelledId = cancelledRunId(req.url) if (cancelledId !== undefined) { cancelled.push(cancelledId) @@ -139,7 +215,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( 'x-run-request-digest': TEST_RUN_DIGEST, }) res.write( - `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2, cost: 0.01 } })}\n\n`, + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2, cost: 0.01, cost_known: true, cost_provenance: 'provider-receipt' } })}\n\n`, ) markFirstRequest() return @@ -154,7 +230,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, }, 'Choose the next experiment.', { @@ -163,7 +239,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', }, budget: { maxIterations: 4, maxTokens: 10_000 }, }, @@ -192,7 +267,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( it('selects heterogeneous leaf completion checks from the exact authorized spawn context', async () => { const requests: BridgeRequest[] = [] - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) requests.push(body) const content = @@ -216,14 +291,13 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( }> = [] let turn = 0 const result = await supervise( - { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run all checks.' } }, + routerTestProfile('root', 'Run all checks.'), 'Compare implementation and evaluation evidence.', { backend: { backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/test', }, budget: { maxIterations: 20, maxTokens: 20_000 }, perWorker: { maxIterations: 4, maxTokens: 2_000 }, @@ -243,7 +317,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( id: 'worker', name: 'spawn_agent', arguments: JSON.stringify({ - profile: { name: 'worker' }, + profile: codexTestProfile('worker'), task: { kind: 'implement', expected: 'WORK=42' }, key: 'worker', }), @@ -252,7 +326,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( id: 'evaluator', name: 'spawn_agent', arguments: JSON.stringify({ - profile: { name: 'evaluator' }, + profile: codexTestProfile('evaluator'), task: { kind: 'evaluate', expected: 'EVAL=pass' }, key: 'evaluator', }), @@ -261,7 +335,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( id: 'fallback', name: 'spawn_agent', arguments: JSON.stringify({ - profile: { name: 'fallback' }, + profile: codexTestProfile('fallback'), task: { kind: 'archive', expected: 'FALLBACK=ready' }, key: 'fallback', }), @@ -343,7 +417,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( it('reuses the recorded per-spawn completion result on durable resume', async () => { let requests = 0 - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) requests += 1 respondWithBridgeStream(res, body, successStream('RESULT=durable')) @@ -363,7 +437,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( id: 'spawn', name: 'spawn_agent', arguments: JSON.stringify({ - profile: { name: 'durable-worker' }, + profile: codexTestProfile('durable-worker'), task: 'produce the durable result', key: 'durable-worker', }), @@ -384,7 +458,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge' as const, bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/test', }, budget: { maxIterations: 8, maxTokens: 10_000 }, perWorker: { maxIterations: 2, maxTokens: 1_000 }, @@ -400,7 +473,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( } }, } - const profile = { name: 'root', harness: 'cli-base' as const } + const profile = routerTestProfile('root') try { const first = await supervise(profile, 'resume the exact result', { ...common, @@ -439,7 +512,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( profile: AgentProfile task: unknown }> = [] - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { try { const body = await readJson(req) requests.push(body) @@ -455,7 +528,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( description: 'Run the discriminating experiment', harness: 'codex', prompt: { systemPrompt: 'Supervise one empirical worker.' }, - model: { default: 'gpt-5.6', reasoningEffort: 'high' }, + model: { provider: 'openai', default: 'gpt-5.6', reasoningEffort: 'high' }, tools: { shell: true }, resources: { skills: [ @@ -484,7 +557,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( description: 'Execute and report the measurement', harness: 'codex', prompt: { systemPrompt: 'Return the exact measured result.' }, - model: { default: 'gpt-5.6', reasoningEffort: 'medium' }, + model: { provider: 'openai', default: 'gpt-5.6', reasoningEffort: 'medium' }, permissions: { shell: 'allow' }, tools: { shell: true, web: false }, resources: { @@ -526,14 +599,13 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', } const rootProfile: AgentProfile = { name: 'pi-leader', description: 'Lead the full pursuit', harness: 'codex', prompt: { systemPrompt: 'Choose and supervise the most informative experiment.' }, - model: { default: 'gpt-5.6', reasoningEffort: 'xhigh' }, + model: { provider: 'openai', default: 'gpt-5.6', reasoningEffort: 'xhigh' }, tools: { web: true }, mcp: { literature: { transport: 'http', url: 'https://papers.example.test/mcp' }, @@ -624,9 +696,9 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( 'experiment-worker', ]) expect(requests.map((request) => request.model)).toEqual([ - 'codex/gpt-5.6', - 'codex/gpt-5.6', - 'codex/gpt-5.6', + 'codex/openai/gpt-5.6', + 'codex/openai/gpt-5.6', + 'codex/openai/gpt-5.6', ]) expect(requests.map((request) => request.session_id)).toEqual([ expect.stringMatching(/^supervised-manager-[a-f0-9]{64}$/), @@ -706,7 +778,9 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( candidateDigest: canonicalCandidateDigest({ candidate: 'pi-leader' }), correlation: { pursuitId: 'pursuit-1', experimentId: 'experiment-root' }, }) - const rootMaterialized = rootEvents?.find((event) => event.kind === 'materialized') + const rootMaterialized = rootEvents?.find( + (event) => event.kind === 'materialized' && event.id === 'identity-run', + ) expect(rootMaterialized).toMatchObject({ kind: 'materialized', id: 'identity-run', @@ -716,7 +790,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( effectiveProfileDigest: canonicalCandidateDigest(rootProfile), runtime: 'cli', backend: 'bridge', - model: { status: 'known', id: 'codex/gpt-5.6' }, + model: { status: 'known', id: 'codex/openai/gpt-5.6' }, execution: { kind: 'session', id: requests[0]!.session_id }, }, }) @@ -753,7 +827,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( it('isolates identical concurrent managers but reuses one durable manager session on restart', async () => { const sessions: string[] = [] - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) sessions.push(body.session_id) respondWithBridgeStream(res, body, successStream('managed')) @@ -764,13 +838,12 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, } as const const backend = { backend: 'bridge' as const, bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', } const task = 'Choose the next experiment.' @@ -811,7 +884,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( it('captures mutable supervision policy, profiles, limits, and callback selection at intake', async () => { const requests: BridgeRequest[] = [] - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) requests.push(body) respondWithBridgeStream(res, body, successStream('RESULT=42')) @@ -843,7 +916,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( profile: { name: 'policy-worker', harness: 'codex', - model: { default: 'safe-model' }, + model: { provider: 'openai', default: 'safe-model' }, mcp: { allowed: { transport: 'http', url: 'https://allowed.test/mcp' }, }, @@ -868,13 +941,13 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( const rootProfile: AgentProfile = { name: 'original-root', harness: 'cli-base', + model: { provider: 'tangle-router', default: 'safe-model' }, prompt: { systemPrompt: 'Use the worker.' }, } const backend = { backend: 'bridge' as const, bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'safe-model', } const profileSecurity = { allowLocalMcp: false, @@ -917,7 +990,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( await firstTurnEntered rootProfile.name = 'mutated-root' backend.bridgeUrl = 'http://127.0.0.1:1' - backend.model = 'mutated-model' profileSecurity.allowedMcpHosts.splice(0, 1, 'mutated.test') perWorker.maxIterations = 0 perWorker.maxTokens = 1 @@ -937,7 +1009,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( expect(callbackCalls).toEqual(['original-authorizer']) expect(requests).toHaveLength(1) expect(requests[0]).toMatchObject({ - model: 'codex/safe-model', + model: 'codex/openai/safe-model', agent_profile: { name: 'policy-worker', mcp: { allowed: { url: 'https://allowed.test/mcp' } }, @@ -946,39 +1018,8 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( expect(requests[0]?.agent_profile.mcp?.['agent-runtime-coordination']).toBeUndefined() }) - it('refuses a backend overlay before it can occupy the coordination alias', () => { - expect(() => - supervise( - { - name: 'pi-leader', - harness: 'codex', - prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, - }, - 'Choose the next experiment.', - { - backend: { - backend: 'bridge', - bridgeUrl: 'http://127.0.0.1:1', - bridgeBearer: 'unused', - model: 'codex/gpt-5.6', - agentProfile: { - mcp: { - 'agent-runtime-coordination': { - transport: 'http', - url: 'http://169.254.169.254/latest/meta-data', - }, - }, - }, - }, - budget: { maxIterations: 2, maxTokens: 10_000 }, - }, - ), - ).toThrow(/backend agentProfile overlays are not allowed/) - }) - it('refuses a manager with unknown cost under a dollar-capped budget', async () => { - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) respondWithBridgeStream(res, body, unknownCostStream('managed')) }) @@ -990,7 +1031,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, }, 'Choose the next experiment.', { @@ -998,7 +1039,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', }, budget: { maxIterations: 2, maxTokens: 10_000, maxUsd: 1 }, }, @@ -1016,7 +1056,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( }) it('records a manager with unknown token usage as unknown telemetry without ending the run', async () => { - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) respondWithBridgeStream(res, body, unknownTokenStream('managed')) }) @@ -1028,7 +1068,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, }, 'Choose the next experiment.', { @@ -1036,7 +1076,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', }, budget: { maxIterations: 2, maxTokens: 10_000 }, }, @@ -1058,7 +1097,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( const runDir = await mkdtemp(join(tmpdir(), 'manager-partial-stream-')) let requests = 0 try { - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const cancelled = cancelledRunId(req.url) if (cancelled !== undefined) { respondWithTerminalCancellation(res, cancelled) @@ -1072,7 +1111,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( 'x-run-request-digest': TEST_RUN_DIGEST, }) res.write( - `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 13, completion_tokens: 5, cost: 0.02 } })}\n\n`, + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 13, completion_tokens: 5, cost: 0.02, cost_known: true, cost_provenance: 'provider-receipt' } })}\n\n`, ) setTimeout(() => res.socket?.destroy(new Error('socket died before terminal receipt')), 10) }) @@ -1082,14 +1121,13 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, } as const const options = { backend: { backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', } as const, budget: { maxIterations: 2, maxTokens: 100, maxUsd: 1 }, runDir, @@ -1123,7 +1161,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( it("preserves a leaf's unknown dollar cost and refuses it under a dollar cap", async () => { const requests: BridgeRequest[] = [] - server = createServer(async (req, res) => { + server = createBridgeServer(async (req, res) => { const body = await readJson(req) requests.push(body) const profile = body.agent_profile @@ -1134,7 +1172,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'worker', harness: 'codex', prompt: { systemPrompt: 'Return the result.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, }, task: 'Return RESULT=42.', }) @@ -1154,7 +1192,7 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( name: 'pi-leader', harness: 'codex', prompt: { systemPrompt: 'Lead the pursuit.' }, - model: { default: 'gpt-5.6' }, + model: { provider: 'openai', default: 'gpt-5.6' }, }, 'Choose the next experiment.', { @@ -1162,7 +1200,6 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( backend: 'bridge', bridgeUrl: `http://127.0.0.1:${port}`, bridgeBearer: 'test-token', - model: 'codex/gpt-5.6', }, budget: { maxIterations: 4, maxTokens: 10_000, maxUsd: 1 }, }, diff --git a/tests/runtime.test.ts b/tests/runtime.test.ts index 9a54179b..6e60f690 100644 --- a/tests/runtime.test.ts +++ b/tests/runtime.test.ts @@ -8,7 +8,6 @@ import { applyRunRecordDefaults, type ControlEvalResult, createIterableBackend, - createOpenAICompatibleBackend, createRuntimeEventCollector, createRuntimeStreamEventCollector, createSandboxPromptBackend, @@ -546,193 +545,6 @@ describe('runAgentTask', () => { ).toEqual(['flat-wins']) }) - it('parses OpenAI-compatible streamed chat completions', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - fetchImpl: async () => - new Response( - 'data: {"choices":[{"delta":{"content":"hel"}}]}\n\n' + - 'data: {"choices":[{"delta":{"content":"lo"}}]}\n\n' + - 'data: [DONE]\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'chat-task', intent: 'say hello', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hello' }, - }), - ) - - expect( - events - .filter((event) => event.type === 'text_delta') - .map((event) => event.text) - .join(''), - ).toBe('hello') - expect(events.at(-1)).toMatchObject({ type: 'final', status: 'completed', text: 'hello' }) - }) - - it('ignores SSE comment frames from OpenAI-compatible streams', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - fetchImpl: async () => - new Response( - ': connected\n\n' + - ': keepalive\n\n' + - 'data: {"choices":[{"delta":{"content":"clean"}}]}\n\n' + - ': keepalive\n\n' + - 'data: [DONE]\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'chat-comments', intent: 'say clean', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hello' }, - }), - ) - - expect( - events - .filter((event) => event.type === 'text_delta') - .map((event) => event.text) - .join(''), - ).toBe('clean') - expect(events.at(-1)).toMatchObject({ type: 'final', status: 'completed', text: 'clean' }) - }) - - it('maps wire artifact and proposal events from a streamed backend', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - fetchImpl: async () => - new Response( - 'data: {"type":"artifact","artifactId":"art-9","name":"brief.md","mimeType":"text/markdown","content":"# Brief body"}\n\n' + - 'data: {"type":"proposal_created","proposalId":"prop-9","title":"File amendment","status":"pending"}\n\n' + - 'data: [DONE]\n\n', - { status: 200 }, - ), - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'wire-task', intent: 'produce a brief', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'go' }, - }), - ) - const artifact = events.find((event) => event.type === 'artifact') - expect(artifact).toMatchObject({ - type: 'artifact', - artifactId: 'art-9', - name: 'brief.md', - mimeType: 'text/markdown', - content: '# Brief body', - }) - const proposal = events.find((event) => event.type === 'proposal_created') - expect(proposal).toMatchObject({ - type: 'proposal_created', - proposalId: 'prop-9', - title: 'File amendment', - status: 'pending', - }) - }) - - it('retries a thrown fetch error and succeeds on a later attempt', async () => { - let calls = 0 - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - retry: { initialBackoffMs: 1, maxBackoffMs: 2 }, - fetchImpl: async () => { - calls += 1 - // First two attempts throw a network error (the `fetch failed` - // shape); the third returns a real stream. - if (calls < 3) throw new TypeError('fetch failed') - return new Response('data: {"choices":[{"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n', { - status: 200, - }) - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'retry-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(calls).toBe(3) - expect(events.at(-1)).toMatchObject({ type: 'final', status: 'completed', text: 'ok' }) - }) - - it('aborts a hung attempt via the per-attempt timeout, then retries', async () => { - let calls = 0 - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - retry: { initialBackoffMs: 1, maxBackoffMs: 2, requestTimeoutMs: 30 }, - fetchImpl: (_url, init) => { - calls += 1 - // First attempt hangs until its per-attempt signal aborts; the - // second returns immediately. - if (calls === 1) { - return new Promise((_resolve, reject) => { - const signal = (init as RequestInit | undefined)?.signal - signal?.addEventListener('abort', () => reject(signal.reason ?? new Error('aborted'))) - }) - } - return Promise.resolve( - new Response( - 'data: {"choices":[{"delta":{"content":"recovered"}}]}\n\ndata: [DONE]\n\n', - { - status: 200, - }, - ), - ) - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'timeout-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - expect(calls).toBe(2) - expect(events.at(-1)).toMatchObject({ type: 'final', status: 'completed', text: 'recovered' }) - }) - - it('throws BackendTransportError when every attempt throws', async () => { - const backend = createOpenAICompatibleBackend({ - apiKey: 'sk-test', - baseUrl: 'https://router.example/v1', - model: 'model-a', - retry: { maxAttempts: 2, initialBackoffMs: 1, maxBackoffMs: 2 }, - fetchImpl: async () => { - throw new TypeError('fetch failed') - }, - }) - const events = await collect( - runAgentTaskStream({ - task: { id: 'dead-task', intent: 'go', requiredKnowledge: [readyReq] }, - backend, - input: { message: 'hi' }, - }), - ) - const final = events.at(-1) - expect(final).toMatchObject({ type: 'final' }) - expect(final?.type === 'final' && final.status).not.toBe('completed') - }) - it('stops a backend and emits failed final event when streaming throws', async () => { const store = new InMemoryRuntimeSessionStore() const stopped: string[] = [] diff --git a/tests/runtime/bridge-executor.test.ts b/tests/runtime/bridge-executor.test.ts index f050c3cf..5be53d14 100644 --- a/tests/runtime/bridge-executor.test.ts +++ b/tests/runtime/bridge-executor.test.ts @@ -1,5 +1,6 @@ import { PassThrough, type Readable } from 'node:stream' import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' +import { type AgentProfile, canonicalAgentProfileDigest } from '@tangle-network/agent-interface' import type { SandboxEvent } from '@tangle-network/sandbox' import { afterEach, describe, expect, it, vi } from 'vitest' import { createExecutor, type ExecutorConfig, inlineSandboxClient } from '../../src/runtime' @@ -14,12 +15,13 @@ import type { Agent, AgentSpec, Executor, UsageEvent } from '../../src/runtime/s // live-but-slow bridge. Tests drive that transport by setting `bridgeHttpHandler`. let bridgeHttpHandler: ((payload: Record) => Readable) | null = null let lastBridgeUrl: URL | null = null +let activeBridgePayload: Record | null = null vi.mock('node:http', async () => { const actual = await vi.importActual('node:http') return { ...actual, - request: (url: URL, _opts: unknown, cb: (res: Readable) => void) => { + request: (url: URL, opts: unknown, cb: (res: Readable) => void) => { lastBridgeUrl = url let body = '' return { @@ -27,12 +29,32 @@ vi.mock('node:http', async () => { body += chunk }, end: () => { + if ((opts as { method?: string }).method === 'GET' && url.pathname === '/') { + const capability = new PassThrough() as Readable & { + statusCode?: number + headers?: Record + } + capability.statusCode = 200 + capability.headers = { 'content-type': 'application/json' } + capability.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + cb(capability) + return + } const payload = JSON.parse(body || '{}') as Record if (!bridgeHttpHandler) throw new Error('bridgeHttpHandler not set') + activeBridgePayload = payload const res = bridgeHttpHandler(payload) as Readable & { statusCode?: number headers?: Record } + activeBridgePayload = null res.statusCode = res.statusCode ?? 200 if (res.statusCode >= 200 && res.statusCode < 300) { res.headers = { @@ -50,13 +72,42 @@ vi.mock('node:http', async () => { }) function sse(content: string, input: number, output: number): Readable { + if (!activeBridgePayload) throw new Error('sse: no active bridge request') + const payload = activeBridgePayload + const profile = payload.agent_profile as AgentProfile + const model = String(payload.model) + const parts = model.split('/') const stream = new PassThrough() stream.end( [ 'id: 1', `data: ${JSON.stringify({ choices: [{ delta: { content } }], - usage: { prompt_tokens: input, completion_tokens: output }, + usage: { + prompt_tokens: input, + completion_tokens: output, + cost: 0.001, + cost_known: true, + cost_provenance: 'provider-receipt', + }, + })}`, + '', + 'id: 2', + `data: ${JSON.stringify({ + profile_materialization: { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: canonicalAgentProfileDigest(profile), + harness: parts[0], + provider: parts.length >= 3 ? parts[1] : (profile.model?.provider ?? null), + model, + reasoningEffort: { + requested: profile.model?.reasoningEffort ?? null, + applied: profile.model?.reasoningEffort ?? null, + }, + workspacePlanDigest: `sha256:${'b'.repeat(64)}`, + files: [], + unsupported: [], + }, })}`, '', 'data: [DONE]', @@ -66,23 +117,39 @@ function sse(content: string, input: number, output: number): Readable { return stream } +function exactBridgeProfile( + name: string, + model = 'kimi-k2.6', + metadata?: Record, +): AgentProfile { + return { + name, + harness: 'kimi-code', + model: { + provider: 'moonshot', + default: model, + ...(metadata ? { metadata } : {}), + }, + } +} + function bridgeClient(model: string) { + const profile = exactBridgeProfile('bridge-test', model) return inlineSandboxClient( createExecutor({ backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model, }), + { profile }, ) } async function runOnce( client: ReturnType, prompt: string, - backend?: unknown, ): Promise<{ finalText: string; tokensIn: number; tokensOut: number }> { - const box = await client.create(backend ? ({ backend } as never) : undefined) + const box = await client.create() let finalText = '' let tokensIn = 0 let tokensOut = 0 @@ -112,12 +179,12 @@ describe('bridgeExecutor over node:http', () => { seen.push(payload) return sse('done from bridge', 7, 11) } - const out = await runOnce(bridgeClient('kimi-code/kimi-k2.6'), 'compute the return') + const out = await runOnce(bridgeClient('kimi-k2.6'), 'compute the return') expect(out.finalText).toBe('done from bridge') expect(out.tokensIn).toBe(7) expect(out.tokensOut).toBe(11) expect(seen).toHaveLength(1) - expect(seen[0]?.model).toBe('kimi-code/kimi-k2.6') + expect(seen[0]?.model).toBe('kimi-code/moonshot/kimi-k2.6') expect(seen[0]?.stream).toBe(true) const msgs = seen[0]?.messages as Array<{ role: string; content: string }> expect(msgs.some((m) => m.content.includes('compute the return'))).toBe(true) @@ -138,9 +205,15 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'pi/tangle-router/deepseek-v4-flash', timeoutMs: 14_400_000, }), + { + profile: { + name: 'timeout-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }, + }, ) await runOnce(client, 'work until the task is complete') @@ -157,9 +230,15 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'pi/tangle-router/deepseek-v4-flash', timeoutMs, }), + { + profile: { + name: 'timeout-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }, + }, ) await expect(runOnce(client, 'do not dispatch')).rejects.toThrow( @@ -169,38 +248,14 @@ describe('bridgeExecutor over node:http', () => { }, ) - it('a per-create backend override targets the cell model as harness/model', async () => { - const seen: Array> = [] - bridgeHttpHandler = (payload) => { - seen.push(payload) - return sse('ok', 1, 2) - } - const client = bridgeClient('kimi-code/default') - await runOnce(client, 'go', { type: 'opencode', model: { model: 'glm-4.6' } }) - expect(seen[0]?.model).toBe('opencode/glm-4.6') - }) - - it('an already-prefixed override model passes through unchanged', async () => { - const seen: Array> = [] - bridgeHttpHandler = (payload) => { - seen.push(payload) - return sse('ok', 1, 2) - } - await runOnce(bridgeClient('kimi-code/default'), 'go', { - type: 'opencode', - model: { model: 'opencode/glm-4.6' }, - }) - expect(seen[0]?.model).toBe('opencode/glm-4.6') - }) - - it('uses the seam model verbatim when no per-create override is given', async () => { + it('uses the exact profile model verbatim', async () => { const seen: Array> = [] bridgeHttpHandler = (payload) => { seen.push(payload) return sse('ok', 1, 2) } - await runOnce(bridgeClient('kimi-code/kimi-k2.6'), 'go') - expect(seen[0]?.model).toBe('kimi-code/kimi-k2.6') + await runOnce(bridgeClient('kimi-k2.6'), 'go') + expect(seen[0]?.model).toBe('kimi-code/moonshot/kimi-k2.6') }) it('captures model and nested profile policy when createExecutor is called', async () => { @@ -213,19 +268,20 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'safe-model', - agentProfile: { - name: 'policy-overlay', - permissions: { shell: 'deny' }, - }, } - const client = inlineSandboxClient(createExecutor(config)) - config.model = 'mutated-model' - if (config.agentProfile?.permissions) config.agentProfile.permissions.shell = 'allow' + const profile: AgentProfile = { + name: 'policy-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'safe-model' }, + permissions: { shell: 'deny' }, + } + const client = inlineSandboxClient(createExecutor(config), { profile }) + profile.model.default = 'mutated-model' + profile.permissions.shell = 'allow' await runOnce(client, 'go') - expect(seen[0]?.model).toBe('safe-model') + expect(seen[0]?.model).toBe('pi/tangle-router/safe-model') expect(seen[0]?.agent_profile).toMatchObject({ permissions: { shell: 'deny' } }) }) @@ -241,15 +297,15 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'safe-model', - maxTurns: 1, } const factory = createExecutor(config) - config.maxTurns = 3 + const profile = exactBridgeProfile('budget-worker', 'safe-model', { maxTurns: 1 }) const executor = factory( - { profile: { name: 'budget-worker' }, harness: null }, + { profile, harness: null }, { signal: new AbortController().signal, seams: {} }, ) + if (!profile.model?.metadata) throw new Error('test profile metadata missing') + profile.model.metadata.maxTurns = 3 deliver = (message) => executor.deliver?.(message) const run = executor.execute('go', new AbortController().signal) if (!isUsageStream(run)) throw new Error('bridge worker must stream usage') @@ -271,19 +327,15 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'safe-model', }) const workers = ['a', 'b'].map( (name, index) => - make( - { name }, - { - assignmentId: `ordinal:${index}`, - budget: { maxIterations: 1, maxTokens: 100 }, - task: 'go', - label: name, - }, - ) as Agent & { + make(exactBridgeProfile(name, 'safe-model'), { + assignmentId: `ordinal:${index}`, + budget: { maxIterations: 1, maxTokens: 100 }, + task: 'go', + label: name, + }) as Agent & { executorSpec: AgentSpec }, ) @@ -321,7 +373,6 @@ describe('bridgeExecutor over node:http', () => { backend: 'bridge' as const, bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'safe-model', } const context = { assignmentId: 'key:stable-experiment', @@ -332,7 +383,7 @@ describe('bridgeExecutor over node:http', () => { } const workers = [workerFromBackend(backend), workerFromBackend(backend)].map( (make) => - make({ name: 'worker' }, context) as Agent & { + make(exactBridgeProfile('worker', 'safe-model'), context) as Agent & { executorSpec: AgentSpec }, ) @@ -362,7 +413,7 @@ describe('bridgeExecutor over node:http', () => { s.end('boom') return s } - await expect(runOnce(bridgeClient('kimi-code/k2'), 'go')).rejects.toThrow(/bridge 500/) + await expect(runOnce(bridgeClient('k2'), 'go')).rejects.toThrow(/bridge 500/) }) }) @@ -673,34 +724,6 @@ describe('bridgeExecutor live observability', () => { } expect((executor.resultArtifact().out as { content: string }).content).toBe('PLATYPUS') }) - - it('reports what it derived about the caller declaration, before any turn runs', () => { - bridgeHttpHandler = () => streamOf(REAL_TOOL_CALL_FRAMES) - const executor = bridgeExecutor( - { profile: { name: 'derived-worker' }, harness: null } as unknown as AgentSpec, - { - signal: new AbortController().signal, - seams: { - bridge: { - bridgeUrl: 'http://bridge.test', - bridgeBearer: 'secret', - model: 'kimi-code/kimi-k2.6', - agentProfile: { name: 'overlay', permissions: { shell: 'deny' } }, - }, - // A per-create backend override resolves a DIFFERENT wire model than the seam default. - createOptions: { backend: { type: 'opencode', model: { model: 'glm-4.6' } } }, - }, - }, - ) - // Readable BEFORE the first turn and never evicted, so a run that dies on turn 40 can still - // answer "which model did this actually run, and whose profile?" — the questions a failure - // raises and that neither the activity ring nor the settled artifact can answer. - const derived = executor.progress?.()?.derived ?? [] - expect(derived).toEqual([ - "profile: merged the bridge seam's agentProfile over the spawn profile before materialization", - 'model: resolved the bridge wire model to opencode/glm-4.6 (seam default kimi-code/kimi-k2.6)', - ]) - }) }) /** One numbered SSE data frame, exactly as cli-bridge writes it. */ @@ -709,8 +732,47 @@ function frame(id: number, payload: unknown): string { } function streamOf(frames: ReadonlyArray): Readable { + if (!activeBridgePayload) throw new Error('streamOf: no active bridge request') + const payload = activeBridgePayload + const profile = payload.agent_profile as AgentProfile + const model = String(payload.model) + const parts = model.split('/') + let maxId = 0 + const normalized = frames.map((raw) => { + const id = /^id: (\d+)/u.exec(raw)?.[1] + if (id) maxId = Math.max(maxId, Number(id)) + const data = /data: (\{.*\})/u.exec(raw)?.[1] + if (!data) return raw + const parsed = JSON.parse(data) as { usage?: Record } + if (!parsed.usage) return raw + parsed.usage = { + ...parsed.usage, + ...(typeof parsed.usage.cost === 'number' + ? { cost_known: true, cost_provenance: 'provider-receipt' } + : { cost_known: false }), + } + return raw.replace(data, JSON.stringify(parsed)) + }) + const receipt = frame(maxId + 1, { + profile_materialization: { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: canonicalAgentProfileDigest(profile), + harness: parts[0], + provider: parts[1] ?? null, + model, + reasoningEffort: { + requested: profile.model?.reasoningEffort ?? null, + applied: profile.model?.reasoningEffort ?? null, + }, + workspacePlanDigest: `sha256:${'b'.repeat(64)}`, + files: [], + unsupported: [], + }, + }) const stream = new PassThrough() - stream.end(frames.join('')) + stream.end( + normalized.map((raw) => (raw === 'data: [DONE]\n\n' ? `${receipt}${raw}` : raw)).join(''), + ) return stream } @@ -723,14 +785,20 @@ function jsonOf(body: unknown): Readable { function observedBridgeExecutor(): Executor { return bridgeExecutor( - { profile: { name: 'observed-worker' }, harness: null } as unknown as AgentSpec, + { + profile: { + name: 'observed-worker', + harness: 'pi', + model: { provider: 'tangle-router', default: 'gpt-5-mini' }, + }, + harness: null, + } as AgentSpec, { signal: new AbortController().signal, seams: { bridge: { bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'pi/tangle-router/gpt-5-mini', }, }, }, @@ -760,10 +828,7 @@ describe('profile-selected model keeps its provider', () => { // pi fell back to its own default provider and died with "No API key found for opencode" — a // credential error naming a provider nobody chose. Measured live against a real cli-bridge: // `pi/tangle-router/glm-5.2` returns 200, `pi/glm-5.2` does not. - async function wireModelFor( - profile: Record, - seamModel = 'pi/seam-default', - ): Promise { + async function wireModelFor(profile: Record): Promise { const seen: Array> = [] bridgeHttpHandler = (payload) => { seen.push(payload) @@ -773,7 +838,6 @@ describe('profile-selected model keeps its provider', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: seamModel, })({ profile, harness: null } as unknown as AgentSpec, { signal: new AbortController().signal, seams: {}, @@ -796,10 +860,10 @@ describe('profile-selected model keeps its provider', () => { ).toBe('pi/tangle-router/glm-5.2') }) - it('leaves a model with no declared provider to the harness own resolution', async () => { - expect(await wireModelFor({ name: 'w', harness: 'pi', model: { default: 'glm-5.2' } })).toBe( - 'pi/glm-5.2', - ) + it('refuses a profile with no declared provider before dispatch', async () => { + await expect( + wireModelFor({ name: 'w', harness: 'pi', model: { default: 'glm-5.2' } }), + ).rejects.toThrow(/model\.provider must be explicit/) }) it('does not double-qualify a model that already carries its provider', async () => { @@ -812,41 +876,13 @@ describe('profile-selected model keeps its provider', () => { ).toBe('pi/tangle-router/glm-5.2') }) - it('keeps the configured bridge model when Eval delegates model selection to runtime', async () => { - expect( - await wireModelFor({ + it('refuses runtime-selected model markers before dispatch', async () => { + await expect( + wireModelFor({ name: 'w', harness: 'pi', model: { provider: 'tangle-router', default: HARNESS_NATIVE_MODEL }, }), - ).toBe('pi/seam-default') - }) - - it('qualifies a bare configured bridge model with the selected harness', async () => { - expect( - await wireModelFor( - { - name: 'w', - harness: 'pi', - model: { provider: 'tangle-router', default: HARNESS_NATIVE_MODEL }, - }, - 'seam-default', - ), - ).toBe('pi/seam-default') - }) - - it('uses the harness configured model when a real per-create override delegates selection', async () => { - const seen: Array> = [] - bridgeHttpHandler = (payload) => { - seen.push(payload) - return sse('ok', 1, 2) - } - - await runOnce(bridgeClient('seam-default'), 'go', { - type: 'pi', - model: { model: HARNESS_NATIVE_MODEL }, - }) - - expect(seen[0]?.model).toBe('pi') + ).rejects.toThrow(/model\.default is runtime-selected/) }) }) diff --git a/tests/runtime/executor-config-snapshot.test.ts b/tests/runtime/executor-config-snapshot.test.ts index cefd2e4d..43896b5d 100644 --- a/tests/runtime/executor-config-snapshot.test.ts +++ b/tests/runtime/executor-config-snapshot.test.ts @@ -12,9 +12,28 @@ import { import type { AgentSpec, ExecutorContext, Runtime } from '../../src/runtime/supervise/types' import type { ExecCtx, SandboxClient } from '../../src/runtime/types' -const spec: AgentSpec = { profile: { name: 'snapshot-worker' }, harness: null } const context: ExecutorContext = { signal: new AbortController().signal, seams: {} } +function specFor(config: ExecutorConfig): AgentSpec { + const harness = + config.backend === 'bridge' || (config.backend === 'cli-worktree' && config.bridge) + ? 'pi' + : config.backend === 'cli-worktree' + ? 'codex' + : 'cli-base' + return { + profile: { + name: 'snapshot-worker', + harness, + model: { + provider: harness === 'codex' ? 'openai' : 'test-provider', + default: 'test-model', + }, + }, + harness: null, + } +} + describe('createExecutor config intake', () => { it('captures every backend variant while retaining only explicit live ports', () => { const executeToolCall = async () => 'tool result' @@ -48,7 +67,6 @@ describe('createExecutor config intake', () => { backend: 'router', routerBaseUrl: 'http://router.test', routerKey: 'key', - model: 'model', }, runtime: 'router', }, @@ -58,7 +76,6 @@ describe('createExecutor config intake', () => { backend: 'router-tools', routerBaseUrl: 'http://router.test', routerKey: 'key', - model: 'model', tools: [], executeToolCall, onToolStep, @@ -71,7 +88,6 @@ describe('createExecutor config intake', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', }, runtime: 'cli', }, @@ -85,7 +101,6 @@ describe('createExecutor config intake', () => { config: { backend: 'cli-worktree', repoRoot: '/repo', - harness: 'codex', runGit, runCommand, }, @@ -110,7 +125,6 @@ describe('createExecutor config intake', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'pi/tangle-router/gpt-5-mini', }, runtime: 'cli', }, @@ -139,6 +153,7 @@ describe('createExecutor config intake', () => { for (const testCase of cases) { const originalBackend = testCase.config.backend const factory = createExecutor(testCase.config) + const spec = specFor(testCase.config) const mutableConfig = testCase.config as { backend: string } mutableConfig.backend = originalBackend === 'router' ? 'cli' : 'router' @@ -168,29 +183,21 @@ describe('createExecutor config intake', () => { }) current = providerB - expect(factory(spec, context).runtime).toBe('provider-a') + const config: ExecutorConfig = { backend: 'provider', provider: providerA } + expect(factory(specFor(config), context).runtime).toBe('provider-a') }) - it('rejects reusable profile overlays and fixed execution ids', () => { + it('rejects fixed execution ids on reusable backends', () => { const invalid: ExecutorConfig[] = [ { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', - agentProfile: { name: 'late-overlay' }, - }, - { - backend: 'bridge', - bridgeUrl: 'http://bridge.test', - bridgeBearer: 'secret', - model: 'model', sessionId: 'SHARED', }, { backend: 'cli-worktree', repoRoot: '/repo', - harness: 'codex', runId: 'SHARED', }, { @@ -199,7 +206,6 @@ describe('createExecutor config intake', () => { bridge: { bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', sessionId: 'SHARED', }, }, @@ -218,19 +224,17 @@ describe('createExecutor config intake', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', sessionId: 'PINNED-ONCE', }, { backend: 'cli-worktree', repoRoot: '/repo', - harness: 'codex', runId: 'PINNED-ONCE', }, ] for (const config of direct) { - const executor = createExecutor(config)(spec, context) + const executor = createExecutor(config)(specFor(config), context) expect(executor.runtime).toBe('cli') } }) @@ -241,7 +245,6 @@ describe('createExecutor config intake', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', }, 'bridge-test', ) @@ -252,7 +255,6 @@ describe('createExecutor config intake', () => { bridge: { bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', }, }, 'worktree-test', diff --git a/tests/runtime/executor-profile-model.test.ts b/tests/runtime/executor-profile-model.test.ts index bab45c17..eab6bfc5 100644 --- a/tests/runtime/executor-profile-model.test.ts +++ b/tests/runtime/executor-profile-model.test.ts @@ -7,7 +7,8 @@ import { type AgentSpec, createExecutor } from '../../src/runtime' const profile: AgentProfile = { name: 'profile-model-worker', - model: { default: 'profile-selected-model' }, + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'profile-selected-model' }, } const spec: AgentSpec = { profile, harness: null } @@ -32,58 +33,48 @@ async function startRouter(onRequest: (body: Record) => void): return `http://127.0.0.1:${port}` } -describe('router executor model precedence', () => { +async function drainExecution(value: unknown): Promise { + if (value !== null && typeof value === 'object' && Symbol.asyncIterator in value) { + for await (const _event of value as AsyncIterable) { + // drain + } + return + } + await value +} + +describe('router executor exact-profile identity', () => { afterEach(async () => { if (server) await new Promise((resolve) => server?.close(() => resolve())) server = undefined }) - it('uses AgentProfile.model.default instead of the router fallback', async () => { - let request: Record | undefined - const routerBaseUrl = await startRouter((body) => { - request = body - }) - const factory = createExecutor({ - backend: 'router', - routerBaseUrl, - routerKey: 'key', - model: 'backend-fallback-model', - }) - const executor = factory(spec, { - signal: new AbortController().signal, - seams: {}, - }) - - await executor.execute('do the task', new AbortController().signal) - - expect(request?.model).toBe('profile-selected-model') - }) - - it('uses AgentProfile.model.default instead of the router-tools fallback', async () => { - let request: Record | undefined - const routerBaseUrl = await startRouter((body) => { - request = body - }) - const factory = createExecutor({ - backend: 'router-tools', - routerBaseUrl, - routerKey: 'key', - model: 'backend-fallback-model', - tools: [], - executeToolCall: async () => '', - }) - const executor = factory(spec, { - signal: new AbortController().signal, - seams: {}, - }) + it.each(['router', 'router-tools'] as const)( + 'uses only the AgentProfile model on the %s backend', + async (backend) => { + let request: Record | undefined + const routerBaseUrl = await startRouter((body) => { + request = body + }) + const factory = createExecutor({ + backend, + routerBaseUrl, + routerKey: 'key', + ...(backend === 'router-tools' ? { tools: [], executeToolCall: async () => '' } : {}), + }) + const executor = factory(spec, { + signal: new AbortController().signal, + seams: {}, + }) - await executor.execute('do the task', new AbortController().signal) + await drainExecution(executor.execute('do the task', new AbortController().signal)) - expect(request?.model).toBe('profile-selected-model') - }) + expect(request?.model).toBe('profile-selected-model') + }, + ) it.each(['router', 'router-tools'] as const)( - 'uses the %s configured model when Eval delegates model selection', + 'refuses runtime-selected model markers on the %s backend before dispatch', async (backend) => { let request: Record | undefined const routerBaseUrl = await startRouter((body) => { @@ -93,23 +84,23 @@ describe('router executor model precedence', () => { backend, routerBaseUrl, routerKey: 'key', - model: 'backend-fallback-model', ...(backend === 'router-tools' ? { tools: [], executeToolCall: async () => '' } : {}), }) - const executor = factory( - { - profile: { - name: 'runtime-selected-model', - model: { default: HARNESS_NATIVE_MODEL }, - }, - harness: null, - }, - { signal: new AbortController().signal, seams: {} }, - ) - - await executor.execute('do the task', new AbortController().signal) - expect(request?.model).toBe('backend-fallback-model') + expect(() => + factory( + { + profile: { + name: 'runtime-selected-model', + harness: 'cli-base', + model: { provider: 'tangle-router', default: HARNESS_NATIVE_MODEL }, + }, + harness: null, + }, + { signal: new AbortController().signal, seams: {} }, + ), + ).toThrow(/model\.default is runtime-selected/) + expect(request).toBeUndefined() }, ) }) diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index 08c20b76..826adfd7 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -2,7 +2,7 @@ import type { ChildProcess } from 'node:child_process' import { createHash } from 'node:crypto' import { EventEmitter } from 'node:events' import { PassThrough, type Readable } from 'node:stream' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalAgentProfileDigest } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' // The bridge transport (`streamBridgeSession`) POSTs over the `node:http` core client, @@ -16,8 +16,8 @@ vi.mock('node:http', async () => { return { ...actual, request: ( - _url: unknown, - _opts: unknown, + url: URL, + opts: unknown, cb: (res: Readable) => void, ): { write: (b: string) => void; end: () => void; on: () => void; destroy: () => void } => { let body = '' @@ -26,6 +26,24 @@ vi.mock('node:http', async () => { body += chunk }, end: () => { + if ((opts as { method?: string }).method === 'GET' && url.pathname === '/') { + const response = new PassThrough() as Readable & { + statusCode?: number + headers?: Record + } + response.statusCode = 200 + response.headers = { 'content-type': 'application/json' } + response.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + cb(response) + return + } const payload = JSON.parse(body || '{}') as Record if (!bridgeHttpHandler) throw new Error('bridgeHttpHandler not set') const res = bridgeHttpHandler(payload) as Readable & { @@ -103,19 +121,47 @@ const authoredProfile: AgentProfile = { const reproducibleCodexProfile: AgentProfile = { name: 'reproducible-codex', + harness: 'codex', prompt: { systemPrompt: 'You are a careful refactorer.', instructions: ['Never search for a public solution.'], }, - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + model: { provider: 'openai', default: 'gpt-5.4', reasoningEffort: 'xhigh' }, } -function bridgeSseResponse(content: string): Readable { +function bridgeSseResponse(content: string, request: Record): Readable { + const profile = request.agent_profile as AgentProfile + const model = String(request.model) + const parts = model.split('/') const payload = [ 'id: 1', `data: ${JSON.stringify({ choices: [{ delta: { content } }], - usage: { prompt_tokens: 3, completion_tokens: 5, cost: 0.02 }, + usage: { + prompt_tokens: 3, + completion_tokens: 5, + cost: 0.02, + cost_known: true, + cost_provenance: 'provider-receipt', + }, + })}`, + '', + 'id: 2', + `data: ${JSON.stringify({ + profile_materialization: { + schema: 'cli-bridge.profile-materialization.v2', + effectiveProfileDigest: canonicalAgentProfileDigest(profile), + harness: parts[0], + provider: parts[1] ?? null, + model, + reasoningEffort: { + requested: profile.model?.reasoningEffort ?? null, + applied: profile.model?.reasoningEffort ?? null, + }, + workspacePlanDigest: `sha256:${'b'.repeat(64)}`, + files: [], + unsupported: [], + }, })}`, '', 'data: [DONE]', @@ -521,7 +567,6 @@ describe('createWorktreeCliExecutor', () => { const factory = createExecutor({ backend: 'cli-worktree', repoRoot: '/workspace', - harness: 'codex', taskPrompt: 'x', codexReproducible: true, }) @@ -674,7 +719,7 @@ describe('createWorktreeCliExecutor', () => { const checks: Array<{ command: string; cwd: string }> = [] bridgeHttpHandler = (payload) => { requests.push(payload as (typeof requests)[number]) - return bridgeSseResponse('done from bridge') + return bridgeSseResponse('done from bridge', payload) } const factory = createExecutor({ @@ -684,7 +729,6 @@ describe('createWorktreeCliExecutor', () => { bridge: { bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'codex/live', sessionId: 'session-live', }, testCmd: 'pnpm test', @@ -694,7 +738,14 @@ describe('createWorktreeCliExecutor', () => { return { exitCode: 0, output: 'tests passed' } }, }) - const spec: AgentSpec = { profile: authoredProfile, harness: null } + const spec: AgentSpec = { + profile: { + ...authoredProfile, + harness: 'codex', + model: { provider: 'tangle-router', default: 'live' }, + }, + harness: null, + } const exec = factory(spec, { signal: new AbortController().signal, seams: {} }) const authorizedTask = { experimentId: 'bridge-exp-9', From 2765837dd89e9070a33ed49cb3685c135c9e3620 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:08:46 -0600 Subject: [PATCH 02/39] fix(runtime): enforce exact model execution evidence --- CHANGELOG.md | 6 + bench/CHANGELOG.md | 4 + bench/package.json | 2 +- bench/src/benchmarks/appworld.test.mts | 86 + bench/src/benchmarks/appworld.ts | 58 +- bench/src/swe-arena/gepa-seat.mts | 2 +- bench/src/swe-jail.test.ts | 32 + bench/src/swe-jail.ts | 9 + docs/agent-bus-protocol.md | 4 +- docs/api/index.md | 546 ++--- docs/api/intelligence.md | 14 +- docs/api/mcp.md | 40 +- docs/api/primeintellect.md | 19 +- docs/api/primitive-catalog.md | 8 +- docs/api/runtime.md | 1845 ++++++----------- docs/canonical-api.md | 6 +- docs/concepts.md | 27 +- examples/chat-handler/README.md | 12 +- examples/chat-handler/chat-handler.ts | 5 +- examples/coding-benchmark/dispatch.ts | 8 +- examples/runtime-run/README.md | 5 +- examples/runtime-run/runtime-run.ts | 5 +- examples/webcode-matrix/webcode-matrix.ts | 6 +- package.json | 2 +- scripts/check-model-execution-boundary.mjs | 82 +- .../check-model-execution-boundary.test.mjs | 20 + src/conversation/run-conversation.ts | 4 +- src/conversation/run-persona.test.ts | 50 + src/conversation/run-persona.ts | 45 +- src/conversation/types.ts | 6 +- src/improvement/official-optimizers.ts | 2 +- src/mcp/bin-helpers.ts | 19 - src/mcp/detached-turn.ts | 13 +- src/mcp/in-process-executor.ts | 62 +- src/mcp/openai-tools.ts | 15 +- src/runtime/in-process-sandbox-client.test.ts | 9 +- src/runtime/index.ts | 3 +- src/runtime/inline-sandbox-client.ts | 4 +- src/runtime/loop-dispatch.ts | 4 +- src/runtime/profile-chat-client.test.ts | 46 + src/runtime/profile-chat-client.ts | 64 +- src/runtime/run-loop.provenance.test.ts | 16 +- src/runtime/run-loop.test.ts | 33 +- src/runtime/run-loop.ts | 4 + src/runtime/sandbox-events.ts | 25 +- src/runtime/stream-agent-turn.test.ts | 34 +- src/runtime/stream-agent-turn.ts | 6 +- src/runtime/tool-loop.ts | 12 +- .../fixtures/agent-improvement-proposal.json | 10 +- .../agent-profile-improvement-proposal.json | 6 +- src/types.ts | 6 +- tests/kernel/composition.test.ts | 6 +- tests/kernel/loop-dispatch.test.ts | 6 +- tests/kernel/run-loop-harden.test.ts | 10 +- tests/kernel/sandbox-events.test.ts | 19 +- tests/kernel/sandbox-lineage.test.ts | 10 +- tests/kernel/validator-tracing.test.ts | 13 +- tests/mcp/detached-turn.test.ts | 40 +- tests/mcp/in-process-detect.test.ts | 27 +- tests/mcp/in-process-executor.test.ts | 89 +- 60 files changed, 1689 insertions(+), 1882 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5616d99..d1ee9da4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.128.1 + +- Require Agent Eval 0.144.1 and route its official-optimizer callback through Runtime's exact `AgentProfile` execution path. +- Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. +- Keep missing token usage and billed cost unknown, and report reasoning-token usage when the provider supplies it. + ## 0.128.0 ### chat-transport executor: workers on a bare chat-completions transport diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 8595750d..a58e85f7 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.2 + +- Consume Runtime 0.128.1, Eval 0.144.1, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. + ## 0.7.1 - Consume Runtime 0.126.0 with Eval 0.143.0 and Knowledge 7.0.8, so campaign cost remains observed, estimated, or explicitly uncaptured across the complete benchmark dependency tree. diff --git a/bench/package.json b/bench/package.json index 167a1b7f..66fc76db 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.7.1", + "version": "0.7.2", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/src/benchmarks/appworld.test.mts b/bench/src/benchmarks/appworld.test.mts index c5fd93bc..af3fd490 100644 --- a/bench/src/benchmarks/appworld.test.mts +++ b/bench/src/benchmarks/appworld.test.mts @@ -12,6 +12,7 @@ import { appworldReactResultWithUsage, appworldReactUsageEvent, appworldSolutionOutput, + appworldToolLoopClient, createAppWorldAdapter, } from './appworld' @@ -81,3 +82,88 @@ test('successful react episode survives unknown catalog dollars without fabricat }) assert.equal(Object.hasOwn(event?.data ?? {}, 'costUsd'), false) }) + +interface TestToolLoopBox { + streamPrompt( + prompt: string, + opts?: { signal?: AbortSignal }, + ): AsyncGenerator> +} + +async function testToolLoopBox(client: unknown): Promise { + return (client as { create(): Promise }).create() +} + +async function drain(stream: AsyncGenerator>): Promise { + for await (const _event of stream) { + // drain + } +} + +test('react client refuses an already-aborted round before a Python session or model call', async () => { + let sessionCalls = 0 + let modelCalls = 0 + const controller = new AbortController() + controller.abort(new Error('already stopped')) + const client = appworldToolLoopClient({ + model: 'offline-model', + routerBaseUrl: 'https://router.invalid', + routerKey: 'offline', + runWorldSession: async () => { + sessionCalls += 1 + return undefined as T + }, + complete: async () => { + modelCalls += 1 + return {} + }, + }) + const box = await testToolLoopBox(client) + + await assert.rejects( + drain(box.streamPrompt('@appworld-react task-1 test_normal\n', { signal: controller.signal })), + /already stopped/, + ) + assert.equal(sessionCalls, 0) + assert.equal(modelCalls, 0) +}) + +test('react client threads late abort to both the world session and Router call', async () => { + const controller = new AbortController() + let sessionSignal: AbortSignal | undefined + let modelSignal: AbortSignal | undefined + let sessionStopped = false + const client = appworldToolLoopClient({ + model: 'offline-model', + routerBaseUrl: 'https://router.invalid', + routerKey: 'offline', + runWorldSession: async (_taskId, _split, signal, fn): Promise => { + sessionSignal = signal + try { + return await fn(async () => ({ success: true, num_tests: 1, passes: 1 }), 'offline task') + } finally { + sessionStopped = signal.aborted + } + }, + complete: async (_body, request) => { + modelSignal = request?.signal + return new Promise((_resolve, reject) => { + request?.signal?.addEventListener( + 'abort', + () => reject(request.signal?.reason ?? new Error('aborted')), + { once: true }, + ) + }) + }, + }) + const box = await testToolLoopBox(client) + const running = drain( + box.streamPrompt('@appworld-react task-1 test_normal\n', { signal: controller.signal }), + ) + setTimeout(() => controller.abort(new Error('late stop')), 0) + + await assert.rejects(running, /aborted|late stop/i) + assert.equal(sessionSignal, controller.signal) + assert.equal(modelSignal?.aborted, true) + assert.equal(sessionStopped, true) +}) diff --git a/bench/src/benchmarks/appworld.ts b/bench/src/benchmarks/appworld.ts index dfa25bef..f76d5804 100644 --- a/bench/src/benchmarks/appworld.ts +++ b/bench/src/benchmarks/appworld.ts @@ -289,11 +289,17 @@ const EXECUTE_TOOL: ToolSpec = { async function withWorldSession( taskId: string, split: string, + signal: AbortSignal, fn: (call: (cmd: Record) => Promise>, instruction: string) => Promise, ): Promise { + signal.throwIfAborted() const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], { cwd: benchRoot, }) + const stopChild = (): void => { + if (!child.killed) child.kill('SIGTERM') + } + signal.addEventListener('abort', stopChild, { once: true }) const rl = createInterface({ input: child.stdout }) const pending: Array<(line: string) => void> = [] const backlog: string[] = [] @@ -308,6 +314,10 @@ async function withWorldSession( }) const nextLine = (timeoutMs: number): Promise => new Promise((resolve, reject) => { + if (signal.aborted) { + reject(signal.reason) + return + } const fromBacklog = backlog.shift() if (fromBacklog !== undefined) return resolve(fromBacklog) const t = setTimeout( @@ -318,14 +328,29 @@ async function withWorldSession( // cap) — remove it on the resolve path. const onExit = (code: number | null): void => { clearTimeout(t) - reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`)) + signal.removeEventListener('abort', onAbort) + reject( + signal.aborted + ? signal.reason + : new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`), + ) } - pending.push((l) => { + const onAbort = (): void => { clearTimeout(t) child.removeListener('exit', onExit) - resolve(l) - }) + const index = pending.indexOf(onLine) + if (index >= 0) pending.splice(index, 1) + reject(signal.reason) + } + const onLine = (line: string): void => { + clearTimeout(t) + child.removeListener('exit', onExit) + signal.removeEventListener('abort', onAbort) + resolve(line) + } + pending.push(onLine) child.once('exit', onExit) + signal.addEventListener('abort', onAbort, { once: true }) }) try { const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string } @@ -338,26 +363,43 @@ async function withWorldSession( } return await fn(call, ready.instruction ?? '') } finally { + signal.removeEventListener('abort', stopChild) child.stdin.end() - child.kill('SIGTERM') + stopChild() } } +type AppWorldWorldSession = typeof withWorldSession +type AppWorldComplete = ( + body: Record, + request?: { + readonly headers: Readonly> + readonly signal?: AbortSignal + }, +) => Promise + /** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */ export function appworldToolLoopClient(cfg: { model: string routerBaseUrl: string routerKey: string maxTurns?: number + /** Offline-test seam; production always uses the Python AppWorld session above. */ + runWorldSession?: AppWorldWorldSession + /** Offline-test seam; production uses Runtime's Router HTTP transport. */ + complete?: AppWorldComplete }): unknown { const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40) + const runWorldSession = cfg.runWorldSession ?? withWorldSession let seq = 0 return { async create() { const id = `appworld-toolloop-${seq++}` return { id, - async *streamPrompt(prompt: string) { + async *streamPrompt(prompt: string, promptOpts?: { signal?: AbortSignal }) { + const signal = promptOpts?.signal ?? new AbortController().signal + signal.throwIfAborted() const m = prompt.match(REACT_HEADER) if (!m) { throw new Error( @@ -366,7 +408,7 @@ export function appworldToolLoopClient(cfg: { } const [, taskId, split] = m const directive = prompt.replace(REACT_HEADER, '').trim() - const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => { + const out = await runWorldSession(taskId as string, split as string, signal, async (call, instruction) => { const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM const transcriptSteps: Array<{ args: string; result: string }> = [] const profile = { @@ -384,6 +426,7 @@ export function appworldToolLoopClient(cfg: { backend: 'router-tools', routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, + ...(cfg.complete ? { complete: cfg.complete } : {}), tools: [EXECUTE_TOOL], executeToolCall: async (name, args) => { if (name !== 'execute_python') return `error: unknown tool ${name}` @@ -398,6 +441,7 @@ export function appworldToolLoopClient(cfg: { streamAgentTurn( { kind: 'executor', factory, profile }, `Task: ${instruction}`, + { signal }, ), ) if (loop.status !== 'completed') { diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 0c2d6c5d..ed262ff4 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -233,7 +233,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.143.0`, then install ' + + 'install `agent-eval-rpc==0.144.1`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/bench/src/swe-jail.test.ts b/bench/src/swe-jail.test.ts index 1e15e077..b652a7ad 100644 --- a/bench/src/swe-jail.test.ts +++ b/bench/src/swe-jail.test.ts @@ -50,6 +50,7 @@ test('zaiChatRaw preserves the SWE transport shape through Runtime', async () => }, { name: 'swe-jail-test-worker', + harness: 'cli-base', model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, tools: { run: true }, }, @@ -94,3 +95,34 @@ test('zaiChatRaw preserves the SWE transport shape through Runtime', async () => globalThis.fetch = originalFetch } }) + +test('zaiChatRaw refuses a local profile error without entering the retry ladder', async () => { + let fetchCalls = 0 + const originalFetch = globalThis.fetch + globalThis.fetch = async () => { + fetchCalls += 1 + throw new Error('fetch must not run') + } + + try { + await assert.rejects( + zaiChatRaw( + { + base: 'http://router.test/v1', + key: 'secret', + timeoutMs: 1_000, + maxAttempts: 7, + }, + { model: 'deepseek-v4-flash', messages: [{ role: 'user', content: 'inspect' }] }, + { + name: 'incomplete-worker', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }, + ), + /AgentProfile\.harness must be explicit/, + ) + assert.equal(fetchCalls, 0) + } finally { + globalThis.fetch = originalFetch + } +}) diff --git a/bench/src/swe-jail.ts b/bench/src/swe-jail.ts index 2d1da733..8662a7f4 100644 --- a/bench/src/swe-jail.ts +++ b/bench/src/swe-jail.ts @@ -161,6 +161,15 @@ export async function zaiChatRaw( } catch (e) { lastErr = e instanceof Error ? e.message : String(e) const status = Number(/router (\d+)/.exec(lastErr)?.[1]) + const transientStatus = + Number.isFinite(status) && + (status === 408 || status === 409 || status === 425 || status === 429 || status >= 500) + const transientTransport = + ctl.signal.aborted || + /fetch failed|network|socket|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|operation was aborted/i.test( + lastErr, + ) + if (!transientStatus && !transientTransport) throw e delayBase = status === 429 ? 60_000 : 2_000 } finally { clearTimeout(timer) diff --git a/docs/agent-bus-protocol.md b/docs/agent-bus-protocol.md index d3ca777d..bdecb21a 100644 --- a/docs/agent-bus-protocol.md +++ b/docs/agent-bus-protocol.md @@ -60,8 +60,8 @@ A `@tangle-network/agent-gateway`-fronted endpoint: A `@tangle-network/agent-runtime` consumer (driver code): - Passes `propagatedHeaders` + `inboundDepth` + `parentTurnId` to `runConversation` / `runConversationStream` from its inbound request context. -- The runtime automatically emits `buildForwardHeaders(...)` on every participant backend call. -- Backends that issue HTTP (e.g. `createOpenAICompatibleBackend`) merge `context.propagatedHeaders` into their outbound request automatically. +- The runtime derives `buildForwardHeaders(...)` and exposes the result as `AgentBackendContext.propagatedHeaders` on every participant backend call. +- A caller-owned HTTP backend MUST merge `context.propagatedHeaders` into its outbound request; Runtime does not issue provider HTTP on its behalf. ## Reference diff --git a/docs/api/index.md b/docs/api/index.md index 429e0ef5..c5ecd7e4 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1169,64 +1169,6 @@ In-memory `RuntimeSessionStore` for single-process use and tests. ## Interfaces -### BackendRetryPolicy - -Retry policy for transient transport errors (rate limits, upstream -timeouts). Defaults to 5 attempts with exponential backoff starting at -1s, ±25% jitter, capped at 30s. Set `maxAttempts: 1` to disable retries. - -Retried status codes: - - 408 Request Timeout - - 425 Too Early - - 429 Too Many Requests - - 500 / 502 / 503 / 504 — upstream transient failures - -Hard failures (401, 403, 4xx other than the above) propagate immediately. - -#### Properties - -##### maxAttempts? - -> `optional` **maxAttempts?**: `number` - -Total attempts including the first try. Default 5. - -##### initialBackoffMs? - -> `optional` **initialBackoffMs?**: `number` - -Initial backoff in ms before the second attempt. Default 1000. - -##### maxBackoffMs? - -> `optional` **maxBackoffMs?**: `number` - -Hard ceiling on backoff in ms. Default 30000. - -##### jitter? - -> `optional` **jitter?**: `number` - -Jitter fraction in [0, 1]. Default 0.25 (±25%). - -##### retryStatuses? - -> `optional` **retryStatuses?**: readonly `number`[] - -Status codes that trigger a retry. Default: 408, 425, 429, 500, 502, 503, 504. - -##### requestTimeoutMs? - -> `optional` **requestTimeoutMs?**: `number` - -Per-attempt wall-clock deadline in ms. If a single fetch attempt does -not return headers within this window the attempt is aborted and -retried. Default 120000 (2 min). Without this a hung upstream blocks -the attempt indefinitely — observed in production as a 15-minute -`fetch failed` that burned an entire eval persona. Set to 0 to disable. - -*** - ### AgentCandidateCodeSurfaceSource The only accepted path from an agent-eval code candidate to executable bytes. @@ -1377,7 +1319,7 @@ Immutable signed identity stored for one execution attempt. ##### retryPolicy -> `readonly` **retryPolicy**: `"none"` \| `"pre-model-infrastructure-only"` +> `readonly` **retryPolicy**: `"pre-model-infrastructure-only"` \| `"none"` ##### bundleDigest @@ -1927,7 +1869,7 @@ Catalog/snapshot resolution stays separate from credential issuance. ###### reasoningEffort -`"medium"` \| `"none"` \| `"high"` \| `"low"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `undefined` +`"medium"` \| `"high"` \| `"low"` \| `"none"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `undefined` ###### Returns @@ -2233,7 +2175,7 @@ any archive encoding, or no-op when the exact workspace is already present. ###### reasoningEffort -`"medium"` \| `"none"` \| `"high"` \| `"low"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `undefined` +`"medium"` \| `"high"` \| `"low"` \| `"none"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `undefined` ###### Returns @@ -3784,12 +3726,12 @@ The agent under test. Metered; its rendered prompt leads its turns. The simulated user driving the dialogue. -##### backendFor +##### executorFor -> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) +> **executorFor**: (`profile`, `role`) => [`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> -Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). - Applied to the worker and to a `profile`-kind persona. +Resolve transport/executable ports for the exact profile. Runtime still materializes the +profile and owns every model call. Applied to the worker and a profile-driven persona. ###### Parameters @@ -3803,23 +3745,7 @@ Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). ###### Returns -[`AgentExecutionBackend`](#agentexecutionbackend) - -##### systemPromptOf - -> **systemPromptOf**: (`profile`) => `string` - -Render a profile's system prompt — prepended to that profile's messages. - -###### Parameters - -###### profile - -`AgentProfile` - -###### Returns - -`string` +[`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> ##### maxTurns? @@ -3899,11 +3825,11 @@ Worker-only spend (the side under test). #### Properties -##### backendFor +##### executorFor -> **backendFor**: (`profile`, `role`) => [`AgentExecutionBackend`](#agentexecutionbackend) +> **executorFor**: (`profile`, `role`) => [`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> -Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). +Resolve transport/executable ports for each exact profile. ###### Parameters @@ -3917,23 +3843,7 @@ Turn an `AgentProfile` into a runnable backend (router / sandbox / fake). ###### Returns -[`AgentExecutionBackend`](#agentexecutionbackend) - -##### systemPromptOf - -> **systemPromptOf**: (`profile`) => `string` - -Render a profile's system prompt. - -###### Parameters - -###### profile - -`AgentProfile` - -###### Returns - -`string` +[`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> ##### personaOf @@ -4069,9 +3979,9 @@ Who pays for THIS participant's outbound calls? - `(state) => AuthSource` — per-turn / per-condition decision, e.g. base sub-services are agent-owned but premium add-ons forward the user. -The agent's own credentials live on the backend (set at construction -time, e.g. `createOpenAICompatibleBackend({ apiKey })`); this field is -purely about *whether to also forward the user's identity downstream*. +The agent's own credentials live on its caller-owned backend or +profile-bound Runtime executor; this field is purely about *whether to +also forward the user's identity downstream*. *** @@ -4481,7 +4391,7 @@ One-based shot number within this candidate. ##### reasoningEffort -> `readonly` **reasoningEffort**: `"medium"` \| `"none"` \| `"high"` \| `"low"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `null` +> `readonly` **reasoningEffort**: `"medium"` \| `"high"` \| `"low"` \| `"none"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` \| `null` ##### promptSha256 @@ -5967,7 +5877,7 @@ Findings to fall back to when the generation had NO failing cells, so a ###### profile -[`SupervisorProfile`](runtime.md#supervisorprofile) +`AgentProfile` ###### task @@ -6293,7 +6203,7 @@ Findings to fall back to when the generation had NO failing cells, so a ###### profile -[`SupervisorProfile`](runtime.md#supervisorprofile) +`AgentProfile` ###### task @@ -7223,152 +7133,6 @@ Idempotency-Key header (e.g. the runId) — safe retries + upsert. *** -### ResolveAgentBackendOptions - -#### Extends - -- `OpenAICompatPassthrough` - -#### Type Parameters - -##### TInput - -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) - -#### Properties - -##### tools? - -> `optional` **tools?**: readonly [`OpenAIChatTool`](#openaichattool)[] - -OpenAI Chat Completions `tools[]` definitions surfaced to the model on -every request. Omit to send a tool-free request (existing behavior). -The runtime makes no assumption about the dispatcher — calls stream out -as `tool_call` events and the caller is responsible for executing them -and feeding `tool_result` messages back on a follow-up turn. - -###### Inherited from - -`OpenAICompatPassthrough.tools` - -##### toolChoice? - -> `optional` **toolChoice?**: [`OpenAIChatToolChoice`](#openaichattoolchoice) - -OpenAI Chat Completions `tool_choice`. Default `undefined` (request -omits the field; provider falls back to its own default — typically -`'auto'`). - -###### Inherited from - -`OpenAICompatPassthrough.toolChoice` - -##### responseFormat? - -> `optional` **responseFormat?**: [`OpenAIChatResponseFormat`](#openaichatresponseformat) - -OpenAI Chat Completions `response_format`. Omit for provider default text. - -###### Inherited from - -`OpenAICompatPassthrough.responseFormat` - -##### temperature? - -> `optional` **temperature?**: `number` - -OpenAI Chat Completions `temperature`. Omit for provider default. - -###### Inherited from - -`OpenAICompatPassthrough.temperature` - -##### maxTokens? - -> `optional` **maxTokens?**: `number` - -Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. - -###### Inherited from - -`OpenAICompatPassthrough.maxTokens` - -##### fetchImpl? - -> `optional` **fetchImpl?**: (`input`, `init?`) => `Promise`\<`Response`\> - -###### Parameters - -###### input - -`string` \| `URL` \| `Request` - -###### init? - -`RequestInit` - -###### Returns - -`Promise`\<`Response`\> - -###### Inherited from - -`OpenAICompatPassthrough.fetchImpl` - -##### retry? - -> `optional` **retry?**: [`BackendRetryPolicy`](#backendretrypolicy) - -###### Inherited from - -`OpenAICompatPassthrough.retry` - -##### kind - -> **kind**: [`AgentBackendKind`](#agentbackendkind) - -The chat transport to resolve. - -##### apiKey - -> **apiKey**: `string` - -Bearer credential for the OpenAI-compat kinds. Empty string is valid for a -loopback-anonymous cli-bridge; a `router`/`tcloud` route with an empty key -is a caller bug the product surfaces before calling in. - -##### baseUrl - -> **baseUrl**: `string` - -Base URL for the OpenAI-compat kinds. cli-bridge's is its `/v1`. - -##### model - -> **model**: `string` - -Model id sent on every request. cli-bridge rejects a request without it. - -##### label? - -> `optional` **label?**: `string` - -`kind` label stamped on the resolved backend + its traces. Defaults to `kind`. - -##### sandboxBackend? - -> `optional` **sandboxBackend?**: () => [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -`sandbox` kind: the product's own domain backend. Required for that kind — -the substrate owns no product sandbox shape, so a `sandbox` resolution with -no seam is a caller bug, not a silent fallback. - -###### Returns - -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -*** - ### RuntimeHookEvent #### Type Parameters @@ -8807,6 +8571,30 @@ readonly [`Iteration`](runtime.md#iteration-1)\<`Task`, `Output`\>[] Sum of every iteration's `costUsd`. +##### costUsdKnown? + +> `optional` **costUsdKnown?**: `false` + +**`Experimental`** + +False when `costUsd` is only the observed subtotal, not a complete bill. + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +**`Experimental`** + +Sum of separately-labelled local/catalog estimates. + +##### promptCache? + +> `optional` **promptCache?**: `Record`\<`string`, `string` \| `number`\> + +**`Experimental`** + +Aggregated provider-reported prompt-cache fields. + ##### tokenUsage > **tokenUsage**: [`LoopTokenUsage`](runtime.md#looptokenusage) @@ -9584,9 +9372,9 @@ Truncated response body (≤2 KiB). Diagnostic only — never machine-parsed. **`Stable`** OpenAI Chat Completions tool descriptor. The shape mirrors the -`/v1/chat/completions` `tools[]` parameter so callers can pass tool -definitions through `createOpenAICompatibleBackend({ tools })` without any -runtime translation. The router proxies this shape verbatim to Anthropic +`/v1/chat/completions` `tools[]` parameter so caller-owned compatible +transports can pass tool definitions without translation. A router can +proxy this shape to Anthropic (translated server-side), DeepSeek, Groq, OpenAI, and Gemini — every model that the eval surface targets. @@ -11261,14 +11049,6 @@ The single shell-command-in-worktree runner seam (replaces the per-executor copi *** -### AgentBackendKind - -> **AgentBackendKind** = `"router"` \| `"tcloud"` \| `"cli-bridge"` \| `"sandbox"` - -The transport a chat backend runs on. - -*** - ### RuntimeHookPhase > **RuntimeHookPhase** = `"before"` \| `"after"` \| `"error"` \| `"event"` @@ -11728,7 +11508,7 @@ pin `{ type: 'function', function: { name } }`. ### RuntimeStreamEvent -> **RuntimeStreamEvent** = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: [`KnowledgeReadinessDecision`](#knowledgereadinessdecision); `timestamp`: `string`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} \| \{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `timestamp`: `string`; \} \| \{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `timestamp`: `string`; \} \| \{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} \| \{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} \| \{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} +> **RuntimeStreamEvent** = \{ `type`: `"task_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `timestamp`: `string`; \} \| \{ `type`: `"readiness_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `knowledge`: `KnowledgeReadinessReport`; `decision`: [`KnowledgeReadinessDecision`](#knowledgereadinessdecision); `timestamp`: `string`; \} \| \{ `type`: `"questions_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `timestamp`: `string`; \} \| \{ `type`: `"questions_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `questions`: `UserQuestion`[]; `userAnswers`: `Record`\<`string`, `string`\>; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `timestamp`: `string`; \} \| \{ `type`: `"acquisition_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `acquisitionPlans`: `DataAcquisitionPlan`[]; `acquiredEvidenceIds`: `string`[]; `timestamp`: `string`; \} \| \{ `type`: `"session_created"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `timestamp`: `string`; \} \| \{ `type`: `"session_resumed"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `timestamp`: `string`; \} \| \{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp`: `string`; \} \| \{ `type`: `"text_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"reasoning_delta"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `text`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `toolName`: `string`; `toolCallId?`: `string`; `args?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"tool_result"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `toolName`: `string`; `toolCallId?`: `string`; `result?`: `unknown`; `timestamp?`: `string`; \} \| \{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `tokensKnown?`: `false`; `costUsd?`: `number`; `usdKnown?`: `false`; `estimatedCostUsd?`: `number`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"artifact"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `artifactId`: `string`; `name?`: `string`; `mimeType?`: `string`; `uri?`: `string`; `content?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp?`: `string`; \} \| \{ `type`: `"proposal_created"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `proposalId`: `string`; `title`: `string`; `status?`: `"pending"` \| `"approved"` \| `"rejected"`; `content?`: `string`; `timestamp?`: `string`; \} \| \{ `type`: `"backend_error"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `message`: `string`; `recoverable`: `boolean`; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} \| \{ `type`: `"backend_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"task_end"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `timestamp`: `string`; \} \| \{ `type`: `"final"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `status`: [`AgentTaskStatus`](#agenttaskstatus); `reason`: `string`; `text?`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `error?`: [`BackendErrorDetail`](#backenderrordetail); `timestamp`: `string`; \} **`Stable`** @@ -11790,7 +11570,35 @@ pin `{ type: 'function', function: { name } }`. ##### Type Literal -\{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `timestamp`: `string`; \} +\{ `type`: `"backend_start"`; `task`: [`AgentTaskSpec`](#agenttaskspec); `session`: [`RuntimeSession`](#runtimesession); `backend`: `string`; `metadata?`: `Record`\<`string`, `unknown`\>; `timestamp`: `string`; \} + +###### type + +> **type**: `"backend_start"` + +###### task + +> **task**: [`AgentTaskSpec`](#agenttaskspec) + +###### session + +> **session**: [`RuntimeSession`](#runtimesession) + +###### backend + +> **backend**: `string` + +###### metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +Canonical execution identity and materialization evidence for this turn, when Runtime + owns the selected executor. Generic metadata keeps the event vocabulary open while the + values use Runtime's existing identity/materialization receipt shapes. + +###### timestamp + +> **timestamp**: `string` *** @@ -11820,7 +11628,71 @@ pin `{ type: 'function', function: { name } }`. ##### Type Literal -\{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `costUsd?`: `number`; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} +\{ `type`: `"llm_call"`; `task?`: [`AgentTaskSpec`](#agenttaskspec); `session?`: [`RuntimeSession`](#runtimesession); `model`: `string`; `tokensIn?`: `number`; `tokensOut?`: `number`; `tokensKnown?`: `false`; `costUsd?`: `number`; `usdKnown?`: `false`; `estimatedCostUsd?`: `number`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `latencyMs?`: `number`; `finishReason?`: `string`; `timestamp?`: `string`; \} + +###### type + +> **type**: `"llm_call"` + +###### task? + +> `optional` **task?**: [`AgentTaskSpec`](#agenttaskspec) + +###### session? + +> `optional` **session?**: [`RuntimeSession`](#runtimesession) + +###### model + +> **model**: `string` + +###### tokensIn? + +> `optional` **tokensIn?**: `number` + +###### tokensOut? + +> `optional` **tokensOut?**: `number` + +###### tokensKnown? + +> `optional` **tokensKnown?**: `false` + +False when the numeric token subtotal is incomplete or absent. + +###### costUsd? + +> `optional` **costUsd?**: `number` + +###### usdKnown? + +> `optional` **usdKnown?**: `false` + +False when `costUsd` is only an observed floor, estimate, or absent. + +###### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +Separately-labelled local/catalog estimate; never billed spend. + +###### promptCache? + +> `optional` **promptCache?**: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\> + +Provider-reported prompt-cache fields; absent fields remain unknown. + +###### latencyMs? + +> `optional` **latencyMs?**: `number` + +###### finishReason? + +> `optional` **finishReason?**: `string` + +###### timestamp? + +> `optional` **timestamp?**: `string` *** @@ -12268,119 +12140,6 @@ Build an `AgentExecutionBackend` backed by a sandbox/sidecar `streamPrompt` call *** -### createOpenAICompatibleBackend() - -> **createOpenAICompatibleBackend**\<`TInput`\>(`options`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -**`Stable`** - -OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any -`POST /chat/completions` endpoint that speaks OpenAI's SSE protocol — -Tangle Router, OpenAI direct, OpenRouter, Groq, DeepSeek, Together. The -router also fronts Anthropic models in Anthropic-native SSE shape; this -backend handles both. - -### Tool calls - -Pass `tools` (and optionally `toolChoice`) to forward an OpenAI Chat -Completions `tools[]` array on every request. Streamed `tool_call` chunks -are buffered until the model finalizes them (either `finish_reason: -'tool_calls'` for OpenAI shape or a `content_block_stop` for Anthropic -`tool_use` blocks proxied through the router), then emitted as a single -`tool_call` RuntimeStreamEvent with the assembled `args`. - -The backend does NOT execute tools — it surfaces calls for the caller's -own dispatcher (typically the product's MCP / sandbox runtime) to fulfill -and feed back as a subsequent `messages` turn. This keeps the transport -thin and lets the agent host own tool dispatch policy. - -### Fail-loud errors - -Non-success HTTP responses (4xx/5xx) and exhausted retry budgets throw -`BackendTransportError` from inside the `stream()` generator. The runtime -catches the throw, yields a `backend_error` with a typed `error` field -(`kind`, `status`, truncated `body`) and a terminal `final` event with -`status: 'failed'` carrying the same detail. Consumers MUST map -`final.error` onto their `RunRecord.error` — silently treating an empty -`finalText` as "agent produced nothing" hides credit exhaustion, auth -failure, and upstream outages. - -#### Type Parameters - -##### TInput - -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) - -#### Parameters - -##### options - -###### apiKey - -`string` - -###### baseUrl - -`string` - -###### model - -`string` - -###### kind? - -`string` - -###### tools? - -readonly [`OpenAIChatTool`](#openaichattool)[] - -OpenAI Chat Completions `tools[]` definitions surfaced to the model on -every request. Omit to send a tool-free request (existing behavior). -The runtime makes no assumption about the dispatcher — calls stream out -as `tool_call` events and the caller is responsible for executing them -and feeding `tool_result` messages back on a follow-up turn. - -###### toolChoice? - -[`OpenAIChatToolChoice`](#openaichattoolchoice) - -OpenAI Chat Completions `tool_choice`. Default `undefined` (request -omits the field; provider falls back to its own default — typically -`'auto'`). - -###### responseFormat? - -[`OpenAIChatResponseFormat`](#openaichatresponseformat) - -OpenAI Chat Completions `response_format`. Omit for provider default text. - -###### temperature? - -`number` - -OpenAI Chat Completions `temperature`. Omit for provider default. - -###### maxTokens? - -`number` - -Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for provider default. - -###### fetchImpl? - -(`input`, `init?`) => `Promise`\<`Response`\> - -###### retry? - -[`BackendRetryPolicy`](#backendretrypolicy) - -#### Returns - -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -*** - ### buildAgentCandidateBundle() > **buildAgentCandidateBundle**(`input`): `AgentCandidateBundle` @@ -14494,31 +14253,6 @@ Map a `KnowledgeReadinessReport` to a three-state branch (`ready` / `blocked` / *** -### resolveAgentBackend() - -> **resolveAgentBackend**\<`TInput`\>(`opts`): [`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead -of hand-rolling the `createOpenAICompatibleBackend` branch in each product. - -#### Type Parameters - -##### TInput - -`TInput` *extends* [`AgentBackendInput`](#agentbackendinput) = [`AgentBackendInput`](#agentbackendinput) - -#### Parameters - -##### opts - -[`ResolveAgentBackendOptions`](#resolveagentbackendoptions)\<`TInput`\> - -#### Returns - -[`AgentExecutionBackend`](#agentexecutionbackend)\<`TInput`\> - -*** - ### applyRunRecordDefaults() > **applyRunRecordDefaults**(`records`, `scenarioId`, `controlFailureClass`): `RunRecord`[] diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index a354ee62..6340f213 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -4743,7 +4743,7 @@ readonly [`AgentImprovementActivationTargetIdentity`](#agentimprovementactivatio ### isAgentImprovementProfileSurface() -> **isAgentImprovementProfileSurface**(`surface`): surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" +> **isAgentImprovementProfileSurface**(`surface`): surface is "mcp" \| "subagents" \| "hooks" \| "prompt" \| "tools" \| "skills" Return whether a measured surface can be delivered through an agent profile. @@ -4755,13 +4755,13 @@ Return whether a measured surface can be delivered through an agent profile. #### Returns -surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" +surface is "mcp" \| "subagents" \| "hooks" \| "prompt" \| "tools" \| "skills" *** ### isAgentProfileMeasuredSurface() -> **isAgentProfileMeasuredSurface**(`surface`): surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile" +> **isAgentProfileMeasuredSurface**(`surface`): surface is "mcp" \| "subagents" \| "hooks" \| "prompt" \| "tools" \| "skills" \| "agent-profile" Return whether a surface is eligible for shared profile measurement. @@ -4773,7 +4773,7 @@ Return whether a surface is eligible for shared profile measurement. #### Returns -surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile" +surface is "mcp" \| "subagents" \| "hooks" \| "prompt" \| "tools" \| "skills" \| "agent-profile" *** @@ -4794,7 +4794,7 @@ same profile inside a candidate bundle. ##### surface -`"tools"` \| `"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"skills"` +`"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"tools"` \| `"skills"` #### Returns @@ -4816,7 +4816,7 @@ Return the `Sha256Digest` of one profile surface using Runtime's canonical candi ##### surface -`"tools"` \| `"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"skills"` +`"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"tools"` \| `"skills"` #### Returns @@ -4838,7 +4838,7 @@ so exact replacement requires a reset record followed by a set record. ###### surface -`"tools"` \| `"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"skills"` +`"mcp"` \| `"subagents"` \| `"hooks"` \| `"prompt"` \| `"tools"` \| `"skills"` ###### desiredInput diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 0491ef04..9e991ad6 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -2134,7 +2134,7 @@ Which harness handled this delegation. ###### Inherited from -[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-12) +[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-13) ##### sandboxId? @@ -4392,10 +4392,6 @@ Parsed `delegate` tool arguments. > **intent**: `string` -##### model? - -> `optional` **model?**: `string` - ##### runId? > `optional` **runId?**: `string` @@ -4428,12 +4424,20 @@ What killed a delegation, projected for the calling agent: the rejection's name ##### router -> **router**: [`RouterConfig`](runtime.md#routerconfig) +> **router**: [`RouterTransportConfig`](runtime.md#routertransportconfig) **`Experimental`** The supervisor brain's router substrate (REQUIRED — the default supervisor is router-brained). +##### supervisorProfile + +> **supervisorProfile**: `AgentProfile` + +**`Experimental`** + +Exact executable supervisor identity selected by the trusted composition root. + ##### backend > **backend**: [`ExecutorConfig`](runtime.md#executorconfig) @@ -4450,14 +4454,6 @@ WHERE the authored workers run. Required for `supervise()` to spawn anything. The completion oracle the authored workers settle against (settled ⟺ delivered). -##### model? - -> `optional` **model?**: `string` - -**`Experimental`** - -Default supervisor brain model when a call omits `model`. - ##### allowedModels? > `optional` **allowedModels?**: readonly `string`[] @@ -6401,7 +6397,7 @@ Human-readable description of the `delegate` MCP tool, injected into the tool ma **`Experimental`** -JSON Schema for `delegate` tool arguments (`intent` + optional `model` and `runId`). +JSON Schema for `delegate` tool arguments (`intent` + optional trace id). #### Type Declaration @@ -6425,18 +6421,6 @@ JSON Schema for `delegate` tool arguments (`intent` + optional `model` and `runI > `readonly` **description**: `"What you want accomplished, as an outcome. The supervisor authors the worker."` = `'What you want accomplished, as an outcome. The supervisor authors the worker.'` -###### properties.model - -> `readonly` **model**: `object` - -###### properties.model.type - -> `readonly` **type**: `"string"` = `'string'` - -###### properties.model.description - -> `readonly` **description**: `"Optional per-call override for the supervisor brain model."` = `'Optional per-call override for the supervisor brain model.'` - ###### properties.runId > `readonly` **runId**: `object` @@ -7114,7 +7098,7 @@ created, against the same table that emits the argv. ##### reasoningEffort -`"medium"` \| `"none"` \| `"high"` \| `"low"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` +`"medium"` \| `"high"` \| `"low"` \| `"none"` \| `"minimal"` \| `"xhigh"` \| `"ultracode"` #### Returns diff --git a/docs/api/primeintellect.md b/docs/api/primeintellect.md index 4f14b2ec..2754f369 100644 --- a/docs/api/primeintellect.md +++ b/docs/api/primeintellect.md @@ -484,12 +484,6 @@ The answer-free task exposed to the caller's runtime program. ## Type Aliases -### PrimeIntellectBackendOptions - -> **PrimeIntellectBackendOptions** = `Omit`\<`Parameters`\<*typeof* [`createOpenAICompatibleBackend`](index.md#createopenaicompatiblebackend)\>\[`0`\], `"apiKey"` \| `"baseUrl"` \| `"model"`\> - -*** - ### PrimeIntellectImportDefaults > **PrimeIntellectImportDefaults** = [`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) @@ -594,11 +588,12 @@ Read and validate the private process contract installed by the generated Prime *** -### createPrimeIntellectBackend() +### primeIntellectExecutorConfig() -> **createPrimeIntellectBackend**(`context`, `options?`): [`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> +> **primeIntellectExecutorConfig**(`context`): `object` & [`RouterSeam`](runtime.md#routerseam) -Build the existing runtime backend against Prime's intercepted model endpoint. +Resolve Prime's intercepted endpoint as transport-only Runtime executor configuration. +The caller's exact `AgentProfile` remains the sole owner of model and behavior. #### Parameters @@ -606,13 +601,9 @@ Build the existing runtime backend against Prime's intercepted model endpoint. [`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) -##### options? - -[`PrimeIntellectBackendOptions`](#primeintellectbackendoptions) = `{}` - #### Returns -[`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> +`object` & [`RouterSeam`](runtime.md#routerseam) *** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index e680c40c..5012f3c3 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.128.0` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.128.1` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -506,7 +506,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 711 exports. +Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | Symbol | Kind | Summary | |---|---|---| @@ -959,6 +959,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 711 exports. | `SurfaceWorkerConfig` | interface | How a worker runs the surface task (its router substrate + per-attempt bounds). | | `SurfaceWorkerOut` | interface | What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is | | `ToolLoopCompaction` | interface | Self-compaction — bound the loop's OWN context window the way a fresh-respawn (dumb-Ralph) loop | +| `ToolLoopToolCall` | interface | One provider-neutral tool request emitted by a tool-loop model. | | `TrajectoryNode` | interface | One node in the reconstructed trajectory tree — a driver OR a leaf, with its OWN spend and the | | `TrajectoryReport` | interface | The whole reconstructed trajectory — the realized tree + its root-rolled-up total. The | | `TrajectoryReportOptions` | interface | `trajectoryReport(journal, blobs, root, { withOutputs? })` — reconstruct the whole tree with | @@ -985,6 +986,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 711 exports. | `AgentEnvironmentProviderRef` | type | Provider object or registry name accepted by runtime provider adapters. | | `AgentProfileRef` | type | Portable profile reference: inline profile or provider catalog id. | | `AgentTurnBackend` | type | The execution substrate one turn runs on — a closed discriminated union over | +| `AgentTurnInput` | type | One prompt or an exact OpenAI-compatible conversation carried as the turn input. | | `AssertTraceDerivedFindings` | type | The firewall assertion contract, re-stated for the reactive seam (PORT of | | `AuthoredProfile` | type | What the supervisor AUTHORS per sub-task: one complete canonical profile whose name and | | `AuthorizeDownMessage` | type | Product decision over an exact continuation before it is durably recorded or delivered. | @@ -1071,7 +1073,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 711 exports. | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge diff --git a/docs/api/runtime.md b/docs/api/runtime.md index b0b80f29..63394f05 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -788,7 +788,7 @@ One flattened node with the journal tree that owns its records. ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`id`](#id-19) +[`NodeSnapshot`](#nodesnapshot).[`id`](#id-18) ##### parent? @@ -906,7 +906,7 @@ Conserved spend so far for this node. ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`outRef`](#outref-5) +[`NodeSnapshot`](#nodesnapshot).[`outRef`](#outref-6) ##### trace? @@ -1769,19 +1769,17 @@ The loop-level purpose (meta-intent): what the WHOLE run is for — lets the aud #### Properties -##### chat - -> **chat**: `ChatClient` +##### profile -##### model? +> **profile**: `AgentProfile` -> `optional` **model?**: `string` +Exact auditor identity. -##### auditorInstruction? +##### executor -> `optional` **auditorInstruction?**: `string` +> **executor**: [`ExecutorConfig`](#executorconfig) -Override the auditor instruction (optimizable like any analyst prompt). +Execution substrate. All behavior comes from the profile. ##### maxTraceLines? @@ -2929,15 +2927,17 @@ The same domain surface in the structural `BenchmarkAdapter` shape. The completed runs to analyze — map your store's rows to `ObserveInput`. -##### chat +##### profile + +> **profile**: `AgentProfile` -> **chat**: `ChatClient` +Exact analyst identity. -The model-call seam (agent-eval `createChatClient`). +##### executor -##### model? +> **executor**: [`ExecutorConfig`](#executorconfig) -> `optional` **model?**: `string` +Execution substrate. All behavior comes from the profile. ##### corpus @@ -2951,12 +2951,6 @@ The durable corpus the facts accrete into. Tags written onto learned facts (the product/domain key the read side queries by). -##### analystInstruction? - -> `optional` **analystInstruction?**: `string` - -Override the analyst instruction (the GEPA-tunable knob). - ##### concurrency? > `optional` **concurrency?**: `number` @@ -3150,7 +3144,7 @@ Resolved secret env. Reaches only the child process; redacted everywhere else. > **router**: `object` -The worker brain: router chat-completions with tool-calling. All three required. +Router endpoint/auth. The exact per-create profile owns model and loop behavior. ###### baseUrl @@ -3160,22 +3154,6 @@ The worker brain: router chat-completions with tool-calling. All three required. > **key**: `string` -###### model - -> **model**: `string` - -##### maxTurns? - -> `optional` **maxTurns?**: `number` - -Tool-loop turns per prompt. Default 8. - -##### temperature? - -> `optional` **temperature?**: `number` - -Brain sampling temperature. Default: `routerBrain`'s (0.4). - ##### profile? > `optional` **profile?**: `AgentProfile` @@ -3569,15 +3547,17 @@ Provenance back to the run. #### Properties -##### chat +##### profile -> **chat**: `ChatClient` +> **profile**: `AgentProfile` -The model-call seam (agent-eval `createChatClient`: router / cli-bridge / …). +Exact analyst identity. -##### model? +##### executor -> `optional` **model?**: `string` +> **executor**: [`ExecutorConfig`](#executorconfig) + +Execution substrate. All behavior comes from the profile. ##### corpus? @@ -3601,16 +3581,6 @@ Tags written onto learned facts + used by the next run's corpus query. Cap the trace lines fed to the observer (keeps the call cheap). Default 80. -##### analystInstruction? - -> `optional` **analystInstruction?**: `string` - -Override the analyst's system instruction — the prompt that turns a trace into - findings + recommended_actions. The analyst IS the steerer, so this is the knob a - prompt optimizer (GEPA) tunes. Omitted ⇒ the default observer instruction. The - firewall (trace-only, never the verdict) is structural (input has no score), so a - custom instruction cannot break it. - *** ### Observation @@ -3633,6 +3603,24 @@ Facts persisted to the corpus (empty when no corpus was supplied). Operator-facing markdown: what the observer noticed + what to change. +##### usage + +> **usage**: `object` + +Measured model usage for this analysis turn. + +###### input + +> **input**: `number` + +###### output + +> **output**: `number` + +###### known + +> **known**: `boolean` + *** ### CreateScopeAnalystOptions @@ -5713,7 +5701,7 @@ The execution transport for the driven loop. > `optional` **bridge?**: `object` -`bridge` backend: local cli-bridge transport. `bearer` + `model` required. +`bridge` backend: local cli-bridge transport. The per-create profile owns the model. ###### url? @@ -5725,12 +5713,6 @@ cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. > **bearer**: `string` -###### model - -> **model**: `string` - -Bridge model id, doubling as the harness selector (e.g. `claude-code/sonnet`). - ###### timeoutMs? > `optional` **timeoutMs?**: `number` @@ -5741,7 +5723,7 @@ Per-turn deadline (ms). > `optional` **router?**: `object` -`router` backend: router chat-completion transport. All three fields required. +`router` backend: endpoint/auth only; the per-create profile owns behavior. ###### baseUrl @@ -5751,10 +5733,6 @@ Per-turn deadline (ms). > **key**: `string` -###### model - -> **model**: `string` - ##### local? > `optional` **local?**: [`LocalSandboxClientOptions`](#localsandboxclientoptions) @@ -5764,7 +5742,12 @@ Per-turn deadline (ms). *** -### RouterConfig +### RouterTransportConfig + +Connection details for Runtime's Router-backed executors. + +This is deliberately transport-only: model, prompt, tools, generation settings, and retry +policy belong to the exact executable `AgentProfile` consumed by `streamAgentTurn`. #### Properties @@ -5776,19 +5759,11 @@ Per-turn deadline (ms). > **routerKey**: `string` -##### model - -> **model**: `string` - ##### complete? -> `optional` **complete?**: (`body`) => `Promise`\<`unknown`\> +> `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> -Optional completion transport. When set, `routerChatWithUsage` / `routerChatWithTools` call it -with the OpenAI-shape request body and use the parsed `/chat/completions` JSON it returns, -INSTEAD of `fetch(routerBaseUrl + '/chat/completions')`. When absent the fetch path runs -unchanged — the live router stays the default. The injection seam an offline benchmark uses to -drive the worker with no network: a deterministic in-process responder satisfies it, no server. +Injectable OpenAI-compatible transport for offline execution. ###### Parameters @@ -5796,172 +5771,19 @@ drive the worker with no network: a deterministic in-process responder satisfies `Record`\<`string`, `unknown`\> -###### Returns - -`Promise`\<`unknown`\> - -##### maxTokens? - -> `optional` **maxTokens?**: `number` - -Ceiling for one completion, forwarded as `max_tokens`. Defaults to 8192. - -A REASONING model spends this budget on hidden thinking BEFORE it emits a visible token, so -the default can truncate one mid-thought and return no content at all — observed live with a -model that spent 8,188 of the 8,192 on reasoning and answered with nothing. Raise it for a -thinking model; the ceiling belongs to the router and model a caller chose, which is why it -lives here rather than on one call site. - -##### stream? - -> `optional` **stream?**: `boolean` - -Take the tool-calling completion over SSE instead of one buffered POST. Off by default — -`routerChatWithTools` never streams, and every existing caller keeps the buffered transport -byte for byte. - -Why it exists: a buffered POST holds one connection idle for the WHOLE completion, and a -supervisor turn is the longest completion in the system. An intermediary gateway with an -idle-read timeout kills that connection mid-completion (the 524/503 family). A streamed -response puts bytes on the wire from the first generated token on, so the connection is only -idle through prefill. It does NOT shorten prefill, so a gateway whose deadline is -time-to-FIRST-byte is unaffected; only an idle-timeout gateway is. - -Mutually exclusive with `complete`: the injected transport returns one parsed JSON body and has -no stream to read, so setting both throws rather than silently taking the buffered path. - -WHICH PATHS CAN OPT IN. This flag is read in exactly one place (the private `chatWithTools` transport switch), so -every entry point that takes a caller-supplied `RouterConfig` honors it: `routerBrain`, -`routerToolLoop`, and `supervisorAgent` (which spreads `deps.router` into the brain's config — -the supervisor turn this exists for). Two production call sites build a `RouterConfig` literal -from their own options and therefore CANNOT express it today: the bench strategy's -`routerToolLoop` config in `strategy.ts` and the local sandbox client's `routerBrain` config in -`local-sandbox-client.ts`. Neither drives a supervisor-length turn; setting `stream` on a -config handed to either has no path to reach them, and they stay buffered. - -*** - -### RouterChatResult - -#### Properties - -##### content - -> **content**: `string` - -The final answer, with any inline `...` block stripped into `reasoning`. - -##### reasoning? - -> `optional` **reasoning?**: `string` - -Thinking-model reasoning, when the provider surfaced it — either as a separate -`reasoning`/`reasoning_content` message field (OpenRouter style) or inlined into -`content` as a `` block (Groq style). Undefined for non-thinking models. -Downstream parsers that match single-token answers must read `content`, which is -clean either way; before this split, Groq-style inlining made the same model look -broken on one provider and fine on another. - -##### usage? - -> `optional` **usage?**: `object` - -REAL usage, or undefined when the provider reported none. - -###### input - -> **input**: `number` - -###### output - -> **output**: `number` - -##### costUsd? - -> `optional` **costUsd?**: `number` - -Derived from usage via `estimateCost` when the model is priced; else undefined. - -*** - -### RouterToolCall - -A tool-call the model emitted (provider-neutral; mirrors the runtime's ToolCallRequest). - -#### Properties - -##### id - -> **id**: `string` - -##### name - -> **name**: `string` - -##### arguments - -> **arguments**: `string` - -Raw JSON arguments string as emitted by the model. - -*** - -### RouterChatToolsResult - -#### Properties - -##### content - -> **content**: `string` \| `null` - -##### toolCalls - -> **toolCalls**: [`RouterToolCall`](#routertoolcall)[] +###### request? -##### usage? +###### headers -> `optional` **usage?**: `object` +`Readonly`\<`Record`\<`string`, `string`\>\> -###### input - -> **input**: `number` - -###### output - -> **output**: `number` - -##### costUsd? - -> `optional` **costUsd?**: `number` - -##### reasoning? - -> `optional` **reasoning?**: `string` - -Thinking-model reasoning, normalized the way `RouterChatResult.reasoning` is (a separate -`reasoning_content`/`reasoning` field, or an inline `` block split out of `content`). -Populated by the STREAMED path only — `routerChatWithTools` discards reasoning today and its -behavior is preserved unchanged, so a buffered turn still leaves this undefined. - -##### finishReason? - -> `optional` **finishReason?**: `string` - -The provider's `finish_reason` for the turn (`'stop'`, `'tool_calls'`, `'length'`, …). -Populated by the STREAMED path only. `'length'` is the truncation signal the buffered path -cannot surface: it says the turn hit `max_tokens`, not that the model chose to stop. - -##### usageUnknown? +###### signal? -> `optional` **usageUnknown?**: `true` +`AbortSignal` -The turn happened and its token usage is UNKNOWN — not zero, not free. Set by the STREAMED -transport when the stream ran to completion without a single usage-bearing chunk, which means -the `stream_options.include_usage` contract was not honored upstream. +###### Returns -It exists so a bare `usage: undefined` cannot read as a free turn: a metering caller branches -on this marker and records an UNKNOWN turn (see the coordination driver's `meteredBrain`), -rather than skipping the turn and letting a conserved budget pool believe it cost nothing. +`Promise`\<`unknown`\> *** @@ -5991,66 +5813,6 @@ rather than skipping the turn and letting a conserved budget pool believe it cos *** -### RouterToolLoopResult - -#### Properties - -##### final - -> **final**: `string` - -The model's final assistant text (the turn where it stopped calling tools, or the budget turn). - -##### turns - -> **turns**: `number` - -Inference turns spent (≤ maxTurns) — the equal-budget unit vs random@k. - -##### toolCalls - -> **toolCalls**: `number` - -##### toolTrace - -> **toolTrace**: `object`[] - -The behavior trace: each tool call + its result, in order. What a trace-analyst - steerer reads (behavior, never the verdict) to diagnose + redirect the next shot. - -###### name - -> **name**: `string` - -###### args - -> **args**: `string` - -###### result - -> **result**: `string` - -##### usage - -> **usage**: `object` - -###### input - -> **input**: `number` - -###### output - -> **output**: `number` - -##### messages - -> **messages**: `Record`\<`string`, `unknown`\>[] - -The full conversation after the loop (seed + every assistant/tool turn). Lets a caller - CARRY the messages into the next shot (depth continuation) and read the trajectory. - -*** - ### BenchmarkConfig #### Properties @@ -6190,6 +5952,10 @@ The progress curve (refine: score per shot; sample: best-so-far per rollout). > **usd**: `number` +##### usdKnown + +> **usdKnown**: `boolean` + ##### ms > **ms**: `number` @@ -6206,6 +5972,10 @@ The progress curve (refine: score per shot; sample: best-so-far per rollout). > **output**: `number` +##### tokensKnown + +> **tokensKnown**: `boolean` + *** ### BenchmarkTaskRow @@ -6260,6 +6030,12 @@ Fraction of tasks fully resolved. Mean cost vector per task. +##### usdKnownRate + +> **usdKnownRate**: `number` + +Fraction of task cells whose billed-dollar total was complete. + ##### ms > **ms**: `number` @@ -7394,21 +7170,23 @@ Kill every spawned server. Idempotent. #### Properties -##### chat +##### profile -> **chat**: `ChatClient` +> **profile**: `AgentProfile` -The model-call seam (agent-eval `createChatClient`). +Exact author identity. Runtime binds it to every authoring turn. -##### model? +##### executor -> `optional` **model?**: `string` +> **executor**: [`ExecutorConfig`](#executorconfig) -##### fallbackModel? +Execution substrate for the author. Behavioral settings are forbidden here. -> `optional` **fallbackModel?**: `string` +##### fallbackProfile? -A NAMED fallback author tried once when the primary call fails or returns no code +> `optional` **fallbackProfile?**: `AgentProfile` + +An exact fallback author tried once when the primary call fails or returns no code block (thinking models time out at the edge on long authoring prompts, or return empty content without `maxTokens`). Opt-in — absent means the primary's failure propagates. @@ -7445,16 +7223,6 @@ The budget the strategy must respect (shots/width). Where the authored module file is written (created if missing). -##### temperature? - -> `optional` **temperature?**: `number` - -##### maxTokens? - -> `optional` **maxTokens?**: `number` - -Completion cap — required by thinking-model authors that stream reasoning first. - ##### signal? > `optional` **signal?**: `AbortSignal` @@ -7483,27 +7251,23 @@ Completion cap — required by thinking-model authors that stream reasoning firs #### Properties -##### chat - -> **chat**: `ChatClient` - -The model-call seam (agent-eval `createChatClient`). +##### profile -##### model? +> **profile**: `AgentProfile` -> `optional` **model?**: `string` +Exact author identity. -##### fallbackModel? +##### executor -> `optional` **fallbackModel?**: `string` +> **executor**: [`ExecutorConfig`](#executorconfig) -##### temperature? +Execution substrate. All behavior comes from the profile. -> `optional` **temperature?**: `number` +##### fallbackProfile? -##### maxTokens? +> `optional` **fallbackProfile?**: `AgentProfile` -> `optional` **maxTokens?**: `number` +Optional exact fallback identity. *** @@ -8034,10 +7798,6 @@ SEARCH TELEMETRY, not evidence: each entry is that generation's own train-slice > `readonly` **id**: `string` -##### systemPrompt - -> `readonly` **systemPrompt**: `string` - ##### userPrompt > `readonly` **userPrompt**: `string` @@ -8232,9 +7992,11 @@ A stateful, checkable environment an agent operates over with tools. Open behind > **routerKey**: `string` -##### model +##### workerProfile -> **model**: `string` +> **workerProfile**: `AgentProfile` + +Exact worker identity. Model and standing instructions are read only from this profile. ##### complete? @@ -8256,36 +8018,11 @@ Optional completion transport (see `RouterConfig.complete`): when set, BOTH legs `Promise`\<`unknown`\> -##### temperature? - -> `optional` **temperature?**: `number` - -##### maxTokens? - -> `optional` **maxTokens?**: `number` - -Completion cap per worker turn — REQUIRED for thinking models (they burn unbounded - budgets on reasoning and return empty content without it). Omitted ⇒ provider default. - -##### innerTurns? - -> `optional` **innerTurns?**: `number` +##### analystProfile? -Turns the agent may take within ONE shot before the driver intervenes. +> `optional` **analystProfile?**: `AgentProfile` -##### analystInstruction? - -> `optional` **analystInstruction?**: `string` - -The depth STEERER's analyst instruction (observe()'s system prompt). The knob a - prompt optimizer (GEPA) tunes — the analyst IS the steerer. Omitted ⇒ the default. - -##### analystModel? - -> `optional` **analystModel?**: `string` - -The critic's model — lets the analyst be a stronger (or cheaper) model than the - worker. Omitted ⇒ the worker's `model`. +Exact critic identity. Omitted means the exact worker profile also runs the critic. ##### corpus? @@ -8408,8 +8145,11 @@ DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-s > **usd**: `number` -The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real - router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. +Observed billed subtotal. `usdKnown:false` means it is incomplete, never a measured zero. + +##### usdKnown + +> **usdKnown**: `boolean` ##### ms @@ -8427,6 +8167,10 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r > **output**: `number` +##### tokensKnown + +> **tokensKnown**: `boolean` + *** ### Strategy @@ -8473,28 +8217,6 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r *** -### ShotPersona - -A role for one shot — multi-agent loops (researcher + engineer, a panel of k - researchers) give each shot its own system prompt and optionally its own model. - -#### Properties - -##### systemPrompt? - -> `optional` **systemPrompt?**: `string` - -Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is - injected as a hand-off message (the transcript's earlier roles stay intact). - -##### model? - -> `optional` **model?**: `string` - -Per-shot model override (e.g. a stronger model for the engineer shot). - -*** - ### ShotSpec #### Properties @@ -8513,9 +8235,11 @@ present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh on > `optional` **steer?**: `string` -##### persona? +##### profile? + +> `optional` **profile?**: `AgentProfile` -> `optional` **persona?**: [`ShotPersona`](#shotpersona) +Exact profile for this shot. Omitted means `AgenticOptions.workerProfile`. ##### tools? @@ -8735,13 +8459,15 @@ The tools THIS artifact's task actually offers (names + descriptions only — ne [`AgenticOptions`](#agenticoptions).[`routerKey`](#routerkey-1) -##### model +##### workerProfile -> **model**: `string` +> **workerProfile**: `AgentProfile` + +Exact worker identity. Model and standing instructions are read only from this profile. ###### Inherited from -[`AgenticOptions`](#agenticoptions).[`model`](#model-7) +[`AgenticOptions`](#agenticoptions).[`workerProfile`](#workerprofile) ##### complete? @@ -8767,56 +8493,15 @@ Optional completion transport (see `RouterConfig.complete`): when set, BOTH legs [`AgenticOptions`](#agenticoptions).[`complete`](#complete-1) -##### temperature? - -> `optional` **temperature?**: `number` - -###### Inherited from - -[`AgenticOptions`](#agenticoptions).[`temperature`](#temperature-3) - -##### maxTokens? - -> `optional` **maxTokens?**: `number` - -Completion cap per worker turn — REQUIRED for thinking models (they burn unbounded - budgets on reasoning and return empty content without it). Omitted ⇒ provider default. - -###### Inherited from - -[`AgenticOptions`](#agenticoptions).[`maxTokens`](#maxtokens-3) - -##### innerTurns? - -> `optional` **innerTurns?**: `number` - -Turns the agent may take within ONE shot before the driver intervenes. - -###### Inherited from - -[`AgenticOptions`](#agenticoptions).[`innerTurns`](#innerturns) - -##### analystInstruction? - -> `optional` **analystInstruction?**: `string` +##### analystProfile? -The depth STEERER's analyst instruction (observe()'s system prompt). The knob a - prompt optimizer (GEPA) tunes — the analyst IS the steerer. Omitted ⇒ the default. +> `optional` **analystProfile?**: `AgentProfile` -###### Inherited from - -[`AgenticOptions`](#agenticoptions).[`analystInstruction`](#analystinstruction-2) - -##### analystModel? - -> `optional` **analystModel?**: `string` - -The critic's model — lets the analyst be a stronger (or cheaper) model than the - worker. Omitted ⇒ the worker's `model`. +Exact critic identity. Omitted means the exact worker profile also runs the critic. ###### Inherited from -[`AgenticOptions`](#agenticoptions).[`analystModel`](#analystmodel) +[`AgenticOptions`](#agenticoptions).[`analystProfile`](#analystprofile) ##### corpus? @@ -8914,6 +8599,22 @@ Wall-clock deadline for the whole turn in ms. An expired deadline aborts the backend and terminates the stream with `final.status: 'failed'` (a blown deadline is a turn failure, not a caller cancellation). +##### callId? + +> `optional` **callId?**: `string` + +**`Experimental`** + +Stable logical paid-call id, forwarded as the provider idempotency key and retained in evidence. + +##### correlationId? + +> `optional` **correlationId?**: `string` + +**`Experimental`** + +Caller trace tag retained in evidence and forwarded when the transport supports it. + ##### preserveToolParts? > `optional` **preserveToolParts?**: `boolean` @@ -8958,9 +8659,9 @@ has no sandbox events. **`Experimental`** Metered usage of one turn, summed over every cost-bearing event the backend -emitted. `input`/`output` are token counts (0 when the backend reported -none — the honest sum, never a fabricated estimate). `costUsd`/`model` are -present only when the backend actually reported them. +emitted. `input`/`output` are token counts and are accompanied by +`tokensKnown: false` when the backend did not report them. `costUsd`/`model` +are present only when the backend actually reported them. #### Properties @@ -8976,12 +8677,52 @@ present only when the backend actually reported them. **`Experimental`** +##### tokensKnown? + +> `optional` **tokensKnown?**: `false` + +**`Experimental`** + +Present when a real turn ran but the provider did not report token usage. + ##### costUsd? > `optional` **costUsd?**: `number` **`Experimental`** +##### usdKnown? + +> `optional` **usdKnown?**: `false` + +**`Experimental`** + +Present when Runtime could not prove the full dollar amount. + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +**`Experimental`** + +Separately-labelled local/catalog estimate; never billed spend. + +##### promptCache? + +> `optional` **promptCache?**: `Readonly`\<`Record`\<`string`, `string` \| `number`\>\> + +**`Experimental`** + +Provider-reported prompt-cache fields; absent fields remain unknown. + +##### reasoningTokens? + +> `optional` **reasoningTokens?**: `number` + +**`Experimental`** + +Provider-reported reasoning-token subset of output, when available. + ##### model? > `optional` **model?**: `string` @@ -9006,12 +8747,46 @@ turn stays inspectable without re-scanning `events`. **`Experimental`** +##### output? + +> `optional` **output?**: `unknown` + +**`Experimental`** + +Exact terminal artifact output from a Runtime-owned executor. + ##### usage > **usage**: [`AgentTurnUsage`](#agentturnusage) **`Experimental`** +##### transportAttempts? + +> `optional` **transportAttempts?**: `number` + +**`Experimental`** + +Exact underlying transport calls when the Runtime-owned executor reports them. + +##### toolCalls + +> **toolCalls**: `object`[] + +**`Experimental`** + +###### id? + +> `optional` **id?**: `string` + +###### name + +> **name**: `string` + +###### arguments + +> **arguments**: `string` + ##### events > **events**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] @@ -9066,12 +8841,6 @@ Model-authored visible checks requested per task; 0 disables authoring. Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. -##### temperature? - -> `optional` **temperature?**: `number` - -Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. - *** ### VisibleCheck @@ -9453,13 +9222,15 @@ How a worker runs the surface task (its router substrate + per-attempt bounds). > `readonly` **routerKey**: `string` -##### model +##### profile -> `readonly` **model**: `string` +> `readonly` **profile**: `AgentProfile` -##### maxTokens? +Exact worker behavior, tools, and model. -> `readonly` `optional` **maxTokens?**: `number` +##### analystProfile? + +> `readonly` `optional` **analystProfile?**: `AgentProfile` ##### innerTurns? @@ -9498,10 +9269,9 @@ The conserved compute pool for the whole supervised run. Default: sized off the ##### router? -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +> `readonly` `optional` **router?**: [`RouterTransportConfig`](#routertransportconfig) -The driver brain's router substrate (its own inference). Default: the worker's router + model — the - driver and workers share one router unless you separate them (e.g. a stronger driver model). +The driver brain's Router endpoint/auth. Model and behavior remain owned by `profile`. ##### analysts? @@ -9829,17 +9599,13 @@ Fail loud if any reservation is still open — the conserved-pool leak detector. ### ChatSessionStore -Conversation history keyed by the settled worker id — the resume substrate. The kernel owns - identity, ordering, ledger truth, and spend continuity; this store owns only the message - lists a `'resume'` spawn continues (`WorkerSpawnContext.resume.ofWorker` is the load key). - PROCESS-LOCAL by the same boundary the kernel documents for resume itself: a prior process's - workers are not resume targets. +Conversation history keyed by the settled Runtime worker id. #### Methods ##### load() -> **load**(`workerId`): readonly `Record`\<`string`, `unknown`\>[] \| `undefined` +> **load**(`workerId`): readonly `Readonly`\<`Record`\<`string`, `unknown`\>\>[] \| `undefined` ###### Parameters @@ -9849,7 +9615,7 @@ Conversation history keyed by the settled worker id — the resume substrate. Th ###### Returns -readonly `Record`\<`string`, `unknown`\>[] \| `undefined` +readonly `Readonly`\<`Record`\<`string`, `unknown`\>\>[] \| `undefined` ##### save() @@ -9863,7 +9629,7 @@ readonly `Record`\<`string`, `unknown`\>[] \| `undefined` ###### messages -readonly `Record`\<`string`, `unknown`\>[] +readonly `Readonly`\<`Record`\<`string`, `unknown`\>\>[] ###### Returns @@ -9873,8 +9639,7 @@ readonly `Record`\<`string`, `unknown`\>[] ### ChatTransportTool -One entry of the caller-provided tool table: the OpenAI function spec the model sees, and the - host-side implementation run when the model calls it. +One profile-authorized function tool and its host implementation. #### Properties @@ -9886,10 +9651,6 @@ One entry of the caller-provided tool table: the OpenAI function spec the model > `readonly` **execute**: (`args`, `task`) => `Promise`\<`string`\> -Runs ON THIS HOST; the returned string folds back as the `tool` message. A throw is fed - back as an error message for the model to correct — a bad tool call is a real outcome, not - an infra fault. - ###### Parameters ###### args @@ -9908,164 +9669,114 @@ Runs ON THIS HOST; the returned string folds back as the `tool` message. A throw ### ChatTransportExecutorOptions -#### Properties - -##### url - -> **url**: `string` - -OpenAI-compatible base URL (with or without `/v1`); the executor POSTs to - `${url}/chat/completions`. Ignored when `complete` is injected. +Transport and session data for one exact profile-driven conversation. +Behavioral controls belong only in `profile.model.metadata`. -##### bearer? - -> `optional` **bearer?**: `string` - -Bearer token for the default transport. Omit for an unauthenticated endpoint. +#### Properties -##### model +##### profile -> **model**: `string` +> `readonly` **profile**: `AgentProfile` -The wire model id sent on every completion. +##### url? -##### system? +> `readonly` `optional` **url?**: `string` -> `optional` **system?**: `string` +##### bearer? -System prompt seeding a FRESH conversation. A resumed conversation keeps the system message - it was recorded with — a session continues; it is not re-primed. +> `readonly` `optional` **bearer?**: `string` ##### tools? -> `optional` **tools?**: readonly [`ChatTransportTool`](#chattransporttool)[] +> `readonly` `optional` **tools?**: readonly [`ChatTransportTool`](#chattransporttool)[] -Tool table. Omitted = a pure conversation (no `tools` field on the wire). +##### complete? -##### temperature? +> `readonly` `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> -> `optional` **temperature?**: `number` +###### Parameters -##### maxTokens? +###### body -> `optional` **maxTokens?**: `number` +`Record`\<`string`, `unknown`\> -Output-token ceiling for ONE completion, sent as `max_tokens` on every request when set. - Omitted = no field on the wire, so the endpoint's own default governs. A harness pairing - this executor against another sampling path (P1 parity) pins BOTH arms to one value. +###### request? -##### maxTurnsPerShot? +###### headers -> `optional` **maxTurnsPerShot?**: `number` +`Readonly`\<`Record`\<`string`, `string`\>\> -Inference-turn cap for ONE shot (one `execute`). Default 200 — a runaway backstop, not a - workflow limit (mirrors `routerToolsInlineExecutor.maxTurns`). +###### signal? -##### complete? +`AbortSignal` -> `optional` **complete?**: [`ChatCompletionsTransport`](#chatcompletionstransport) +###### Returns -Injected buffered transport — the offline seam (mirrors `RouterConfig.complete`). When set, - `url`/`bearer` are unused and NO network is touched. +`Promise`\<`unknown`\> ##### sessions? -> `optional` **sessions?**: [`ChatSessionStore`](#chatsessionstore) - -Session store backing continuity. Required to record this conversation (with `sessionKey`) - or to continue a prior one (with `resume`). +> `readonly` `optional` **sessions?**: [`ChatSessionStore`](#chatsessionstore) ##### sessionKey? -> `optional` **sessionKey?**: `string` - -The id this worker's conversation is recorded under at settle — the kernel node id when - spawned through a scope, so a later `'resume'` spawn's `resume.ofWorker` finds it. +> `readonly` `optional` **sessionKey?**: `string` ##### resume? -> `optional` **resume?**: [`WorkerResumeContext`](#workerresumecontext) - -The resume lineage from `WorkerSpawnContext.resume`: this shot continues `ofWorker`'s - recorded message list. Requires `sessions` holding that conversation — fails loud before - any spend when it does not. - -##### profile? - -> `optional` **profile?**: `AgentProfile` - -Profile this executor materializes, for the kernel's materialization receipt. Omitted = - the node's receipt reads `executor-did-not-report` (a direct, unsupervised use). - -##### attemptId? - -> `optional` **attemptId?**: `string` - -Kernel-minted attempt id (`ExecutorNodeContext.attemptId`) binding the receipt to this - exact spawn. +> `readonly` `optional` **resume?**: [`WorkerResumeContext`](#workerresumecontext) *** ### ChatWorkerSeamOptions -#### Properties +Transport/session configuration shared by every spawned exact profile. -##### url +#### Properties -> **url**: `string` +##### url? -OpenAI-compatible base URL every spawned worker speaks. Unused when `complete` is set. +> `readonly` `optional` **url?**: `string` ##### bearer? -> `optional` **bearer?**: `string` - -##### model? +> `readonly` `optional` **bearer?**: `string` -> `optional` **model?**: `string` +##### tools? -Fallback wire model when a spawned profile carries none (`profile.model.default` wins). +> `readonly` `optional` **tools?**: readonly [`ChatTransportTool`](#chattransporttool)[] -##### tools? +##### complete? -> `optional` **tools?**: readonly [`ChatTransportTool`](#chattransporttool)[] +> `readonly` `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> -##### temperature? +###### Parameters -> `optional` **temperature?**: `number` +###### body -##### maxTokens? +`Record`\<`string`, `unknown`\> -> `optional` **maxTokens?**: `number` +###### request? -Per-completion `max_tokens` for every spawned worker (see - [ChatTransportExecutorOptions.maxTokens](#maxtokens-6)). +###### headers -##### maxTurnsPerShot? +`Readonly`\<`Record`\<`string`, `string`\>\> -> `optional` **maxTurnsPerShot?**: `number` +###### signal? -##### complete? +`AbortSignal` -> `optional` **complete?**: [`ChatCompletionsTransport`](#chatcompletionstransport) +###### Returns -Injected buffered transport — the offline seam; no network is touched when set. +`Promise`\<`unknown`\> ##### sessions? -> `optional` **sessions?**: [`ChatSessionStore`](#chatsessionstore) - -Session store backing continuity. Default: one fresh in-memory store PER SEAM, matching the - kernel's process-local resume boundary (one seam = one run's sessions). +> `readonly` `optional` **sessions?**: [`ChatSessionStore`](#chatsessionstore) ##### deliverable? -> `optional` **deliverable?**: [`DeliverableSpec`](#deliverablespec)\<`unknown`\> - -The completion oracle: each worker settles `valid` ⟺ this check passes on its final - assistant text (`gateOnDeliverable` — settled ⟺ DELIVERED, exactly how `workerFromBackend` - composes it). Pass the graph's deliverable so a keep-best driver can pick a winner; omitted, - workers settle unverdicted and only a driver `submit_result` can win. +> `readonly` `optional` **deliverable?**: [`DeliverableSpec`](#deliverablespec)\<`unknown`\> *** @@ -10108,6 +9819,30 @@ What the spawn was supposed to produce — surfaced in traces/reports. *** +### ExecutorResultMapping + +#### Type Parameters + +##### Out + +`Out` + +#### Properties + +##### outRef + +> **outRef**: `string` + +##### out + +> **out**: `Out` + +##### verdict? + +> `optional` **verdict?**: `DefaultVerdict` + +*** + ### DriverAgentOptions #### Properties @@ -10582,7 +10317,7 @@ readonly [`SettledWorker`](mcp.md#settledworker)[] ### DelegateOptions Inputs to [delegate](#delegate). The intent is the first positional arg; everything here is optional - with sensible defaults, so the common call is `delegate(intent, { backend, router })`. + with explicit execution identity, so the common call names one exact supervisor profile. #### Type Parameters @@ -10614,41 +10349,17 @@ WHERE the authored workers run — the worker-execution backend (`router-tools` The conserved compute pool for the whole delegation. Defaults to [defaultDelegateBudget](#defaultdelegatebudget). -##### model? - -> `readonly` `optional` **model?**: `string` - -The model the supervisor BRAIN runs on (the router model). The brain must tool-call - (`spawn_agent` / `await_event`), so a delegator model, not a hidden-reasoning model. - -##### router? - -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) - -The supervisor brain's router substrate. REQUIRED for the default router-brained supervisor - (the brain is resolved from this), unless a test injects `brain` directly. `model` overrides - `router.model`. (Design delta vs the bare `supervise()` profile: the brain needs a router.) - -##### brain? - -> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) - -Inject the supervisor brain directly (tests / advanced) instead of resolving it from `router`. - -##### supervisor? +##### supervisorProfile -> `readonly` `optional` **supervisor?**: `object` +> `readonly` **supervisorProfile**: `AgentProfile` -Override the default authoring-supervisor profile (name / extra system-prompt stance). The - default already carries the authoring skill; override only to add a goal or rename. - -###### name? +Exact executable authoring supervisor. Model, prompt, harness, and provider live here. -> `readonly` `optional` **name?**: `string` +##### router -###### systemPrompt? +> `readonly` **router**: [`RouterTransportConfig`](#routertransportconfig) -> `readonly` `optional` **systemPrompt?**: `string` +Router endpoint/auth for a `cli-base` supervisor; contains no behavioral settings. ##### allowedModels? @@ -11220,7 +10931,7 @@ Leaf-execution override (offline tests / advanced). `runGraph` still owns node p ##### router? -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +> `readonly` `optional` **router?**: [`RouterTransportConfig`](#routertransportconfig) The driver brain's router substrate (`profile.harness` omitted or `cli-base`). @@ -12147,9 +11858,7 @@ The worker LABEL the request targets (already resolved by the caller). ### RouterSeam -Router/inline connection seam. A direct OpenAI-compatible Router endpoint — -the cheapest leaf, no box, no tools. `model` overrides the profile's model -hint when present; otherwise the profile's `model.default` is required. +Router/inline transport seam. The profile owns model, prompt, and generation behavior. #### Properties @@ -12161,9 +11870,37 @@ hint when present; otherwise the profile's `model.default` is required. > **routerKey**: `string` -##### model? +##### complete? -> `optional` **model?**: `string` +> `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> + +Injectable transport for offline/local execution; still passes through Runtime metering. + +###### Parameters + +###### body + +`Record`\<`string`, `unknown`\> + +###### request? + +###### headers + +`Readonly`\<`Record`\<`string`, `string`\>\> + +###### signal? + +`AbortSignal` + +###### Returns + +`Promise`\<`unknown`\> + +##### tools? + +> `optional` **tools?**: readonly [`ToolSpec`](#toolspec)[] + +When present, return one turn's requested tool calls without executing them. *** @@ -12263,7 +12000,7 @@ Working directory for the subprocess. cli-worktree seam. A supervisor-authored `AgentProfile` driving a local coding-harness CLI (claude / codex / opencode) on its own git worktree — the leaf `createWorktreeCliExecutor` -named as data. `harness` + `repoRoot` are required; the task comes from `Executor.execute`. +named as data. `repoRoot` is transport data; `AgentProfile.harness` selects the CLI. `taskPrompt` remains an optional direct-call fallback for callers that execute with `undefined`. The authored `profile.prompt.systemPrompt` + `profile.model.default` reach the harness via the §1.5 @@ -12275,12 +12012,6 @@ The authored > **repoRoot**: `string` -##### harness? - -> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) - -Local CLI harness transport. Omit when `bridge` is set. - ##### taskPrompt? > `optional` **taskPrompt?**: `string` @@ -12362,18 +12093,6 @@ Test seam — forwarded to verification checks. > **bridgeBearer**: `string` -##### model? - -> `optional` **model?**: `string` - -Bridge model/harness id. Defaults to the profile's model hint when omitted. - -##### agentProfile? - -> `optional` **agentProfile?**: `AgentProfile` - -Canonical profile overlay merged over the spawned profile. - ##### timeoutMs? > `optional` **timeoutMs?**: `number` @@ -12387,20 +12106,20 @@ Caller-owned deadline for each bridge turn. Runtime enforces it locally and send Stable cli-bridge session id. Defaults to `bridge-worktree-${runId}`. -##### maxTurns? +##### maxReconnects? + +> `optional` **maxReconnects?**: `number` -> `optional` **maxTurns?**: `number` +Transport reconnects allowed after the first POST. Default 3; set 0 to disable. *** ### BridgeSeam cli-bridge seam. A local OpenAI-compatible bridge that fronts harness CLIs -(claude-code / opencode / kimi / pi) behind one HTTP surface; `model` doubles -as the harness selector (e.g. `claude-code/sonnet`, `opencode//`). -`agentProfile` is the bridge-dialect profile (metadata.disallowedTools, mcp) -forwarded verbatim per request — how an arm disables native tools or injects -a provider search MCP. +(claude-code / opencode / kimi / pi) behind one HTTP surface. The spawned +`AgentProfile` is the sole harness/provider/model and behavioral authority and +is forwarded verbatim per request; this seam carries transport data only. The executor opens a resumable cli-bridge session. `sessionId` identifies the harness conversation across turns; each turn also receives its own durable run id. @@ -12414,7 +12133,7 @@ context files, or prompt templates — because ambient state is how a paired exp loses its pairing: an installed extension that persists memory across runs carries arm A's state into arm B, and nothing reports it. -That is what the `AgentProfile` on this seam (and on the spawn spec) is FOR. `agent_profile` +That is what the spawned `AgentProfile` is FOR. `agent_profile` rides every request verbatim, and cli-bridge maps it onto each harness's own native controls: - Materializing any profile at all already starts the harness isolated from ambient @@ -12451,24 +12170,12 @@ new harness capability there. > **bridgeBearer**: `string` -##### model? - -> `optional` **model?**: `string` - -Fallback bridge wire id. A spawned profile may select its own harness and model. - ##### cwd? > `optional` **cwd?**: `string` Optional working directory forwarded to cli-bridge and persisted with the session. -##### agentProfile? - -> `optional` **agentProfile?**: `AgentProfile` - -Canonical profile overlay merged over the spawned profile. - ##### timeoutMs? > `optional` **timeoutMs?**: `number` @@ -12483,12 +12190,11 @@ Caller-owned deadline for each bridge turn. Runtime enforces it locally and send Stable, caller-owned cli-bridge session id for harness-side resume. Defaults to a freshly minted per-spawn id so each worker is its own resumable session. -##### maxTurns? +##### maxReconnects? -> `optional` **maxTurns?**: `number` +> `optional` **maxReconnects?**: `number` -Per-resume-turn inference cap before the worker settles on its last output. - Mirrors `routerToolsInlineExecutor.maxTurns`; default 200 (runaway backstop). +Transport reconnects allowed after the first POST. Default 3; set 0 to disable. ##### activityWindow? @@ -12636,15 +12342,35 @@ surfaces (e.g. a gym keyed by task) can dispatch correctly. > **routerKey**: `string` -##### model? +##### complete? -> `optional` **model?**: `string` +> `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> -##### tools +###### Parameters -> **tools**: readonly [`ToolSpec`](#toolspec)[] +###### body -##### executeToolCall +`Record`\<`string`, `unknown`\> + +###### request? + +###### headers + +`Readonly`\<`Record`\<`string`, `string`\>\> + +###### signal? + +`AbortSignal` + +###### Returns + +`Promise`\<`unknown`\> + +##### tools + +> **tools**: readonly [`ToolSpec`](#toolspec)[] + +##### executeToolCall > **executeToolCall**: (`name`, `args`, `task`) => `Promise`\<`string`\> @@ -12666,6 +12392,28 @@ surfaces (e.g. a gym keyed by task) can dispatch correctly. `Promise`\<`string`\> +##### initialMessages? + +> `optional` **initialMessages?**: readonly `Readonly`\<`Record`\<`string`, `unknown`\>\>[] + +Exact conversation to continue. Runtime validates its system message against the profile. + +##### onMessages? + +> `optional` **onMessages?**: (`messages`) => `void` \| `Promise`\<`void`\> + +Observe the detached final conversation for session persistence. + +###### Parameters + +###### messages + +readonly `Readonly`\<`Record`\<`string`, `unknown`\>\>[] + +###### Returns + +`void` \| `Promise`\<`void`\> + ##### onToolStep? > `optional` **onToolStep?**: (`step`) => `void` @@ -12707,14 +12455,6 @@ Online observer of each tool step — the seam a `DetectorMonitor` taps to watch `void` -##### maxTurns? - -> `optional` **maxTurns?**: `number` - -Max inference turns. Default 200 (runaway backstop — set far above any - legitimate workflow). For tighter per-workflow limits use a cost budget - or wall-clock deadline at the call site. - *** ### SandboxSteeringOptions @@ -13663,7 +13403,7 @@ Decide whether an authorized child becomes another supervisor. By default only ##### router? -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +> `readonly` `optional` **router?**: [`RouterTransportConfig`](#routertransportconfig) The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's model wins. @@ -14040,89 +13780,9 @@ Exact trusted context after a manager-authored spawn has passed product authoriz *** -### SupervisorProfile - -The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. -`harness` is the backend-as-data discriminant; `systemPrompt` is the standing instruction. - -A canonical `AgentProfile` from `@tangle-network/agent-interface` satisfies this interface -structurally: its `model` is a hints OBJECT and its system prompt lives at `prompt.systemPrompt`, -so both spellings are accepted here and reduced by [resolveSupervisorProfile](#resolvesupervisorprofile). Before that, -a canonical profile's model object reached `RouterConfig.model` (a string) as an object and its -`prompt.systemPrompt` was dropped — a request the provider rejects, and a supervisor running the -default strategy while its profile named another. - -WHAT EACH ARM HONORS — the two brains read different amounts of a profile, so state it rather -than let a caller infer that a field took effect: - - - ROUTER arm (`harness` null): only `name`, the resolved model id (`model`, or - `model.default`), and the resolved system prompt (`prompt.systemPrompt`/`systemPrompt` plus - `prompt.instructions` and `resources.instructions`) reach the brain. A full `AgentProfile`'s - `tools`, `mcp`, `permissions`, `resources.skills`/`files`, `hooks`, `modes`, `subagents`, - `model.provider`, `model.small` and `model.reasoningEffort` are NOT honored here: the router - brain is one `ToolLoopChat` over the coordination verbs, and neither of its two tool-calling - transports (`routerChatWithTools` buffered, `streamRouterChatWithTools` when - `RouterConfig.stream` is set) has a parameter for any of them. - - HARNESS arm (`harness` set): the WHOLE profile object is handed to `deps.driveHarness` - untouched, plus the resolved system prompt as a separate argument. Everything the profile - declares is the harness's to materialize; this module changes none of it. - -#### Properties - -##### name? - -> `readonly` `optional` **name?**: `string` - -##### harness? - -> `readonly` `optional` **harness?**: `string` \| `null` - -null/undefined/`cli-base` → router brain (in-process tool-loop); a coding-CLI harness → an - external harness brain. - -##### model? - -> `readonly` `optional` **model?**: `string` \| `AgentProfileModelHints` - -The router model when the brain is router-driven: a model id, or a canonical profile's model - hints whose `default` IS the id. Absent (including a hints object with no `default`) → the - deps router config's model applies. Other hints (`small`, `provider`, `reasoningEffort`) are - harness-arm material only. - -##### prompt? - -> `readonly` `optional` **prompt?**: `AgentProfilePrompt` - -Canonical `AgentProfile` prompt shaping. `prompt.systemPrompt` and the top-level `systemPrompt` - are the same standing instruction in two spellings; disagreeing values are a fault, not a pick. - `prompt.instructions` lines are appended to the resolved prompt, one per line. - -##### resources? - -> `readonly` `optional` **resources?**: `AgentProfileResources` - -Canonical `AgentProfile` resources. Only `instructions` shapes the brain here (appended to the - resolved system prompt); every other resource is the harness's to materialize. - -##### systemPrompt? - -> `readonly` `optional` **systemPrompt?**: `string` - -The standing instructions ("you delegate, you do not solve"). - -*** - ### ResolvedSupervisorProfile -A `SupervisorProfile` reduced to the scalars the two brain arms consume. `modelId`/`systemPrompt` - stay `undefined` when the profile named none — the caller's fallback (`deps.router.model`, - the built-in default supervisor prompt) then applies, and this type cannot hide which happened. - - There is deliberately no `reasoningEffort` here: the router brain runs on `chatWithTools` (the - buffered/streamed switch in the router client), and neither transport has a `reasoning_effort` - parameter — only the chat-only `routerChatWithUsage` does — so a field carrying it would be a - public promise nothing keeps. `model.reasoningEffort` still reaches the harness arm inside the - profile. +The exact profile fields consumed by supervisor materialization. #### Properties @@ -14134,9 +13794,9 @@ A `SupervisorProfile` reduced to the scalars the two brain arms consume. `modelI > `readonly` **harness**: `string` \| `null` -##### modelId? +##### modelId -> `readonly` `optional` **modelId?**: `string` +> `readonly` **modelId**: `string` ##### systemPrompt? @@ -14217,7 +13877,7 @@ Assignment identity within the parent manager; absent only for the root. ##### profile -> `readonly` **profile**: [`SupervisorProfile`](#supervisorprofile) +> `readonly` **profile**: `AgentProfile` ##### task @@ -14304,11 +13964,11 @@ Assignment identity within the parent manager; absent only for the root. ##### profile -> `readonly` **profile**: [`SupervisorProfile`](#supervisorprofile) +> `readonly` **profile**: `AgentProfile` ###### Inherited from -[`SupervisorNodeContext`](#supervisornodecontext).[`profile`](#profile-7) +[`SupervisorNodeContext`](#supervisornodecontext).[`profile`](#profile-15) ##### task @@ -14400,7 +14060,7 @@ How to run an external harness as the DRIVER, with the coordination verbs mounte ###### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` The caller's profile, EXACTLY as passed to `supervisorAgent` — never rewritten. A canonical `AgentProfile` stays schema-valid here (the canonical schema rejects unknown top-level keys, @@ -14501,7 +14161,7 @@ Hard cap on simultaneously-LIVE workers across both arms — `spawn_agent` fails ##### router? -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +> `readonly` `optional` **router?**: [`RouterTransportConfig`](#routertransportconfig) Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The profile's model wins. @@ -16689,6 +16349,28 @@ returning an incomplete reproducibility receipt. *** +### ToolLoopToolCall + +One provider-neutral tool request emitted by a tool-loop model. + +#### Properties + +##### id + +> **id**: `string` + +##### name + +> **name**: `string` + +##### arguments + +> **arguments**: `string` + +Raw JSON arguments emitted by the model. + +*** + ### ToolLoopCompaction Self-compaction — bound the loop's OWN context window the way a fresh-respawn (dumb-Ralph) loop @@ -17021,6 +16703,12 @@ campaign dispatch settles real usage instead of appearing as a stub. > **output**: `number` +##### tokensKnown? + +> `optional` **tokensKnown?**: `false` + +False when the subtotal is incomplete. + *** ### MountManifestEntry @@ -17241,6 +16929,30 @@ Raw sandbox event stream collected for this iteration. **`Experimental`** +##### costUsdKnown? + +> `optional` **costUsdKnown?**: `false` + +**`Experimental`** + +False when `costUsd` is only the observed subtotal, not a complete bill. + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +**`Experimental`** + +Local/catalog estimates remain separate from billed spend. + +##### promptCache? + +> `optional` **promptCache?**: `Record`\<`string`, `string` \| `number`\> + +**`Experimental`** + +Provider-reported prompt-cache fields; absent fields remain unknown. + ##### tokenUsage > **tokenUsage**: [`LoopTokenUsage`](#looptokenusage) @@ -17794,6 +17506,18 @@ Iteration this one was planned from; `undefined` ⇒ root. **`Experimental`** +##### costUsdKnown? + +> `optional` **costUsdKnown?**: `false` + +**`Experimental`** + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +**`Experimental`** + ##### durationMs > **durationMs**: `number` @@ -17874,6 +17598,18 @@ Truncated string preview of the parsed output — for a viewer's drawer. **`Experimental`** +##### costUsdKnown? + +> `optional` **costUsdKnown?**: `false` + +**`Experimental`** + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + +**`Experimental`** + ##### durationMs > **durationMs**: `number` @@ -19058,65 +18794,32 @@ One provider-neutral conversation record carried between strategy shots. ### AgentTurnBackend -> **AgentTurnBackend** = \{ `kind`: `"box"`; `box`: `SandboxInstance`; `options?`: `Omit`\<`PromptOptions`, `"signal"`\>; `agentRunName?`: `string`; \} \| \{ `kind`: `"executor"`; `factory`: [`ExecutorFactory`](#executorfactory)\<`unknown`\>; `agentRunName?`: `string`; \} \| \{ `kind`: `"chat"`; `backend`: [`AgentExecutionBackend`](index.md#agentexecutionbackend); \} +> **AgentTurnBackend** = `object` **`Experimental`** The execution substrate one turn runs on — a closed discriminated union over the three stream surfaces the runtime already owns. -#### Union Members - -##### Type Literal - -\{ `kind`: `"box"`; `box`: `SandboxInstance`; `options?`: `Omit`\<`PromptOptions`, `"signal"`\>; `agentRunName?`: `string`; \} - -###### kind - -> **kind**: `"box"` - -A live sandbox box: the turn is one `box.streamPrompt(prompt)` call. - -###### box - -> **box**: `SandboxInstance` - -###### options? - -> `optional` **options?**: `Omit`\<`PromptOptions`, `"signal"`\> - -Per-turn `PromptOptions` forwarded verbatim to `streamPrompt` -(`sessionId`, `turnId`, `model`, `backend` profile, `timeoutMs`, …). -The turn's derived abort signal (caller `signal` + `timeoutMs` -deadline) is always installed as `signal` — pass cancellation through -`StreamAgentTurnOptions`, not here. - -###### agentRunName? - -> `optional` **agentRunName?**: `string` - -Model label stamped on cost-only `llm_call` events. Default `'agent'`. +#### Properties -*** +##### kind -##### Type Literal +> **kind**: `"executor"` -\{ `kind`: `"executor"`; `factory`: [`ExecutorFactory`](#executorfactory)\<`unknown`\>; `agentRunName?`: `string`; \} +A Runtime-owned executor factory materialized from this exact canonical profile. -###### kind +##### factory -> **kind**: `"executor"` +> **factory**: [`ExecutorFactory`](#executorfactory)\<`unknown`\> -A one-shot `Executor` (cli-bridge / router / BYO): the factory is -instantiated fresh for the turn via `inlineSandboxClient`, run once on -the prompt, and torn down — the same per-spawn lifecycle the supervise -runtime gives it. +##### profile -###### factory +> **profile**: `AgentProfile` -> **factory**: [`ExecutorFactory`](#executorfactory)\<`unknown`\> +Exact canonical identity materialized by the executor. -###### agentRunName? +##### agentRunName? > `optional` **agentRunName?**: `string` @@ -19124,20 +18827,11 @@ Model label stamped on cost-only `llm_call` events. Default `'agent'`. *** -##### Type Literal - -\{ `kind`: `"chat"`; `backend`: [`AgentExecutionBackend`](index.md#agentexecutionbackend); \} - -###### kind - -> **kind**: `"chat"` +### AgentTurnInput -An in-process `AgentExecutionBackend` (`resolveAgentBackend` output or -any custom backend): the turn is one `backend.stream()` call. +> **AgentTurnInput** = `string` \| \{ `messages`: `ReadonlyArray`\<`Readonly`\<`Record`\<`string`, `unknown`\>\>\>; \} -###### backend - -> **backend**: [`AgentExecutionBackend`](index.md#agentexecutionbackend) +One prompt or an exact OpenAI-compatible conversation carried as the turn input. *** @@ -19207,26 +18901,9 @@ Why a reservation was refused. `budget-exhausted` means the pool ran out of a ch ### ChatCompletionsTransport -> **ChatCompletionsTransport** = (`body`, `signal?`) => `Promise`\<`unknown`\> - -One buffered chat-completions call: the OpenAI-shape request body in, the parsed completion - JSON out. The ONE wire function of this module — the executor's default transport is built - from it, and a harness that must prove two arms share a substrate (P1 parity) drives BOTH - through the same instance. - -#### Parameters - -##### body +> **ChatCompletionsTransport** = `NonNullable`\<[`RouterToolsSeam`](#routertoolsseam)\[`"complete"`\]\> -`Record`\<`string`, `unknown`\> - -##### signal? - -`AbortSignal` - -#### Returns - -`Promise`\<`unknown`\> +Buffered OpenAI-compatible completion port used only for offline execution. *** @@ -19435,6 +19112,14 @@ Exact trusted context for selecting one backend-derived leaf's completion check. *** +### SupervisorProfile + +> **SupervisorProfile** = `AgentProfile` + +A supervisor is an exact canonical AgentProfile; no looser model/prompt shape exists. + +*** + ### SupervisorNodeContextSeed > **SupervisorNodeContextSeed** = `Omit`\<[`SupervisorNodeContext`](#supervisornodecontext), `"nodeId"` \| `"profile"` \| `"task"`\> @@ -19516,27 +19201,39 @@ Resolve an external harness for one exact Runtime-owned manager identity. ### UsageEvent -> **UsageEvent** = \{ `kind`: `"tokens"`; `input`: `number`; `output`: `number`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} \| \{ `kind`: `"iteration"`; \} +> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} \| \{ `kind`: `"iteration"`; \} Normalized usage event — the single channel every executor reports through, so the conserved pool meters all runtimes identically. `tokens` carries `LoopTokenUsage`'s `{ input, output }`; `usd` is a SEPARATE channel (never folded into tokens). -KNOWN LIMITATION (pre-existing): the `cost` variant can say its dollars are a subtotal -(`usdKnown: false`), and the `tokens` variant has NO twin — there is no way to report "this turn -happened and its token count is unknown". `Spend.tokensKnown` exists downstream, but nothing -upstream of `foldStream` (`scope.ts`) can ever set it, so a STREAMING executor whose provider -omitted usage reports the turn as costing zero tokens rather than as unmeasured. Only the -non-streaming path, which returns a whole `Spend`, can carry the marker today. Closing it means -widening this union (a `tokensKnown: false` field on `tokens`, or an `unknown` variant) and -threading it through `foldStream` — a change to the metering contract every executor implements, -which is why it is not folded into a streaming-transport fix. Filed separately. +Either channel can explicitly say its numeric subtotal is incomplete. A missing provider receipt +therefore remains unknown through live metering and terminal reconciliation instead of becoming +a fabricated zero. #### Union Members ##### Type Literal -\{ `kind`: `"tokens"`; `input`: `number`; `output`: `number`; \} +\{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; \} + +###### kind + +> **kind**: `"tokens"` + +###### tokensKnown? + +> `optional` **tokensKnown?**: `false` + +Known token subtotal. When false, these counts are only the observed/estimated floor. + +###### input + +> **input**: `number` + +###### output + +> **output**: `number` *** @@ -19585,7 +19282,7 @@ A named model carried into an execution, or an explicit reason the exact model i ### UnknownMaterializationReason -> **UnknownMaterializationReason** = `"executor-did-not-report"` \| `"invalid-executor-report"` \| `"root-agent-did-not-report"` +> **UnknownMaterializationReason** = `"executor-did-not-report"` \| `"executor-receipt-pending"` \| `"invalid-executor-report"` \| `"root-agent-did-not-report"` Why exact materialization evidence is unavailable for a node. @@ -20327,7 +20024,7 @@ Provider-neutral conversation record accepted by a tool-loop brain. ### ToolLoopChat -> **ToolLoopChat** = (`messages`, `tools`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`RouterToolCall`](#routertoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `usageUnknown?`: `true`; \}\> +> **ToolLoopChat** = (`messages`, `tools`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; \}\> One inference turn over the running conversation + the tool specs → the model's text, any tool calls, and token usage. The seam every brain satisfies. @@ -20344,7 +20041,7 @@ One inference turn over the running conversation + the tool specs → the model' #### Returns -`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`RouterToolCall`](#routertoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `usageUnknown?`: `true`; \}\> +`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; \}\> *** @@ -20482,6 +20179,14 @@ The compressed consumable a skill carries: everything an author needs to emit a *** +### strategyAuthorSystemPrompt + +> `const` **strategyAuthorSystemPrompt**: `string` + +Standing behavior callers put in the strategy-author AgentProfile. + +*** + ### sample > `const` **sample**: [`Strategy`](#strategy-3) @@ -21329,7 +21034,7 @@ this function, so call sites stay cast-free. ### inlineSandboxClient() -> **inlineSandboxClient**(`factory`): [`SandboxClient`](#sandboxclient-5) +> **inlineSandboxClient**(`factory`, `defaults?`): [`SandboxClient`](#sandboxclient-5) Adapt an `ExecutorFactory` into a `SandboxClient` for `runAgentRounds`. The factory is instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle): @@ -21341,6 +21046,12 @@ run once on the prompt, emit the terminal result event, tear down. [`ExecutorFactory`](#executorfactory)\<`unknown`\> +##### defaults? + +###### profile? + +`AgentProfile` + #### Returns [`SandboxClient`](#sandboxclient-5) @@ -22230,292 +21941,105 @@ readonly [`EqualKArm`](#equalkarm)[] *** -### promotionGate() - -> **promotionGate**(`opts`): [`PromotionVerdict`](#promotionverdict) - -Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. - -#### Parameters - -##### opts - -[`PromotionGateOptions`](#promotiongateoptions) - -#### Returns - -[`PromotionVerdict`](#promotionverdict) - -*** - -### resolveSandboxClient() +### profileChatClient() -> **resolveSandboxClient**(`opts`): [`SandboxClient`](#sandboxclient-5) +> **profileChatClient**(`args`): `ChatClient` -Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core -that `resolveBenchClient` builds on — reuse this instead of hand-rolling the -`createExecutor`/`inlineSandboxClient` branch in each product. +Profile-exact adapter for packages that consume agent-eval's ChatClient contract. +Every call still enters Runtime through createExecutor -> streamAgentTurn, and every +behavioral field is checked against the exact AgentProfile before any transport runs. #### Parameters -##### opts - -[`ResolveSandboxClientOptions`](#resolvesandboxclientoptions) - -#### Returns - -[`SandboxClient`](#sandboxclient-5) - -*** - -### routerChatWithUsage() - -> **routerChatWithUsage**(`cfg`, `messages`, `opts?`): `Promise`\<[`RouterChatResult`](#routerchatresult)\> - -One OpenAI-compatible chat completion through the Tangle router, returning text + REAL token usage (`undefined` when the provider omits it — never a fabricated 0). - -#### Parameters - -##### cfg - -[`RouterConfig`](#routerconfig) - -##### messages - -`object`[] - -##### opts? - -###### temperature? - -`number` - -###### signal? - -`AbortSignal` - -###### maxTokens? - -`number` - -###### reasoningEffort? - -`"medium"` \| `"none"` \| `"high"` \| `"low"` - -Reasoning control for thinking models, forwarded as `reasoning_effort`. -'none' is the load-bearing value: binary/single-token decisions (routing, -gating) on a thinking model otherwise burn the whole token budget inside -the think block — on slow backends (CPU-local) that turns into a client -timeout, not just waste. Providers that ignore the field are handled by -the reasoning/content split in `parseChatResult`. - -#### Returns - -`Promise`\<[`RouterChatResult`](#routerchatresult)\> - -*** - -### routerChatWithTools() - -> **routerChatWithTools**(`cfg`, `messages`, `tools`, `opts?`): `Promise`\<[`RouterChatToolsResult`](#routerchattoolsresult)\> - -A router completion WITH tool-calling — the operator driver's LLM seam. Passes OpenAI-shape -`messages` (system/user/assistant-with-tool_calls/tool roles) + function `tools`, and returns the -assistant text plus the tool calls the model wants run. Same fail-loud + real-usage discipline as -`routerChatWithUsage`. `tool_choice: 'auto'` lets the model decide; the driver loops on the result. - -#### Parameters - -##### cfg - -[`RouterConfig`](#routerconfig) - -##### messages - -readonly `Record`\<`string`, `unknown`\>[] - -##### tools - -readonly `object`[] - -##### opts? - -###### temperature? - -`number` +##### args -###### signal? +###### profile -`AbortSignal` +`AgentProfile` -###### toolChoice? +###### executor -`"auto"` \| `"none"` \| `"required"` +[`ExecutorConfig`](#executorconfig) -###### maxTokens? +###### context -`number` +`string` #### Returns -`Promise`\<[`RouterChatToolsResult`](#routerchattoolsresult)\> +`ChatClient` *** -### streamRouterChatWithTools() - -> **streamRouterChatWithTools**(`cfg`, `messages`, `tools`, `opts?`): `Promise`\<[`RouterChatToolsResult`](#routerchattoolsresult)\> - -The SAME completion as `routerChatWithTools`, taken over SSE (`stream: true`) and reassembled -into the identical `RouterChatToolsResult`. Opt in with `RouterConfig.stream` — the buffered -function is untouched and stays the default for every existing caller. +### profileOptimizerModelCall() -What it buys: a buffered POST holds one connection idle for the whole completion, and that idle -window is what an intermediary gateway kills (524/503). Streaming puts bytes on the wire from the -first generated token, so the connection is only idle through prefill. +> **profileOptimizerModelCall**(`args`): `ExternalOptimizerModelCall` -Usage accounting is preserved exactly: `stream_options.include_usage` asks the provider for a -terminal usage chunk, and those tokens run through the same `meterTurn` the buffered path uses. - -When NO chunk reported usage, `usage`/`costUsd` stay undefined (never a fabricated 0) AND -`usageUnknown: true` is set. A stream that finishes with no usage chunk means the -`include_usage` request was not honored upstream, and returning a quiet `undefined` for it is -indistinguishable from a free turn — the marker is what lets a metering caller record an UNKNOWN -turn instead. Streaming raises the odds of this (one dropped terminal frame is enough), which is -why the streamed transport says so explicitly and the buffered one has no equivalent claim to make. +Profile-exact adapter for agent-eval's external optimizer callback. +Eval validates and freezes the provider-neutral request; Runtime owns the exact +AgentProfile, execution route, retries, usage, and finite execution evidence. #### Parameters -##### cfg - -[`RouterConfig`](#routerconfig) - -##### messages - -readonly `Record`\<`string`, `unknown`\>[] - -##### tools - -readonly [`ToolSpec`](#toolspec)[] - -##### opts? +##### args -###### temperature? +###### profile -`number` +`AgentProfile` -###### signal? +###### executor -`AbortSignal` +[`ExecutorConfig`](#executorconfig) -###### toolChoice? +###### context -`"auto"` \| `"none"` \| `"required"` +`string` -###### maxTokens? +###### pricing? -`number` +`CustomTokenPricing` #### Returns -`Promise`\<[`RouterChatToolsResult`](#routerchattoolsresult)\> +`ExternalOptimizerModelCall` *** -### routerToolLoop() - -> **routerToolLoop**(`cfg`, `system`, `user`, `tools`, `execute`, `opts?`): `Promise`\<[`RouterToolLoopResult`](#routertoolloopresult)\> +### promotionGate() -The tool-using router backend: a real agentic loop OVER the Tangle router (which -supports tool-calling), off-box — no sandbox. Each turn is one router completion -with `tools`; if the model emits tool_calls, `execute` runs them on the host and -their results are folded back as `tool` messages; the loop repeats until the -model answers without a tool call or the turn budget is hit. One turn = one -inference call, so `maxTurns` is the equal-compute unit against random@k. +> **promotionGate**(`opts`): [`PromotionVerdict`](#promotionverdict) -This is the depth substrate for agentic gates (the worker ACTS, observes the real -result, and continues) that the chat-only `routerChatWithUsage` cannot express. +Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. #### Parameters -##### cfg - -[`RouterConfig`](#routerconfig) - -##### system - -`string` - -##### user - -`string` - -##### tools - -readonly [`ToolSpec`](#toolspec)[] - -##### execute - -(`name`, `args`) => `Promise`\<`string`\> - -##### opts? - -###### maxTurns? - -`number` - -###### temperature? - -`number` - -###### signal? - -`AbortSignal` - -###### maxTokens? - -`number` - -###### initialMessages? - -readonly `Record`\<`string`, `unknown`\>[] +##### opts -Seed the loop with an existing conversation (depth continuation) instead of - `[system, user]`. When set, `system`/`user` are ignored. The array is copied. +[`PromotionGateOptions`](#promotiongateoptions) #### Returns -`Promise`\<[`RouterToolLoopResult`](#routertoolloopresult)\> +[`PromotionVerdict`](#promotionverdict) *** -### routerBrain() - -> **routerBrain**(`cfg`, `opts?`): [`ToolLoopChat`](#toolloopchat) +### resolveSandboxClient() -The router as a supervisor BRAIN: the canonical `ToolLoopChat` seam backed by the router's -tool-calling. The driver's spawn/observe/steer/await/stop turns become real router tool-calls. -The turnkey production brain — tests script a mock `ToolLoopChat`; production passes -`routerBrain(cfg)`. No message translation: the loop already speaks the router's OpenAI shape. +> **resolveSandboxClient**(`opts`): [`SandboxClient`](#sandboxclient-5) -Transport follows `cfg.stream`: buffered by default, SSE when the caller opts in. A supervisor -turn is the longest completion in the system, so it is the call site streaming exists for. +Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core +that `resolveBenchClient` builds on — reuse this instead of hand-rolling the +`createExecutor`/`inlineSandboxClient` branch in each product. #### Parameters -##### cfg - -[`RouterConfig`](#routerconfig) - -##### opts? - -###### temperature? +##### opts -`number` +[`ResolveSandboxClientOptions`](#resolvesandboxclientoptions) #### Returns -[`ToolLoopChat`](#toolloopchat) +[`SandboxClient`](#sandboxclient-5) *** @@ -22726,7 +22250,10 @@ event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`: receipt: (turn) => { const u = sumSandboxUsage(turn.events) return { model, inputTokens: u.input, outputTokens: u.output, - ...(u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}) } + ...(u.tokensKnown ? {} : { usageUnknown: true }), + ...(u.usdKnown && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), + ...(u.usdKnown ? {} : { costUnknown: true }), + ...(u.estimatedCostUsd !== undefined ? { estimatedCostUsd: u.estimatedCostUsd } : {}) } } Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the @@ -22758,6 +22285,18 @@ readonly `SandboxEvent`[] > **costUsd**: `number` +##### tokensKnown + +> **tokensKnown**: `boolean` + +##### usdKnown + +> **usdKnown**: `boolean` + +##### estimatedCostUsd? + +> `optional` **estimatedCostUsd?**: `number` + *** ### createSandboxToolPartState() @@ -23250,7 +22789,7 @@ Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved- ### streamAgentTurn() -> **streamAgentTurn**(`backend`, `prompt`, `opts?`): `AsyncGenerator`\<[`RuntimeStreamEvent`](index.md#runtimestreamevent)\> +> **streamAgentTurn**(`backend`, `input`, `opts?`): `AsyncGenerator`\<[`RuntimeStreamEvent`](index.md#runtimestreamevent)\> **`Experimental`** @@ -23267,9 +22806,9 @@ timeout alike. The generator never throws; failures surface in-band as [`AgentTurnBackend`](#agentturnbackend) -##### prompt +##### input -`string` +[`AgentTurnInput`](#agentturninput) ##### opts? @@ -23606,7 +23145,7 @@ Drive a team of agents (spawned + steered by `profile`) to solve a graded `Agent ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### task @@ -23641,36 +23180,6 @@ Narrow an untyped `spawn_agent` profile argument to an `AuthoredProfile`, or nul *** -### canonicalizeAuthoredProfile() - -> **canonicalizeAuthoredProfile**(`raw`): `AgentProfile` - -Lift a profile the supervisor AUTHORED into the canonical shape every executor reads. - -The skill asks for `systemPrompt` and `model` as flat fields — the vocabulary a model writes -well — while `AgentProfile` carries them as `prompt.systemPrompt` and `model.default`. Nothing -downstream reads the flat form: the router and cli-bridge leaves read `profile.prompt -.systemPrompt`, and the sandbox leaf hands the profile to a strict schema that REJECTS the flat -key outright (`Unrecognized key: "systemPrompt"`), which fails the worker's every round. Lift -both here, once, so what the supervisor writes is what the worker runs. - -Purely additive: a profile already canonical is returned untouched, and a flat field is dropped -only after its canonical slot is filled. Both spellings of the same standing instruction, set to -DIFFERENT text, is a contradiction with no safe reading — it fails loud, matching -`resolveSupervisorProfile`'s rule for the supervisor's own profile. - -#### Parameters - -##### raw - -`unknown` - -#### Returns - -`AgentProfile` - -*** - ### supervisorInstructions() > **supervisorInstructions**(`opts?`): `string` @@ -23697,43 +23206,6 @@ The supervisor SKILL — the how-to the supervisor reads (its system prompt). TH *** -### authoredWorker() - -> **authoredWorker**(`profile`, `opts`): [`Agent`](#agent-2)\<`unknown`, `unknown`\> - -Build a router-only worker from an authored profile. This helper executes the prompt/model axes; - use `workerFromBackend` for full materialization of tools, MCP, resources, hooks, and subagents. - -#### Parameters - -##### profile - -[`AuthoredProfile`](#authoredprofile) - -##### opts - -###### cfg - -[`RouterConfig`](#routerconfig) - -###### taskPrompt - -`string` - -###### deliverable - -[`DeliverableSpec`](#deliverablespec) - -###### temperature? - -`number` - -#### Returns - -[`Agent`](#agent-2)\<`unknown`, `unknown`\> - -*** - ### assessAuthoredProfile() > **assessAuthoredProfile**(`profile`, `opts?`): [`ProfileRichness`](#profilerichness) @@ -23843,37 +23315,11 @@ wall-clock limit. The readout is an absolute instant, not a shrinking remainder. *** -### chatCompletionsTransport() - -> **chatCompletionsTransport**(`opts`): [`ChatCompletionsTransport`](#chatcompletionstransport) - -The default transport: POST `${url}/chat/completions` with an optional bearer. Fail-loud on - any non-2xx — the status and body head become the settle reason. - -#### Parameters - -##### opts - -###### url - -`string` - -###### bearer? - -`string` - -#### Returns - -[`ChatCompletionsTransport`](#chatcompletionstransport) - -*** - ### createChatSessionStore() > **createChatSessionStore**(): [`ChatSessionStore`](#chatsessionstore) -In-memory `ChatSessionStore`. Entries are detached copies — a caller mutating a saved array - cannot corrupt a recorded session. +In-memory, process-local conversation store with detached reads and writes. #### Returns @@ -23885,16 +23331,8 @@ In-memory `ChatSessionStore`. Entries are detached copies — a caller mutating > **chatTransportExecutor**(`opts`): [`Executor`](index.md#executor-2)\<`string`\> -Build the chat-transport `Executor`: one `execute` = one conversation SHOT — seed (fresh system -prompt, or the resumed session's recorded history) + the task as the next user message, then -loop completion → host tool calls → tool messages until the model answers without a tool call -(or the turn cap). Settles with the final assistant text as `out`. - -Fail-loud contract: transport failures (non-2xx, network faults, malformed completions) throw -`ValidationError`, which the scope settles as an INFRA failure (`Settled.down.infra`) — never a -fake success. The accumulated conversation is still recorded before the throw when a store is -configured, because the inference HAPPENED and a resume may continue a failed session (the -kernel deliberately allows resume-after-failure; the seam decides). +Build one exact profile-driven chat executor through `createExecutor`. +Prefer `chatWorkerSeam` for supervised work because it supplies trusted node identity. #### Parameters @@ -23912,12 +23350,7 @@ kernel deliberately allows resume-after-failure; the seam decides). > **chatWorkerSeam**(`opts`): [`MakeWorkerAgent`](#makeworkeragent) -The `makeWorkerAgent` seam over [chatTransportExecutor](#chattransportexecutor) — the continuity consumer -`workerFromBackend` refuses to be. Every spawn becomes one conversation shot: the spawned -profile's system prompt + instructions (which is where a graph's delegates directive lands) -seed a fresh session, and a `'resume'` spawn re-attaches by loading `resume.ofWorker`'s -recorded message list from the seam's session store. Conversations are recorded under the -kernel node id, which is exactly what a later `resume.ofWorker` names. +Session-owning worker factory for graph continuity. #### Parameters @@ -23962,6 +23395,41 @@ executor has produced its output. The inner `score` is preserved; only `valid` i *** +### mapExecutorResult() + +> **mapExecutorResult**\<`In`, `Out`\>(`inner`, `map`): [`Executor`](index.md#executor-2)\<`Out`\> + +Transform a Runtime executor's terminal artifact without losing its private +profile-materialization attestation or altering its measured spend. This is +the composition point for deterministic post-processing and grading; callers +must not rebuild an Executor around a model transport merely to change `out`. + +#### Type Parameters + +##### In + +`In` + +##### Out + +`Out` + +#### Parameters + +##### inner + +[`Executor`](index.md#executor-2)\<`In`\> + +##### map + +(`result`, `task`) => [`ExecutorResultMapping`](#executorresultmapping)\<`Out`\> \| `Promise`\<[`ExecutorResultMapping`](#executorresultmapping)\<`Out`\>\> + +#### Returns + +[`Executor`](index.md#executor-2)\<`Out`\> + +*** + ### driverAgent() > **driverAgent**(`opts`): [`Agent`](#agent-2)\<`unknown`, `unknown`\> @@ -24150,7 +23618,7 @@ Product-selected tools already bound to this exact supervisor node. They share t ### delegate() -> **delegate**\<`Out`\>(`intent`, `opts?`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<`Out`\>\> +> **delegate**\<`Out`\>(`intent`, `opts`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<`Out`\>\> Delegate an INTENT to a default authoring supervisor and return its `SupervisedResult` unchanged. @@ -24170,9 +23638,9 @@ authored worker's delivered output; a `no-winner` result names why (never a fabr `string` -##### opts? +##### opts -[`DelegateOptions`](#delegateoptions)\<`Out`\> = `{}` +[`DelegateOptions`](#delegateoptions)\<`Out`\> #### Returns @@ -25317,7 +24785,7 @@ One-call supervisor: build + run a supervisor from its profile with sensible def ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### task @@ -25337,22 +24805,13 @@ One-call supervisor: build + run a supervisor from its profile with sensible def > **resolveSupervisorProfile**(`profile`): [`ResolvedSupervisorProfile`](#resolvedsupervisorprofile) -Reduce either profile spelling — a hand-written `SupervisorProfile` or a canonical `AgentProfile` -— to the scalars the brain arms consume: - - - `modelId`: a string `model` verbatim, else `model.default`. Absent or unresolvable → the - router config's own model applies unchanged. - - `systemPrompt`: the system prompt plus the `prompt.instructions` and `resources.instructions` - lines, one per line. - -`supervisorAgent` resolves each piece only where it is consumed (the model id on the router arm -only); this whole-profile reduction is the caller-facing view of the same rules. +Reduce one canonical executable profile to the scalars the two brain arms consume. #### Parameters ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` #### Returns @@ -25392,7 +24851,7 @@ Build a supervisor `Agent` from its profile: the brain resolves from `profile.ha ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### deps diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 0650d816..2e6601ba 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.128.0.** +> **Version 0.128.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.143.0 <0.144.0`. +> `agent-eval` must satisfy `>=0.144.1 <0.145.0`. > `sandbox` must satisfy `>=0.17.2 <0.18.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.0 <0.44.0`. > @@ -133,7 +133,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Run + **resume** ONE persistent box across turns | `openSandboxRun(client, opts, deliverable)`: `/kernel` | a per-domain `new Sandbox`+`box.fs.read`+delete copy | | Run **ONE agent turn** on any substrate: box (`streamPrompt`), cli-bridge/router `Executor`, or in-process chat backend: as ONE normalized `RuntimeStreamEvent` stream with a guaranteed terminal result+usage event; opt into in-stream `tool_call`/`tool_result` with `preserveToolParts`, or tap the raw sandbox events with `onRawEvent` | `streamAgentTurn(backend, prompt, { signal, timeoutMs, preserveToolParts?, onRawEvent? })` + `collectAgentTurn(stream)`: `/kernel` | a per-provider stream→event mapper zoo, a hand-faked box around a non-box executor, or raw fetch leaking through the turn abstraction | | Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })`: `/kernel` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | -| Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })`: root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | +| Adapt an exact `AgentProfile` to agent-eval's `ChatClient` without moving credentials or execution policy into Eval | `profileChatClient({ profile, executor, context })`: `/kernel` | a provider fetch configured separately from the profile, or request fields that override the profile's model policy | | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor`: `/kernel` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | | Run a worker as a **conversation on a bare `/v1/chat/completions` endpoint** (no sandbox), with session continuity for `continuity: 'resume'` graphs | `chatTransportExecutor(options)` + `chatWorkerSeam({ url, sessions?, deliverable? })` + `createChatSessionStore()`: `/kernel` | a leaf-seam fake of a chat worker, a multishot transcript loop outside the kernel (no ledger, no conserved pool), or a resume that re-primes a fresh session | | Optimize text or named components with upstream GEPA | `officialGepa({ recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | diff --git a/docs/concepts.md b/docs/concepts.md index b0845894..4fe50bbb 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -3,7 +3,7 @@ > **In plain terms:** This is the one-page mental model of agent-runtime — > read it first if you're meeting the package cold. agent-runtime is a small > shared foundation that handles the plumbing every AI agent needs — running a -> task, streaming a chat reply, reconnecting a dropped connection, picking a +> task, streaming a chat reply, reconnecting a dropped connection, executing an > model — so you only write the parts unique to your agent. The one takeaway: > it owns a handful of reusable building blocks and leaves all the > domain-specific work — your tools, prompts, and scoring rules — to you. @@ -42,14 +42,15 @@ place. └───────────────────────────────────────┬─────────────────┘ │ ┌───────────────────────────────────────┴─────────────────┐ - │ Backends + catalog │ - │ createOpenAICompatibleBackend, createSandboxPromptBackend, + │ Profile-bound model execution + catalog │ + │ profileChatClient, profileOptimizerModelCall, │ + │ createSandboxPromptBackend, │ │ getModels / resolveChatModel / validateChatModelId │ └─────────────────────────────────────────────────────────┘ ``` Each layer composes the one below it. You can use the bottom layers -alone (a raw backend + the model catalog), or the whole stack +alone (a profile-bound adapter + the model catalog), or the whole stack (`defineAgent` → `handleChatTurn`) — they're the same primitives nested. @@ -129,13 +130,17 @@ answers wrong (or differently). Substrate primitive: This module has **no React, no `process.env` assumption** — it runs unchanged in Node and in Cloudflare Workers. -## Backends - -`createOpenAICompatibleBackend({ baseUrl, model, apiKey })` and -`createSandboxPromptBackend({ ... })` are the two production backends. -Both stream. `policy.fallbackModels: [...]` rotates through a named list -on transient failure — that's the only fallback you should ever wire, -and it's explicit. +## Model execution + +`profileChatClient({ profile, executor, context })` adapts one exact +`AgentProfile` to agent-eval's chat contract. +`profileOptimizerModelCall({ profile, executor, context })` does the same for +external optimizer calls, while `createSandboxPromptBackend({ ... })` +normalizes a caller-owned sandbox stream. +The Runtime executor owns credentials, routing, retries, and usage evidence; +request fields cannot override the profile's model policy. +An explicit `policy.fallbackModels: [...]` list may rotate through named models +on transient failure; no unnamed fallback is allowed. The doctrine is in `AGENTS.md`: **no silent fallbacks**. Required fields fail loud; named rotations are opt-in. diff --git a/examples/chat-handler/README.md b/examples/chat-handler/README.md index c913195a..19c4422e 100644 --- a/examples/chat-handler/README.md +++ b/examples/chat-handler/README.md @@ -53,14 +53,18 @@ self-contained — that reader is illustrative, not something to copy. In your p ## From offline to production — one swap -The only scripted part is `produce()`. In a real product it wraps your agent backend: +The only scripted part is `produce()`. In a real product it wraps a profile-bound Runtime turn: ```ts -produce: () => runAgentTaskStream({ task, backend, input }) -// backend = createOpenAICompatibleBackend(...) — any OpenAI-compatible model -// or createSandboxPromptBackend(...) — a sandboxed coding agent +produce: () => streamAgentTurn( + { kind: 'executor', profile, factory: createExecutor(executorConfig) }, + input, +) ``` +The exact `AgentProfile` selects the model and tools; the Runtime executor owns credentials, +routing, retries, and usage evidence. + Everything else — the NDJSON framing, the `session.run.*` envelope, the after-drain persist — stays identical. That framing is the whole point of `handleChatTurn`. diff --git a/examples/chat-handler/chat-handler.ts b/examples/chat-handler/chat-handler.ts index 14065250..27b2e6dd 100644 --- a/examples/chat-handler/chat-handler.ts +++ b/examples/chat-handler/chat-handler.ts @@ -3,9 +3,8 @@ * chat handler implements. `handleChatTurn` frames events with NDJSON + * `session.run.*` envelope and calls product hooks after drain. * - * In a real product, `produce()` calls `runAgentTaskStream({ task, - * backend, input })` against a real backend - * (`createOpenAICompatibleBackend` / `createSandboxPromptBackend`). + * In a real product, `produce()` adapts `streamAgentTurn(...)` from a + * profile-bound Runtime executor into the product's chat events. * Here we yield a small scripted stream so the example runs offline. * * Run with: diff --git a/examples/coding-benchmark/dispatch.ts b/examples/coding-benchmark/dispatch.ts index 864265d7..c96c7394 100644 --- a/examples/coding-benchmark/dispatch.ts +++ b/examples/coding-benchmark/dispatch.ts @@ -143,7 +143,13 @@ export function codingDispatch( model, inputTokens: usage.input, outputTokens: usage.output, - ...(usage.costUsd > 0 ? { actualCostUsd: usage.costUsd } : {}), + ...(usage.tokensKnown === false ? { usageUnknown: true } : {}), + ...(usage.usdKnown === false + ? { costUnknown: true } + : { actualCostUsd: usage.costUsd }), + ...(usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: usage.estimatedCostUsd } + : {}), } }, }) diff --git a/examples/runtime-run/README.md b/examples/runtime-run/README.md index baa9de8c..0f9abb04 100644 --- a/examples/runtime-run/README.md +++ b/examples/runtime-run/README.md @@ -65,8 +65,9 @@ Persisted row: { ## Make it real -- **Real work:** replace the toy backend with `createOpenAICompatibleBackend` (any OpenAI-style - model API), `createSandboxPromptBackend` (a cloud sandbox), or any `AgentExecutionBackend`. +- **Real work:** replace the toy backend with `createSandboxPromptBackend` (a cloud sandbox) or + another caller-owned `AgentExecutionBackend`. Paid model work uses `streamAgentTurn` with an + exact `AgentProfile` and Runtime executor instead of a provider transport in this lifecycle. - **Real store:** implement `RuntimeRunPersistenceAdapter` — one `upsert(row)` method — against D1, Postgres, or your existing runs table. The row shape doesn't change. diff --git a/examples/runtime-run/runtime-run.ts b/examples/runtime-run/runtime-run.ts index 4f2a0adf..e7cc722b 100644 --- a/examples/runtime-run/runtime-run.ts +++ b/examples/runtime-run/runtime-run.ts @@ -29,8 +29,9 @@ const readyTask: AgentTaskSpec = { } // Toy backend that yields a couple of llm_call events so the cost ledger has -// real input. Real consumers plug in `createOpenAICompatibleBackend`, -// `createSandboxPromptBackend`, or any `AgentExecutionBackend`. +// real input. Real consumers plug in `createSandboxPromptBackend` or another +// caller-owned `AgentExecutionBackend`. Paid model work enters Runtime through +// `streamAgentTurn` with an exact `AgentProfile` and executor. const backend = createIterableBackend({ kind: 'demo', async *stream(_input, ctx) { diff --git a/examples/webcode-matrix/webcode-matrix.ts b/examples/webcode-matrix/webcode-matrix.ts index 60d57c72..529b3d83 100644 --- a/examples/webcode-matrix/webcode-matrix.ts +++ b/examples/webcode-matrix/webcode-matrix.ts @@ -118,7 +118,11 @@ function webcodeDispatch( model, inputTokens: usage.input, outputTokens: usage.output, - ...(usage.costUsd > 0 ? { actualCostUsd: usage.costUsd } : {}), + ...(usage.tokensKnown === false ? { usageUnknown: true } : {}), + ...(usage.usdKnown === false ? { costUnknown: true } : { actualCostUsd: usage.costUsd }), + ...(usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: usage.estimatedCostUsd } + : {}), } }, }) diff --git a/package.json b/package.json index cea9c843..453728b6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.128.0", + "version": "0.128.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index 3e90652d..0abb4749 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -18,6 +18,35 @@ const directTransportOwners = new Set([ 'src/runtime/supervise/supervisor-agent.ts', ]) +// These files use caller-supplied URLs for non-inference HTTP (OAuth, telemetry, MCP, search, or +// public benchmark data). Everywhere else a global fetch target must be statically readable so a +// computed provider endpoint cannot hide from this check. +const dynamicNonModelFetchOwners = new Set([ + 'src/model-resolution.ts', + 'src/otel-export.ts', + 'src/platform/auth.ts', + 'src/platform/integrations.ts', + 'src/runtime/mcp-environment.ts', + 'bench/src/research-shot.ts', + 'bench/src/search-tool.ts', + 'bench/src/benchmarks/aec-bench.ts', + 'bench/src/benchmarks/commit0.ts', + 'bench/src/benchmarks/enterpriseops-gym.ts', + 'bench/src/benchmarks/finsearchcomp.ts', + 'bench/src/benchmarks/humaneval.ts', + 'bench/src/benchmarks/programbench.ts', +]) + +const providerSdkModules = new Set([ + 'openai', + '@anthropic-ai/sdk', + '@google/genai', + '@google/generative-ai', + 'cohere-ai', + 'groq-sdk', + '@mistralai/mistralai', +]) + const lowLevelModelCalls = new Set([ 'chatCompletionsTransport', 'createChatClient', @@ -103,7 +132,14 @@ export function checkJavaScript(path, text) { collect(source) for (const statement of source.statements) { - if (!ts.isImportDeclaration(statement) || !statement.importClause?.namedBindings) continue + if (!ts.isImportDeclaration(statement)) continue + const moduleName = ts.isStringLiteral(statement.moduleSpecifier) + ? statement.moduleSpecifier.text + : undefined + if (moduleName !== undefined && providerSdkModules.has(moduleName)) { + failures.push({ node: statement, detail: `provider SDK import ${JSON.stringify(moduleName)}` }) + } + if (!statement.importClause?.namedBindings) continue const bindings = statement.importClause.namedBindings if (ts.isNamespaceImport(bindings)) { lowLevelNamespaces.add(bindings.name.text) @@ -126,6 +162,13 @@ export function checkJavaScript(path, text) { function collectCommonJs(node) { if (ts.isVariableDeclaration(node) && node.initializer !== undefined) { if (isRequireCall(node.initializer)) { + const moduleArg = node.initializer.arguments[0] + if (ts.isStringLiteral(moduleArg) && providerSdkModules.has(moduleArg.text)) { + failures.push({ + node: node.initializer, + detail: `provider SDK require ${JSON.stringify(moduleArg.text)}`, + }) + } if (ts.isIdentifier(node.name)) lowLevelNamespaces.add(node.name.text) if (ts.isObjectBindingPattern(node.name)) { for (const element of node.name.elements) { @@ -148,6 +191,34 @@ export function checkJavaScript(path, text) { } collectCommonJs(source) + function staticString(node, seen = new Set()) { + if (ts.isStringLiteralLike(node)) return node.text + if (ts.isIdentifier(node)) { + if (seen.has(node.text)) return undefined + const initializer = initializers.get(node.text) + if (initializer === undefined) return undefined + const nextSeen = new Set(seen) + nextSeen.add(node.text) + return staticString(initializer, nextSeen) + } + if (ts.isParenthesizedExpression(node)) return staticString(node.expression, seen) + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.PlusToken) { + const left = staticString(node.left, seen) + const right = staticString(node.right, seen) + return left === undefined || right === undefined ? undefined : left + right + } + if (ts.isTemplateExpression(node)) { + let value = node.head.text + for (const span of node.templateSpans) { + const expression = staticString(span.expression, seen) + if (expression === undefined) return undefined + value += expression + span.literal.text + } + return value + } + return undefined + } + function expressionText(node, seen = new Set()) { if (ts.isIdentifier(node)) { if (seen.has(node.text)) return node.getText(source) @@ -165,11 +236,16 @@ export function checkJavaScript(path, text) { const callee = node.expression.getText(source) const first = node.arguments[0] const target = first === undefined ? '' : expressionText(first) + const resolvedTarget = first === undefined ? undefined : staticString(first) const call = node.getText(source) + const isGlobalFetch = callee === 'fetch' const directFetch = - callee === 'fetch' && namesModelEndpoint(target) && !isLocalTestTarget(path, target) + isGlobalFetch && + (namesModelEndpoint(resolvedTarget ?? target) || + (resolvedTarget === undefined && !dynamicNonModelFetchOwners.has(path))) && + !isLocalTestTarget(path, resolvedTarget ?? target) const providerSdk = - /(?:^|\.)(?:chat\.completions\.create|responses\.create|messages\.create|generateContent)$/.test( + /(?:^|\.)(?:chat\.completions\.create|responses\.(?:create|stream)|messages\.(?:create|stream)|generateContent)$/.test( callee, ) const rawHttp = diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index 7a37c34a..2bbb1c3c 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -24,6 +24,26 @@ describe('model execution boundary source check', () => { ).toHaveLength(1) }) + it('rejects aliased provider SDK clients and responses.stream', () => { + const violations = checkJavaScript( + 'examples/direct.ts', + `import Client from 'openai'\nconst client = new Client({ apiKey: 'x' })\nawait client.responses.stream({ model: 'x' })`, + ) + expect(violations.length).toBeGreaterThan(0) + expect(violations.some((violation) => violation.detail.includes('provider SDK import'))).toBe( + true, + ) + }) + + it('rejects computed model URLs passed to fetch', () => { + const violations = checkJavaScript( + 'examples/direct.ts', + `const base = process.env.PROVIDER_URL\nconst route = '/v1/chat/' + 'completions'\nawait fetch(base + route)`, + ) + expect(violations).toHaveLength(1) + expect(violations[0]?.location).toBe('3:7') + }) + it('rejects low-level Runtime model clients outside Runtime-owned adapters', () => { expect( checkJavaScript( diff --git a/src/conversation/run-conversation.ts b/src/conversation/run-conversation.ts index ac8080fd..c3bcb7cb 100644 --- a/src/conversation/run-conversation.ts +++ b/src/conversation/run-conversation.ts @@ -287,8 +287,8 @@ export async function* runConversationStream( // When the participant elects to pay for its own outbound calls, // drop the forwarded user identity so the downstream gateway // bills the participant's own credentials instead. The backend - // brings its own `Authorization` header at construction time - // (e.g. `createOpenAICompatibleBackend({ apiKey: sk-tan-AGENT })`); + // brings its own credentials through its caller-owned backend or + // profile-bound Runtime executor; // omitting the forwarded header is what flips the billing target. forwardedAuthorization: resolveAuthForwarding(speaker, { transcript, diff --git a/src/conversation/run-persona.test.ts b/src/conversation/run-persona.test.ts index 6111979b..781f50b1 100644 --- a/src/conversation/run-persona.test.ts +++ b/src/conversation/run-persona.test.ts @@ -46,6 +46,18 @@ function fakeExecutor( }) } +function unknownUsageExecutor(): ExecutorFactory { + return createExecutor({ + backend: 'router', + routerBaseUrl: 'https://router.test/v1', + routerKey: 'test-key', + complete: async (body) => ({ + model: body.model, + choices: [{ message: { content: 'answer without a provider receipt' } }], + }), + }) +} + const PROFILE = { name: 'worker', harness: 'cli-base', @@ -183,6 +195,22 @@ describe('runPersonaConversation', () => { }), ).rejects.toThrow(/no turns/) }) + + it('marks missing worker usage and cost as unknown instead of measured zero', async () => { + const result = await runPersonaConversation({ + worker: PROFILE, + persona: { kind: 'scripted', turns: ['q1'] }, + executorFor: () => unknownUsageExecutor(), + }) + + expect(result).toMatchObject({ + tokensIn: 0, + tokensOut: 0, + costUsd: 0, + tokensKnown: false, + costUsdKnown: false, + }) + }) }) describe('runPersonaDispatch (matrix adapter)', () => { @@ -244,4 +272,26 @@ describe('runPersonaDispatch (matrix adapter)', () => { expect.objectContaining({ maximumCostUsd: 0.05, actualCostUsd: 0.02, costUsd: 0.02 }), ]) }) + + it('carries missing Router usage and cost into the campaign receipt as unknown', async () => { + const dispatch = runPersonaDispatch({ + executorFor: () => unknownUsageExecutor(), + personaOf: (scenario) => ({ kind: 'scripted', turns: scenario.turns }), + artifactOf: (transcript) => transcript.length, + }) + const ctx = fakeCtx() + + await expect( + dispatch(PROFILE, { id: 'unknown-usage', kind: 'persona', turns: ['q1'] }, ctx), + ).resolves.toBe(2) + expect(ctx.ledger.list()).toEqual([ + expect.objectContaining({ + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + costUnknown: true, + costUsd: 0, + }), + ]) + }) }) diff --git a/src/conversation/run-persona.ts b/src/conversation/run-persona.ts index f618edf0..ca13d48b 100644 --- a/src/conversation/run-persona.ts +++ b/src/conversation/run-persona.ts @@ -64,12 +64,19 @@ export interface PersonaConversationResult { costUsd: number tokensIn: number tokensOut: number + /** Absent means every worker call reported complete token usage. */ + tokensKnown?: false + /** Absent means every worker call reported provider-billed cost, including a known zero. */ + costUsdKnown?: false } interface UsageCounter { tokensIn: number tokensOut: number costUsd: number + sawLlmCall: boolean + tokensKnown: boolean + usdKnown: boolean } /** Adapt one exact profile + Runtime executor into the conversation stream protocol. */ @@ -90,9 +97,20 @@ function profileRuntimeBackend( ...(context.runId ? { correlationId: context.runId } : {}), })) { if (counter && event.type === 'llm_call') { + counter.sawLlmCall = true counter.tokensIn += event.tokensIn ?? 0 counter.tokensOut += event.tokensOut ?? 0 counter.costUsd += event.costUsd ?? 0 + if ( + event.tokensKnown === false || + event.tokensIn === undefined || + event.tokensOut === undefined + ) { + counter.tokensKnown = false + } + if (event.usdKnown === false || event.costUsd === undefined) { + counter.usdKnown = false + } } yield event } @@ -132,7 +150,14 @@ function scriptedPersonaBackend(turns: readonly string[]): AgentExecutionBackend export async function runPersonaConversation( opts: RunPersonaConversationOptions, ): Promise { - const counter: UsageCounter = { tokensIn: 0, tokensOut: 0, costUsd: 0 } + const counter: UsageCounter = { + tokensIn: 0, + tokensOut: 0, + costUsd: 0, + sawLlmCall: false, + tokensKnown: true, + usdKnown: true, + } const workerName = opts.workerName ?? 'agent' const worker = profileRuntimeBackend( opts.worker, @@ -178,12 +203,11 @@ export async function runPersonaConversation( // profile-driven persona the aggregate also includes the persona-driver's // spend, so attributing it to the worker would over-count; report the // worker's metered spend (0 if its backend reported none) instead. - const costUsd = - counter.costUsd > 0 - ? counter.costUsd - : opts.persona.kind === 'scripted' - ? result.spentCreditsCents / 100 - : 0 + const fallbackCostKnown = + !counter.sawLlmCall && opts.persona.kind === 'scripted' && result.spentCreditsCents > 0 + const costUsd = fallbackCostKnown ? result.spentCreditsCents / 100 : counter.costUsd + const tokensKnown = counter.sawLlmCall && counter.tokensKnown + const costUsdKnown = counter.sawLlmCall ? counter.usdKnown : fallbackCostKnown return { transcript: result.transcript, turns: result.turns, @@ -191,6 +215,8 @@ export async function runPersonaConversation( costUsd, tokensIn: counter.tokensIn, tokensOut: counter.tokensOut, + ...(tokensKnown ? {} : { tokensKnown: false }), + ...(costUsdKnown ? {} : { costUsdKnown: false }), } } @@ -251,7 +277,10 @@ export function runPersonaDispatch( model, inputTokens: result.tokensIn, outputTokens: result.tokensOut, - ...(result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}), + ...(result.tokensKnown === false ? { usageUnknown: true } : {}), + ...(result.costUsdKnown === false + ? { costUnknown: true } + : { actualCostUsd: result.costUsd }), }), }) if (!paid.succeeded) throw paid.error diff --git a/src/conversation/types.ts b/src/conversation/types.ts index 222cdf38..04583505 100644 --- a/src/conversation/types.ts +++ b/src/conversation/types.ts @@ -57,9 +57,9 @@ export interface ConversationParticipant { * - `(state) => AuthSource` — per-turn / per-condition decision, e.g. base * sub-services are agent-owned but premium add-ons forward the user. * - * The agent's own credentials live on the backend (set at construction - * time, e.g. `createOpenAICompatibleBackend({ apiKey })`); this field is - * purely about *whether to also forward the user's identity downstream*. + * The agent's own credentials live on its caller-owned backend or + * profile-bound Runtime executor; this field is purely about *whether to + * also forward the user's identity downstream*. */ authSource?: AuthSource } diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index 6e370aa9..b9968efa 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -25,7 +25,7 @@ import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.143.0"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.1"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' diff --git a/src/mcp/bin-helpers.ts b/src/mcp/bin-helpers.ts index 6ede3b96..b7698cd4 100644 --- a/src/mcp/bin-helpers.ts +++ b/src/mcp/bin-helpers.ts @@ -15,7 +15,6 @@ import { type FleetHandle, } from './executor' import { createInProcessExecutor } from './in-process-executor' -import { LOCAL_HARNESSES, type LocalHarness } from './local-harness' /** @experimental */ export interface DetectExecutorArgs { @@ -60,7 +59,6 @@ export async function detectExecutor(args: DetectExecutorArgs): Promise | undefined { - if (!raw) return undefined - const parts = raw - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - if (parts.length === 0) return undefined - for (const part of parts) { - if (!LOCAL_HARNESSES.includes(part as LocalHarness)) { - throw new Error( - `agent-runtime-mcp: AGENT_RUNTIME_LOCAL_HARNESSES contains unknown harness "${part}". Expected: ${LOCAL_HARNESSES.join(', ')}.`, - ) - } - } - return parts as LocalHarness[] -} - interface FleetsApi { get(fleetId: string): Promise } diff --git a/src/mcp/detached-turn.ts b/src/mcp/detached-turn.ts index 8427b413..ff46dc75 100644 --- a/src/mcp/detached-turn.ts +++ b/src/mcp/detached-turn.ts @@ -26,8 +26,8 @@ * token/cost aggregation is not produced for that turn. The trace sinks still * observe detached work — `runDetachedTurn` synthesizes a single-iteration * loop event stream (see `RunDetachedTurnOptions.traceEmitter`) so the span - * topology joins the inherited trace context, with cost/tokens reported as 0 - * under the `'detached-turn'` driver tag. Multi-variant fanout stays on the + * topology joins the inherited trace context, with zero observed subtotals + * explicitly marked incomplete under the `'detached-turn'` driver tag. Multi-variant fanout stays on the * streaming `runAgentRounds` path — N concurrent sessions cannot be expressed as one * resume key, and winner selection needs every candidate. * @@ -191,8 +191,8 @@ export interface RunDetachedTurnOptions { * `'detached-turn'`) so trace-context inheritance survives the detached * path — the same events the streaming `runAgentRounds` path would emit, minus * per-token telemetry: `driveTurn` yields one terminal payload, so token - * and cost figures are structurally unavailable and reported as 0 under - * this driver tag. + * and cost figures are structurally unavailable; zero observed subtotals are + * marked incomplete under this driver tag. */ traceEmitter?: LoopTraceEmitter /** Physical placement stamped on the synthesized dispatch event. Default `'sibling'`. */ @@ -274,7 +274,7 @@ export async function runDetachedTurn(options: RunDetachedTurnOptions): Promise< * the trace sinks (OTEL exporter, delegation journal) observe detached work * exactly like a streamed `runAgentRounds` run. `runId` = the deterministic session * id; cost/token figures are structurally unavailable on the `driveTurn` - * surface and emitted as 0 under the `'detached-turn'` driver tag. + * surface, so zero observed subtotals carry explicit unknown flags. */ function createDetachedTurnTrace(options: RunDetachedTurnOptions): { started(): void @@ -337,7 +337,9 @@ function createDetachedTurnTrace(options: RunDetachedTurnOptions): { iterationIndex: 0, agentRunName, costUsd: 0, + costUsdKnown: false, durationMs: endMs - startMs, + tokenUsage: { input: 0, output: 0, tokensKnown: false }, ...(error !== undefined ? { error } : {}), }, }) @@ -348,6 +350,7 @@ function createDetachedTurnTrace(options: RunDetachedTurnOptions): { payload: { ...(error === undefined ? { winnerIterationIndex: 0 } : {}), totalCostUsd: 0, + costUsdKnown: false, durationMs: endMs - startMs, iterations: 1, }, diff --git a/src/mcp/in-process-executor.ts b/src/mcp/in-process-executor.ts index 88971e20..254de80a 100644 --- a/src/mcp/in-process-executor.ts +++ b/src/mcp/in-process-executor.ts @@ -4,7 +4,7 @@ * carries the local coding-harness CLIs (claude / codex / opencode), delegations spawn the harness * AS A SUBPROCESS against a git worktree on the SAME filesystem instead of provisioning a sibling * sandbox. Zero provisioning latency; worker diffs land in-place; multi-harness fanout = N parallel - * subprocesses in N parallel worktrees (round-robin `harnesses`). + * subprocesses in N parallel worktrees. Each authored profile selects its own harness. * * This is a THIN adapter over `runWorktreeHarness` (`./worktree-harness`) — the SAME core the * `Scope` leaf `createWorktreeCliExecutor` uses. It only adapts the core to the `SandboxClient` @@ -18,11 +18,13 @@ */ import { randomUUID } from 'node:crypto' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { ConfigError } from '../errors' import type { LoopSandboxPlacement, SandboxClient } from '../runtime' +import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import type { DelegationExecutor } from './executor' -import { DEFAULT_LOCAL_HARNESS, type LocalHarness } from './local-harness' +import { LOCAL_HARNESSES, type LocalHarness } from './local-harness' import type { GitRunner, WorktreeHandle } from './worktree' import { runWorktreeHarness } from './worktree-harness' @@ -30,8 +32,6 @@ import { runWorktreeHarness } from './worktree-harness' export interface InProcessExecutorOptions { /** Absolute path to the git repo (the workspace). Worktrees go under `/.agent-worktrees/`. */ repoRoot: string - /** Harnesses to round-robin across `create()` calls. One entry = no fanout. Default `['claude-code']`. */ - harnesses?: ReadonlyArray /** Optional per-delegation test command run in the worktree after the harness exits. */ testCmd?: string /** Optional per-delegation typecheck command. Same shape as `testCmd`. */ @@ -82,10 +82,6 @@ const DEFAULT_POSTCHECK_TIMEOUT_MS = 2 * 60 * 1000 * @experimental */ export function createInProcessExecutor(options: InProcessExecutorOptions): DelegationExecutor { - const harnesses = - options.harnesses && options.harnesses.length > 0 - ? [...options.harnesses] - : [DEFAULT_LOCAL_HARNESS] const runPostCheck = options.runPostCheck ?? defaultRunPostCheck // The core speaks one `runCommand` seam ({exitCode, output}); adapt the post-check seam // ({exitCode, stdout, stderr}) onto it, folding a throw into a non-fatal failure signal so a @@ -108,24 +104,35 @@ export function createInProcessExecutor(options: InProcessExecutorOptions): Dele } } - let callIndex = 0 - const client: SandboxClient = { async create(opts?: CreateSandboxOptions): Promise { + // The authored profile is the sole execution authority. Refuse before creating a run id or + // worktree: ambient executor configuration must never choose a worker's harness or model. + const rawProfile = (opts?.backend as { profile?: unknown } | undefined)?.profile + if (rawProfile === undefined) { + throw new ConfigError( + 'in-process executor: backend.profile is required and must select the exact harness, provider, and model', + ) + } + const profile = agentProfileSchema.parse(rawProfile) as AgentProfile + assertExecutableAgentProfile(profile, 'in-process executor') + const harness = profile.harness + if (!LOCAL_HARNESSES.includes(harness as LocalHarness)) { + throw new ConfigError( + `in-process executor: AgentProfile.harness ${JSON.stringify(harness)} is not a local harness; expected ${LOCAL_HARNESSES.join(', ')}`, + ) + } + if (opts?.backend?.type !== undefined && opts.backend.type !== harness) { + throw new ConfigError( + `in-process executor: backend.type ${JSON.stringify(opts.backend.type)} conflicts with AgentProfile.harness ${JSON.stringify(harness)}`, + ) + } + const localHarness = harness as LocalHarness const runId = randomUUID() - const harness = harnesses[callIndex % harnesses.length] as LocalHarness - callIndex += 1 - // §1.5: the authored profile rides in `backend.profile` (set by `buildBackendOptions`). - // Without one (a direct test `create()`), fall back to a name-only profile → the harness - // sees the task prompt with no system prepend, the pre-fix behavior. - const profile = - ((opts?.backend as { profile?: AgentProfile } | undefined)?.profile as - | AgentProfile - | undefined) ?? ({ name: `in-process-${harness}` } as AgentProfile) const virtual: VirtualSandbox = { id: `in-process-${runId}`, - __inProcess: { runId, harness }, + __inProcess: { runId, harness: localHarness }, async *streamPrompt( this: VirtualSandbox, message: string | unknown[], @@ -147,7 +154,7 @@ export function createInProcessExecutor(options: InProcessExecutorOptions): Dele const run = await runWorktreeHarness({ repoRoot: options.repoRoot, profile, - harness, + harness: localHarness, taskPrompt, runId, harnessTimeoutMs: options.harnessTimeoutMs ?? DEFAULT_HARNESS_TIMEOUT_MS, @@ -164,7 +171,12 @@ export function createInProcessExecutor(options: InProcessExecutorOptions): Dele try { yield { type: 'in_process.harness.started', - data: { runId, harness, worktreePath: run.worktree.path, command: harness }, + data: { + runId, + harness: localHarness, + worktreePath: run.worktree.path, + command: localHarness, + }, } const h = run.result.harness yield { @@ -184,7 +196,7 @@ export function createInProcessExecutor(options: InProcessExecutorOptions): Dele data: { result: run.result, source: 'in-process-executor', - harness, + harness: localHarness, runId, }, } @@ -212,7 +224,7 @@ export function createInProcessExecutor(options: InProcessExecutorOptions): Dele client, placement: 'in-process', describe(): string { - return `in-process (repoRoot=${options.repoRoot}, harnesses=[${harnesses.join(',')}]${ + return `in-process (repoRoot=${options.repoRoot}, harness=profile-selected${ options.testCmd ? `, testCmd="${options.testCmd}"` : '' }${options.typecheckCmd ? `, typecheckCmd="${options.typecheckCmd}"` : ''})` }, diff --git a/src/mcp/openai-tools.ts b/src/mcp/openai-tools.ts index a649f650..6558a3bc 100644 --- a/src/mcp/openai-tools.ts +++ b/src/mcp/openai-tools.ts @@ -3,12 +3,11 @@ * OpenAI Chat Completions `tools[]` projection of the queue-bound agent-runtime * MCP delegation tools. * - * Use when configuring `createOpenAICompatibleBackend({ tools: ... })` so the - * model can call `delegate_feedback`, `delegation_status`, and - * `delegation_history` through the OpenAI-compat transport (tcloud, OpenRouter, - * OpenAI direct, cli-bridge). The runtime surfaces tool calls as `tool_call` - * stream events — execution is the caller's responsibility (typically the - * parent sandbox runtime's MCP mount). + * Use when a caller-owned OpenAI-compatible transport needs the model to call + * `delegate_feedback`, `delegation_status`, and `delegation_history`. Execution + * is the caller's responsibility (typically the parent sandbox runtime's MCP + * mount); Runtime's profile-bound model path materializes profile tools through + * its executor instead. * * Sandbox-SDK callers do NOT need this helper: the sandbox runtime mounts * MCP servers natively and the in-sandbox harness discovers tools via the @@ -44,8 +43,8 @@ function buildTool( parameters: Readonly>, ): OpenAIChatTool { // `parameters` arrives as a deeply-readonly `as const` literal. The - // OpenAI-compat backend JSON-serializes the body so a shallow copy - // into a plain object is sufficient — and shields callers that mutate + // OpenAI-compatible transports JSON-serialize the body, so a shallow copy + // into a plain object is sufficient and shields callers that mutate // the returned descriptor from corrupting the source constant. return { type: 'function', diff --git a/src/runtime/in-process-sandbox-client.test.ts b/src/runtime/in-process-sandbox-client.test.ts index 471aa58f..7366b9c7 100644 --- a/src/runtime/in-process-sandbox-client.test.ts +++ b/src/runtime/in-process-sandbox-client.test.ts @@ -28,7 +28,14 @@ describe('inProcessSandboxClient', () => { const result = await runAgentRounds({ driver, - agentRun: { profile: { name: 'echo' }, taskToPrompt: (t) => t }, + agentRun: { + profile: { + name: 'echo', + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + taskToPrompt: (t) => t, + }, output, validator: { async validate() { diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 745e6525..d6835134 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -167,7 +167,6 @@ export { type AgentSession, type AgentSessionRef, type AgentSessionStatus, - type AgentTurnInput, type AgentTurnResult, type CheckpointRef, type CheckpointRequest, @@ -461,6 +460,7 @@ export { } from './strategy-evolution' export { type AgentTurnBackend, + type AgentTurnInput, type AgentTurnUsage, type CollectedAgentTurn, collectAgentTurn, @@ -961,6 +961,7 @@ export type { ToolLoopCompaction, ToolLoopCompactionOptions, ToolLoopMessageRecord, + ToolLoopToolCall, } from './tool-loop' export type { AgentRunSpec, diff --git a/src/runtime/inline-sandbox-client.ts b/src/runtime/inline-sandbox-client.ts index d62fe85d..5234eb66 100644 --- a/src/runtime/inline-sandbox-client.ts +++ b/src/runtime/inline-sandbox-client.ts @@ -116,10 +116,10 @@ export function inlineSandboxClient( type: 'llm_call', data: { ...(artifact.spent.tokensKnown === false ? {} : { tokensIn, tokensOut }), - ...(costUsd > 0 ? { costUsd } : {}), + ...(artifact.spent.usdKnown !== false ? { costUsd } : {}), ...(artifact.spent.tokensKnown === false ? { tokensKnown: false } : {}), ...(artifact.spent.usdKnown === false ? { costKnown: false } : {}), - ...(artifact.spent.usdKnown !== false && costUsd > 0 + ...(artifact.spent.usdKnown !== false ? { costKnown: true, costProvenance: 'provider-receipt', diff --git a/src/runtime/loop-dispatch.ts b/src/runtime/loop-dispatch.ts index 463c8399..ec83beba 100644 --- a/src/runtime/loop-dispatch.ts +++ b/src/runtime/loop-dispatch.ts @@ -170,9 +170,7 @@ function loopCostReceipt( inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output, ...(result.tokenUsage.tokensKnown === false ? { usageUnknown: true } : {}), - ...(result.costUsdKnown !== false && result.costUsd > 0 - ? { actualCostUsd: result.costUsd } - : {}), + ...(result.costUsdKnown !== false ? { actualCostUsd: result.costUsd } : {}), ...(result.costUsdKnown === false ? { costUnknown: true } : {}), ...(result.estimatedCostUsd !== undefined ? { estimatedCostUsd: result.estimatedCostUsd } : {}), } diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts index 76c92c7a..e28aa255 100644 --- a/src/runtime/profile-chat-client.test.ts +++ b/src/runtime/profile-chat-client.test.ts @@ -226,6 +226,52 @@ describe('profileChatClient exact Runtime adapter', () => { }) expect(complete).toHaveBeenCalledOnce() }) + + it('keeps paid-call evidence when cache classification fails after transport', async () => { + const complete = vi.fn(async () => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'paid response' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 3, + completion_tokens: 2, + cost: 0.001, + prompt_cache_hit_tokens: 4, + }, + })) + const call = profileOptimizerModelCall({ + profile, + context: 'profile optimizer post-call evidence test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) + + const result = await call({ + callId: 'optimizer-paid-invalid-cache', + request: { ...request, model: 'deepseek-v4-flash' }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(false) + if (result.succeeded) throw new Error('expected post-call receipt failure') + expect(result.error).toMatch(/receipt normalization failed after execution/u) + expect(result.receipt).toMatchObject({ + model: 'deepseek-v4-flash', + inputTokens: 3, + outputTokens: 2, + actualCostUsd: 0.001, + }) + expect(result.execution).toMatchObject({ + executed: true, + succeeded: false, + model: 'deepseek-v4-flash', + }) + expect(complete).toHaveBeenCalledOnce() + }) }) describe('terminalDurationMs', () => { diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts index 3fa61212..f4333520 100644 --- a/src/runtime/profile-chat-client.ts +++ b/src/runtime/profile-chat-client.ts @@ -66,21 +66,13 @@ export function profileOptimizerModelCall(args: { request: request.request, endpointFormat: request.endpointFormat ?? null, }) + let run: ProfileChatRun try { - const run = await runBoundProfileChat( - binding, - structuredClone(request.request) as ChatRequest, - { - signal: request.signal, - idempotencyKey: request.callId, - correlationId: request.callId, - }, - ) - const receipt = optimizerReceipt(binding.model, run, args.pricing) - const execution = optimizerExecution(profileDigest, requestDigest, request, run) - return run.succeeded - ? { succeeded: true, response: run.response, receipt, execution } - : { succeeded: false, error: run.error, receipt, execution } + run = await runBoundProfileChat(binding, structuredClone(request.request) as ChatRequest, { + signal: request.signal, + idempotencyKey: request.callId, + correlationId: request.callId, + }) } catch (error) { return { succeeded: false, @@ -97,6 +89,25 @@ export function profileOptimizerModelCall(args: { }, } } + const execution = optimizerExecution(profileDigest, requestDigest, request, run) + try { + const receipt = optimizerReceipt(binding.model, run, args.pricing) + return run.succeeded + ? { succeeded: true, response: run.response, receipt, execution } + : { succeeded: false, error: run.error, receipt, execution } + } catch (error) { + const message = `profile optimizer receipt normalization failed after execution: ${errorMessage(error)}` + return { + succeeded: false, + error: message, + receipt: rawOptimizerReceipt(binding.model, run, args.pricing), + execution: { + ...execution, + succeeded: false, + postCallError: message, + }, + } + } } } @@ -378,6 +389,31 @@ function optimizerReceipt( } } +/** Preserve the observed call totals when finer receipt classification is inconsistent. */ +function rawOptimizerReceipt( + model: string, + run: ProfileChatRun, + pricing: CustomTokenPricing | undefined, +): CostReceiptInput { + const usage = run.turn.usage + const tokensKnown = usage.tokensKnown !== false + const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd + return { + model, + inputTokens: tokensKnown ? usage.input : 0, + outputTokens: tokensKnown ? usage.output : 0, + ...(tokensKnown ? {} : { usageUnknown: true }), + ...(usage.reasoningTokens !== undefined ? { reasoningTokens: usage.reasoningTokens } : {}), + ...(actualCostUsd !== undefined + ? { actualCostUsd } + : usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: usage.estimatedCostUsd } + : pricing + ? { customTokenPricing: pricing } + : { costUnknown: true }), + } +} + function optimizerExecution( profileDigest: string, requestDigest: string, diff --git a/src/runtime/run-loop.provenance.test.ts b/src/runtime/run-loop.provenance.test.ts index fcaf01c0..6c02e555 100644 --- a/src/runtime/run-loop.provenance.test.ts +++ b/src/runtime/run-loop.provenance.test.ts @@ -21,6 +21,14 @@ const output: OutputAdapter = { }, } +function exactProfile(name: string) { + return { + name, + harness: 'opencode' as const, + model: { provider: 'offline', default: 'offline-test-model' }, + } +} + describe('runAgentRounds provenance', () => { it('surfaces a mount manifest recorded during prepareBox', async () => { const bytes = Buffer.from('fixture-contents') @@ -31,7 +39,7 @@ describe('runAgentRounds provenance', () => { }, } const agentRun: AgentRunSpec = { - profile: { name: 'mounting-agent' }, + profile: exactProfile('mounting-agent'), taskToPrompt: (task) => task, // The caller owns the bytes it writes; it declares each mount via the // recorder. The kernel never reads the box, so this is the only path the @@ -86,7 +94,7 @@ describe('runAgentRounds provenance', () => { const result = await runAgentRounds({ driver, - agentRun: { profile: { name: 'plain-agent' }, taskToPrompt: (t) => t }, + agentRun: { profile: exactProfile('plain-agent'), taskToPrompt: (t) => t }, output, task: 'hello', maxIterations: 1, @@ -116,7 +124,7 @@ describe('runAgentRounds provenance', () => { const result = await runAgentRounds({ driver, - agentRun: { profile: { name: 'fanout-agent' }, taskToPrompt: (t) => t }, + agentRun: { profile: exactProfile('fanout-agent'), taskToPrompt: (t) => t }, output, validator: { async validate(_out, ctx) { @@ -180,7 +188,7 @@ describe('runAgentRounds provenance', () => { const result = await runAgentRounds({ driver, - agentRun: { profile: { name: 'caller-select-agent' }, taskToPrompt: (t) => t }, + agentRun: { profile: exactProfile('caller-select-agent'), taskToPrompt: (t) => t }, output, task: 'go', maxIterations: 2, diff --git a/src/runtime/run-loop.test.ts b/src/runtime/run-loop.test.ts index acc606dc..9ef09baf 100644 --- a/src/runtime/run-loop.test.ts +++ b/src/runtime/run-loop.test.ts @@ -1,8 +1,16 @@ import type { SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { runAgentRounds } from './run-loop' import type { AgentRunSpec, Driver, OutputAdapter, SandboxClient } from './types' +function exactProfile(name: string) { + return { + name, + harness: 'opencode' as const, + model: { provider: 'offline', default: 'offline-test-model' }, + } +} + describe('runAgentRounds sandbox preparation', () => { it('runs AgentRunSpec.prepareBox before the first prompt is streamed', async () => { const order: string[] = [] @@ -23,7 +31,7 @@ describe('runAgentRounds sandbox preparation', () => { }, } const agentRun: AgentRunSpec = { - profile: { name: 'prepared-agent' }, + profile: exactProfile('prepared-agent'), taskToPrompt: (task) => task, async prepareBox() { order.push('prepare') @@ -64,6 +72,25 @@ describe('runAgentRounds sandbox preparation', () => { expect(validatorBox).toBe(box) expect(order).toEqual(['create', 'prepare', 'stream:hello']) }) + + it('refuses an incomplete profile before planning or creating a sandbox', async () => { + const plan = vi.fn(async () => ['hello']) + const create = vi.fn() + + await expect( + runAgentRounds({ + driver: { plan, decide: () => 'done' as const }, + agentRun: { profile: { name: 'incomplete' }, taskToPrompt: (task) => task }, + output: { parse: () => 'unused' }, + task: 'hello', + maxIterations: 1, + ctx: { sandboxClient: { create } }, + }), + ).rejects.toThrow(/AgentProfile\.harness must be explicit/) + + expect(plan).not.toHaveBeenCalled() + expect(create).not.toHaveBeenCalled() + }) }) describe('runAgentRounds onSandboxEvent tee', () => { @@ -102,7 +129,7 @@ describe('runAgentRounds onSandboxEvent tee', () => { }, } const agentRun: AgentRunSpec = { - profile: { name: 'tee-agent' }, + profile: exactProfile('tee-agent'), taskToPrompt: (task) => task, } const output: OutputAdapter = { diff --git a/src/runtime/run-loop.ts b/src/runtime/run-loop.ts index 32634537..b6ea8b18 100644 --- a/src/runtime/run-loop.ts +++ b/src/runtime/run-loop.ts @@ -37,6 +37,7 @@ import { type SandboxLineage, type SandboxLineageHandle, } from './sandbox-lineage' +import { assertExecutableAgentProfile } from './supervise/model-policy' import type { AgentRunSpec, Driver, @@ -152,6 +153,9 @@ export async function runAgentRounds( options: RunAgentRoundsOptions, ): Promise> { const specs = resolveAgentRuns(options) + for (const [index, spec] of specs.entries()) { + assertExecutableAgentProfile(spec.profile, `runAgentRounds: agent spec ${index}`) + } const maxIterations = options.maxIterations ?? DEFAULT_MAX_ITERATIONS if (!Number.isFinite(maxIterations) || maxIterations <= 0) { throw new ValidationError('runAgentRounds: maxIterations must be > 0') diff --git a/src/runtime/sandbox-events.ts b/src/runtime/sandbox-events.ts index 52dfb796..150d503d 100644 --- a/src/runtime/sandbox-events.ts +++ b/src/runtime/sandbox-events.ts @@ -131,9 +131,9 @@ export function extractLlmCallEvent( * receipt: (turn) => { * const u = sumSandboxUsage(turn.events) * return { model, inputTokens: u.input, outputTokens: u.output, - * ...(u.tokensKnown ? {} : { usageUnknown: true }), - * ...(u.usdKnown && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), - * ...(u.usdKnown ? {} : { costUnknown: true }), + * ...(u.tokensKnown === false ? { usageUnknown: true } : {}), + * ...(u.usdKnown !== false && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), + * ...(u.usdKnown === false ? { costUnknown: true } : {}), * ...(u.estimatedCostUsd !== undefined ? { estimatedCostUsd: u.estimatedCostUsd } : {}) } * } * @@ -147,8 +147,8 @@ export function sumSandboxUsage( input: number output: number costUsd: number - tokensKnown: boolean - usdKnown: boolean + tokensKnown?: false + usdKnown?: false estimatedCostUsd?: number } { let input = 0 @@ -177,8 +177,8 @@ export function sumSandboxUsage( input, output, costUsd, - tokensKnown: sawCall && tokensKnown, - usdKnown: sawCall && usdKnown, + ...(sawCall && tokensKnown ? {} : { tokensKnown: false as const }), + ...(sawCall && usdKnown ? {} : { usdKnown: false as const }), ...(sawEstimate ? { estimatedCostUsd } : {}), } } @@ -205,10 +205,15 @@ function buildLlmCall( const promptCache = finiteMetadata(data.promptCache ?? data.prompt_cache) const tokensKnown = explicitTokensKnown !== false && tokensIn !== undefined && tokensOut !== undefined + // Sandbox's canonical terminal `totalCostUsd` is already the provider-reported receipt. The + // current SDK carries no provenance tag, so absence means the canonical receipt rather than + // "unknown". Explicit estimates and explicit false-known markers remain unknown. const usdKnown = - explicitCostKnown === true && - (costProvenance === 'provider-receipt' || costProvenance === 'billing-receipt') && - costUsd !== undefined + explicitCostKnown !== false && + costUsd !== undefined && + (costProvenance === undefined || + costProvenance === 'provider-receipt' || + costProvenance === 'billing-receipt') if ( tokensIn === undefined && tokensOut === undefined && diff --git a/src/runtime/stream-agent-turn.test.ts b/src/runtime/stream-agent-turn.test.ts index e89a27ec..5b4fb5cd 100644 --- a/src/runtime/stream-agent-turn.test.ts +++ b/src/runtime/stream-agent-turn.test.ts @@ -17,7 +17,8 @@ import type { Executor, ExecutorFactory, ExecutorResult } from './supervise/type const TEST_PROFILE = { name: 'stream-agent-turn-test', - model: { default: 'offline-test-model' }, + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, } as const function finalOf(events: RuntimeStreamEvent[]): RuntimeStreamEvent & { type: 'final' } { @@ -57,7 +58,7 @@ describe('streamAgentTurn: box backend', () => { expect(final.text).toBe('Hello world') expect(final.metadata).toMatchObject({ tokenUsage: { input: 100, output: 40 }, - usdKnown: false, + costUsd: 0.02, model: 'kimi-k2', }) }) @@ -135,7 +136,12 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { if (start?.type !== 'backend_start') throw new Error('expected backend_start') expect(start.backend).toBe('box') expect(turn.finalText).toBe('task output') - expect(turn.usage).toEqual({ input: 9, output: 4, usdKnown: false, model: 'kimi-k2' }) + expect(turn.usage).toEqual({ + input: 9, + output: 4, + costUsd: 0.01, + model: 'kimi-k2', + }) expect(turn.status).toBe('completed') }) @@ -468,6 +474,28 @@ describe('streamAgentTurn: executor backend', () => { } } + it('rejects an incomplete execution profile before constructing an executor', async () => { + let factoryCalls = 0 + const factory: ExecutorFactory = (...args) => { + factoryCalls += 1 + return stubFactory()(...args) + } + + await expect( + collectAgentTurn( + streamAgentTurn( + { + kind: 'executor', + factory, + profile: { name: 'incomplete', model: { default: 'offline-test-model' } }, + }, + 'must not run', + ), + ), + ).rejects.toThrow(/AgentProfile\.harness must be explicit/u) + expect(factoryCalls).toBe(0) + }) + it('runs the factory once and terminates with the executor usage', async () => { let toreDown = 0 const stream = streamAgentTurn( diff --git a/src/runtime/stream-agent-turn.ts b/src/runtime/stream-agent-turn.ts index 547fa01c..b68f7a6b 100644 --- a/src/runtime/stream-agent-turn.ts +++ b/src/runtime/stream-agent-turn.ts @@ -78,6 +78,7 @@ import { unknownMaterializationReceipt, } from './supervise/materialization' import { + assertExecutableAgentProfile, concreteModelId, profileBridgeWireModel, profileProviderModel, @@ -128,8 +129,8 @@ type ObservedAgentTurnBackend = } | { /** - * An in-process `AgentExecutionBackend` (`resolveAgentBackend` output or - * any custom backend): the turn is one `backend.stream()` call. + * A caller-supplied in-process `AgentExecutionBackend`: the turn is one + * `backend.stream()` call. */ kind: 'chat' backend: AgentExecutionBackend @@ -476,6 +477,7 @@ async function* streamAgentTurnInternal( const label = backend.kind === 'chat' ? backend.backend.kind : backend.kind const profile = backend.kind === 'executor' ? agentProfileSchema.parse(backend.profile) : undefined + if (profile) assertExecutableAgentProfile(profile, 'streamAgentTurn') const profileDigest = profile ? authoredProfileDigest(profile) : undefined assertTurnIdentity(opts.callId, 'callId') assertTurnIdentity(opts.correlationId, 'correlationId') diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index 135482e6..e7b995d5 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -10,11 +10,19 @@ */ import { ValidationError } from '../errors' -import type { RouterToolCall, ToolSpec } from './router-client' +import type { ToolSpec } from './router-client' /** Provider-neutral conversation record accepted by a tool-loop brain. */ export type ToolLoopMessageRecord = Record +/** One provider-neutral tool request emitted by a tool-loop model. */ +export interface ToolLoopToolCall { + id: string + name: string + /** Raw JSON arguments emitted by the model. */ + arguments: string +} + /** One inference turn over the running conversation + the tool specs → the model's text, any * tool calls, and token usage. The seam every brain satisfies. */ export type ToolLoopChat = ( @@ -22,7 +30,7 @@ export type ToolLoopChat = ( tools: ReadonlyArray, ) => Promise<{ content?: string | null - toolCalls: RouterToolCall[] + toolCalls: ToolLoopToolCall[] usage?: { input: number; output: number } /** Dollar value reported for the turn. It is not billed spend unless provenance says so. */ costUsd?: number diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 42532b50..5d5ed3af 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:5feb51aa14c16c29e4b06a4f344dec42ecc7c13f0a876c17ed743014580dc93a", + "digest": "sha256:ae45e6d96a4931bf38572396e968de5c4b753b05c9e3a5608c0d3b0f27e1d4c7", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.128.0" + "runtimeVersion": "0.128.1" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:21aec498ac5cf1fec87690d5216cf4edb811e0db19b841f8be84c1bea8a9365e", - "runId": "agent-runtime-0.128.0-proposal-fixture", + "recordDigest": "sha256:f6030055252c51ab3c035cc2dded16b24d98cc6ecbb0630e0cd5b8f5be4db34e", + "runId": "agent-runtime-0.128.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.128.0-proposal-fixture" + "runId": "agent-runtime-0.128.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index a02b52f5..42d4c917 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:481b9db8d4e8e1834114e3b922876eaa0d68ac007c094ce9b2bd9aa7a14d812e", + "digest": "sha256:f5f780797333cfb8abfe9bfefe46d70faa412d804b997626d8a1d1316d9c2b68", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.128.0" + "runtimeVersion": "0.128.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:9c0bf8c61092567c90d1d40e48ba3f44f70433c80dcd6d28cc14b14a80bcd295", + "recordDigest": "sha256:6c9c0e3a1cf1e01b2ee759dc67e0ac6d61fde0da14fe22d15d11532bdc63870e", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/src/types.ts b/src/types.ts index 340bd0c4..7ab74e0d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -226,9 +226,9 @@ export interface BackendErrorDetail { /** * * OpenAI Chat Completions tool descriptor. The shape mirrors the - * `/v1/chat/completions` `tools[]` parameter so callers can pass tool - * definitions through `createOpenAICompatibleBackend({ tools })` without any - * runtime translation. The router proxies this shape verbatim to Anthropic + * `/v1/chat/completions` `tools[]` parameter so caller-owned compatible + * transports can pass tool definitions without translation. A router can + * proxy this shape to Anthropic * (translated server-side), DeepSeek, Groq, OpenAI, and Gemini — every model * that the eval surface targets. * diff --git a/tests/kernel/composition.test.ts b/tests/kernel/composition.test.ts index 8173c727..4045dfd0 100644 --- a/tests/kernel/composition.test.ts +++ b/tests/kernel/composition.test.ts @@ -22,7 +22,11 @@ interface Outer { best: number } -const profile: AgentProfile = { name: 'compose-stub' } +const profile: AgentProfile = { + name: 'compose-stub', + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, +} const innerOutput: OutputAdapter = { parse(events) { diff --git a/tests/kernel/loop-dispatch.test.ts b/tests/kernel/loop-dispatch.test.ts index 035a6253..467eebdc 100644 --- a/tests/kernel/loop-dispatch.test.ts +++ b/tests/kernel/loop-dispatch.test.ts @@ -23,7 +23,11 @@ interface FakeScenario { kind: string } -const sandboxProfile: SandboxAgentProfile = { name: 'stub', model: { default: 'm' } } +const sandboxProfile: SandboxAgentProfile = { + name: 'stub', + harness: 'opencode', + model: { provider: 'offline', default: 'm' }, +} function spec(): AgentRunSpec { return { profile: sandboxProfile, name: 'agent', taskToPrompt: (t) => t.goal } diff --git a/tests/kernel/run-loop-harden.test.ts b/tests/kernel/run-loop-harden.test.ts index bad0ed65..779d9a53 100644 --- a/tests/kernel/run-loop-harden.test.ts +++ b/tests/kernel/run-loop-harden.test.ts @@ -24,7 +24,15 @@ const output: OutputAdapter = { } function spec(name: string, taskToPrompt = (t: Task) => JSON.stringify(t)): AgentRunSpec { - return { profile: { name }, name, taskToPrompt } + return { + profile: { + name, + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + name, + taskToPrompt, + } } describe('runAgentRounds — abort short-circuits before launching a fresh batch', () => { diff --git a/tests/kernel/sandbox-events.test.ts b/tests/kernel/sandbox-events.test.ts index 60f1ed1f..ef4c6274 100644 --- a/tests/kernel/sandbox-events.test.ts +++ b/tests/kernel/sandbox-events.test.ts @@ -24,6 +24,8 @@ describe('sumSandboxUsage — meter an openSandboxRun turn', () => { input: 0, output: 0, costUsd: 0, + tokensKnown: false, + usdKnown: false, }) }) }) @@ -92,7 +94,13 @@ describe('mapSandboxEvent — SandboxEvent → RuntimeStreamEvent', () => { { type: 'result', data: { usage: { inputTokens: 3, outputTokens: 4 } } }, { agentRunName: 'r' }, ), - ).toEqual({ type: 'llm_call', model: 'r', tokensIn: 3, tokensOut: 4 }) + ).toEqual({ + type: 'llm_call', + model: 'r', + tokensIn: 3, + tokensOut: 4, + usdKnown: false, + }) expect(mapSandboxEvent({ type: 'result', data: { finalText: 'done' } })).toBeUndefined() }) @@ -117,7 +125,13 @@ describe('extractLlmCallEvent — strict numeric coercion', () => { { type: 'usage', data: { prompt_tokens: 100, completion_tokens: 20 } }, 'agent', ), - ).toEqual({ type: 'llm_call', model: 'agent', tokensIn: 100, tokensOut: 20 }) + ).toEqual({ + type: 'llm_call', + model: 'agent', + tokensIn: 100, + tokensOut: 20, + usdKnown: false, + }) }) it.each([ @@ -135,6 +149,7 @@ describe('extractLlmCallEvent — strict numeric coercion', () => { model: 'agent', tokensIn: 2, tokensOut: 10, + usdKnown: false, }) }) diff --git a/tests/kernel/sandbox-lineage.test.ts b/tests/kernel/sandbox-lineage.test.ts index cf4b0d81..edd96378 100644 --- a/tests/kernel/sandbox-lineage.test.ts +++ b/tests/kernel/sandbox-lineage.test.ts @@ -24,7 +24,15 @@ const output: OutputAdapter = { } function spec(name: string): AgentRunSpec { - return { profile: { name }, name, taskToPrompt: (t) => JSON.stringify(t) } + return { + profile: { + name, + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + name, + taskToPrompt: (t) => JSON.stringify(t), + } } /** One `streamPrompt` call's recorded arguments — the box id it ran on, the diff --git a/tests/kernel/validator-tracing.test.ts b/tests/kernel/validator-tracing.test.ts index f987a983..6e7cea81 100644 --- a/tests/kernel/validator-tracing.test.ts +++ b/tests/kernel/validator-tracing.test.ts @@ -21,6 +21,13 @@ function makeSandboxClient() { } } +const profile = { + name: 'test', + harness: 'opencode' as const, + model: { provider: 'offline', default: 'offline-test-model' }, + systemPrompt: 'test', +} + describe('validator tracing', () => { it('validator receives traceEmitter in ctx when kernel has one', async () => { let receivedEmitter: LoopTraceEmitter | undefined @@ -50,7 +57,7 @@ describe('validator tracing', () => { }, }, agentRun: { - profile: { name: 'test', systemPrompt: 'test' }, + profile, taskToPrompt: (t: string) => t, }, output: { parse: () => 'parsed-output' }, @@ -88,7 +95,7 @@ describe('validator tracing', () => { }, }, agentRun: { - profile: { name: 'test', systemPrompt: 'test' }, + profile, taskToPrompt: (t: string) => t, }, output: { parse: () => 'parsed-output' }, @@ -138,7 +145,7 @@ describe('validator tracing', () => { }, }, agentRun: { - profile: { name: 'test', systemPrompt: 'test' }, + profile, taskToPrompt: (t: string) => t, }, output: { parse: () => 'parsed-output' }, diff --git a/tests/mcp/detached-turn.test.ts b/tests/mcp/detached-turn.test.ts index dee4f650..32bce1a4 100644 --- a/tests/mcp/detached-turn.test.ts +++ b/tests/mcp/detached-turn.test.ts @@ -24,6 +24,11 @@ import type { DelegateCodeArgs } from '../../src/mcp/types' import type { LoopTraceEvent, SandboxClient } from '../../src/runtime' const codeArgs: DelegateCodeArgs = { goal: 'fix bug', repoRoot: '/repo' } +const exactWorkerProfile = { + name: 'coder-test', + harness: 'claude-code' as const, + model: { provider: 'anthropic', default: 'offline-test-model' }, +} /** * Submit a single-variant coder delegation to the queue exactly as the bin's `delegate` dispatch @@ -152,7 +157,7 @@ describe('detached session ref codec', () => { describe('runDetachedTurn', () => { const spec = { - profile: { name: 'coder-test' }, + profile: exactWorkerProfile, taskToPrompt: () => 'do the thing', } as never @@ -288,9 +293,26 @@ describe('runDetachedTurn', () => { expect(started.agentRunNames).toEqual(['coder-test']) const dispatch = events[2]!.payload as { placement: string; sandboxId: string } expect(dispatch).toMatchObject({ placement: 'sibling', sandboxId: 'sandbox_t1' }) - const ended = events[4]!.payload as { winnerIterationIndex?: number; iterations: number } + const iterationEnded = events[3]!.payload as { + costUsd: number + costUsdKnown?: false + tokenUsage?: { input: number; output: number; tokensKnown?: false } + } + expect(iterationEnded).toMatchObject({ + costUsd: 0, + costUsdKnown: false, + tokenUsage: { input: 0, output: 0, tokensKnown: false }, + }) + const ended = events[4]!.payload as { + winnerIterationIndex?: number + iterations: number + totalCostUsd: number + costUsdKnown?: false + } expect(ended.winnerIterationIndex).toBe(0) expect(ended.iterations).toBe(1) + expect(ended.totalCostUsd).toBe(0) + expect(ended.costUsdKnown).toBe(false) }) it('records the error on the synthesized stream when the turn fails', async () => { @@ -585,7 +607,11 @@ describe('detachedSessionDelegate detached path', () => { id: 'sandbox_77', }) const executor = createSiblingSandboxExecutor({ client: fakeClient(fake.box) }) - const delegate = detachedSessionDelegate({ executor, detachedTickIntervalMs: 1 }) + const delegate = detachedSessionDelegate({ + executor, + workerProfile: exactWorkerProfile, + detachedTickIntervalMs: 1, + }) const rebinds: string[] = [] const sessionId = 'dlg-turn-coder-deadbeef' const output = await delegate(codeArgs, { @@ -606,7 +632,11 @@ describe('detachedSessionDelegate detached path', () => { id: 'sandbox_88', }) const executor = createSiblingSandboxExecutor({ client: fakeClient(fake.box) }) - const delegate = detachedSessionDelegate({ executor, detachedTickIntervalMs: 1 }) + const delegate = detachedSessionDelegate({ + executor, + workerProfile: exactWorkerProfile, + detachedTickIntervalMs: 1, + }) const queue = new DelegationTaskQueue() const { taskId } = submitCoder( queue, @@ -645,7 +675,7 @@ describe('detachedSessionDelegate detached path', () => { }) const box = { ...fake.box, streamPrompt } const executor = createSiblingSandboxExecutor({ client: fakeClient(box) }) - const delegate = detachedSessionDelegate({ executor }) + const delegate = detachedSessionDelegate({ executor, workerProfile: exactWorkerProfile }) const output = await delegate(codeArgs, { signal: new AbortController().signal, report: () => {}, diff --git a/tests/mcp/in-process-detect.test.ts b/tests/mcp/in-process-detect.test.ts index 19e9fac5..36de36c9 100644 --- a/tests/mcp/in-process-detect.test.ts +++ b/tests/mcp/in-process-detect.test.ts @@ -11,7 +11,7 @@ describe('detectExecutor — in-process selection', () => { }) expect(exec.describe()).toMatch(/in-process/) expect(exec.describe()).toContain('/workspace') - expect(exec.describe()).toContain('harnesses=[claude-code]') + expect(exec.describe()).toContain('harness=profile-selected') // In-process placement has no sandbox session — the bin keys detached // dispatch off this tag, so it must never read session-backed here. expect(exec.placement).toBe('in-process') @@ -31,31 +31,6 @@ describe('detectExecutor — in-process selection', () => { ).rejects.toThrow(/AGENT_RUNTIME_REPO_ROOT/) }) - it('passes through configured harnesses list', async () => { - const exec = await detectExecutor({ - sandboxClient: stubClient, - env: { - AGENT_RUNTIME_IN_SANDBOX: '1', - AGENT_RUNTIME_REPO_ROOT: '/wk', - AGENT_RUNTIME_LOCAL_HARNESSES: 'claude-code,codex,opencode', - }, - }) - expect(exec.describe()).toContain('harnesses=[claude-code,codex,opencode]') - }) - - it('rejects unknown harness name', async () => { - await expect( - detectExecutor({ - sandboxClient: stubClient, - env: { - AGENT_RUNTIME_IN_SANDBOX: '1', - AGENT_RUNTIME_REPO_ROOT: '/wk', - AGENT_RUNTIME_LOCAL_HARNESSES: 'claude-code,gemini', - }, - }), - ).rejects.toThrow(/unknown harness "gemini"/) - }) - it('threads test + typecheck commands into the executor description', async () => { const exec = await detectExecutor({ sandboxClient: stubClient, diff --git a/tests/mcp/in-process-executor.test.ts b/tests/mcp/in-process-executor.test.ts index 4b3391d4..20beaddf 100644 --- a/tests/mcp/in-process-executor.test.ts +++ b/tests/mcp/in-process-executor.test.ts @@ -32,6 +32,29 @@ function makeFakeGit(state: FakeGitState): GitRunner { } } +type TestHarness = 'claude-code' | 'codex' | 'opencode' + +function exactProfile(harness: TestHarness = 'claude-code') { + return { + name: `worker-${harness}`, + harness, + model: { + provider: + harness === 'codex' ? 'openai' : harness === 'claude-code' ? 'anthropic' : 'offline', + default: 'offline-test-model', + }, + } as const +} + +async function createBox( + exec: ReturnType, + harness: TestHarness = 'claude-code', +) { + return exec.client.create({ + backend: { type: harness, profile: exactProfile(harness) }, + } as unknown as Parameters[0]) +} + describe('createInProcessExecutor', () => { it('streamPrompt emits started → ended → result events with the raw patch artifact', async () => { const state: FakeGitState = { @@ -44,7 +67,6 @@ describe('createInProcessExecutor', () => { } const exec = createInProcessExecutor({ repoRoot: '/workspace', - harnesses: ['claude-code'], runGit: makeFakeGit(state), runHarness: vi.fn(async () => ({ exitCode: 0, @@ -56,7 +78,7 @@ describe('createInProcessExecutor', () => { })), }) - const box = await exec.client.create() + const box = await createBox(exec) const events: Array<{ type: string; data: Record }> = [] for await (const event of ( box as unknown as { @@ -87,7 +109,7 @@ describe('createInProcessExecutor', () => { expect(state.worktreesRemoved.length).toBe(1) }) - it('rotates harnesses round-robin across create() calls', async () => { + it('uses each exact profile harness across create() calls', async () => { const state: FakeGitState = { worktreesCreated: [], worktreesRemoved: [], @@ -105,13 +127,20 @@ describe('createInProcessExecutor', () => { })) const exec = createInProcessExecutor({ repoRoot: '/w', - harnesses: ['claude-code', 'codex', 'opencode'], runGit: makeFakeGit(state), runHarness, }) - for (let i = 0; i < 6; i++) { - const box = await exec.client.create() + const authoredHarnesses: TestHarness[] = [ + 'claude-code', + 'codex', + 'opencode', + 'claude-code', + 'codex', + 'opencode', + ] + for (const [i, harness] of authoredHarnesses.entries()) { + const box = await createBox(exec, harness) for await (const _ of ( box as unknown as { streamPrompt: (m: string) => AsyncGenerator } ).streamPrompt(`task ${i}`)) { @@ -119,14 +148,7 @@ describe('createInProcessExecutor', () => { } } const harnesses = runHarness.mock.calls.map((c) => (c[0] as { harness: string }).harness) - expect(harnesses).toEqual([ - 'claude-code', - 'codex', - 'opencode', - 'claude-code', - 'codex', - 'opencode', - ]) + expect(harnesses).toEqual(authoredHarnesses) }) it('runs testCmd + typecheckCmd against the worktree and folds results into the artifact checks', async () => { @@ -144,7 +166,6 @@ describe('createInProcessExecutor', () => { })) const exec = createInProcessExecutor({ repoRoot: '/w', - harnesses: ['claude-code'], testCmd: 'pnpm test', typecheckCmd: 'pnpm typecheck', runGit: makeFakeGit(state), @@ -159,7 +180,7 @@ describe('createInProcessExecutor', () => { runPostCheck, }) - const box = await exec.client.create() + const box = await createBox(exec) const events: Array<{ type: string; data: Record }> = [] for await (const event of ( box as unknown as { @@ -200,7 +221,7 @@ describe('createInProcessExecutor', () => { timedOut: false, })), }) - const box = await exec.client.create() + const box = await createBox(exec) const events: Array<{ type: string; data: Record }> = [] for await (const event of ( box as unknown as { @@ -231,7 +252,7 @@ describe('createInProcessExecutor', () => { throw new Error('boom') }), }) - const box = await exec.client.create() + const box = await createBox(exec) await expect( (async () => { for await (const _ of ( @@ -255,7 +276,6 @@ describe('createInProcessExecutor', () => { } const exec = createInProcessExecutor({ repoRoot: '/w', - harnesses: ['codex'], runGit: makeFakeGit(state), runHarness: vi.fn(async () => ({ exitCode: 0, @@ -266,7 +286,7 @@ describe('createInProcessExecutor', () => { timedOut: false, })), }) - const box = await exec.client.create() + const box = await createBox(exec, 'codex') for await (const _ of ( box as unknown as { streamPrompt: (m: string) => AsyncGenerator } ).streamPrompt('x')) { @@ -300,7 +320,6 @@ describe('createInProcessExecutor', () => { })) const exec = createInProcessExecutor({ repoRoot: '/w', - harnesses: ['claude-code'], runGit: makeFakeGit(state), runHarness, }) @@ -310,8 +329,9 @@ describe('createInProcessExecutor', () => { type: 'claude-code', profile: { name: 'w', + harness: 'claude-code', prompt: { systemPrompt: 'BE RIGOROUS' }, - model: { default: 'deepseek-v4-flash' }, + model: { provider: 'anthropic', default: 'deepseek-v4-flash' }, }, }, } as unknown as Parameters[0]) @@ -328,4 +348,29 @@ describe('createInProcessExecutor', () => { expect(args).toContain('add util(a,b)') expect(args).toContain('deepseek-v4-flash') }) + + it('refuses missing or conflicting profile authority before starting a harness', async () => { + const runHarness = vi.fn() + const exec = createInProcessExecutor({ repoRoot: '/w', runHarness }) + + await expect(exec.client.create()).rejects.toThrow(/backend\.profile is required/) + await expect( + exec.client.create({ + backend: { + type: 'codex', + profile: exactProfile('claude-code'), + }, + } as unknown as Parameters[0]), + ).rejects.toThrow(/conflicts with AgentProfile\.harness/) + await expect( + exec.client.create({ + backend: { + type: 'claude-code', + profile: { name: 'incomplete', harness: 'claude-code' }, + }, + } as unknown as Parameters[0]), + ).rejects.toThrow(/model\.default/) + + expect(runHarness).not.toHaveBeenCalled() + }) }) From 9e3435b1a80630a17057945004fd21f5becb7357 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:06:46 -0600 Subject: [PATCH 03/39] fix(bench): make package test timeout caller-controlled --- .github/workflows/ci.yml | 2 + .github/workflows/publish.yml | 4 ++ bench/package.json | 2 +- bench/scripts/run-package-tests.mjs | 92 ++++++++++++++---------- bench/scripts/run-package-tests.test.mjs | 33 +++++++++ 5 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 bench/scripts/run-package-tests.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b92d7acf..a9d66a96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,8 @@ jobs: run: pnpm run build - name: Verify agent-bench against current agent-runtime + env: + AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: 900000 run: pnpm run verify:bench official-optimizers: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4e0b5192..e0fe8cfb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -169,6 +169,8 @@ jobs: run: pnpm run verify:primeintellect - name: Verify agent-bench against this release + env: + AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: 900000 run: pnpm run verify:bench - name: Package and verify the exact Runtime archive @@ -296,6 +298,8 @@ jobs: run: pnpm --filter @tangle-network/agent-bench run wait:published-dependencies - name: Verify packed agent-bench against published dependencies + env: + AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: 900000 run: pnpm run verify:bench:published - name: Verify tag/version lock diff --git a/bench/package.json b/bench/package.json index 66fc76db..f8b9fc4b 100644 --- a/bench/package.json +++ b/bench/package.json @@ -33,7 +33,7 @@ "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", "terminal-compare": "tsx src/terminal-compare.ts", - "test": "node scripts/run-package-tests.mjs && node --test scripts/wait-for-published-dependencies.test.mjs", + "test": "node scripts/run-package-tests.mjs && node --test scripts/run-package-tests.test.mjs scripts/wait-for-published-dependencies.test.mjs", "typecheck:public": "tsc -p tsconfig.public.json", "verify:package": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs", "verify:package:local-runtime": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs --local-runtime", diff --git a/bench/scripts/run-package-tests.mjs b/bench/scripts/run-package-tests.mjs index f215d241..c0ab48f2 100644 --- a/bench/scripts/run-package-tests.mjs +++ b/bench/scripts/run-package-tests.mjs @@ -8,6 +8,19 @@ const execFileAsync = promisify(execFile) const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const sourceDir = path.join(benchDir, 'src') +export function resolvePackageTestTimeoutMs(env = process.env) { + const raw = env.AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS + if (raw === undefined) return undefined + if (!/^(?:0|[1-9]\d*)$/.test(raw)) { + throw new Error('AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS must be a non-negative safe integer') + } + const timeoutMs = Number(raw) + if (!Number.isSafeInteger(timeoutMs)) { + throw new Error('AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS must be a non-negative safe integer') + } + return timeoutMs +} + async function collectTests(dir) { const files = [] for (const entry of await readdir(dir, { withFileTypes: true })) { @@ -18,13 +31,14 @@ async function collectTests(dir) { return files.sort() } -async function run(command, args, env = process.env) { +export async function run(command, args, env = process.env) { + const timeoutMs = resolvePackageTestTimeoutMs(env) try { await execFileAsync(command, args, { cwd: benchDir, env, maxBuffer: 10 * 1024 * 1024, - timeout: 120_000, + ...(timeoutMs === undefined ? {} : { timeout: timeoutMs }), }) } catch (error) { if (error?.stdout) process.stdout.write(error.stdout) @@ -35,44 +49,50 @@ async function run(command, args, env = process.env) { } } -const python = path.join(benchDir, '.venv', 'bin', 'python') -try { - await access(python) -} catch { - await run('python3', ['-m', 'venv', '.venv']) -} +async function main() { + const python = path.join(benchDir, '.venv', 'bin', 'python') + try { + await access(python) + } catch { + await run('python3', ['-m', 'venv', '.venv']) + } -const tests = await collectTests(sourceDir) -if (tests.length === 0) throw new Error('no package tests found under src/') + const tests = await collectTests(sourceDir) + if (tests.length === 0) throw new Error('no package tests found under src/') -// Two test runtimes coexist under src/: node:test files run under `node --test`; -// vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest -// worker), so partition by the framework each file actually imports. -const nodeTests = [] -const vitestTests = [] -for (const file of tests) { - const body = await readFile(file, 'utf8') - if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file) - else nodeTests.push(file) -} + // Two test runtimes coexist under src/: node:test files run under `node --test`; + // vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest + // worker), so partition by the framework each file actually imports. + const nodeTests = [] + const vitestTests = [] + for (const file of tests) { + const body = await readFile(file, 'utf8') + if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file) + else nodeTests.push(file) + } + + if (nodeTests.length > 0) { + await run( + process.execPath, + ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))], + { + ...process.env, + TSX_TSCONFIG_PATH: 'tsconfig.public.json', + }, + ) + } + + if (vitestTests.length > 0) { + await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))]) + } + + await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py']) -if (nodeTests.length > 0) { - await run( - process.execPath, - ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))], - { - ...process.env, - TSX_TSCONFIG_PATH: 'tsconfig.public.json', - }, + console.log( + `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`, ) } -if (vitestTests.length > 0) { - await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))]) +if (path.resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) { + await main() } - -await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py']) - -console.log( - `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`, -) diff --git a/bench/scripts/run-package-tests.test.mjs b/bench/scripts/run-package-tests.test.mjs new file mode 100644 index 00000000..031ceb15 --- /dev/null +++ b/bench/scripts/run-package-tests.test.mjs @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { resolvePackageTestTimeoutMs, run } from './run-package-tests.mjs' + +test('package test timeout is optional and caller-controlled', () => { + assert.equal(resolvePackageTestTimeoutMs({}), undefined) + assert.equal(resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '0' }), 0) + assert.equal(resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '900000' }), 900_000) + + for (const value of ['', '-1', '1.5', 'Infinity', '9007199254740992']) { + assert.throws( + () => resolvePackageTestTimeoutMs({ AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: value }), + /must be a non-negative safe integer/, + ) + } +}) + +test('package test timeout reaches the child process', async () => { + const env = { + ...process.env, + AGENT_BENCH_PACKAGE_TEST_TIMEOUT_MS: '100', + } + const startedAt = Date.now() + + await assert.rejects( + run(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], env), + (error) => { + assert.equal(error.cause?.killed, true) + return true + }, + ) + assert.ok(Date.now() - startedAt < 2_000) +}) From 753c0a0893f4a6887213059b1bd7b928919f0282 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:24:17 -0600 Subject: [PATCH 04/39] fix(runtime): migrate exact execution consumers --- CHANGELOG.md | 3 +- bench/CHANGELOG.md | 2 +- examples/graphs/analyst-agent-review.ts | 8 +- examples/graphs/best-of-n.ts | 8 +- examples/graphs/collaborates-review-loop.ts | 8 +- examples/graphs/shared.ts | 10 ++ examples/graphs/shot-loop-resumed.ts | 6 +- examples/graphs/shot-loop.ts | 6 +- examples/graphs/user-sim-conversation.ts | 13 +- examples/graphs/watchdog-steer.ts | 6 +- pnpm-lock.yaml | 33 ++--- pnpm-workspace.yaml | 2 +- src/runtime/define-leaderboard.test.ts | 129 ++++++++++---------- src/runtime/define-leaderboard.ts | 64 +++------- src/runtime/environment-provider.test.ts | 61 +++++---- src/runtime/stdio-mcp-client.test.ts | 26 +++- src/runtime/structural-rollout.test.ts | 7 +- tests/fixtures/stdio-mcp-server.cjs | 40 ++++++ tests/helpers/improvement-method-fixture.ts | 7 +- tests/helpers/resume-driver-child.ts | 14 ++- tests/optimization-receipt.test.ts | 16 ++- tests/profile-improvement-stack.test.ts | 9 +- tests/runtime-observe.test.ts | 92 +++++++------- tests/runtime/cli-executor-metering.test.ts | 23 ++-- tests/runtime/mid-flight-steering.test.ts | 73 +++++------ tests/runtime/sandbox-run.test.ts | 16 ++- tests/runtime/supervisor-finalizer.test.ts | 20 ++- tests/sandbox-act.test.ts | 4 +- 28 files changed, 396 insertions(+), 310 deletions(-) create mode 100644 tests/fixtures/stdio-mcp-server.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index d1ee9da4..8902a061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## 0.128.1 -- Require Agent Eval 0.144.1 and route its official-optimizer callback through Runtime's exact `AgentProfile` execution path. +- Require Agent Eval 0.144.1 and Agent Knowledge 7.0.9, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. - Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. +- Require `defineLeaderboard` callers to supply an exact `baseProfile`; remove its `modelBackend` override so each matrix cell's profile remains the only model authority. - Keep missing token usage and billed cost unknown, and report reasoning-token usage when the provider supplies it. ## 0.128.0 diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index a58e85f7..4c2507e4 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.7.2 -- Consume Runtime 0.128.1, Eval 0.144.1, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. +- Consume Runtime 0.128.1, Eval 0.144.1, Knowledge 7.0.9, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. ## 0.7.1 diff --git a/examples/graphs/analyst-agent-review.ts b/examples/graphs/analyst-agent-review.ts index 6ddf9b98..91bec09e 100644 --- a/examples/graphs/analyst-agent-review.ts +++ b/examples/graphs/analyst-agent-review.ts @@ -22,7 +22,7 @@ import { type RunGraphOptions, runGraph, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger, scriptedBrain } from './shared' +import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') const report = promptHandle('analyzes/findings-report/v1') @@ -31,11 +31,11 @@ export function analystAgentReview(): { graph: AgentGraph; opts: RunGraphOptions // ── The topology: plain data (the analyst is the 'reviewer' NODE, not a registry lens) ── const graph: AgentGraph = { nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Drive the build.' } } }, - { id: 'implementer', profile: { name: 'implementer', prompt: { systemPrompt: 'Build.' } } }, + { id: 'driver', profile: offlineProfile('driver', 'Drive the build.') }, + { id: 'implementer', profile: offlineProfile('implementer', 'Build.') }, { id: 'reviewer', - profile: { name: 'reviewer', prompt: { systemPrompt: 'Review the trace evidence.' } }, + profile: offlineProfile('reviewer', 'Review the trace evidence.'), }, ], edges: [ diff --git a/examples/graphs/best-of-n.ts b/examples/graphs/best-of-n.ts index 0559b31c..c8b66db8 100644 --- a/examples/graphs/best-of-n.ts +++ b/examples/graphs/best-of-n.ts @@ -17,7 +17,7 @@ import { type RunGraphOptions, runGraph, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger, scriptedBrain } from './shared' +import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -25,9 +25,9 @@ export function bestOfN(): { graph: AgentGraph; opts: RunGraphOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ - { id: 'lead', profile: { name: 'lead', prompt: { systemPrompt: 'Keep the best.' } } }, - { id: 'coder-a', profile: { name: 'coder-a', prompt: { systemPrompt: 'Minimal diff.' } } }, - { id: 'coder-b', profile: { name: 'coder-b', prompt: { systemPrompt: 'Full rewrite.' } } }, + { id: 'lead', profile: offlineProfile('lead', 'Keep the best.') }, + { id: 'coder-a', profile: offlineProfile('coder-a', 'Minimal diff.') }, + { id: 'coder-b', profile: offlineProfile('coder-b', 'Full rewrite.') }, ], edges: [ { kind: 'delegates', from: 'lead', to: 'coder-a', directive: brief }, diff --git a/examples/graphs/collaborates-review-loop.ts b/examples/graphs/collaborates-review-loop.ts index 1c40e5ad..af033615 100644 --- a/examples/graphs/collaborates-review-loop.ts +++ b/examples/graphs/collaborates-review-loop.ts @@ -26,7 +26,7 @@ import { type RunGraphOptions, runGraph, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger, scriptedBrain } from './shared' +import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') const report = promptHandle('analyzes/findings-report/v1') @@ -47,9 +47,9 @@ export function collaboratesReviewLoop(): { graph: AgentGraph; opts: RunGraphOpt // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Drive the loop.' } } }, - { id: 'implementer', profile: { name: 'implementer', prompt: { systemPrompt: 'Build.' } } }, - { id: 'reviewer', profile: { name: 'reviewer', prompt: { systemPrompt: 'Review.' } } }, + { id: 'driver', profile: offlineProfile('driver', 'Drive the loop.') }, + { id: 'implementer', profile: offlineProfile('implementer', 'Build.') }, + { id: 'reviewer', profile: offlineProfile('reviewer', 'Review.') }, ], edges: [ { kind: 'delegates', from: 'driver', to: 'implementer', directive: brief }, diff --git a/examples/graphs/shared.ts b/examples/graphs/shared.ts index 2b8cb124..6a0d56de 100644 --- a/examples/graphs/shared.ts +++ b/examples/graphs/shared.ts @@ -28,6 +28,16 @@ import { type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' +/** Exact profile identity for the examples' injected, deterministic executors. */ +export function offlineProfile(name: string, systemPrompt: string): AgentProfile { + return { + name, + harness: 'cli-base', + model: { provider: 'offline', default: `offline/${name}` }, + prompt: { systemPrompt }, + } +} + // ── The scripted driver brain ────────────────────────────────────────────────── /** A scripted driver turn in the easy-to-write form (parsed tool args). */ diff --git a/examples/graphs/shot-loop-resumed.ts b/examples/graphs/shot-loop-resumed.ts index e28eee4f..ddad4015 100644 --- a/examples/graphs/shot-loop-resumed.ts +++ b/examples/graphs/shot-loop-resumed.ts @@ -22,7 +22,7 @@ import { runGraph, type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger, scriptedBrain } from './shared' +import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -34,8 +34,8 @@ export function shotLoopResumed(): { // ── The topology: plain data — continuity is one field on the edge ── const graph: AgentGraph = { nodes: [ - { id: 'reviewer', profile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } } }, - { id: 'coder', profile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } } }, + { id: 'reviewer', profile: offlineProfile('reviewer', 'Verify.') }, + { id: 'coder', profile: offlineProfile('coder', 'Make tests pass.') }, ], edges: [ { diff --git a/examples/graphs/shot-loop.ts b/examples/graphs/shot-loop.ts index 0bca6d11..6373ff6c 100644 --- a/examples/graphs/shot-loop.ts +++ b/examples/graphs/shot-loop.ts @@ -22,7 +22,7 @@ import { type RunGraphOptions, runGraph, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger, scriptedBrain } from './shared' +import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') const report = promptHandle('analyzes/findings-report/v1') @@ -37,8 +37,8 @@ export function shotLoop(): { graph: AgentGraph; opts: RunGraphOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ - { id: 'reviewer', profile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } } }, - { id: 'coder', profile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } } }, + { id: 'reviewer', profile: offlineProfile('reviewer', 'Verify.') }, + { id: 'coder', profile: offlineProfile('coder', 'Make tests pass.') }, ], edges: [ { kind: 'delegates', from: 'reviewer', to: 'coder', directive: brief, maxTraversals: 3 }, diff --git a/examples/graphs/user-sim-conversation.ts b/examples/graphs/user-sim-conversation.ts index 388d7ac0..c186aae7 100644 --- a/examples/graphs/user-sim-conversation.ts +++ b/examples/graphs/user-sim-conversation.ts @@ -24,7 +24,7 @@ import { runGraph, type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' -import { printLedger, scriptedBrain } from './shared' +import { offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -56,13 +56,10 @@ export function userSimConversation(): { nodes: [ { id: 'user-sim', - profile: { - name: 'user-sim', - prompt: { - systemPrompt: - 'You are Ada, a busy founder buying a team plan. Terse. SSO is non-negotiable.', - }, - }, + profile: offlineProfile( + 'user-sim', + 'You are Ada, a busy founder buying a team plan. Terse. SSO is non-negotiable.', + ), }, { id: 'product-agent', diff --git a/examples/graphs/watchdog-steer.ts b/examples/graphs/watchdog-steer.ts index 9776d1e4..b377537c 100644 --- a/examples/graphs/watchdog-steer.ts +++ b/examples/graphs/watchdog-steer.ts @@ -21,7 +21,7 @@ import { runGraph, type ToolLoopChat, } from '@tangle-network/agent-runtime/kernel' -import { leafSeam, printLedger } from './shared' +import { leafSeam, offlineProfile, printLedger } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -51,8 +51,8 @@ export function watchdogSteer(): { graph: AgentGraph; opts: RunGraphOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Watch and steer.' } } }, - { id: 'builder', profile: { name: 'builder', prompt: { systemPrompt: 'Build.' } } }, + { id: 'driver', profile: offlineProfile('driver', 'Watch and steer.') }, + { id: 'builder', profile: offlineProfile('builder', 'Build.') }, ], edges: [{ kind: 'delegates', from: 'driver', to: 'builder', directive: brief }], deliverable: { describe: 'the built artifact', check: (out) => out !== undefined }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 648ef453..20eb779c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,8 +16,8 @@ catalogs: specifier: 0.43.0 version: 0.43.0 '@tangle-network/agent-knowledge': - specifier: 7.0.8 - version: 7.0.8 + specifier: 7.0.9 + version: 7.0.9 '@tangle-network/agent-profile-materialize': specifier: 0.10.2 version: 0.10.2 @@ -49,7 +49,7 @@ importers: dependencies: '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.8 + version: 7.0.9 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.10.2(@tangle-network/agent-interface@0.43.0) @@ -125,7 +125,7 @@ importers: version: 0.43.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.8 + version: 7.0.9 '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1108,11 +1108,6 @@ packages: '@tangle-network/agent-core@0.4.33': resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-eval@0.143.0': - resolution: {integrity: sha512-Vx26rz0+qhb1QYd8YkSfoTgvdcoWtqAkR+4H7de4iu4mXKjLlefdWm5iRd4Nwqxgrd4R3Zrgm17N/b282mMCLQ==} - engines: {node: '>=20'} - hasBin: true - '@tangle-network/agent-eval@0.144.1': resolution: {integrity: sha512-OGwF96faPiiGzcb5QQnCvdzYdTCybJ4JBPLIjgTs21pBkJZkjb0Jqylcto4P31b3e3YWjryts0CALBSDr+0oLA==} engines: {node: '>=20'} @@ -1121,8 +1116,8 @@ packages: '@tangle-network/agent-interface@0.43.0': resolution: {integrity: sha512-t35nGOJ3sWouHoUM/5A8eUmsR+8JcuRF6VFQbT9cPzxIMUV3NsFMJgYXPGmyIj65vtvs73O6Mvijp47FAAGbuQ==} - '@tangle-network/agent-knowledge@7.0.8': - resolution: {integrity: sha512-FXA8KGCraUwX+2aNGWI8hZYAJ8f2T0iLBPbtzwbUoYl4t6h87munYoDQOh9YjarrAmkp6tcuGIGPKnw0FdCydg==} + '@tangle-network/agent-knowledge@7.0.9': + resolution: {integrity: sha512-Otlf8JqyKj+71m817cSDgjDtBZZalGw2F5fnK+YO5yyhc/EtP3s+aLf/9fkIxuq8dZwIlKFRDCJQJmBoq8o4cQ==} engines: {node: '>=20.19.0'} hasBin: true @@ -2803,18 +2798,6 @@ snapshots: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.143.0': - dependencies: - '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) - '@hono/node-server': 2.0.12(hono@4.12.32) - '@tangle-network/agent-core': 0.4.33 - '@tangle-network/agent-interface': 0.43.0 - '@tangle-network/agent-trace-contract': 1.0.2 - hono: 4.12.32 - linear-sum-assignment: 1.0.9 - re2js: 2.8.6 - zod: 4.4.3 - '@tangle-network/agent-eval@0.144.1': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) @@ -2833,9 +2816,9 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@7.0.8': + '@tangle-network/agent-knowledge@7.0.9': dependencies: - '@tangle-network/agent-eval': 0.143.0 + '@tangle-network/agent-eval': 0.144.1 '@tangle-network/agent-interface': 0.43.0 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 563b1d50..9e498731 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,7 +20,7 @@ catalog: '@types/node': 26.1.1 '@tangle-network/agent-eval': 0.144.1 '@tangle-network/agent-interface': 0.43.0 - '@tangle-network/agent-knowledge': 7.0.8 + '@tangle-network/agent-knowledge': 7.0.9 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': 0.17.3 diff --git a/src/runtime/define-leaderboard.test.ts b/src/runtime/define-leaderboard.test.ts index fded59a0..fb9f803d 100644 --- a/src/runtime/define-leaderboard.test.ts +++ b/src/runtime/define-leaderboard.test.ts @@ -1,9 +1,8 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { type DefaultVerdict, HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' -import type { AgentProfile } from '@tangle-network/agent-interface' -import type { CreateSandboxOptions, SandboxEvent } from '@tangle-network/sandbox' +import type { DefaultVerdict } from '@tangle-network/agent-eval' +import type { SandboxEvent } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import { defineLeaderboard, @@ -45,6 +44,11 @@ function board(overrides: Partial> cases: CASES, prompt: async (c) => `solve the task. answer=${c.answer}`, score: (output, c) => (output.includes(c.answer) ? 1 : 0), + baseProfile: { + name: 'fake-board', + harness: 'opencode', + model: { provider: 'offline', default: 'test-model@2026-01-01' }, + }, backends: { inproc: fakeBackend }, export: async () => {}, // silence the default table print in tests ...overrides, @@ -165,10 +169,7 @@ describe('defineLeaderboard', () => { expect(shots[1]?.info).toEqual({ index: 1, verdict: { score: 1 } }) }) - it('pins HARNESS_NATIVE_MODEL-snapped cells via the resolveModel seam', async () => { - // claude-code is vendor-locked to anthropic/*; a moonshot model snaps the - // axis to the 'default' sentinel, and the RunRecord then REQUIRES a - // dispatch-reported served model. + it('refuses a harness/model cell that expands to a runtime-selected marker before backend work', async () => { const snappedAxis = [ '--backend', 'inproc', @@ -177,69 +178,50 @@ describe('defineLeaderboard', () => { '--models', 'moonshot/kimi-k2@2026-01-01', ] - await expect(board().run([...snappedAxis, '--cases', 'case-alpha'])).rejects.toThrow( - /paid-call receipt/, - ) - - const creates: CreateSandboxOptions[] = [] - const result = await board({ - backends: { - inproc: () => { - const inner = fakeBackend() - return { - ...inner, - async create(options?: CreateSandboxOptions) { - creates.push(options ?? {}) - return inner.create(options) - }, - } + let backendFactories = 0 + let prompts = 0 + await expect( + board({ + backends: { + inproc: () => { + backendFactories += 1 + return fakeBackend() + }, }, - }, - resolveModel: (events) => { - // The served model rides the backend's own usage events — here the fake - // backend's llm_call stands in for the harness's terminal event. - const call = events.find((e) => (e as { type: string }).type === 'llm_call') - return call ? 'kimi-k2@2026-01-01' : undefined - }, - }).run([...snappedAxis, '--cases', 'case-alpha']) - expect(result.records[0]?.model).toBe('kimi-k2@2026-01-01') - expect(creates).toHaveLength(1) - expect(creates[0]?.backend?.model).toBeUndefined() - const executionProfile = creates[0]?.backend?.profile as AgentProfile - expect(executionProfile.model?.default).toBeUndefined() - expect(executionProfile.model?.default).not.toBe(HARNESS_NATIVE_MODEL) + prompt: (c) => { + prompts += 1 + return `solve the task. answer=${c.answer}` + }, + resolveModel: () => 'kimi-k2@2026-01-01', + }).run([...snappedAxis, '--cases', 'case-alpha']), + ).rejects.toThrow(/runtime-selected/) + expect(backendFactories).toBe(0) + expect(prompts).toBe(0) }) - it('sends the runtime-selected marker only to cli-bridge cells', async () => { - const creates: CreateSandboxOptions[] = [] - const inner = fakeBackend() - const result = await board({ - backends: { - 'cli-bridge': () => ({ - ...inner, - async create(options?: CreateSandboxOptions) { - creates.push(options ?? {}) - return inner.create(options) + it('does not make runtime-selected cells executable through the cli-bridge backend', async () => { + let backendFactories = 0 + await expect( + board({ + backends: { + 'cli-bridge': () => { + backendFactories += 1 + return fakeBackend() }, - }), - }, - resolveModel: () => 'kimi-k2@2026-01-01', - }).run([ - '--backend', - 'cli-bridge', - '--harnesses', - 'claude-code', - '--models', - 'moonshot/kimi-k2@2026-01-01', - '--cases', - 'case-alpha', - ]) - - expect(result.records[0]?.model).toBe('kimi-k2@2026-01-01') - expect(creates).toHaveLength(1) - expect(creates[0]?.backend?.model?.model).toBe(HARNESS_NATIVE_MODEL) - const executionProfile = creates[0]?.backend?.profile as AgentProfile - expect(executionProfile.model?.default).toBeUndefined() + }, + resolveModel: () => 'kimi-k2@2026-01-01', + }).run([ + '--backend', + 'cli-bridge', + '--harnesses', + 'claude-code', + '--models', + 'moonshot/kimi-k2@2026-01-01', + '--cases', + 'case-alpha', + ]), + ).rejects.toThrow(/runtime-selected/) + expect(backendFactories).toBe(0) }) it('flows a structured TArtifact through parseOutput → score → records natively', async () => { @@ -251,6 +233,11 @@ describe('defineLeaderboard', () => { name: 'structured-board', cases: CASES, prompt: async (c) => `solve the task. answer=${c.answer}`, + baseProfile: { + name: 'structured-board', + harness: 'opencode', + model: { provider: 'offline', default: 'test-model@2026-01-01' }, + }, parseOutput: (events): Structured => { const final = events.find((e) => (e as { type: string }).type === 'result') as | { data?: { finalText?: string } } @@ -286,6 +273,11 @@ describe('defineLeaderboard', () => { cases: CASES, prompt: (c) => c.id, score: () => 0, + baseProfile: { + name: 'no-backend', + harness: 'opencode', + model: { provider: 'offline', default: 'm@1' }, + }, }).run(['--models', 'm@1']), ).rejects.toThrow(/backends\.sandbox/) }) @@ -314,6 +306,11 @@ describe('defineLeaderboard', () => { cases: [CASES[0] as FakeCase, CASES[0] as FakeCase], prompt: (c) => c.id, score: () => 0, + baseProfile: { + name: 'dup', + harness: 'opencode', + model: { provider: 'offline', default: 'test-model@2026-01-01' }, + }, }).toBenchmarkAdapter() await expect(dup.preflight()).rejects.toThrow(/duplicate case id/) }) diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index 4a44226a..faa5bfd9 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -15,7 +15,7 @@ * * - LEVEL 0 (declarative): `cases` / `prompt` / `score` / `axis`. * - LEVEL 1 (seams): `backends`, `flags`, `parseOutput`, `onCellEvents`, - * `resolveModel`, `setup`/`teardown`, `export`, `modelBackend`, `matrix` + * `resolveModel`, `setup`/`teardown`, `export`, `matrix` * passthrough. * - LEVEL 2 (replacement): `dispatch` and `judges` swap out the whole * loop wiring or scoring; `runProfileMatrix` itself stays public as the @@ -38,7 +38,6 @@ import { CODING_HARNESSES, expandProfileAxes, type HarnessType, - harnessAxisOf, type MaximumCharge, } from '@tangle-network/agent-eval' import { @@ -53,7 +52,7 @@ import { collectAgentResponseText, type SandboxEvent } from '@tangle-network/san import { leaderboard, renderLeaderboardMarkdown } from './benchmark-report' import { loopDispatch } from './loop-dispatch' import { resolveSandboxClient } from './resolve-sandbox-client' -import { isHarnessNativeModel } from './supervise/model-policy' +import { assertExecutableAgentProfile } from './supervise/model-policy' import type { Driver, LoopResult, SandboxClient } from './types' /** Structured per-case verdict a `score` function may return (a bare number is @@ -200,9 +199,10 @@ export interface LeaderboardSpec { * `CODING_HARNESSES` × the base profile's `model.default`. `--harnesses` / * `--models` override per run. */ axis?: { harnesses?: readonly HarnessType[]; models?: readonly string[] } - /** Base profile the axes expand over (prompt/tools/skills held fixed). - * Default: a minimal `{ name, model: { default: } }`. */ - baseProfile?: AgentProfile + /** Exact base profile the axes expand over (prompt/tools/skills held fixed). + * Its provider remains authoritative while each axis cell replaces the + * harness and concrete model. */ + baseProfile: AgentProfile /** * Execution-backend registry: `--backend ` picks the factory that * yields the `SandboxClient` every cell runs on. Merged over the defaults: @@ -215,10 +215,6 @@ export interface LeaderboardSpec { backends?: Record SandboxClient) | undefined> /** Extra `--flag value` CLI args `run()` parses and surfaces via `ctx.args`. */ flags?: Record - /** Extra fields merged into each cell's `backend.model` create override — - * e.g. `{ provider: 'openai-compat', apiKey, baseUrl }` for a router-backed - * sandbox. The cell's bare model id is set by the facade from the axis. */ - modelBackend?: Record /** Runs once before the matrix (fetch fixtures, warm caches). */ setup?: (ctx: LeaderboardRunContext) => Promise | void /** Runs once after the matrix, even on failure (reap boxes, close handles). */ @@ -240,13 +236,10 @@ export interface LeaderboardSpec { * this (or a LEVEL-2 `dispatch`). */ parseOutput?: (events: readonly SandboxEvent[], c: TCase) => TArtifact /** - * Resolve the model the backend ACTUALLY served off a shot's raw events. - * Required for HARNESS_NATIVE_MODEL-snapped cells (a vendor-locked harness × - * an out-of-family model expands to the `default` sentinel): the RunRecord - * must pin a real snapshot-bearing model id, which only the dispatch — - * reading the backend's usage/terminal events — can know. When this returns - * a value the default dispatch records it on the paid-call receipt; - * in-family cells (concrete declared model) never need it. + * Resolve the model the backend actually served from a shot's raw events. + * When this returns a value the default dispatch records it on the paid-call + * receipt. It cannot complete an inexact planning profile: every expanded + * cell must already declare a concrete model before backend work starts. */ resolveModel?: (events: readonly SandboxEvent[]) => string | undefined /** Result export. Default: write `matrix-result.json` under the run dir and @@ -316,11 +309,6 @@ function withSnapshot(model: string, snapshot: string): string { return model.includes('@') ? model : `${model}@${snapshot}` } -/** The bare model id the backend actually serves (identity snapshot stripped). */ -function bareModel(model: string): string { - return model.split('@')[0] ?? model -} - function gitSha(): string { try { return execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() @@ -438,10 +426,10 @@ export function defineLeaderboard( } const models = rawModels.map((m) => withSnapshot(m, snapshot)) - const base: AgentProfile = - spec.baseProfile ?? - ({ name: spec.name, model: { default: bareModel(models[0] ?? '') } } as AgentProfile) - const profiles = expandProfileAxes({ base, harnesses, models }) + const profiles = expandProfileAxes({ base: spec.baseProfile, harnesses, models }) + for (const profile of profiles) { + assertExecutableAgentProfile(profile, `defineLeaderboard(${spec.name})`) + } const ctx: LeaderboardRunContext = { name: spec.name, @@ -511,8 +499,7 @@ export function defineLeaderboard( // per-cell resource cost) so the loop's finished iterations can be joined // with the campaign ctx: onCellEvents gets EVERY shot's outcome (a thrown // shot never reaches parse, so parse-time tapping would hide it), and a - // the cost receipt records the spec-resolved served model (the only way to - // pin HARNESS_NATIVE_MODEL-snapped cells to a real model). + // the cost receipt records the spec-resolved served model. const maximumCharge = spec.maximumCharge const dispatch: ProfileDispatchFn, TArtifact> = spec.dispatch ?? ((profile, scenario, dispatchCtx) => { @@ -543,17 +530,6 @@ export function defineLeaderboard( return [...served][0] ?? cellProfile.model?.default }, toLoopOptions: (cellScenario, cellProfile) => { - // The cell's harness + model come off the profile's axis stamp set - // by expandProfileAxes; the sandbox create override carries them to - // whichever backend client runs the cell. - const axis = harnessAxisOf(cellProfile) - const modelId = bareModel(axis?.model ?? models[0] ?? '') - const backendModel = { - ...spec.modelBackend, - ...(!isHarnessNativeModel(modelId) || backendName === 'cli-bridge' - ? { model: modelId } - : {}), - } return { // The no-signal retry floor: re-run the same case as an independent attempt until // one scores (>0) or the shot cap. The task is re-run verbatim — no grader findings @@ -562,16 +538,6 @@ export function defineLeaderboard( agentRun: { profile: cellProfile, taskToPrompt: (s) => `${promptOf(s)}\n\n`, - ...(axis - ? { - sandboxOverrides: { - backend: { - type: axis.harness, - ...(Object.keys(backendModel).length > 0 ? { model: backendModel } : {}), - }, - } as never, - } - : {}), }, output: { parse: (events) => diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index b50a647d..4f628540 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -619,7 +619,11 @@ describe('environment provider adapters', () => { }, }) const spec: AgentSpec = { - profile: { name: 'pi-worker', harness: 'pi' }, + profile: { + name: 'pi-worker', + harness: 'pi', + model: { provider: 'offline', default: 'offline-test-model' }, + }, harness: null, } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } @@ -773,6 +777,7 @@ describe('environment provider adapters', () => { instructions: ['Keep exact evidence.'], }, model: { + provider: 'zai', default: 'zai/glm-5.2', reasoningEffort: 'high', }, @@ -845,11 +850,12 @@ describe('environment provider adapters', () => { }) expect(usage).toEqual([ { kind: 'tokens', input: 2, output: 7 }, - { kind: 'cost', usd: 0.1 }, + { kind: 'cost', usd: 0.1, usdKnown: false }, { kind: 'iteration' }, { kind: 'tokens', input: 5, output: 13 }, - { kind: 'cost', usd: 0.2 }, + { kind: 'cost', usd: 0.2, usdKnown: false }, { kind: 'iteration' }, + { kind: 'cost', usd: 0, usdKnown: false }, ]) expect(turns).toHaveLength(3) expect(turns[0]).toMatchObject({ prompt: 'investigate' }) @@ -952,7 +958,7 @@ describe('environment provider adapters', () => { profile: { name: 'normalized-worker', harness: 'pi', - metadata: { backendType: 'codex' }, + model: { provider: 'offline', default: 'offline-test-model' }, }, harness: null, } @@ -1017,13 +1023,14 @@ describe('environment provider adapters', () => { const spec: AgentSpec = { profile: { name: 'missing-harness', + model: { provider: 'offline', default: 'offline-test-model' }, metadata: { backendType: 'pi' }, }, harness: null, } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } - expect(() => factory(spec, ctx)).toThrow(/AgentProfile\.harness is required/) + expect(() => factory(spec, ctx)).toThrow(/AgentProfile\.harness must be explicit/) expect({ capabilityCalls, createCalls }).toEqual({ capabilityCalls: 0, createCalls: 0 }) }) @@ -1055,7 +1062,11 @@ describe('environment provider adapters', () => { }, }) const spec: AgentSpec = { - profile: { name: 'pi-worker', harness: 'pi' }, + profile: { + name: 'pi-worker', + harness: 'pi', + model: { provider: 'offline', default: 'offline-test-model' }, + }, harness: 'specHarness' in conflict ? conflict.specHarness : null, } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } @@ -1077,7 +1088,14 @@ describe('environment provider adapters', () => { }, } const factory = createExecutor({ backend: 'provider', provider }) - const spec: AgentSpec = { profile: { name: 'worker' } as AgentProfile, harness: null } + const spec: AgentSpec = { + profile: { + name: 'worker', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + harness: null, + } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } const executor = factory(spec, ctx) @@ -1086,7 +1104,7 @@ describe('environment provider adapters', () => { expect(executor.resultArtifact().out).toMatchObject({ content: 'from-package' }) }) - it('keeps the runtime-selected model marker out of provider.create', async () => { + it('refuses the runtime-selected model marker before provider.create', () => { let createdProfile: AgentProfile | string | undefined let taskProfile: AgentProfile | undefined const provider: AgentEnvironmentProvider = { @@ -1104,7 +1122,6 @@ describe('environment provider adapters', () => { const factory = createExecutor({ backend: 'provider', provider, - profileForCreate: (profile) => ({ ...profile, description: 'create-only transform' }), taskToTurn: (task, profile) => { taskProfile = profile return { prompt: String(task) } @@ -1112,6 +1129,7 @@ describe('environment provider adapters', () => { }) const profile: AgentProfile = { name: 'runtime-model-worker', + harness: 'pi', model: { provider: 'tangle-router', default: ` ${HARNESS_NATIVE_MODEL} `, @@ -1120,17 +1138,9 @@ describe('environment provider adapters', () => { } const spec: AgentSpec = { profile, harness: null } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } - const executor = factory(spec, ctx) - - await collect(executor.execute('task', ctx.signal) as AsyncIterable) - - expect(createdProfile).toMatchObject({ - name: 'runtime-model-worker', - description: 'create-only transform', - model: { provider: 'tangle-router', reasoningEffort: 'high' }, - }) - expect((createdProfile as AgentProfile).model?.default).toBeUndefined() - expect(taskProfile).toBe(profile) + expect(() => factory(spec, ctx)).toThrow(/model\.default is runtime-selected/) + expect(createdProfile).toBeUndefined() + expect(taskProfile).toBeUndefined() }) it('resolves a named provider through the runtime registry', async () => { @@ -1157,14 +1167,21 @@ describe('environment provider adapters', () => { workspace: { cwd: '/repo' }, }, }) - const spec: AgentSpec = { profile: { name: 'worker' } as AgentProfile, harness: null } + const spec: AgentSpec = { + profile: { + name: 'worker', + harness: 'codex', + model: { provider: 'openai', default: 'offline-test-model' }, + }, + harness: null, + } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } const executor = factory(spec, ctx) await collect(executor.execute('task', ctx.signal) as AsyncIterable) expect(created).toMatchObject({ - profile: { name: 'worker' }, + profile: spec.profile, backend: 'codex', workspace: { cwd: '/repo' }, }) diff --git a/src/runtime/stdio-mcp-client.test.ts b/src/runtime/stdio-mcp-client.test.ts index 19475448..edc3bb61 100644 --- a/src/runtime/stdio-mcp-client.test.ts +++ b/src/runtime/stdio-mcp-client.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { fileURLToPath } from 'node:url' import { type AgentProfile, defineAgentProfilePublicConfig, @@ -35,6 +36,10 @@ require('node:fs').writeFileSync(process.env.MARKER_PATH, process.env.TEST_TOKEN ${HELLO_SERVER} ` +const PUBLIC_MCP_SERVER = fileURLToPath( + new URL('../../tests/fixtures/stdio-mcp-server.cjs', import.meta.url), +) + const ENV_BOUNDARY_SERVER = ` require('node:fs').writeFileSync(process.env.MARKER_PATH, JSON.stringify({ declared: process.env.DECLARED_VALUE, @@ -205,13 +210,16 @@ describe('materializeLocalMcp', () => { const marker = join(dir, 'received.txt') const get = vi.fn(async (name: string) => (name === 'GREETER_TOKEN' ? 'test-token' : undefined)) const profile: AgentProfile = { + name: 'secret-mcp-worker', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, mcp: { greeter: { transport: 'stdio', command: 'node', - args: [pub('-e'), pub(SECRET_SERVER)], - env: { MARKER_PATH: pub(marker) }, - metadata: { secretEnv: { TEST_TOKEN: 'GREETER_TOKEN' } }, + args: [pub(PUBLIC_MCP_SERVER)], + env: { MARKER_PATH: pub(marker), WRITE_MARKER: pub('1') }, + metadata: { secretEnv: { MESSAGE: 'GREETER_TOKEN' } }, }, }, } @@ -304,11 +312,14 @@ describe('materializeLocalMcp', () => { const dir = mkdtempSync(join(tmpdir(), 'local-mcp-dynamic-')) const marker = join(dir, 'must-not-exist.txt') const trustedProfile: AgentProfile = { + name: 'trusted-mcp-worker', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, mcp: { trusted: { transport: 'stdio', command: 'node', - args: [pub('-e'), pub(HELLO_SERVER)], + args: [pub(PUBLIC_MCP_SERVER)], }, }, } @@ -318,12 +329,15 @@ describe('materializeLocalMcp', () => { profileSecurityPolicy: TRUSTED_LOCAL_MCP_POLICY, }) const dynamicProfile: AgentProfile = { + name: 'dynamic-mcp-worker', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, mcp: { untrusted: { transport: 'stdio', command: 'node', - args: [pub('-e'), pub(SECRET_SERVER)], - env: { MARKER_PATH: pub(marker) }, + args: [pub(PUBLIC_MCP_SERVER)], + env: { MARKER_PATH: pub(marker), WRITE_MARKER: pub('1') }, }, }, } diff --git a/src/runtime/structural-rollout.test.ts b/src/runtime/structural-rollout.test.ts index 8629cf78..c0d8ebce 100644 --- a/src/runtime/structural-rollout.test.ts +++ b/src/runtime/structural-rollout.test.ts @@ -281,7 +281,12 @@ describe('structuralRollout — the strategy, end to end (offline transport, fak routerKey: 'k', workerProfile: { name: 'starved-worker', - model: { default: 'stub-model', metadata: { maxTokens: 64 } }, + harness: 'cli-base', + model: { + provider: 'offline', + default: 'stub-model', + metadata: { maxTokens: 64 }, + }, prompt: { systemPrompt: 'Solve it.' }, tools: {}, }, diff --git a/tests/fixtures/stdio-mcp-server.cjs b/tests/fixtures/stdio-mcp-server.cjs new file mode 100644 index 00000000..2147382a --- /dev/null +++ b/tests/fixtures/stdio-mcp-server.cjs @@ -0,0 +1,40 @@ +const fs = require('node:fs') +const readline = require('node:readline') + +if (process.env.WRITE_MARKER === '1') { + fs.writeFileSync(process.env.MARKER_PATH, process.env.MESSAGE || 'missing') +} + +const rl = readline.createInterface({ input: process.stdin }) +rl.on('line', (line) => { + let message + try { + message = JSON.parse(line) + } catch { + return + } + if (message.id === undefined) return + const reply = (result) => { + process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', id: message.id, result })}\n`) + } + if (message.method === 'initialize') { + reply({ + protocolVersion: '2024-11-05', + capabilities: {}, + serverInfo: { name: 'hello', version: '0' }, + }) + } else if (message.method === 'tools/list') { + reply({ + tools: [ + { + name: 'hello', + description: 'say hello', + inputSchema: { type: 'object', properties: { name: { type: 'string' } } }, + }, + ], + }) + } else if (message.method === 'tools/call') { + const name = message.params?.arguments?.name || 'world' + reply({ content: [{ type: 'text', text: `hello ${name}` }] }) + } +}) diff --git a/tests/helpers/improvement-method-fixture.ts b/tests/helpers/improvement-method-fixture.ts index 28293878..1b9b98c5 100644 --- a/tests/helpers/improvement-method-fixture.ts +++ b/tests/helpers/improvement-method-fixture.ts @@ -93,7 +93,12 @@ export const improvementMethod: ImproveMethodFactory, } @@ -119,7 +127,7 @@ const brain: ToolLoopChat = async (messages) => { id: `spawn-${i}`, name: 'spawn_agent', arguments: JSON.stringify({ - profile: { name: w.key }, + profile: offlineProfile(w.key), task: `do ${w.key}`, label: w.key, key: w.key, @@ -139,7 +147,7 @@ const brain: ToolLoopChat = async (messages) => { } } -const result = await supervise({ name: 'root', harness: 'cli-base' }, 'five assignments', { +const result = await supervise(offlineProfile('root'), 'five assignments', { budget: { maxIterations: 200, maxTokens: 500_000 }, // Explicit per-worker ceiling: the default is a quarter of the pool, which would starve the // fifth spawn and make this a four-worker test. diff --git a/tests/optimization-receipt.test.ts b/tests/optimization-receipt.test.ts index d505381c..0f89f401 100644 --- a/tests/optimization-receipt.test.ts +++ b/tests/optimization-receipt.test.ts @@ -30,7 +30,12 @@ const officialImprovementMethod: ImproveMethodFactory ({ name: 'incident-responder', + harness: 'cli-base', + model: { provider: 'offline', default: 'deterministic-profile-stack' }, prompt: { systemPrompt: 'Handle the task directly.' }, }) @@ -49,7 +51,12 @@ const improvingMethod: OptimizationMethod = return { winnerSurface: 'Handle the task directly.\n\nREPAIR_ON_FAILURE: after a failed draft, revise once using the failure signal.', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } diff --git a/tests/runtime-observe.test.ts b/tests/runtime-observe.test.ts index ddc21a92..270a3883 100644 --- a/tests/runtime-observe.test.ts +++ b/tests/runtime-observe.test.ts @@ -1,32 +1,40 @@ -import type { ChatClient } from '@tangle-network/agent-eval' import { describe, expect, it } from 'vitest' import { observe } from '../src/runtime/observe' +const observerProfile = { + name: 'test-observer', + harness: 'cli-base' as const, + model: { provider: 'offline', default: 'observer-test' }, +} + +function observerExecutor(content: string) { + return { + backend: 'router' as const, + routerBaseUrl: 'http://offline.invalid/v1', + routerKey: 'offline-test', + complete: async () => ({ + model: 'observer-test', + choices: [{ message: { content }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 5, cost: 0 }, + }), + } +} + describe('runtime observe', () => { it('marks observed production behavior as proposal input from production', async () => { - const chat: ChatClient = { - transport: 'mock', - chat: async () => ({ - content: JSON.stringify({ - findings: [ - { - area: 'verification', - severity: 'high', - claim: 'The worker returned before running the requested check.', - recommended_action: 'Run the requested check before returning.', - audience: 'agent', - confidence: 0.9, - }, - ], - }), - usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, - costUsd: 0, - model: 'observer-test', - durationMs: 1, - raw: {}, - }), - } + const content = JSON.stringify({ + findings: [ + { + area: 'verification', + severity: 'high', + claim: 'The worker returned before running the requested check.', + recommended_action: 'Run the requested check before returning.', + audience: 'agent', + confidence: 0.9, + }, + ], + }) const result = await observe( { @@ -35,7 +43,7 @@ describe('runtime observe', () => { trace: [{ type: 'status', data: { status: 'completed' } }], runId: 'production-run-1', }, - { chat }, + { profile: observerProfile, executor: observerExecutor(content) }, ) expect(result.findings).toHaveLength(1) @@ -48,28 +56,18 @@ describe('runtime observe', () => { }) it('rejects malformed model findings before returning them to callers', async () => { - const chat: ChatClient = { - transport: 'mock', - chat: async () => ({ - content: JSON.stringify({ - findings: [ - { - area: 'verification', - severity: 'urgent', - claim: 'The worker skipped the requested check.', - recommended_action: 'Run the requested check.', - audience: 'agent', - confidence: 0.9, - }, - ], - }), - usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, - costUsd: 0, - model: 'observer-test', - durationMs: 1, - raw: {}, - }), - } + const content = JSON.stringify({ + findings: [ + { + area: 'verification', + severity: 'urgent', + claim: 'The worker skipped the requested check.', + recommended_action: 'Run the requested check.', + audience: 'agent', + confidence: 0.9, + }, + ], + }) await expect( observe( @@ -78,7 +76,7 @@ describe('runtime observe', () => { output: 'Changed the code.', trace: [{ type: 'status', data: { status: 'completed' } }], }, - { chat }, + { profile: observerProfile, executor: observerExecutor(content) }, ), ).rejects.toThrow(/observe findings: every finding must match AnalystFinding/) }) diff --git a/tests/runtime/cli-executor-metering.test.ts b/tests/runtime/cli-executor-metering.test.ts index 66bef446..61281745 100644 --- a/tests/runtime/cli-executor-metering.test.ts +++ b/tests/runtime/cli-executor-metering.test.ts @@ -3,7 +3,14 @@ import { createBudgetPool } from '../../src/runtime/supervise/budget' import { createExecutor } from '../../src/runtime/supervise/runtime' import type { AgentSpec, ExecutorContext, UsageEvent } from '../../src/runtime/supervise/types' -const spec: AgentSpec = { profile: { name: 'raw-cli-worker' }, harness: null } +const spec: AgentSpec = { + profile: { + name: 'raw-cli-worker', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + harness: null, +} const context: ExecutorContext = { signal: new AbortController().signal, seams: {} } async function drain(stream: AsyncIterable): Promise { @@ -54,13 +61,7 @@ describe('cli backend reports unmetered work as unknown, never as measured zero' await executor.teardown('brutalKill') }) - it('does NOT mark the dollar channel, which would refuse the exemption it was granted', async () => { - // A deliberate boundary, pinned so it reads as a decision rather than a missed field. - // `usdKnown: false` is not a marker on this channel but a REFUSAL: under a dollar-capped root - // `budget.ts` treats it as a reconcile violation and fails the child. `backend: 'cli'` is - // `budgetExempt`, i.e. the kernel already agreed to settle it OUT of the conserved pool, so - // marking it would make an explicitly-exempt worker fail after its work had already burned. - // Changing that is a policy decision about allowed configurations, not a reporting fix. + it('marks unknown dollars so a dollar-capped pool refuses an unmetered CLI result', async () => { const pool = createBudgetPool({ maxIterations: 4, maxTokens: 1_000, maxUsd: 5 }) const reservation = pool.reserve({ maxIterations: 1, maxTokens: 100, maxUsd: 1 }) if (!reservation.ok) throw new Error(`reservation rejected: ${reservation.reason}`) @@ -70,10 +71,8 @@ describe('cli backend reports unmetered work as unknown, never as measured zero' ) const { spent } = executor.resultArtifact() - expect(spent.usdKnown).toBeUndefined() - expect(() => pool.reconcile(reservation.ticket, spent)).not.toThrow() - // The token taint still reaches the readout, so the accounting is not silently trusted. - expect(pool.readout().tokensKnown).toBe(false) + expect(spent.usdKnown).toBe(false) + expect(() => pool.reconcile(reservation.ticket, spent)).toThrow(/unknown dollar cost/) await executor.teardown('brutalKill') }) }) diff --git a/tests/runtime/mid-flight-steering.test.ts b/tests/runtime/mid-flight-steering.test.ts index 21812a03..c476d0d1 100644 --- a/tests/runtime/mid-flight-steering.test.ts +++ b/tests/runtime/mid-flight-steering.test.ts @@ -38,6 +38,17 @@ const STEER = `stop editing ${WRONG} — the change belongs in ${RIGHT}` const ANSWER = `continue in ${RIGHT}` const budget: Budget = { maxIterations: 200, maxTokens: 400_000 } +const rootProfile = { + name: 'root', + harness: 'cli-base' as const, + model: { provider: 'offline', default: 'offline/supervisor' }, + prompt: { systemPrompt: 'drive one coder and correct it' }, +} +const coderProfile = { + name: 'coder', + harness: 'opencode' as const, + model: { provider: 'offline', default: 'offline/coder' }, +} /** A promise a test resolves by hand — how the fake harness is held mid-turn until the driver * has actually observed it and sent its steer, so the assertion is about the worker's behavior @@ -161,7 +172,7 @@ function steeringBrain(harness: FakeHarness, record: BrainRecord): ToolLoopChat turn += 1 if (turn === 1) { - return call('spawn_agent', { profile: { name: 'coder' }, task: 'make the change' }) + return call('spawn_agent', { profile: coderProfile, task: 'make the change' }) } if (turn === 2) { workerId = String(parsed?.workerId ?? 'w') @@ -201,7 +212,7 @@ function missingMessageAuthorityBrain( turn += 1 if (turn === 1) { - return call('spawn_agent', { profile: { name: 'coder' }, task: 'make the change' }) + return call('spawn_agent', { profile: coderProfile, task: 'make the change' }) } if (turn === 2) { workerId = String(parsed?.workerId ?? workerId) @@ -266,38 +277,30 @@ interface AuthorityRecord { async function runSupervisedSteer(steerable: boolean, authority?: AuthorityRecord) { const harness = createFakeHarness() const record: BrainRecord = {} - const result = await supervise( - { - name: 'root', - harness: 'cli-base', - prompt: { systemPrompt: 'drive one coder and correct it' }, - }, - 'change the right module', - { - budget, - backend: backend(harness, steerable), - brain: steeringBrain(harness, record), - maxTurns: 8, - ...(authority - ? { - authorizeSpawn(input) { - authority.spawnCalls += 1 - return { profile: input.profile } - }, - authorizeMessage(input) { - authority.messages.push({ - instruction: input.instruction, - frozen: Object.isFrozen(input) && Object.isFrozen(input.workerIdentity), - hasIdentity: - input.workerIdentity.profileDigest !== undefined && - input.workerIdentity.taskDigest !== undefined, - }) - return { instruction: input.instruction } - }, - } - : {}), - }, - ) + const result = await supervise(rootProfile, 'change the right module', { + budget, + backend: backend(harness, steerable), + brain: steeringBrain(harness, record), + maxTurns: 8, + ...(authority + ? { + authorizeSpawn(input) { + authority.spawnCalls += 1 + return { profile: input.profile } + }, + authorizeMessage(input) { + authority.messages.push({ + instruction: input.instruction, + frozen: Object.isFrozen(input) && Object.isFrozen(input.workerIdentity), + hasIdentity: + input.workerIdentity.profileDigest !== undefined && + input.workerIdentity.taskDigest !== undefined, + }) + return { instruction: input.instruction } + }, + } + : {}), + }) return { harness, record, result } } @@ -307,7 +310,7 @@ describe('mid-flight steering — a supervisor observes a live worker and change const harness = createFakeHarness() const record: { steer?: Record; answer?: Record } = {} try { - await supervise({ name: 'root', harness: 'cli-base' }, 'change the right module', { + await supervise(rootProfile, 'change the right module', { budget, backend: backend(harness, true), brain: missingMessageAuthorityBrain(harness, record), diff --git a/tests/runtime/sandbox-run.test.ts b/tests/runtime/sandbox-run.test.ts index f2355de6..0f7d22d6 100644 --- a/tests/runtime/sandbox-run.test.ts +++ b/tests/runtime/sandbox-run.test.ts @@ -109,12 +109,24 @@ function createFakeClient(opts: FakeOpts = {}) { } function spec(name = 'w'): AgentRunSpec { - return { profile: { name }, name, taskToPrompt: (t) => t } + return { + profile: { + name, + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + name, + taskToPrompt: (t) => t, + } } function backendSpec(name = 'w'): AgentRunSpec { return { - profile: { name }, + profile: { + name, + harness: 'opencode', + model: { provider: 'offline', default: 'offline-test-model' }, + }, name, taskToPrompt: (t) => t, sandboxOverrides: { backend: { type: 'opencode' } }, diff --git a/tests/runtime/supervisor-finalizer.test.ts b/tests/runtime/supervisor-finalizer.test.ts index 8fb42728..e7160d99 100644 --- a/tests/runtime/supervisor-finalizer.test.ts +++ b/tests/runtime/supervisor-finalizer.test.ts @@ -37,6 +37,14 @@ import { scriptedBrain } from '../kernel/scripted-brain' const budget: Budget = { maxIterations: 100, maxTokens: 100_000 } +function offlineProfile(name: string): AgentProfile { + return { + name, + harness: 'cli-base', + model: { provider: 'offline', default: `offline/${name}` }, + } +} + const emptyTree: TreeView = { root: 'r', nodes: [], inFlight: 0, waiting: 0 } const poolReadout: Scope['budget'] = { tokensLeft: budget.maxTokens, @@ -217,7 +225,7 @@ function leaf(name: string, out: unknown, score: number, valid: boolean): Agent< spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 }, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: offlineProfile(name), harness: null, executor: ex } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -230,11 +238,11 @@ const twoWorkerScript = () => toolCalls: [ { name: 'spawn_agent', - arguments: { profile: { name: 'good' }, task: 'go', label: 'good' }, + arguments: { profile: offlineProfile('good'), task: 'go', label: 'good' }, }, { name: 'spawn_agent', - arguments: { profile: { name: 'unchecked' }, task: 'go', label: 'unchecked' }, + arguments: { profile: offlineProfile('unchecked'), task: 'go', label: 'unchecked' }, }, ], }, @@ -253,7 +261,7 @@ const makeWorker = (profile: unknown) => { describe('SupervisorFinalizer — end to end through supervise()', () => { it('the default keeps the delivered answer over a higher-scoring unchecked one', async () => { - const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { + const result = await supervise(offlineProfile('root'), 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: makeWorker, @@ -264,7 +272,7 @@ describe('SupervisorFinalizer — end to end through supervise()', () => { }) it('an opted-in collectDelivered changes the SHAPE without ever widening eligibility', async () => { - const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { + const result = await supervise(offlineProfile('root'), 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: makeWorker, @@ -278,7 +286,7 @@ describe('SupervisorFinalizer — end to end through supervise()', () => { }) it('a run whose only high scorer is unchecked is a no-winner, not a rescued output', async () => { - const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { + const result = await supervise(offlineProfile('root'), 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: () => leaf('unchecked', 'UNCHECKED-PROSE', 0.99, false), diff --git a/tests/sandbox-act.test.ts b/tests/sandbox-act.test.ts index 62d74f70..ddb91307 100644 --- a/tests/sandbox-act.test.ts +++ b/tests/sandbox-act.test.ts @@ -6,6 +6,8 @@ import type { OutputAdapter, SandboxClient } from '../src/runtime' const BASE = { name: 'demo-agent', + harness: 'opencode' as const, + model: { provider: 'offline', default: 'offline-test-model' }, prompt: { systemPrompt: 'base' }, mcp: { domain: { transport: 'stdio' as const, command: 'domain-mcp', enabled: true } }, } @@ -67,7 +69,7 @@ describe('createSandboxAct — prod-profile eval parity', () => { expect(events).toEqual([ { type: 'text_delta', text: 'Hel' }, { type: 'text_delta', text: 'lo' }, - { type: 'llm_call', model: 'gpt', tokensIn: 5, tokensOut: 3 }, + { type: 'llm_call', model: 'gpt', tokensIn: 5, tokensOut: 3, usdKnown: false }, ]) }) From a24e67580c088adcbb8ff23a2d57ae442c22e312 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:24:57 -0600 Subject: [PATCH 05/39] test(runtime): update examples for exact agent profiles --- tests/delegate-example.test.ts | 39 +++++++++++++++++++--- tests/profile-materialization.test.ts | 48 ++++++++++++++++----------- tests/supervisor-loop-example.test.ts | 16 ++++++--- 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/tests/delegate-example.test.ts b/tests/delegate-example.test.ts index 7f72db63..a05ba285 100644 --- a/tests/delegate-example.test.ts +++ b/tests/delegate-example.test.ts @@ -9,9 +9,10 @@ * green offline. The offline assertion (delegate fails loud without a brain/router) always runs. */ import { existsSync, readFileSync } from 'node:fs' +import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { fileDeliverable, makeWriteFileBackend, scratchTarget } from '../examples/delegate/shared' -import { delegate } from '../src/runtime/index' +import { delegate, supervisorInstructions } from '../src/runtime/index' const routerKey = process.env.TANGLE_API_KEY const routerBaseUrl = process.env.TANGLE_ROUTER_URL ?? 'https://router.tangle.tools/v1' @@ -20,14 +21,42 @@ const brainModel = process.env.MODEL ?? process.env.BRAIN_MODEL ?? model describe('delegate example', () => { it('fails loud without a supervisor brain or router (offline)', async () => { - await expect(delegate('do something', {})).rejects.toThrow(/router|brain/) + const supervisorProfile: AgentProfile = { + name: 'offline-delegator', + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'offline-model' }, + } + + await expect( + delegate('do something', { + backend: { + backend: 'router', + routerBaseUrl: 'https://router.invalid/v1', + routerKey: 'unused', + }, + supervisorProfile, + router: undefined as never, + }), + ).rejects.toThrow(/router|brain/) }) it.skipIf(!routerKey)( 'authors a worker that delivers the file on disk; cost rides through (live)', async () => { const { workDir, target, targetAbs } = scratchTarget() - const backend = makeWriteFileBackend({ workDir, routerBaseUrl, routerKey: routerKey!, model }) + const backend = makeWriteFileBackend({ workDir, routerBaseUrl, routerKey: routerKey! }) + const supervisorProfile: AgentProfile = { + name: 'file-delegator', + harness: 'cli-base', + model: { provider: 'tangle-router', default: brainModel }, + prompt: { + systemPrompt: + supervisorInstructions() + + `\nFor this run, every worker you author must use harness="cli-base", ` + + `model.provider="tangle-router", model.default=${JSON.stringify(model)}, and ` + + 'model.metadata.maxTurns=8.', + }, + } const result = await delegate( `Create a file named ${target} containing exactly the word hello (lowercase, no quotes). ` + @@ -35,8 +64,8 @@ describe('delegate example', () => { `reply with the single word DONE and STOP — do not call any more tools after the file is written.`, { backend, - router: { routerBaseUrl, routerKey: routerKey!, model: brainModel }, - model: brainModel, + router: { routerBaseUrl, routerKey: routerKey! }, + supervisorProfile, deliverable: fileDeliverable(targetAbs, target), budget: { maxIterations: 40, maxTokens: 200_000, maxUsd: 0.5 }, }, diff --git a/tests/profile-materialization.test.ts b/tests/profile-materialization.test.ts index 9b838933..a8985332 100644 --- a/tests/profile-materialization.test.ts +++ b/tests/profile-materialization.test.ts @@ -242,7 +242,7 @@ describe('profile materialization contracts', () => { contract: promptModelProfileMaterialization, changedAxes: requested, }).map((issue) => issue.axis), - ).toEqual(['modelSmall', 'modelReasoningEffort']) + ).toEqual(['modelSmall']) expect( validateProfileMaterialization({ contract: controlProfileMaterialization, @@ -352,23 +352,27 @@ describe('profile materialization contracts', () => { it('pins the resolver behavior the harness claim rests on', () => { // The contract claims `harness` only because buildBackendOptions resolves the runner from - // it. The order is sandboxOverrides, then metadata.backendType, then profile.harness — the - // metadata hint still wins so existing callers are unaffected. - expect(buildBackendOptions({ name: 'a', harness: 'codex' }, undefined).backend?.type).toBe( - 'codex', - ) + // it. A complete profile is mandatory, and its optional backendType selects a deliberate + // execution implementation while model identity remains fixed by the profile. + const executable = { + name: 'a', + harness: 'codex', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + } as const satisfies AgentProfile + expect(buildBackendOptions(executable, undefined).backend?.type).toBe('codex') expect( - buildBackendOptions( - { name: 'a', harness: 'codex', metadata: { backendType: 'amp' } }, - undefined, - ).backend?.type, + buildBackendOptions({ ...executable, metadata: { backendType: 'amp' } }, undefined).backend + ?.type, ).toBe('amp') - expect(buildBackendOptions({ name: 'a' }, undefined).backend?.type).toBe('opencode') + expect(() => buildBackendOptions({ ...executable, harness: undefined }, undefined)).toThrow( + /harness must be explicit/, + ) }) - it('keeps the runtime-selected model marker out of sandbox execution profiles', () => { + it('refuses a runtime-selected model marker before sandbox execution', () => { const profile: AgentProfile = { name: 'runtime-selected', + harness: 'codex', model: { default: HARNESS_NATIVE_MODEL, provider: 'tangle-router', @@ -376,18 +380,24 @@ describe('profile materialization contracts', () => { }, } - const executable = buildBackendOptions(profile, undefined).backend?.profile as AgentProfile - expect(executable).not.toBe(profile) - expect(executable.model).toEqual({ provider: 'tangle-router', reasoningEffort: 'high' }) - expect(profile.model?.default).toBe(HARNESS_NATIVE_MODEL) + expect(() => buildBackendOptions(profile, undefined)).toThrow( + /model\.default is runtime-selected/, + ) }) it('refuses a declared harness the sandbox cannot run', () => { // Falling through to opencode would run a gemini profile on a different harness and // report success, so the mismatch has to surface as a failure. - expect(() => buildBackendOptions({ name: 'a', harness: 'gemini' }, undefined)).toThrow( - /no backend for/, - ) + expect(() => + buildBackendOptions( + { + name: 'a', + harness: 'gemini', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }, + undefined, + ), + ).toThrow(/no backend for/) }) it('deduplicates axes while preserving first-seen order', () => { diff --git a/tests/supervisor-loop-example.test.ts b/tests/supervisor-loop-example.test.ts index c7fe3b1d..0572c3d2 100644 --- a/tests/supervisor-loop-example.test.ts +++ b/tests/supervisor-loop-example.test.ts @@ -1,4 +1,4 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { demoCheck, @@ -45,10 +45,10 @@ function answerExecutor(): Executor { /** The worker seam the example's scripted brain spawns into: an offline leaf gated on the * SAME deployable check (`demoCheck`) the real runners use — settled ⟺ delivered. */ function makeWorkerAgent(rawProfile: unknown): Agent { - const p = (rawProfile ?? {}) as { name?: unknown } - const name = typeof p.name === 'string' && p.name.length > 0 ? p.name : 'worker' + const profile = agentProfileSchema.parse(rawProfile) + const name = profile.name ?? 'worker' const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile, harness: null, executor: gateOnDeliverable(answerExecutor(), { check: demoCheck, @@ -62,10 +62,16 @@ function makeWorkerAgent(rawProfile: unknown): Agent { describe('supervisor-loop example — supervise() on the scripted brain (offline, $0)', () => { it('the surviving example path drives a worker to a CHECKED delivery and returns a winner', async () => { + const workerProfile: AgentProfile = { + name: 'worker', + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'offline-worker-model' }, + } const result = await supervise( { name: 'supervisor', harness: 'cli-base', + model: { provider: 'tangle-router', default: 'offline-supervisor-model' }, prompt: { systemPrompt: 'You are a supervisor. Spawn a worker, await it, and stop on delivery.', }, @@ -74,7 +80,7 @@ describe('supervisor-loop example — supervise() on the scripted brain (offline { // The example's offline brain (a fixed spawn → await → stop plan) + an injected worker // seam — exactly the no-creds wiring the runners default to. - brain: scriptedSupervisorChat(1, 'solver'), + brain: scriptedSupervisorChat(1, 'solver', workerProfile), makeWorkerAgent, budget: { maxIterations: 50, maxTokens: 500_000 }, runId: 'supervisor-loop-example-test', From 8c7835074e16a3f34328907d4cd9c3c38fbaf5d8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:26:04 -0600 Subject: [PATCH 06/39] docs(runtime): refresh exact execution API --- docs/api/index.md | 12 ++++++++++++ docs/api/mcp.md | 12 ++---------- docs/api/runtime.md | 42 ++++++++++++++++-------------------------- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index c5ecd7e4..e6fc0f6d 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -3809,6 +3809,18 @@ Worker-only spend (the side under test). > **tokensOut**: `number` +##### tokensKnown? + +> `optional` **tokensKnown?**: `false` + +Absent means every worker call reported complete token usage. + +##### costUsdKnown? + +> `optional` **costUsdKnown?**: `false` + +Absent means every worker call reported provider-billed cost, including a known zero. + *** ### RunPersonaConfig diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 9e991ad6..45ab42bf 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -1572,8 +1572,8 @@ single-iteration loop span tree (`runId` = `sessionId`, driver `'detached-turn'`) so trace-context inheritance survives the detached path — the same events the streaming `runAgentRounds` path would emit, minus per-token telemetry: `driveTurn` yields one terminal payload, so token -and cost figures are structurally unavailable and reported as 0 under -this driver tag. +and cost figures are structurally unavailable; zero observed subtotals are +marked incomplete under this driver tag. ##### placement? @@ -1982,14 +1982,6 @@ across all namespaces. Returns events in insertion order. Absolute path to the git repo (the workspace). Worktrees go under `/.agent-worktrees/`. -##### harnesses? - -> `optional` **harnesses?**: readonly [`LocalHarness`](#localharness)[] - -**`Experimental`** - -Harnesses to round-robin across `create()` calls. One entry = no fanout. Default `['claude-code']`. - ##### testCmd? > `optional` **testCmd?**: `string` diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 63394f05..962d95a8 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -2663,12 +2663,13 @@ Harness × model axes for `expandProfileAxes`. Defaults: the canonical > `optional` **models?**: readonly `string`[] -##### baseProfile? +##### baseProfile -> `optional` **baseProfile?**: `AgentProfile` +> **baseProfile**: `AgentProfile` -Base profile the axes expand over (prompt/tools/skills held fixed). - Default: a minimal `{ name, model: { default: } }`. +Exact base profile the axes expand over (prompt/tools/skills held fixed). + Its provider remains authoritative while each axis cell replaces the + harness and concrete model. ##### backends? @@ -2688,14 +2689,6 @@ yields the `SandboxClient` every cell runs on. Merged over the defaults: Extra `--flag value` CLI args `run()` parses and surfaces via `ctx.args`. -##### modelBackend? - -> `optional` **modelBackend?**: `Record`\<`string`, `unknown`\> - -Extra fields merged into each cell's `backend.model` create override — - e.g. `{ provider: 'openai-compat', apiKey, baseUrl }` for a router-backed - sandbox. The cell's bare model id is set by the facade from the axis. - ##### setup? > `optional` **setup?**: (`ctx`) => `void` \| `Promise`\<`void`\> @@ -2784,13 +2777,10 @@ readonly `SandboxEvent`[] > `optional` **resolveModel?**: (`events`) => `string` \| `undefined` -Resolve the model the backend ACTUALLY served off a shot's raw events. -Required for HARNESS_NATIVE_MODEL-snapped cells (a vendor-locked harness × -an out-of-family model expands to the `default` sentinel): the RunRecord -must pin a real snapshot-bearing model id, which only the dispatch — -reading the backend's usage/terminal events — can know. When this returns -a value the default dispatch records it on the paid-call receipt; -in-family cells (concrete declared model) never need it. +Resolve the model the backend actually served from a shot's raw events. +When this returns a value the default dispatch records it on the paid-call +receipt. It cannot complete an inexact planning profile: every expanded +cell must already declare a concrete model before backend work starts. ###### Parameters @@ -22250,9 +22240,9 @@ event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`: receipt: (turn) => { const u = sumSandboxUsage(turn.events) return { model, inputTokens: u.input, outputTokens: u.output, - ...(u.tokensKnown ? {} : { usageUnknown: true }), - ...(u.usdKnown && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), - ...(u.usdKnown ? {} : { costUnknown: true }), + ...(u.tokensKnown === false ? { usageUnknown: true } : {}), + ...(u.usdKnown !== false && u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}), + ...(u.usdKnown === false ? { costUnknown: true } : {}), ...(u.estimatedCostUsd !== undefined ? { estimatedCostUsd: u.estimatedCostUsd } : {}) } } @@ -22285,13 +22275,13 @@ readonly `SandboxEvent`[] > **costUsd**: `number` -##### tokensKnown +##### tokensKnown? -> **tokensKnown**: `boolean` +> `optional` **tokensKnown?**: `false` -##### usdKnown +##### usdKnown? -> **usdKnown**: `boolean` +> `optional` **usdKnown?**: `false` ##### estimatedCostUsd? From 603bbd6eed536e32c1c2928b2c1e17df411ed708 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:43:17 -0600 Subject: [PATCH 07/39] fix(runtime): reject ambient execution authority --- scripts/check-model-execution-boundary.mjs | 7 +- .../check-model-execution-boundary.test.mjs | 19 ++ src/agent/profile-materialization.ts | 8 +- src/knowledge/improvement-job.ts | 12 +- src/knowledge/supervised-update.ts | 21 +- src/mcp/bin.ts | 13 +- src/mcp/delegate-supervisor-provisioning.ts | 75 ++----- src/mcp/delegates.ts | 69 +++---- src/mcp/detached-coder.ts | 91 ++------- src/profiles/researcher.ts | 64 +++--- src/runtime/sandbox-backend.ts | 26 +-- src/runtime/supervise/runtime.ts | 16 +- src/runtime/supervise/supervise.ts | 4 +- tests/knowledge-improvement-job.test.ts | 9 + tests/knowledge-supervised-update.test.ts | 8 + tests/mcp/coder-delegate-selection.test.ts | 44 +++- .../delegate-supervisor-provisioning.test.ts | 191 ++++++------------ tests/mcp/delegate.test.ts | 41 ++-- tests/profile-materialization.test.ts | 9 +- tests/researcher-profile.test.ts | 50 +++++ 20 files changed, 347 insertions(+), 430 deletions(-) create mode 100644 tests/researcher-profile.test.ts diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index 0abb4749..5fd6134c 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -63,6 +63,7 @@ const lowLevelModelCalls = new Set([ const sourceExtensions = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.py', '.sh']) const ignoredDirectories = new Set([ '.git', + '.venv', 'coverage', 'dist', 'fixtures', @@ -75,14 +76,14 @@ function extension(path) { return match?.[0] ?? '' } -function walk(path) { +export function findSourceFiles(path) { const entries = readdirSync(path) const files = [] for (const name of entries) { if (ignoredDirectories.has(name)) continue const child = resolve(path, name) const stats = statSync(child) - if (stats.isDirectory()) files.push(...walk(child)) + if (stats.isDirectory()) files.push(...findSourceFiles(child)) else if (sourceExtensions.has(extension(name))) files.push(child) } return files @@ -342,7 +343,7 @@ export function scanRepository() { const violations = [] for (const sourceRoot of sourceRoots) { const path = resolve(root, sourceRoot) - for (const file of walk(path)) { + for (const file of findSourceFiles(path)) { const repoPath = relative(root, file).replaceAll('\\', '/') if (directTransportOwners.has(repoPath)) continue const text = readFileSync(file, 'utf8') diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index 2bbb1c3c..0355a219 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -1,8 +1,12 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { checkJavaScript, checkPython, checkShell, + findSourceFiles, } from './check-model-execution-boundary.mjs' describe('model execution boundary source check', () => { @@ -122,4 +126,19 @@ describe('model execution boundary source check', () => { expect(checkShell(`# curl https://api.openai.com/v1/chat/completions`)).toEqual([]) expect(checkShell(`curl https://api.openai.com/v1/chat/completions`)).toHaveLength(1) }) + + it('does not scan generated Python virtual environments', () => { + const fixture = mkdtempSync(join(tmpdir(), 'model-boundary-')) + try { + const virtualEnvironment = join(fixture, '.venv', 'lib') + mkdirSync(virtualEnvironment, { recursive: true }) + writeFileSync(join(virtualEnvironment, 'provider.py'), `client.responses.create(model='x')`) + const source = join(fixture, 'source.ts') + writeFileSync(source, `export const value = 'source'\n`) + + expect(findSourceFiles(fixture)).toEqual([source]) + } finally { + rmSync(fixture, { force: true, recursive: true }) + } + }) }) diff --git a/src/agent/profile-materialization.ts b/src/agent/profile-materialization.ts index f26d23f4..9d4824e7 100644 --- a/src/agent/profile-materialization.ts +++ b/src/agent/profile-materialization.ts @@ -177,10 +177,10 @@ export const promptControlProfileMaterialization = defineProfileMaterializationC * Materialization contract for `createSandboxAct`. * * `createSandboxAct` hands the whole `AgentProfile` to the sandbox as `backend.profile`, so every - * profile leaf crosses the boundary. `buildBackendOptions` resolves the runner from an explicit - * `sandboxOverrides.backend.type`, then `profile.metadata.backendType`, then `profile.harness`, - * so a candidate that changes only `harness` runs on the harness it declares — and one declaring - * a harness the sandbox cannot run throws rather than running elsewhere and reporting success. + * profile leaf crosses the boundary. `buildBackendOptions` resolves the runner only from + * `profile.harness`; an explicit `sandboxOverrides.backend.type` may confirm that choice but cannot + * replace it. A candidate declaring a harness the sandbox cannot run throws rather than running + * elsewhere and reporting success. */ export const sandboxActProfileMaterialization = defineProfileMaterializationContract({ name: 'createSandboxAct', diff --git a/src/knowledge/improvement-job.ts b/src/knowledge/improvement-job.ts index 2ef062ce..35b9ec3c 100644 --- a/src/knowledge/improvement-job.ts +++ b/src/knowledge/improvement-job.ts @@ -43,9 +43,7 @@ export interface RunKnowledgeImprovementJobOptions readinessCheck?: KnowledgeReadinessCheck backend?: ExecutorConfig makeWorkerAgent?: SuperviseOptions['makeWorkerAgent'] - harness?: string - supervisorModel?: string - supervisorSystemPrompt?: string + supervisorProfile: SupervisorProfile superviseOptions?: Partial< Omit< SuperviseOptions, @@ -146,13 +144,11 @@ export async function runKnowledgeImprovementJob( backend, budget, candidateArtifacts, - harness, makeWorkerAgent, onMeasurement, readinessCheck, runSupervised, - supervisorModel, - supervisorSystemPrompt, + supervisorProfile, superviseOptions, ...knowledgeOptions } = options @@ -173,9 +169,7 @@ export async function runKnowledgeImprovementJob( budget, backend, makeWorkerAgent, - harness, - supervisorModel, - supervisorSystemPrompt, + supervisorProfile, superviseOptions, allowedModels, runSupervised, diff --git a/src/knowledge/supervised-update.ts b/src/knowledge/supervised-update.ts index 46f8fd2e..7eeab25d 100644 --- a/src/knowledge/supervised-update.ts +++ b/src/knowledge/supervised-update.ts @@ -1,6 +1,8 @@ +import { agentProfileSchema } from '@tangle-network/agent-interface' import type { RagKnowledgeUpdateResult } from '@tangle-network/agent-knowledge' -import { researcherProfile } from '../profiles/researcher' +import { RESEARCHER_SYSTEM_PROMPT } from '../profiles/researcher' import type { DeliverableSpec } from '../runtime/supervise/completion-gate' +import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import type { ExecutorConfig } from '../runtime/supervise/runtime' import { type SuperviseOptions, supervise } from '../runtime/supervise/supervise' import type { SupervisorProfile } from '../runtime/supervise/supervisor-agent' @@ -67,9 +69,8 @@ export interface SupervisedKnowledgeUpdateOptions { budget: Budget backend?: ExecutorConfig makeWorkerAgent?: SuperviseOptions['makeWorkerAgent'] - harness?: string - supervisorModel?: string - supervisorSystemPrompt?: string + /** Caller-owned exact supervisor harness/provider/model identity. */ + supervisorProfile: SupervisorProfile superviseOptions?: Partial< Omit< SuperviseOptions, @@ -128,17 +129,17 @@ export function createSupervisedKnowledgeUpdater( export async function runSupervisedKnowledgeUpdate( options: SupervisedKnowledgeUpdateOptions, ): Promise { - const { profile: workerProfile } = researcherProfile({ harness: options.harness }) - const baseInstructions = options.supervisorSystemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT - const workerContract = workerProfile.prompt?.systemPrompt + const exactSupervisor = agentProfileSchema.parse(options.supervisorProfile) as SupervisorProfile + assertExecutableAgentProfile(exactSupervisor, 'runSupervisedKnowledgeUpdate') + const baseInstructions = exactSupervisor.prompt?.systemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT + const workerContract = RESEARCHER_SYSTEM_PROMPT const systemPrompt = workerContract ? `${baseInstructions}\n\nEach researcher worker you spawn follows this contract:\n${workerContract}` : baseInstructions const profile: SupervisorProfile = { - name: 'knowledge-research-supervisor', - ...(options.supervisorModel ? { model: { default: options.supervisorModel } } : {}), - prompt: { systemPrompt }, + ...exactSupervisor, + prompt: { ...exactSupervisor.prompt, systemPrompt }, } const run = options.runSupervised ?? supervise const task = formatSupervisedKnowledgeTask(options) diff --git a/src/mcp/bin.ts b/src/mcp/bin.ts index a1a4c851..2ba8e049 100644 --- a/src/mcp/bin.ts +++ b/src/mcp/bin.ts @@ -16,16 +16,9 @@ * MCP_ENABLE_DELEGATE set to `1` to serve the generic `delegate` verb. Its authoring * supervisor runs the brain on the router and spawns authored * workers as sub-sandboxes via the same client; needs TANGLE_API_KEY. - * MCP_SUPERVISOR_MODEL supervisor brain model id (falls back to MCP_WORKER_MODEL, then - * WORKER_MODEL, then TANGLE_ROUTER_MODEL). Must be a tool-calling - * model the router serves; with none named the bin exits 2. - * MCP_SUPERVISOR_ROUTER_KEY router key for the supervisor brain (falls back to the box's - * inference credential TANGLE_INFERENCE_KEY, then its vendor - * spelling OPENAI_API_KEY, then TANGLE_API_KEY — which - * authenticates the sandbox API, not the router) - * MCP_SUPERVISOR_ROUTER_BASE_URL router base for the supervisor brain (defaults to the repo's - * resolveRouterBaseUrl, normalized to `/v1`) - * MCP_DELEGATE_WORKER_HARNESS harness the authored workers run on (default `opencode`) + * MCP_SUPERVISOR_MODEL required exact supervisor brain model id + * MCP_SUPERVISOR_ROUTER_KEY required router key for the supervisor brain + * MCP_SUPERVISOR_ROUTER_BASE_URL required router base, normalized to `/v1` * AGENT_RUNTIME_DELEGATION_STATE_FILE * optional — absolute path of a JSON state * file. When set, delegation records persist diff --git a/src/mcp/delegate-supervisor-provisioning.ts b/src/mcp/delegate-supervisor-provisioning.ts index 0832086e..ae64a349 100644 --- a/src/mcp/delegate-supervisor-provisioning.ts +++ b/src/mcp/delegate-supervisor-provisioning.ts @@ -5,35 +5,33 @@ * invocation a consumer already mounts. * * `delegate` is wired into `createMcpServer` via `McpServerOptions.delegateSupervisor`, which needs a - * router (the supervisor brain's substrate) and a backend (WHERE the authored workers run). Inside a + * router (the supervisor brain's transport) and a backend (WHERE the authored workers run). Inside a * sandbox child the natural backend is `sandbox`: authored workers run as sub-sandboxes through the - * SAME `SandboxClient` the bin already loads from `TANGLE_API_KEY`. The brain's router reuses the - * repo's `resolveRouterBaseUrl` convention - * (`TANGLE_ROUTER_URL` / `TANGLE_ROUTER_BASE_URL`), normalised to an OpenAI-compatible `/v1` endpoint, - * keyed by `TANGLE_API_KEY`. + * SAME `SandboxClient` the bin already loads from `TANGLE_API_KEY`. Dedicated MCP variables name + * the supervisor's router connection and exact model; inherited worker/model aliases are rejected. * * @experimental */ import type { AgentProfile } from '@tangle-network/agent-interface' -import type { BackendType } from '@tangle-network/sandbox' import { ConfigError } from '../errors' -import { type RouterEnv, resolveRouterBaseUrl } from '../model-resolution.js' import type { SandboxClient } from '../runtime' import type { RouterTransportConfig } from '../runtime/router-client' import { supervisorInstructions } from '../runtime/supervise/authoring' import type { ExecutorConfig } from '../runtime/supervise/runtime' import type { DelegateHandlerOptions } from './tools/delegate' -// Composition-root default: a worker must run on SOME harness, and the router-backed one is the -// least vendor-locked choice. Not a capability claim — nothing here branches on the name. -const DEFAULT_WORKER_HARNESS = 'opencode' - function trimmed(value: string | undefined): string | undefined { const v = value?.trim() return v ? v : undefined } +function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = trimmed(env[name]) + if (value) return value + throw new ConfigError(`agent-runtime-mcp: ${name} is required when MCP_ENABLE_DELEGATE=1`) +} + /** True when the operator opted the generic `delegate` verb in (`MCP_ENABLE_DELEGATE=1`). Default off: * the wiring is additive, so consumers that do not enable it are unaffected. */ export function delegateEnabled(env: NodeJS.ProcessEnv = process.env): boolean { @@ -41,56 +39,20 @@ export function delegateEnabled(env: NodeJS.ProcessEnv = process.env): boolean { } /** - * Resolve the supervisor brain's router substrate from env. The base reuses `resolveRouterBaseUrl`, - * normalised to `/v1`. - * - * The KEY is the one for INFERENCE, which inside a sandbox is not the one for the sandbox. A box - * carries both: an inference credential for the OpenAI-compatible endpoint the brain calls, and - * `TANGLE_API_KEY`, which authenticates the sandbox CONTROL API — create a box, run a process — - * and which the router answers `403`. The brain therefore prefers the inference credential and - * keeps the platform key as the last rung, for a host that runs one key for both. - * - * `TANGLE_INFERENCE_KEY` leads because it is Tangle's own name for that credential. `OPENAI_API_KEY` - * holds the identical value and stays on the ladder for boxes provisioned before the Tangle-native - * name existed, and for a BYO OpenAI key — but it is a vendor spelling of a Tangle credential, and - * naming it first made this ladder read as "prefer OpenAI over Tangle" to the people who own the - * product (agent-dev-container#4614). A name that needs a paragraph to disclaim is the wrong name. - * - * The order is not a preference between vendors. It is: what the caller explicitly set, then the - * inference credential, then the control-plane key as a single-key-host fallback. - * - * The MODEL is NAMED, never guessed. Which ids a given router serves is deployment state this - * package cannot know, and a wrong guess is invisible until the supervisor's first completion - * rejects — after the tool has been advertised to an agent, from inside a child process whose - * stderr nobody reads. `TANGLE_ROUTER_MODEL` is on the ladder because the sandbox platform sets it - * on every box: a delegate child running there inherits the model its own host declared. When - * nothing names one, fail at startup with the ladder in the message. + * Resolve the supervisor brain from dedicated MCP configuration. The model is part of the exact + * profile; the URL and key are transport-only. No generic worker/model/key variable may silently + * select this paid execution path. */ function resolveRouterSupervisor(env: NodeJS.ProcessEnv): { router: RouterTransportConfig profile: AgentProfile } { - const routerKey = - trimmed(env.MCP_SUPERVISOR_ROUTER_KEY) ?? - trimmed(env.TANGLE_INFERENCE_KEY) ?? - trimmed(env.OPENAI_API_KEY) ?? - trimmed(env.TANGLE_API_KEY) ?? - '' - const base = trimmed(env.MCP_SUPERVISOR_ROUTER_BASE_URL) ?? resolveRouterBaseUrl(env as RouterEnv) + const routerKey = requiredEnv(env, 'MCP_SUPERVISOR_ROUTER_KEY') + const base = requiredEnv(env, 'MCP_SUPERVISOR_ROUTER_BASE_URL') const routerBaseUrl = /\/v\d+\/?$/.test(base) ? base.replace(/\/$/, '') : `${base.replace(/\/$/, '')}/v1` - const model = - trimmed(env.MCP_SUPERVISOR_MODEL) ?? - trimmed(env.MCP_WORKER_MODEL) ?? - trimmed(env.WORKER_MODEL) ?? - trimmed(env.TANGLE_ROUTER_MODEL) - if (!model) { - throw new ConfigError( - 'agent-runtime-mcp: no supervisor brain model — set MCP_SUPERVISOR_MODEL (or MCP_WORKER_MODEL / ' + - 'WORKER_MODEL / TANGLE_ROUTER_MODEL) to a tool-calling model the router serves.', - ) - } + const model = requiredEnv(env, 'MCP_SUPERVISOR_MODEL') return { router: { routerBaseUrl, routerKey }, profile: { @@ -105,8 +67,8 @@ function resolveRouterSupervisor(env: NodeJS.ProcessEnv): { /** * Build the `delegateSupervisor` substrate for `createMcpServer` from env + the bin's loaded * `SandboxClient`. Returns `undefined` when `delegate` is not opted in, so the caller mounts it only - * when asked. The worker backend is `sandbox` (authored workers run as sub-sandboxes via the same - * client) on the harness named by `MCP_DELEGATE_WORKER_HARNESS` (default `opencode`). + * when asked. The worker backend is `sandbox`; every authored worker's exact profile selects its + * own harness/provider/model. */ export function resolveDelegateSupervisor( sandboxClient: SandboxClient, @@ -114,11 +76,8 @@ export function resolveDelegateSupervisor( ): DelegateHandlerOptions | undefined { if (!delegateEnabled(env)) return undefined const supervisor = resolveRouterSupervisor(env) - const harness = (trimmed(env.MCP_DELEGATE_WORKER_HARNESS) ?? - DEFAULT_WORKER_HARNESS) as BackendType const backend: ExecutorConfig = { backend: 'sandbox', - harness, sandboxClient, } return { diff --git a/src/mcp/delegates.ts b/src/mcp/delegates.ts index 0d52213b..18172001 100644 --- a/src/mcp/delegates.ts +++ b/src/mcp/delegates.ts @@ -1,7 +1,7 @@ /** * * `detachedSessionDelegate` — the sandbox-session coder delegate: a closure that drives `runAgentRounds` - * against a `SandboxClient` + a caller-supplied (or minimal model-only default) worker profile, to a + * against a `SandboxClient` + a caller-supplied exact worker profile, to a * mechanically-validated `CoderOutput`. The caller invokes the returned delegate directly with its * coder args; when wired into a durable queue it also settles cross-restart-resumed records. * @@ -11,15 +11,15 @@ * CHOSEN backend (sandbox OR cli-bridge, via `createExecutor({ backend })`) with observe/steer/resume * + recursion, use `delegate()` / the coordination MCP. * - * The worker profile is a parameter the caller supplies (§1.5: the system authors profiles). When - * none is passed, a minimal model-only default profile is materialized in `./detached-coder` — no - * hardcoded skills or tools. For NEW local-repo coding use `worktreeFanout` / `worktreeLoopRunner` + * The worker profile is a parameter the caller supplies (§1.5: the system authors profiles). + * For NEW local-repo coding use `worktreeFanout` / `worktreeLoopRunner` * (author one `AgentProfile` per harness → `createWorktreeCliExecutor` leaves → `gateOnDeliverable`). * * @experimental */ import type { AgentProfile } from '@tangle-network/agent-interface' +import { ConfigError } from '../errors' import type { CoderTask } from '../profiles/coder' import type { AgentRunSpec, @@ -149,27 +149,13 @@ export interface DetachedSessionDelegateOptions { */ sandboxClient?: SandboxClient /** - * The worker's authored `AgentProfile` (§1.5: the system authors profiles). Spread onto the - * sandbox-session run spec → `runAgentRounds` → the executor's `harnessInvocation`, so the harness runs - * under the caller's stance. Omit to use a minimal model-only default (no hardcoded skills/tools); - * `harness` / `model` / `systemPrompt` below are convenience overrides layered onto whichever - * profile is used. + * The worker's exact authored `AgentProfile` (§1.5: the system authors profiles). It is the sole + * harness/provider/model/prompt authority for the single-coder path and the default identity for + * repeated fanout shots. */ - workerProfile?: AgentProfile - /** Backend harness for the single-coder path (sets `metadata.backendType`). Default `claude-code`. */ - harness?: string - /** Model override for the single-coder path. */ - model?: string - /** - * The worker's authored system prompt (§1.5). Flows onto the run spec's - * `profile.prompt.systemPrompt` → through `runAgentRounds` → the executor's `harnessInvocation`, so the - * harness runs under this stance. Omit to keep the profile's own prompt. - */ - systemPrompt?: string - /** Default `['claude-code', 'codex', 'opencode/zai-coding-plan/glm-5.1']` when variants > 1. */ - fanoutHarnesses?: string[] - /** Optional per-harness model override for `variants > 1`. */ - fanoutModels?: (string | undefined)[] + workerProfile: AgentProfile + /** Optional exact identities for heterogeneous fanout. Omit to repeat `workerProfile`. */ + fanoutProfiles?: ReadonlyArray /** Hard cap on the kernel's per-batch concurrency. Default 4. */ maxConcurrency?: number /** @@ -217,23 +203,34 @@ export interface DetachedSessionDelegateOptions { * @experimental */ export function detachedSessionDelegate(options: DetachedSessionDelegateOptions): CoderDelegate { + if (!options.workerProfile) { + throw new ConfigError('detachedSessionDelegate: workerProfile is required') + } + const workerProfile = coderRunSpec({ profile: options.workerProfile }).profile + const fanoutProfiles = options.fanoutProfiles?.map((profile) => coderRunSpec({ profile }).profile) + if (fanoutProfiles?.length === 0) { + throw new ConfigError('detachedSessionDelegate: fanoutProfiles must not be empty') + } const executor = resolveExecutor(options) const sandboxClient = executor.client - const fanoutHarnesses = options.fanoutHarnesses const maxConcurrency = options.maxConcurrency ?? 4 const traceEmitter = options.traceEmitter return async (args, ctx) => { const task = coderTaskFromArgs(args) const variants = Math.max(1, Math.trunc(args.variants ?? 1)) + const selectedFanoutProfiles = + variants <= 1 + ? undefined + : (fanoutProfiles ?? Array.from({ length: variants }, () => workerProfile)) + if (selectedFanoutProfiles && selectedFanoutProfiles.length < variants) { + throw new ConfigError( + `detachedSessionDelegate: ${variants} variants requested but only ${selectedFanoutProfiles.length} exact fanout profiles were configured`, + ) + } const loopEmitter = composeLoopTraceEmitters(traceEmitter, ctx.traceEmitter) ctx.report({ iteration: 0, phase: 'starting' }) if (variants <= 1) { - const agentRunSpec = coderRunSpec({ - ...(options.workerProfile ? { profile: options.workerProfile } : {}), - ...(options.harness ? { harness: options.harness } : {}), - ...(options.model ? { model: options.model } : {}), - ...(options.systemPrompt ? { systemPrompt: options.systemPrompt } : {}), - }) + const agentRunSpec = coderRunSpec({ profile: workerProfile }) const output = coderOutputAdapter const validator = createCoderValidator(task) // Detached dispatch: one session on one box, driven by `driveTurn` ticks @@ -265,8 +262,6 @@ export function detachedSessionDelegate(options: DetachedSessionDelegateOptions) task, sessionId, signal: ctx.signal, - ...(options.harness ? { harness: options.harness } : {}), - ...(options.model ? { model: options.model } : {}), ...(options.reviewer ? { reviewer: options.reviewer } : {}), }) ctx.report({ iteration: 1, phase: 'completed' }) @@ -298,11 +293,7 @@ export function detachedSessionDelegate(options: DetachedSessionDelegateOptions) return chosen } const fanout = multiHarnessCoderFanout({ - ...(options.workerProfile ? { profile: options.workerProfile } : {}), - ...(fanoutHarnesses && fanoutHarnesses.length > 0 - ? { harnesses: fanoutHarnesses.slice(0, variants) } - : {}), - ...(options.fanoutModels ? { models: options.fanoutModels.slice(0, variants) } : {}), + profiles: selectedFanoutProfiles!.slice(0, variants), }) const agentRuns = fanout.agentRuns.slice(0, variants) const result = await runAgentRounds({ @@ -442,8 +433,6 @@ export interface SettleDetachedCoderTurnOptions { /** Session id of the detached turn — used as the synthesized event id. */ sessionId: string signal: AbortSignal - harness?: string - model?: string /** Same gate as the streaming path: an unapproved candidate cannot win. */ reviewer?: CoderReviewer } diff --git a/src/mcp/detached-coder.ts b/src/mcp/detached-coder.ts index 374d16c0..0ad386a6 100644 --- a/src/mcp/detached-coder.ts +++ b/src/mcp/detached-coder.ts @@ -2,7 +2,7 @@ * * Sandbox-session coder decode layer. The sandbox-session delegate (`./delegates`) and the * cross-restart resume driver run the in-box harness over a `SandboxClient` and need to - * (a) build an `AgentRunSpec` from the caller-authored (or minimal model-only default) worker + * (a) build an `AgentRunSpec` from the caller-authored exact worker * profile, (b) decode the harness event stream into a structured `CoderOutput`, and (c) gate it with * the shared mechanical checks. This sandbox-session path is kept separate from the generic recursive * path: `worktreeFanout` instead settles the raw `WorktreePatchArtifact` and gates via @@ -17,24 +17,16 @@ * @experimental */ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import type { SandboxEvent } from '@tangle-network/sandbox' +import { ConfigError } from '../errors' import { type CoderTask, coderTaskToPrompt } from '../profiles/coder' +import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import { type CoderCheckConstraints, runCoderChecks } from '../runtime/supervise/patch-checks' import type { AgentRunSpec, Driver, OutputAdapter, Validator } from '../runtime/types' const DEFAULT_MAX_DIFF_LINES = 400 -/** - * The minimal default worker profile (§1.5: the system authors profiles — there is no hardcoded - * coder). Model-only by construction: no skills, no tool grants, no standing prompt. Callers that - * want a richer worker pass their own `AgentProfile` via `CoderRunSpecOptions.profile`. `harness` / - * `model` / `systemPrompt` are layered onto whichever profile is used. - */ -function minimalCoderProfile(): AgentProfile { - return { name: 'coder' } -} - /** @experimental The structured coder result the sandbox-session path decodes + gates. */ export interface CoderOutput { /** Branch the agent wrote the patch on. */ @@ -48,46 +40,21 @@ export interface CoderOutput { reviewerNotes?: string } -/** @experimental Overrides for one authored coder run on the sandbox-session path. */ +/** @experimental Inputs for one authored coder run on the sandbox-session path. */ export interface CoderRunSpecOptions { - /** - * The caller-authored worker `AgentProfile` (§1.5). When omitted, a minimal model-only default is - * used (no hardcoded skills/tools/prompt). `harness` / `model` / `systemPrompt` are layered onto it. - */ - profile?: AgentProfile - /** Sandbox-SDK backend.type. Default `'claude-code'`. */ - harness?: string - /** Default model id passed in `AgentProfile.model.default`. */ - model?: string - /** Custom system prompt replacement. Default = the supplied profile's own prompt (or none). */ - systemPrompt?: string - /** Stable name for `AgentRunSpec.name`. Default = `coder-${harness}`. */ + /** Caller-authored exact worker identity. It is the sole harness/provider/model authority. */ + profile: AgentProfile + /** Stable name for `AgentRunSpec.name`. Default = `profile.name`. */ name?: string } -/** Build the authored `AgentProfile` for one harness on the sandbox-session path: the caller's - * profile (or the minimal model-only default), with the per-run harness/model/prompt overrides. */ -function coderRunProfile(options: CoderRunSpecOptions): AgentProfile { - // A composition-root default on the SANDBOX-BACKEND axis (`HarnessType`), not the local-CLI - // axis that `DEFAULT_LOCAL_HARNESS` covers. Behavior varies on the profile's declared harness - // downstream, never on this name. - const harness = options.harness ?? 'claude-code' - const name = options.name ?? `coder-${harness}` - const base = options.profile ?? minimalCoderProfile() - return { - ...base, - name, - ...(options.systemPrompt ? { prompt: { systemPrompt: options.systemPrompt } } : {}), - model: options.model ? { default: options.model } : base.model, - metadata: { ...base.metadata, backendType: harness }, - } -} - /** @experimental Build the `AgentRunSpec` the sandbox-session `runAgentRounds` path drives. */ -export function coderRunSpec(options: CoderRunSpecOptions = {}): AgentRunSpec { +export function coderRunSpec(options: CoderRunSpecOptions): AgentRunSpec { + const profile = agentProfileSchema.parse(options.profile) as AgentProfile + assertExecutableAgentProfile(profile, 'coderRunSpec') return { - name: options.name ?? `coder-${options.harness ?? 'claude-code'}`, - profile: coderRunProfile(options), + name: options.name ?? profile.name, + profile, taskToPrompt: coderTaskToPrompt, } } @@ -97,18 +64,8 @@ export const coderOutputAdapter: OutputAdapter = { parse: parseCode /** @experimental */ export interface MultiHarnessCoderFanoutOptions { - /** - * The caller-authored worker `AgentProfile` (§1.5), shared across every parallel harness. When - * omitted, the minimal model-only default is used. - */ - profile?: AgentProfile - /** - * Sandbox-SDK backend.type identifiers, one per parallel agent. Default: - * `['claude-code', 'codex', 'opencode/zai-coding-plan/glm-5.1']`. - */ - harnesses?: string[] - /** Optional per-harness model override. Indexed parallel to `harnesses`. */ - models?: (string | undefined)[] + /** Exact worker identities, one per parallel agent. Must contain at least one profile. */ + profiles: ReadonlyArray } /** @@ -117,24 +74,16 @@ export interface MultiHarnessCoderFanoutOptions { * * @experimental */ -export function multiHarnessCoderFanout(options: MultiHarnessCoderFanoutOptions = {}): { +export function multiHarnessCoderFanout(options: MultiHarnessCoderFanoutOptions): { agentRuns: AgentRunSpec[] output: OutputAdapter validator: Validator driver: Driver } { - const harnesses = - options.harnesses && options.harnesses.length > 0 - ? options.harnesses - : ['claude-code', 'codex', 'opencode/zai-coding-plan/glm-5.1'] - const models = options.models ?? [] - const agentRuns = harnesses.map((harness, i) => - coderRunSpec({ - ...(options.profile ? { profile: options.profile } : {}), - harness, - model: models[i], - }), - ) + if (options.profiles.length === 0) { + throw new ConfigError('multiHarnessCoderFanout: at least one exact profile is required') + } + const agentRuns = options.profiles.map((profile) => coderRunSpec({ profile })) const driver: Driver = { name: 'fanout', plan: async (task, history) => (history.length === 0 ? agentRuns.map(() => task) : []), diff --git a/src/profiles/researcher.ts b/src/profiles/researcher.ts index 3d88791b..1d972d96 100644 --- a/src/profiles/researcher.ts +++ b/src/profiles/researcher.ts @@ -7,9 +7,8 @@ * - emit `proposedWrites[]` — never call materialize itself * - describe `gaps` it could not answer * - * The profile is stateless and agent-agnostic. `harness` selects the - * sandbox-SDK backend. For heterogeneous fanout, use - * `multiHarnessResearcherFanout`. + * The profile is stateless and agent-agnostic. The caller supplies the exact execution profile; + * this preset adds only the researcher prompt, tools, parser, and validator. * * Propose-don't-apply: the profile NEVER writes to the knowledge base. * It produces `proposedWrites: KnowledgeUpdate[]` in the output. The @@ -23,8 +22,10 @@ * @experimental */ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import type { SandboxEvent } from '@tangle-network/sandbox' +import { ConfigError } from '../errors' +import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import type { AgentRunSpec, DefaultVerdict, @@ -106,13 +107,11 @@ export interface ResearchOutput { /** Options for the source-grounded researcher profile preset. @experimental */ export interface ResearcherProfileOptions { - /** Sandbox-SDK backend.type. Default `'opencode/zai-coding-plan/glm-5.1'`. */ - harness?: string - /** Default model id passed in `AgentProfile.model.default`. */ - model?: string + /** Caller-owned exact harness/provider/model identity. */ + profile: AgentProfile /** Custom system prompt replacement. Default = built-in researcher preset. */ systemPrompt?: string - /** Stable name for `AgentRunSpec.name`. Default = `researcher-${harness}`. */ + /** Stable name for `AgentRunSpec.name`. Default = `profile.name`. */ name?: string /** * Default 0.7. Minimum (citations with quote) / items ratio for `valid=true`. @@ -121,30 +120,28 @@ export interface ResearcherProfileOptions { citationDensityMin?: number } -const DEFAULT_HARNESS = 'opencode/zai-coding-plan/glm-5.1' const DEFAULT_CITATION_DENSITY_MIN = 0.7 /** Build a source-grounded researcher profile with output parsing and validation. @experimental */ -export function researcherProfile( - options: ResearcherProfileOptions & { task?: ResearchTask } = {}, -): { +export function researcherProfile(options: ResearcherProfileOptions & { task?: ResearchTask }): { profile: AgentProfile taskToPrompt: (task: ResearchTask) => string output: OutputAdapter validator: Validator agentRunSpec: AgentRunSpec } { - const harness = options.harness ?? DEFAULT_HARNESS - const name = options.name ?? `researcher-${harness}` - const systemPrompt = options.systemPrompt ?? DEFAULT_RESEARCHER_SYSTEM_PROMPT + const base = agentProfileSchema.parse(options.profile) as AgentProfile + assertExecutableAgentProfile(base, 'researcherProfile') + const name = options.name ?? base.name + const systemPrompt = options.systemPrompt ?? RESEARCHER_SYSTEM_PROMPT const citationDensityMin = options.citationDensityMin ?? DEFAULT_CITATION_DENSITY_MIN const profile: AgentProfile = { + ...base, name, - description: "Source-grounded research agent. Propose-don't-apply.", - prompt: { systemPrompt }, - model: options.model ? { default: options.model } : undefined, - tools: { web_search: true, fs: true, shell: true }, - metadata: { backendType: harness, role: 'researcher' }, + description: base.description ?? "Source-grounded research agent. Propose-don't-apply.", + prompt: { ...base.prompt, systemPrompt }, + tools: { web_search: true, fs: true, shell: true, ...base.tools }, + metadata: { ...base.metadata, role: 'researcher' }, } const output: OutputAdapter = { parse: parseResearcherEvents } const validator: Validator = options.task @@ -163,10 +160,8 @@ export function researcherProfile( /** @experimental */ export interface MultiHarnessResearcherFanoutOptions { - /** Backend.type identifiers, one per parallel agent. */ - harnesses?: string[] - /** Optional per-harness model override. Indexed parallel to `harnesses`. */ - models?: (string | undefined)[] + /** Exact execution profiles, one per parallel researcher. */ + profiles: ReadonlyArray /** Default citation density floor for the shared validator. */ citationDensityMin?: number /** Optional task — narrows the validator's namespace check. */ @@ -180,22 +175,21 @@ export interface MultiHarnessResearcherFanoutOptions { * * @experimental */ -export function multiHarnessResearcherFanout(options: MultiHarnessResearcherFanoutOptions = {}): { +export function multiHarnessResearcherFanout(options: MultiHarnessResearcherFanoutOptions): { agentRuns: AgentRunSpec[] output: OutputAdapter validator: Validator driver: Driver } { - const harnesses = - options.harnesses && options.harnesses.length > 0 - ? options.harnesses - : ['opencode/zai-coding-plan/glm-5.1', 'claude-code', 'codex'] - const models = options.models ?? [] - const agentRuns = harnesses.map((harness, i) => { - const { agentRunSpec } = researcherProfile({ harness, model: models[i] }) + if (options.profiles.length === 0) { + throw new ConfigError('multiHarnessResearcherFanout: at least one exact profile is required') + } + const agentRuns = options.profiles.map((profile) => { + const { agentRunSpec } = researcherProfile({ profile }) return agentRunSpec }) const { output, validator } = researcherProfile({ + profile: options.profiles[0]!, citationDensityMin: options.citationDensityMin, task: options.task, }) @@ -205,7 +199,7 @@ export function multiHarnessResearcherFanout(options: MultiHarnessResearcherFano const driver: Driver = { name: 'researcher-fanout', async plan(task, history) { - return history.length === 0 ? Array.from({ length: harnesses.length }, () => task) : [] + return history.length === 0 ? Array.from({ length: agentRuns.length }, () => task) : [] }, // 'done' is a terminal decision in the kernel; the loop finalizes after // the single fanout round and selects the winner via `defaultSelectWinner`. @@ -349,7 +343,7 @@ function recencyMatchScore(items: KnowledgeItem[], window: ResearchTask['recency return total === 0 ? 0 : hits / total } -const DEFAULT_RESEARCHER_SYSTEM_PROMPT = [ +export const RESEARCHER_SYSTEM_PROMPT = [ 'You are a research agent. Your job is to answer a research question with', 'source-grounded knowledge items that the caller will choose whether to', 'persist to a multi-tenant knowledge base.', diff --git a/src/runtime/sandbox-backend.ts b/src/runtime/sandbox-backend.ts index 1339bdee..48ae47f7 100644 --- a/src/runtime/sandbox-backend.ts +++ b/src/runtime/sandbox-backend.ts @@ -45,9 +45,7 @@ function harnessAsBackendType(harness: HarnessType): BackendType | undefined { } /** - * Resolve the backend `type`: the profile's `metadata.backendType` hint, then - * its declared `harness`, else the - * SDK's profile-driven default (`'opencode'` on the platform side). + * Resolve the backend `type` from the exact profile's declared `harness`. * * A declared `harness` the sandbox cannot run throws rather than falling * through: silently running a `gemini` profile on opencode returns a result @@ -57,29 +55,19 @@ function resolveBackendType( profile: AgentProfile, override: Partial | undefined, ): BackendType { - const explicit = profile.metadata?.backendType - let selected: BackendType | undefined - if (typeof explicit === 'string') selected = explicit as BackendType const declared = profile.harness - if (selected === undefined && declared !== undefined) { - const backend = harnessAsBackendType(declared) - if (backend === undefined) { - throw new Error( - `buildBackendOptions: profile declares harness "${declared}", which the sandbox has no backend for. ` + - `Runnable harnesses: ${harnessBackends.join(', ')}. ` + - 'Set metadata.backendType to run it on a different backend deliberately.', - ) - } - selected = backend + const selected = declared === undefined ? undefined : harnessAsBackendType(declared) + if (selected === undefined) { + throw new Error( + `buildBackendOptions: profile declares harness ${JSON.stringify(declared)}, which the sandbox has no backend for. ` + + `Runnable harnesses: ${harnessBackends.join(', ')}.`, + ) } if (override?.type && selected && override.type !== selected) { throw new Error( `buildBackendOptions: backend override ${JSON.stringify(override.type)} conflicts with AgentProfile backend ${JSON.stringify(selected)}`, ) } - if (selected === undefined) { - throw new Error('buildBackendOptions: AgentProfile.harness must select the sandbox backend') - } return selected } diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index fbc28b02..4f8817a6 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -3491,7 +3491,7 @@ export type ExecutorConfig = | ({ backend: 'cli' } & CliSeam) | ({ backend: 'cli-worktree' } & CliWorktreeSeam) | ({ backend: 'provider' } & ProviderSeam) - | ({ backend: 'sandbox'; harness?: BackendType } & SandboxSeam) + | ({ backend: 'sandbox' } & SandboxSeam) function assertExactConfigKeys( value: Readonly>, @@ -3561,6 +3561,11 @@ export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { }) } case 'sandbox': { + assertExactConfigKeys( + config as unknown as Readonly>, + new Set(['backend', 'lineage', 'loopCtx', 'maxIterations', 'sandboxClient', 'steering']), + 'createExecutor sandbox config', + ) const { sandboxClient, loopCtx, ...decisionData } = config if (loopCtx === undefined) { const snapshot = detachedSnapshot(decisionData, 'createExecutor sandbox config') @@ -3740,9 +3745,12 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory })(spec, seamed) } case 'sandbox': { - // The sandbox executor requires a concrete harness; a spec-level harness - // wins, else the config names it (fail-loud inside if both are absent). - const harness = spec.harness ?? captured.harness ?? null + const harness = spec.profile.harness as BackendType + if (spec.harness != null && spec.harness !== harness) { + throw new ValidationError( + `createExecutor(sandbox): AgentSpec.harness ${JSON.stringify(spec.harness)} conflicts with AgentProfile.harness ${JSON.stringify(harness)}`, + ) + } return sandboxExecutor({ ...spec, harness }, seamed) } } diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 9bbd6d6a..a743f780 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -354,9 +354,7 @@ function driveHarnessFromBackend( const spec: AgentSpec = { profile: effectiveProfile, harness: - boundBackend.backend === 'sandbox' - ? ((effectiveProfile.harness ?? boundBackend.harness ?? null) as BackendType | null) - : null, + boundBackend.backend === 'sandbox' ? (effectiveProfile.harness as BackendType) : null, } const executor = baseFactory(spec, { signal: scope.signal, diff --git a/tests/knowledge-improvement-job.test.ts b/tests/knowledge-improvement-job.test.ts index b24cad95..92dd4667 100644 --- a/tests/knowledge-improvement-job.test.ts +++ b/tests/knowledge-improvement-job.test.ts @@ -41,6 +41,12 @@ import { createCandidateExperimentFixture, } from './helpers/candidate-experiment-fixture' +const supervisorProfile: SupervisorProfile = { + name: 'knowledge-research-supervisor', + harness: 'cli-base', + model: { provider: 'test-provider', default: 'test-supervisor' }, +} + afterEach(() => { cleanupCandidateExperimentFixtures() cleanupCandidateFixtures() @@ -166,6 +172,7 @@ describe('runKnowledgeImprovementJob', () => { runId: 'runtime-job', strict: true, budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile, readinessCheck: async ({ root: candidateRoot }) => { try { const text = await readFile( @@ -267,6 +274,7 @@ describe('runKnowledgeImprovementJob', () => { strict: true, readinessSpecs: [readinessSpec], budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile, runSupervised: async (_profile, task, opts) => { await writeRuntimeJobPage(rootFromTask(task)) await expect(opts.deliverable?.check({})).resolves.toBe(true) @@ -314,6 +322,7 @@ describe('runKnowledgeImprovementJob', () => { runId: 'runtime-job-approved', strict: true, budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile, readinessCheck: async ({ root: candidateRoot }) => ({ ready: await readFile(join(candidateRoot, 'knowledge', 'runtime-job.md'), 'utf8') .then((text) => text.includes('source-backed evidence')) diff --git a/tests/knowledge-supervised-update.test.ts b/tests/knowledge-supervised-update.test.ts index 885f18a9..f8222f60 100644 --- a/tests/knowledge-supervised-update.test.ts +++ b/tests/knowledge-supervised-update.test.ts @@ -9,6 +9,12 @@ import type { SuperviseOptions } from '../src/runtime/supervise/supervise' import type { SupervisorProfile } from '../src/runtime/supervise/supervisor-agent' import type { SupervisedResult } from '../src/runtime/supervise/types' +const supervisorProfile: SupervisorProfile = { + name: 'knowledge-research-supervisor', + harness: 'cli-base', + model: { provider: 'test-provider', default: 'test-supervisor' }, +} + function winner(): SupervisedResult { return { kind: 'winner', @@ -58,6 +64,7 @@ describe('knowledge supervisor integration', () => { goal: 'base goal', readiness: ({ root }) => root === '/kb/candidate', budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile, runSupervised: async (profile, task, opts) => { captured = { profile, task, opts } await expect(opts.deliverable?.check({})).resolves.toBe(true) @@ -102,6 +109,7 @@ describe('knowledge supervisor integration', () => { goal: 'blocked goal', readiness: () => false, budget: { maxIterations: 2, maxTokens: 1000 }, + supervisorProfile, runSupervised: async () => ({ kind: 'no-winner', diff --git a/tests/mcp/coder-delegate-selection.test.ts b/tests/mcp/coder-delegate-selection.test.ts index a38c17e8..3819611c 100644 --- a/tests/mcp/coder-delegate-selection.test.ts +++ b/tests/mcp/coder-delegate-selection.test.ts @@ -1,3 +1,4 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import { @@ -55,6 +56,17 @@ function candidateClient() { const ctx = { signal: new AbortController().signal, report() {} } const args = { goal: 'fix it', repoRoot: '/repo', variants: 2 } +function workerProfile(harness: AgentProfile['harness']): AgentProfile { + return { + name: `coder-${harness}`, + harness, + model: { provider: 'test-provider', default: `test-${harness}` }, + } +} + +const claudeWorker = workerProfile('claude-code') +const codexWorker = workerProfile('codex') + // Reviewer that approves both but rates the BIG candidate more ready. const readinessReviewer: CoderReviewer = (output) => ({ approved: true, @@ -66,7 +78,8 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { it('smallest-diff selects the smaller valid patch', async () => { const delegate = detachedSessionDelegate({ sandboxClient: candidateClient(), - fanoutHarnesses: ['claude-code', 'codex'], + workerProfile: claudeWorker, + fanoutProfiles: [claudeWorker, codexWorker], winnerSelection: 'smallest-diff' satisfies DetachedWinnerSelection, }) const out = await delegate(args, ctx) @@ -76,7 +89,8 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { it('highest-readiness selects by the reviewer score, diverging from diff size', async () => { const delegate = detachedSessionDelegate({ sandboxClient: candidateClient(), - fanoutHarnesses: ['claude-code', 'codex'], + workerProfile: claudeWorker, + fanoutProfiles: [claudeWorker, codexWorker], reviewer: readinessReviewer, winnerSelection: 'highest-readiness', }) @@ -92,7 +106,8 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { }) const delegate = detachedSessionDelegate({ sandboxClient: candidateClient(), - fanoutHarnesses: ['claude-code', 'codex'], + workerProfile: claudeWorker, + fanoutProfiles: [claudeWorker, codexWorker], reviewer: rejectAll, }) await expect(delegate(args, ctx)).rejects.toThrow(/validation \+ review/) @@ -101,14 +116,15 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { it('default highest-score (no reviewer) still returns a valid winner', async () => { const delegate = detachedSessionDelegate({ sandboxClient: candidateClient(), - fanoutHarnesses: ['claude-code', 'codex'], + workerProfile: claudeWorker, + fanoutProfiles: [claudeWorker, codexWorker], }) const out = await delegate(args, ctx) // smaller diff → higher diffSize score → highest-score favors it; either way a valid winner. expect(['small', 'big']).toContain(out.branch) }) - it('applies harness and model overrides on the single-coder path', async () => { + it('passes the exact worker profile through without execution overlays', async () => { let createOptions: CreateSandboxOptions | undefined const delegate = detachedSessionDelegate({ sandboxClient: { @@ -121,17 +137,22 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { } as unknown as SandboxInstance }, }, - harness: 'opencode', - model: 'zai/glm-4.7', + workerProfile: { + name: 'exact-opencode-worker', + harness: 'opencode', + model: { provider: 'zai', default: 'glm-4.7' }, + }, }) await delegate({ goal: 'fix it', repoRoot: '/repo' }, ctx) const profile = createOptions?.backend?.profile as - | { model?: { default?: string }; metadata?: Record } + | { harness?: string; model?: { default?: string; provider?: string } } | undefined - expect(profile?.model?.default).toBe('zai/glm-4.7') - expect(profile?.metadata?.backendType).toBe('opencode') + expect(profile).toMatchObject({ + harness: 'opencode', + model: { provider: 'zai', default: 'glm-4.7' }, + }) }) }) @@ -143,7 +164,8 @@ describe('detachedSessionDelegate — trace emitter wiring (MCP → OTEL sink)', const traceEmitter: LoopTraceEmitter = { emit: (e) => void events.push(e) } const delegate = detachedSessionDelegate({ sandboxClient: candidateClient(), - fanoutHarnesses: ['claude-code', 'codex'], + workerProfile: claudeWorker, + fanoutProfiles: [claudeWorker, codexWorker], traceEmitter, }) await delegate(args, ctx) diff --git a/tests/mcp/delegate-supervisor-provisioning.test.ts b/tests/mcp/delegate-supervisor-provisioning.test.ts index a383a673..8e465c33 100644 --- a/tests/mcp/delegate-supervisor-provisioning.test.ts +++ b/tests/mcp/delegate-supervisor-provisioning.test.ts @@ -5,157 +5,88 @@ import { } from '../../src/mcp/delegate-supervisor-provisioning' import type { SandboxClient } from '../../src/runtime' -// The resolver only stores the client reference into the backend config; it never invokes it, so a -// bare cast is a faithful stub for these assertions. const stubClient = {} as SandboxClient +const exactEnv = { + MCP_ENABLE_DELEGATE: '1', + MCP_SUPERVISOR_MODEL: 'brain-model', + MCP_SUPERVISOR_ROUTER_BASE_URL: 'https://router.example.com', + MCP_SUPERVISOR_ROUTER_KEY: 'router-key', +} as unknown as NodeJS.ProcessEnv + describe('delegateEnabled', () => { it('is off by default and on only at MCP_ENABLE_DELEGATE=1', () => { expect(delegateEnabled({} as NodeJS.ProcessEnv)).toBe(false) - expect(delegateEnabled({ MCP_ENABLE_DELEGATE: '0' } as unknown as NodeJS.ProcessEnv)).toBe( - false, - ) - expect(delegateEnabled({ MCP_ENABLE_DELEGATE: '1' } as unknown as NodeJS.ProcessEnv)).toBe(true) + expect(delegateEnabled({ MCP_ENABLE_DELEGATE: '0' } as NodeJS.ProcessEnv)).toBe(false) + expect(delegateEnabled({ MCP_ENABLE_DELEGATE: '1' } as NodeJS.ProcessEnv)).toBe(true) }) }) describe('resolveDelegateSupervisor', () => { - it('returns undefined when delegate is not opted in (fail-closed)', () => { + it('returns undefined when delegate is not opted in', () => { expect(resolveDelegateSupervisor(stubClient, {} as NodeJS.ProcessEnv)).toBeUndefined() }) - it('wires a router-brained supervisor over a sandbox backend when enabled', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - TANGLE_API_KEY: 'tk', - MCP_SUPERVISOR_MODEL: 'brain-model', - } as unknown as NodeJS.ProcessEnv) - expect(opts).toBeDefined() - expect(opts!.router).toEqual({ - routerBaseUrl: 'https://router.tangle.tools/v1', - routerKey: 'tk', - model: 'brain-model', + it('requires the dedicated supervisor model and router connection', () => { + for (const missing of [ + 'MCP_SUPERVISOR_MODEL', + 'MCP_SUPERVISOR_ROUTER_BASE_URL', + 'MCP_SUPERVISOR_ROUTER_KEY', + ] as const) { + const env = { ...exactEnv } + delete env[missing] + expect(() => resolveDelegateSupervisor(stubClient, env)).toThrow(missing) + } + }) + + it('builds one exact supervisor profile and a transport-only sandbox backend', () => { + const options = resolveDelegateSupervisor(stubClient, exactEnv) + + expect(options?.router).toEqual({ + routerBaseUrl: 'https://router.example.com/v1', + routerKey: 'router-key', + }) + expect(options?.supervisorProfile).toMatchObject({ + name: 'delegate-supervisor', + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'brain-model' }, }) - expect(opts!.model).toBe('brain-model') - expect(opts!.backend).toMatchObject({ backend: 'sandbox', harness: 'opencode' }) - expect((opts!.backend as { sandboxClient: unknown }).sandboxClient).toBe(stubClient) + expect(options?.backend).toEqual({ backend: 'sandbox', sandboxClient: stubClient }) }) - // Which ids a router serves is deployment state this package cannot know. A guess is invisible - // until the supervisor's first completion rejects — after the verb has been advertised to an - // agent — so an unnamed brain is a startup error, not a silently broken tool. - it('refuses to resolve a supervisor whose brain model nobody named', () => { + it('does not revive model, key, URL, or harness choices from legacy environment aliases', () => { expect(() => resolveDelegateSupervisor(stubClient, { MCP_ENABLE_DELEGATE: '1', - TANGLE_API_KEY: 'tk', - } as unknown as NodeJS.ProcessEnv), - ).toThrow(/MCP_SUPERVISOR_MODEL/) - }) - - // Inside a sandbox the platform key is for the sandbox, not for inference: the router answers it - // 403, and the whole delegation dies on the supervisor's first completion with no worker spawned. - it('brains on the box INFERENCE credential, not the sandbox control key beside it', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - OPENAI_API_KEY: 'inference-key', - TANGLE_API_KEY: 'sandbox-control-key', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerKey).toBe('inference-key') - }) - - // TANGLE_INFERENCE_KEY is Tangle's own name for the same value OPENAI_API_KEY carries. It leads - // the ladder because a vendor spelling of a Tangle credential made this resolution read - // backwards to the people who own the product (agent-dev-container#4614). - it('prefers the Tangle-native inference name over its vendor spelling', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - TANGLE_INFERENCE_KEY: 'tangle-inference-key', - OPENAI_API_KEY: 'vendor-spelling', - TANGLE_API_KEY: 'sandbox-control-key', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerKey).toBe('tangle-inference-key') - }) - - // A box provisioned before the Tangle-native name existed, or a BYO OpenAI key, must keep working. - it('still resolves the vendor spelling when the Tangle name is absent', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - OPENAI_API_KEY: 'vendor-spelling', - TANGLE_API_KEY: 'sandbox-control-key', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerKey).toBe('vendor-spelling') - }) + MCP_SUPERVISOR_ROUTER_BASE_URL: 'https://router.example.com', + MCP_SUPERVISOR_ROUTER_KEY: 'router-key', + TANGLE_ROUTER_MODEL: 'legacy-model', + WORKER_MODEL: 'legacy-worker-model', + } as NodeJS.ProcessEnv), + ).toThrow('MCP_SUPERVISOR_MODEL') - // An explicit caller override still beats every inherited credential. - it('lets MCP_SUPERVISOR_ROUTER_KEY win over the Tangle inference name', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - MCP_SUPERVISOR_ROUTER_KEY: 'explicit', - TANGLE_INFERENCE_KEY: 'tangle-inference-key', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerKey).toBe('explicit') - }) - - it('keeps the platform key for a host that runs one key for both', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - TANGLE_API_KEY: 'one-key', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerKey).toBe('one-key') - }) - - it('reads the model its sandbox host declared when no override names one', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - TANGLE_API_KEY: 'tk', - TANGLE_ROUTER_MODEL: 'host-declared', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.model).toBe('host-declared') - }) - - it('honors the supervisor model + worker harness + router overrides', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - TANGLE_API_KEY: 'tk', - MCP_SUPERVISOR_MODEL: 'deepseek-chat', - MCP_SUPERVISOR_ROUTER_KEY: 'override', - MCP_SUPERVISOR_ROUTER_BASE_URL: 'https://example.com/v2', - MCP_DELEGATE_WORKER_HARNESS: 'claude-code', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router).toEqual({ - routerBaseUrl: 'https://example.com/v2', - routerKey: 'override', - model: 'deepseek-chat', - }) - expect(opts!.backend).toMatchObject({ backend: 'sandbox', harness: 'claude-code' }) - }) + expect(() => + resolveDelegateSupervisor(stubClient, { + MCP_ENABLE_DELEGATE: '1', + MCP_SUPERVISOR_MODEL: 'brain-model', + MCP_SUPERVISOR_ROUTER_BASE_URL: 'https://router.example.com', + OPENAI_API_KEY: 'legacy-key', + TANGLE_API_KEY: 'control-key', + } as NodeJS.ProcessEnv), + ).toThrow('MCP_SUPERVISOR_ROUTER_KEY') - it('falls back through MCP_WORKER_MODEL then WORKER_MODEL for the brain', () => { - const a = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - WORKER_MODEL: 'wm', - } as unknown as NodeJS.ProcessEnv) - expect(a!.router.model).toBe('wm') - const b = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_WORKER_MODEL: 'mwm', - WORKER_MODEL: 'wm', - } as unknown as NodeJS.ProcessEnv) - expect(b!.router.model).toBe('mwm') + const options = resolveDelegateSupervisor(stubClient, { + ...exactEnv, + MCP_DELEGATE_WORKER_HARNESS: 'codex', + } as NodeJS.ProcessEnv) + expect(options?.backend).not.toHaveProperty('harness') }) - it('normalizes a router base without a version suffix to /v1', () => { - const opts = resolveDelegateSupervisor(stubClient, { - MCP_ENABLE_DELEGATE: '1', - MCP_SUPERVISOR_MODEL: 'brain-model', - TANGLE_ROUTER_BASE_URL: 'https://r.example.com', - } as unknown as NodeJS.ProcessEnv) - expect(opts!.router.routerBaseUrl).toBe('https://r.example.com/v1') + it('preserves an explicit versioned router base', () => { + const options = resolveDelegateSupervisor(stubClient, { + ...exactEnv, + MCP_SUPERVISOR_ROUTER_BASE_URL: 'https://router.example.com/v2/', + } as NodeJS.ProcessEnv) + expect(options?.router.routerBaseUrl).toBe('https://router.example.com/v2') }) }) diff --git a/tests/mcp/delegate.test.ts b/tests/mcp/delegate.test.ts index 03fe8cef..e7e554bb 100644 --- a/tests/mcp/delegate.test.ts +++ b/tests/mcp/delegate.test.ts @@ -1,3 +1,4 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' import { beforeEach, describe, expect, it, vi } from 'vitest' // The `delegate` tool routes through `delegate()` → `supervise()`. Mock the ONE front door so the @@ -21,25 +22,24 @@ import { DELEGATE_TOOL_NAME, validateDelegateArgs, } from '../../src/mcp/tools/delegate' +import type { RouterTransportConfig } from '../../src/runtime/router-client' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import type { - RouterConfig, - Spend, - SupervisedResult, - TreeView, -} from '../../src/runtime/supervise/types' +import type { Spend, SupervisedResult, TreeView } from '../../src/runtime/supervise/types' -const router: RouterConfig = { +const router: RouterTransportConfig = { routerBaseUrl: 'http://localhost/v1', routerKey: 'k', - model: 'deepseek-v4-flash', +} +const supervisorProfile: AgentProfile = { + name: 'delegate-supervisor', + harness: 'cli-base', + model: { provider: 'test-provider', default: 'deepseek-v4-flash' }, } const backend: ExecutorConfig = { backend: 'router-tools', routerBaseUrl: 'http://localhost/v1', routerKey: 'k', - model: 'deepseek-v4-flash', -} as ExecutorConfig +} const emptyTree = { id: 'root', children: [] } as unknown as TreeView const spentTotal: Spend = { @@ -69,7 +69,7 @@ describe('delegate MCP tool — generic delegation verb that returns cost', () = }) it('routes the intent to delegate()/supervise() and returns the delivered output WITH spentTotal', async () => { - const handler = createDelegateHandler({ router, backend }) + const handler = createDelegateHandler({ router, backend, supervisorProfile }) const result = (await handler({ intent: 'fix the bug' })) as { status: string out: unknown @@ -94,7 +94,7 @@ describe('delegate MCP tool — generic delegation verb that returns cost', () = downCount: 2, spentTotal, }) - const handler = createDelegateHandler({ router, backend }) + const handler = createDelegateHandler({ router, backend, supervisorProfile }) const result = (await handler({ intent: 'do x' })) as { status: string reason: string @@ -118,7 +118,7 @@ describe('delegate MCP tool — generic delegation verb that returns cost', () = spentTotal, error: { name: 'Error', message: 'router 503: no provider configured for this model' }, } as unknown as SupervisedResult) - const handler = createDelegateHandler({ router, backend }) + const handler = createDelegateHandler({ router, backend, supervisorProfile }) const result = (await handler({ intent: 'do x' })) as { status: string reason: string @@ -140,23 +140,24 @@ describe('delegate MCP tool — generic delegation verb that returns cost', () = downCount: 2, spentTotal, }) - const handler = createDelegateHandler({ router, backend }) + const handler = createDelegateHandler({ router, backend, supervisorProfile }) const result = (await handler({ intent: 'do x' })) as Record expect('error' in result).toBe(false) }) - it('applies a per-call model override', async () => { - const handler = createDelegateHandler({ router, backend, model: 'deepseek-v4-flash' }) - await handler({ intent: 'do x', model: 'glm-5.2' }) - const [profile] = superviseSpy.mock.calls[0] as [{ model?: { default?: string } }] - expect(profile.model?.default).toBe('glm-5.2') + it('rejects per-call execution overrides before supervision', async () => { + const handler = createDelegateHandler({ router, backend, supervisorProfile }) + await expect(handler({ intent: 'do x', model: 'glm-5.2' })).rejects.toThrow(/unknown.*model/) + expect(superviseSpy).not.toHaveBeenCalled() }) it('createMcpServer registers `delegate` only when delegateSupervisor is wired', () => { const without = createMcpServer({}) expect(without.tools.has(DELEGATE_TOOL_NAME)).toBe(false) - const withSupervisor = createMcpServer({ delegateSupervisor: { router, backend } }) + const withSupervisor = createMcpServer({ + delegateSupervisor: { router, backend, supervisorProfile }, + }) expect(withSupervisor.tools.has(DELEGATE_TOOL_NAME)).toBe(true) }) }) diff --git a/tests/profile-materialization.test.ts b/tests/profile-materialization.test.ts index a8985332..f2fb7255 100644 --- a/tests/profile-materialization.test.ts +++ b/tests/profile-materialization.test.ts @@ -352,8 +352,8 @@ describe('profile materialization contracts', () => { it('pins the resolver behavior the harness claim rests on', () => { // The contract claims `harness` only because buildBackendOptions resolves the runner from - // it. A complete profile is mandatory, and its optional backendType selects a deliberate - // execution implementation while model identity remains fixed by the profile. + // it. A complete profile is mandatory, and neither metadata nor a sandbox override may select + // a different execution implementation. const executable = { name: 'a', harness: 'codex', @@ -363,7 +363,10 @@ describe('profile materialization contracts', () => { expect( buildBackendOptions({ ...executable, metadata: { backendType: 'amp' } }, undefined).backend ?.type, - ).toBe('amp') + ).toBe('codex') + expect(() => buildBackendOptions(executable, { backend: { type: 'amp' } })).toThrow( + /conflicts with AgentProfile/, + ) expect(() => buildBackendOptions({ ...executable, harness: undefined }, undefined)).toThrow( /harness must be explicit/, ) diff --git a/tests/researcher-profile.test.ts b/tests/researcher-profile.test.ts new file mode 100644 index 00000000..dfa3e42a --- /dev/null +++ b/tests/researcher-profile.test.ts @@ -0,0 +1,50 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { multiHarnessResearcherFanout, researcherProfile } from '../src/profiles/researcher' + +const executionProfile: AgentProfile = { + name: 'researcher-exact', + harness: 'opencode', + model: { provider: 'zai', default: 'glm-5.1' }, + tools: { shell: false }, +} + +describe('researcher profile execution authority', () => { + it('adds researcher content while preserving the caller-owned execution identity', () => { + const { profile, agentRunSpec } = researcherProfile({ profile: executionProfile }) + + expect(profile).toMatchObject({ + name: 'researcher-exact', + harness: 'opencode', + model: { provider: 'zai', default: 'glm-5.1' }, + metadata: { role: 'researcher' }, + tools: { web_search: true, fs: true, shell: false }, + }) + expect(profile.prompt?.systemPrompt).toContain('source-grounded knowledge items') + expect(agentRunSpec.profile).toBe(profile) + }) + + it('refuses an incomplete execution profile before returning a run spec', () => { + expect(() => researcherProfile({ profile: { name: 'incomplete' } })).toThrow( + /harness must be explicit/, + ) + }) + + it('requires exact profiles for every fanout arm', () => { + expect(() => multiHarnessResearcherFanout({ profiles: [] })).toThrow( + /at least one exact profile/, + ) + + const second: AgentProfile = { + name: 'researcher-second', + harness: 'codex', + model: { provider: 'openai', default: 'gpt-5.2-codex' }, + } + const fanout = multiHarnessResearcherFanout({ profiles: [executionProfile, second] }) + expect(fanout.agentRuns.map((run) => run.profile.harness)).toEqual(['opencode', 'codex']) + expect(fanout.agentRuns.map((run) => run.profile.model?.default)).toEqual([ + 'glm-5.1', + 'gpt-5.2-codex', + ]) + }) +}) From 290d67befc5f7014bcbdbd6e877b42ebe184e3fc Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:39:43 -0600 Subject: [PATCH 08/39] test(runtime): use exact agent profiles --- tests/kernel/completion.test.ts | 3 +- tests/kernel/delegate.test.ts | 57 +- .../kernel/driver-inference-metering.test.ts | 82 ++- tests/kernel/graph.test.ts | 66 ++- tests/kernel/inbox.test.ts | 30 +- tests/kernel/materialization-evidence.test.ts | 38 +- .../nested-coordination-durability.test.ts | 36 +- tests/kernel/sandbox-leaf-settle.test.ts | 8 +- tests/kernel/strategy-evolution.test.ts | 83 ++- tests/kernel/strategy-suite.test.ts | 67 ++- tests/kernel/supervise-convenience.test.ts | 383 ++++++------- .../supervise-global-concurrency.test.ts | 44 +- tests/kernel/supervise-otel-spans.test.ts | 47 +- tests/kernel/supervise-worker-trace.test.ts | 72 +-- tests/kernel/supervisor-agent.test.ts | 533 ++++++++---------- tests/kernel/supervisor-authoring.test.ts | 85 +-- tests/kernel/test-agent-profile.ts | 19 + tests/mcp/fleet-executor.test.ts | 4 +- tests/mcp/worktree-harness.test.ts | 2 +- 19 files changed, 886 insertions(+), 773 deletions(-) create mode 100644 tests/kernel/test-agent-profile.ts diff --git a/tests/kernel/completion.test.ts b/tests/kernel/completion.test.ts index 75854680..5dd30299 100644 --- a/tests/kernel/completion.test.ts +++ b/tests/kernel/completion.test.ts @@ -13,6 +13,7 @@ import { type Validator, } from '../../src/runtime' import { type ScriptedPlanner, scriptedDriver } from './refine-driver' +import { testAgentProfile } from './test-agent-profile' const output: OutputAdapter = { parse(events) { @@ -32,7 +33,7 @@ const validator: Validator = { }, } const agentRuns: AgentRunSpec[] = [ - { profile: { name: 'a' }, name: 'a', taskToPrompt: (t) => t }, + { profile: testAgentProfile('a'), name: 'a', taskToPrompt: (t) => t }, ] function echoClient() { return { diff --git a/tests/kernel/delegate.test.ts b/tests/kernel/delegate.test.ts index f46f4461..20fdab4c 100644 --- a/tests/kernel/delegate.test.ts +++ b/tests/kernel/delegate.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { RouterTransportConfig } from '../../src/runtime/router-client' // Mock the ONE front door so these unit tests prove delegate's CONTRACT — it routes the intent to // `supervise()` with a DEFAULT authoring-supervisor profile and returns supervise()'s result @@ -16,27 +17,25 @@ vi.mock('../../src/runtime/supervise/supervise', () => ({ supervise: (profile: unknown, task: unknown, opts: unknown) => superviseSpy(profile, task, opts), })) -import { supervisorInstructions } from '../../src/runtime/supervise/authoring' import { defaultDelegateBudget, delegate } from '../../src/runtime/supervise/delegate' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import type { - RouterConfig, - Spend, - SupervisedResult, - TreeView, -} from '../../src/runtime/supervise/types' - -const router: RouterConfig = { +import type { Spend, SupervisedResult, TreeView } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' + +const router: RouterTransportConfig = { routerBaseUrl: 'http://localhost/v1', routerKey: 'k', - model: 'deepseek-v4-flash', } const backend: ExecutorConfig = { backend: 'router-tools', routerBaseUrl: 'http://localhost/v1', routerKey: 'k', - model: 'deepseek-v4-flash', } as ExecutorConfig +const supervisorProfile = testAgentProfile('authoring-supervisor', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + prompt: { systemPrompt: 'author exact worker profiles for the intent' }, +}) const emptyTree = { id: 'root', children: [] } as unknown as TreeView @@ -57,8 +56,8 @@ beforeEach(() => { }) describe('delegate — the one generic delegation verb over supervise()', () => { - it('routes to supervise() with the DEFAULT authoring-supervisor profile (no hardcoded worker)', async () => { - await delegate('fix the failing auth test', { backend, router }) + it('routes to supervise() with the exact authoring-supervisor profile', async () => { + await delegate('fix the failing auth test', { backend, router, supervisorProfile }) expect(superviseSpy).toHaveBeenCalledTimes(1) const [profile, task, opts] = superviseSpy.mock.calls[0] as [ @@ -66,10 +65,9 @@ describe('delegate — the one generic delegation verb over supervise()', () => unknown, { backend?: unknown; router?: unknown; budget?: unknown }, ] - // A router-brained AUTHORING supervisor: its standing instruction IS the authoring skill, so it - // writes its own worker profile from the intent — no worker profile is baked into delegate. + expect(profile).toBe(supervisorProfile) expect(profile.harness).toBe('cli-base') - expect(profile.prompt?.systemPrompt).toBe(supervisorInstructions()) + expect(profile.prompt?.systemPrompt).toBe('author exact worker profiles for the intent') // The intent is handed through verbatim as the task. expect(task).toBe('fix the failing auth test') // The injected substrate (where workers run + the brain) is forwarded. @@ -82,7 +80,7 @@ describe('delegate — the one generic delegation verb over supervise()', () => const canned = winner({ patch: 'diff' }) superviseSpy.mockResolvedValue(canned) - const result = await delegate('refactor the parser', { backend, router }) + const result = await delegate('refactor the parser', { backend, router, supervisorProfile }) expect(result).toBe(canned) expect(result.kind).toBe('winner') @@ -104,7 +102,7 @@ describe('delegate — the one generic delegation verb over supervise()', () => } superviseSpy.mockResolvedValue(noWinner) - const result = await delegate('do the thing', { backend, router }) + const result = await delegate('do the thing', { backend, router, supervisorProfile }) expect(result.kind).toBe('no-winner') if (result.kind === 'no-winner') { // A budget-exhausted delegation still cost real compute; the spend rides back unchanged. @@ -112,7 +110,7 @@ describe('delegate — the one generic delegation verb over supervise()', () => } }) - it('forwards deliverable, model, budget, allowedModels, runId to supervise()', async () => { + it('forwards deliverable, exact profile, budget, allowedModels, and runId to supervise()', async () => { const deliverable = { check: () => true, describe: 'always delivered' } const budget = { maxIterations: 7, maxTokens: 9000 } @@ -120,7 +118,10 @@ describe('delegate — the one generic delegation verb over supervise()', () => backend, router, deliverable, - model: 'glm-5.2', + supervisorProfile: testAgentProfile('glm-supervisor', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'glm-5.2' }, + }), budget, allowedModels: ['glm-5.2', 'deepseek-v4-flash'], runId: 'run-7', @@ -138,26 +139,26 @@ describe('delegate — the one generic delegation verb over supervise()', () => expect(opts.runId).toBe('run-7') }) - it('lets the caller override only the supervisor name/stance', async () => { + it('passes the caller-authored supervisor profile through unchanged', async () => { + const exactProfile = testAgentProfile('my-supervisor', { + harness: 'cli-base', + prompt: { systemPrompt: 'custom stance' }, + }) await delegate('intent', { backend, router, - supervisor: { name: 'my-supervisor', systemPrompt: 'custom stance' }, + supervisorProfile: exactProfile, }) const [profile] = superviseSpy.mock.calls[0] as [ { name?: string; prompt?: { systemPrompt?: string } }, ] expect(profile.name).toBe('my-supervisor') expect(profile.prompt?.systemPrompt).toBe('custom stance') + expect(profile).toBe(exactProfile) }) it('fails loud on an empty intent', async () => { - await expect(delegate(' ', { backend, router })).rejects.toThrow(/intent/) - expect(superviseSpy).not.toHaveBeenCalled() - }) - - it('fails loud when neither router nor brain is provided (no supervisor brain)', async () => { - await expect(delegate('intent', { backend })).rejects.toThrow(/router|brain/) + await expect(delegate(' ', { backend, router, supervisorProfile })).rejects.toThrow(/intent/) expect(superviseSpy).not.toHaveBeenCalled() }) }) diff --git a/tests/kernel/driver-inference-metering.test.ts b/tests/kernel/driver-inference-metering.test.ts index d584bdc6..6ef50bec 100644 --- a/tests/kernel/driver-inference-metering.test.ts +++ b/tests/kernel/driver-inference-metering.test.ts @@ -25,6 +25,7 @@ import type { import type { ToolLoopChat } from '../../src/runtime/tool-loop' import type { RuntimeHookEvent } from '../../src/runtime-hooks' import { type ScriptedTurn, scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // ── A worker leaf with a known, fixed spend (no network/LLM) ───────────────────── function workerLeaf( @@ -37,6 +38,7 @@ function workerLeaf( return (async function* (): AsyncGenerator { yield { kind: 'iteration' } yield { kind: 'tokens', input: tokens.input, output: tokens.output } + yield { kind: 'cost', usd: 0 } })() }, teardown: () => Promise.resolve({ destroyed: true }), @@ -49,7 +51,11 @@ function workerLeaf( } }, } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + const spec: AgentSpec = { + profile: testAgentProfile(name, { harness: 'cli-base' }), + harness: null, + executor, + } return { name, act: async () => ({ worker: name }), executorSpec: spec } as Agent< unknown, unknown @@ -85,7 +91,10 @@ describe("driver inference metering — the driver's own tokens count against th }, } const nested = driverChild( - { name: 'nested', metadata: { role: 'driver' } }, + testAgentProfile('nested', { + harness: 'cli-base', + metadata: { role: 'driver' }, + }), nestedDriver, journal, ) @@ -126,7 +135,12 @@ describe("driver inference metering — the driver's own tokens count against th // 3 driver turns, each with REAL usage: spawn → await → stop. const chat = meteredChat([ { - toolCalls: [{ name: 'spawn_agent', arguments: { profile: {}, task: 'go' } }], + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: testAgentProfile('worker'), task: 'go' }, + }, + ], usage: { input: 100, output: 50 }, costUsd: 0.01, }, @@ -200,14 +214,21 @@ describe("driver inference metering — the driver's own tokens count against th toolCalls: [ { name: 'spawn_agent', - arguments: { profile: { metadata: { kind: 'worker' } }, task: 'sub' }, + arguments: { + profile: testAgentProfile('worker', { metadata: { kind: 'worker' } }), + task: 'sub', + }, }, ], usage: { input: 60, output: 40 }, costUsd: 0.05, }, - { toolCalls: [{ name: 'await_event', arguments: {} }], usage: { input: 30, output: 20 } }, - { content: 'mid done', usage: { input: 10, output: 5 } }, + { + toolCalls: [{ name: 'await_event', arguments: {} }], + usage: { input: 30, output: 20 }, + costUsd: 0, + }, + { content: 'mid done', usage: { input: 10, output: 5 }, costUsd: 0 }, ] function makeAgent( profile: AgentProfile, @@ -221,14 +242,17 @@ describe("driver inference metering — the driver's own tokens count against th ...(context.budget.maxUsd !== undefined ? { maxUsd: context.budget.maxUsd / 4 } : {}), } return driverChild( - 'mid', + testAgentProfile('mid', { + harness: 'cli-base', + metadata: { kind: 'driver' }, + }), driverAgent(driverOf('mid', meteredChat(midTurns), childBudget)), journal, ) } return worker } - const midProfile: AgentProfile = { name: 'mid', metadata: { kind: 'driver' } } + const midProfile = testAgentProfile('mid', { metadata: { kind: 'driver' } }) // root driver inference = 100/50 + 50/30 + 20/10 = 170/90 tokens, $0.02. const rootChat = meteredChat([ { @@ -236,8 +260,12 @@ describe("driver inference metering — the driver's own tokens count against th usage: { input: 100, output: 50 }, costUsd: 0.02, }, - { toolCalls: [{ name: 'await_event', arguments: {} }], usage: { input: 50, output: 30 } }, - { content: 'root done', usage: { input: 20, output: 10 } }, + { + toolCalls: [{ name: 'await_event', arguments: {} }], + usage: { input: 50, output: 30 }, + costUsd: 0, + }, + { content: 'root done', usage: { input: 20, output: 10 }, costUsd: 0 }, ]) const result = await createSupervisor().run( @@ -283,7 +311,10 @@ describe("driver inference metering — the driver's own tokens count against th throw new Error('sub-driver network crash') } return driverChild( - 'mid', + testAgentProfile('mid', { + harness: 'cli-base', + metadata: { kind: 'driver' }, + }), driverAgent({ name: 'mid', brain: crashingChat, @@ -303,7 +334,10 @@ describe("driver inference metering — the driver's own tokens count against th toolCalls: [ { name: 'spawn_agent', - arguments: { profile: { metadata: { kind: 'driver' } }, task: 'go' }, + arguments: { + profile: testAgentProfile('mid', { metadata: { kind: 'driver' } }), + task: 'go', + }, }, ], usage: { input: 100, output: 50 }, @@ -448,12 +482,21 @@ describe("driver inference metering — the driver's own tokens count against th const turnEvents: RuntimeHookEvent[] = [] const chat = meteredChat([ { - toolCalls: [{ name: 'spawn_agent', arguments: { profile: {}, task: 'go' } }], + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: testAgentProfile('worker'), task: 'go' }, + }, + ], usage: { input: 100, output: 50 }, costUsd: 0.01, }, - { toolCalls: [{ name: 'await_event', arguments: {} }], usage: { input: 80, output: 40 } }, - { content: 'done', usage: { input: 30, output: 10 } }, + { + toolCalls: [{ name: 'await_event', arguments: {} }], + usage: { input: 80, output: 40 }, + costUsd: 0, + }, + { content: 'done', usage: { input: 30, output: 10 }, costUsd: 0 }, ]) const opts: DriverAgentOptions = { name: 'root', @@ -464,7 +507,7 @@ describe("driver inference metering — the driver's own tokens count against th systemPrompt: 'drive', maxTurns: 8, } - await createSupervisor().run(driverAgent(opts), 'task', { + const result = await createSupervisor().run(driverAgent(opts), 'task', { budget: { maxIterations: 100, maxTokens: 100_000, maxUsd: 10 }, runId: 'meter-obs', journal, @@ -669,7 +712,12 @@ describe('unmetered turns are impossible — a turn with unknown usage is record // its usage block (or a stream that lost its terminal usage chunk) produces. const chat = meteredChat([ { - toolCalls: [{ name: 'spawn_agent', arguments: { profile: {}, task: 'go' } }], + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: testAgentProfile('worker'), task: 'go' }, + }, + ], usage: { input: 100, output: 50 }, costUsd: 0.01, }, diff --git a/tests/kernel/graph.test.ts b/tests/kernel/graph.test.ts index 5959214c..754d8f34 100644 --- a/tests/kernel/graph.test.ts +++ b/tests/kernel/graph.test.ts @@ -62,6 +62,7 @@ import type { } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // ── Leaf fixtures (the offline execution seam; the graph machinery around them is real) ──────── @@ -177,11 +178,16 @@ const twoNodeGraph = (over?: Partial): AgentGraph => ({ nodes: [ { id: 'driver', - profile: { name: 'driver', prompt: { systemPrompt: 'Drive the worker until it delivers.' } }, + profile: testAgentProfile('driver', { + harness: 'cli-base', + prompt: { systemPrompt: 'Drive the worker until it delivers.' }, + }), }, { id: 'worker', - profile: { name: 'worker', prompt: { systemPrompt: 'You build what the driver asks.' } }, + profile: testAgentProfile('worker', { + prompt: { systemPrompt: 'You build what the driver asks.' }, + }), }, ], edges: [ @@ -202,8 +208,17 @@ describe('runGraph — the 2-node cyclic case over supervise()', () => { // ── The authored topology: 14 lines of plain data (the ≤20 LOC acceptance bar) ── const graph: AgentGraph = { nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Drive.' } } }, - { id: 'worker', profile: { name: 'worker', prompt: { systemPrompt: 'Build.' } } }, + { + id: 'driver', + profile: testAgentProfile('driver', { + harness: 'cli-base', + prompt: { systemPrompt: 'Drive.' }, + }), + }, + { + id: 'worker', + profile: testAgentProfile('worker', { prompt: { systemPrompt: 'Build.' } }), + }, ], edges: [ { @@ -557,9 +572,21 @@ describe('runGraph — analyzes edges (analysts are environment, findings get a // authorized steer machinery a driver steer uses — not hardwired to the spawning driver. const graph = twoNodeGraph({ nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Drive both.' } } }, - { id: 'builder', profile: { name: 'builder', prompt: { systemPrompt: 'Build.' } } }, - { id: 'fixer', profile: { name: 'fixer', prompt: { systemPrompt: 'Fix.' } } }, + { + id: 'driver', + profile: testAgentProfile('driver', { + harness: 'cli-base', + prompt: { systemPrompt: 'Drive both.' }, + }), + }, + { + id: 'builder', + profile: testAgentProfile('builder', { prompt: { systemPrompt: 'Build.' } }), + }, + { + id: 'fixer', + profile: testAgentProfile('fixer', { prompt: { systemPrompt: 'Fix.' } }), + }, ], edges: [ { @@ -807,12 +834,29 @@ describe('runGraph — analyst NODES (the analyzes lens as a tool-equipped agent * delegates edge — it is spawned by the settle hook, with its own pinned profile. */ const inspectorGraph = (to: 'driver' | 'fixer'): AgentGraph => ({ nodes: [ - { id: 'driver', profile: { name: 'driver', prompt: { systemPrompt: 'Drive.' } } }, - { id: 'worker', profile: { name: 'worker', prompt: { systemPrompt: 'Build.' } } }, + { + id: 'driver', + profile: testAgentProfile('driver', { + harness: 'cli-base', + prompt: { systemPrompt: 'Drive.' }, + }), + }, + { + id: 'worker', + profile: testAgentProfile('worker', { prompt: { systemPrompt: 'Build.' } }), + }, ...(to === 'fixer' - ? [{ id: 'fixer', profile: { name: 'fixer', prompt: { systemPrompt: 'Fix.' } } }] + ? [ + { + id: 'fixer', + profile: testAgentProfile('fixer', { prompt: { systemPrompt: 'Fix.' } }), + }, + ] : []), - { id: 'inspector', profile: { name: 'inspector', prompt: { systemPrompt: 'Inspect.' } } }, + { + id: 'inspector', + profile: testAgentProfile('inspector', { prompt: { systemPrompt: 'Inspect.' } }), + }, ], edges: [ { diff --git a/tests/kernel/inbox.test.ts b/tests/kernel/inbox.test.ts index 75e11792..b8ffeb08 100644 --- a/tests/kernel/inbox.test.ts +++ b/tests/kernel/inbox.test.ts @@ -1,6 +1,6 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' import { type AgentSpec, createBudgetPool, createExecutor, createInbox } from '../../src/runtime' +import { testAgentProfile } from './test-agent-profile' describe('worker inbox (down-leg receive end)', () => { it('parses the down-message shapes; ignores malformed', () => { @@ -85,7 +85,11 @@ describe('router-tools executor drains the inbox', () => { executeToolCall: async () => '', }) const spec: AgentSpec = { - profile: { name: 'w', prompt: { systemPrompt: 'sys' } } as unknown as AgentProfile, + profile: testAgentProfile('w', { + harness: 'cli-base', + model: { provider: 'test', default: 'test-model' }, + prompt: { systemPrompt: 'sys' }, + }), harness: null, } as AgentSpec const exec = factory(spec, { signal: new AbortController().signal, seams: {} }) @@ -127,7 +131,11 @@ describe('router-tools executor drains the inbox', () => { executeToolCall: async () => '', }) const spec: AgentSpec = { - profile: { name: 'w', prompt: { systemPrompt: 'sys' } } as unknown as AgentProfile, + profile: testAgentProfile('w', { + harness: 'cli-base', + model: { provider: 'test', default: 'test-model' }, + prompt: { systemPrompt: 'sys' }, + }), harness: null, } as AgentSpec const exec = factory(spec, { signal: new AbortController().signal, seams: {} }) @@ -158,7 +166,13 @@ describe('router-tools executor drains the inbox', () => { executeToolCall: async () => '', }) const exec = factory( - { profile: { name: 'w' }, harness: null }, + { + profile: testAgentProfile('w', { + harness: 'cli-base', + model: { provider: 'test', default: 'unpriced-test-model' }, + }), + harness: null, + }, { signal: new AbortController().signal, seams: {} }, ) @@ -196,7 +210,13 @@ describe('router-tools executor drains the inbox', () => { executeToolCall: async () => '', }) const exec = factory( - { profile: { name: 'w' }, harness: null }, + { + profile: testAgentProfile('w', { + harness: 'cli-base', + model: { provider: 'openai', default: 'gpt-4o' }, + }), + harness: null, + }, { signal: new AbortController().signal, seams: {} }, ) diff --git a/tests/kernel/materialization-evidence.test.ts b/tests/kernel/materialization-evidence.test.ts index 125a7458..ea97d9e9 100644 --- a/tests/kernel/materialization-evidence.test.ts +++ b/tests/kernel/materialization-evidence.test.ts @@ -9,10 +9,9 @@ import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/dri import { attestRuntimeOwnedExecutor, attestRuntimeOwnedScopeOwner, - knownExecutionBindingReceipt, - knownMaterializationReceipt, runtimeOwnedExecutorExecutionBinding, runtimeOwnedExecutorMaterialization, + runtimeOwnedPendingExecutorMaterialization, } from '../../src/runtime/supervise/materialization' import { bridgeExecutor, createExecutorRegistry } from '../../src/runtime/supervise/runtime' import { createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -25,13 +24,16 @@ import type { Scope, SpawnEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' const budget = { maxIterations: 4, maxTokens: 1_000 } const spent = { iterations: 1, tokens: { input: 2, output: 3 }, usd: 0, ms: 1 } function leafAgent(name: string, factory: ExecutorFactory): Agent { const executorSpec: AgentSpec = { - profile: { name, model: { default: 'test/model' } }, + profile: testAgentProfile(name, { + model: { provider: 'offline', default: 'test/model' }, + }), harness: null, executorFactory: factory, } @@ -218,10 +220,12 @@ describe('kernel-owned materialization evidence', () => { }) it('keeps the built-in bridge profile identity stable while endpoints change per attempt', () => { - const profile = { name: 'manager', model: { default: 'test/model' } } + const profile = testAgentProfile('manager', { + model: { provider: 'test', default: 'model' }, + }) const executorFor = (bridgeUrl: string, attemptId: string) => bridgeExecutor( - { profile, harness: null }, + { profile, harness: 'opencode' }, { signal: new AbortController().signal, node: { @@ -242,27 +246,19 @@ describe('kernel-owned materialization evidence', () => { ) const receiptFor = (bridgeUrl: string, attemptId: string) => { const executor = executorFor(bridgeUrl, attemptId) - const declaration = runtimeOwnedExecutorMaterialization(executor) - const binding = runtimeOwnedExecutorExecutionBinding(executor) - expect(declaration).toBeDefined() - expect(binding).toBeDefined() - const materialization = knownMaterializationReceipt({ - authoredProfileDigest: canonicalCandidateDigest(profile), - runtime: 'cli', - declaration: declaration!, - }) + const pending = runtimeOwnedPendingExecutorMaterialization(executor) + expect(runtimeOwnedExecutorMaterialization(executor)).toBeUndefined() + expect(runtimeOwnedExecutorExecutionBinding(executor)).toBeUndefined() + expect(pending).toBeDefined() return { - materialization, - binding: knownExecutionBindingReceipt(materialization, binding!), + declarationDigest: canonicalCandidateDigest(pending!.declaration), + bindingDigest: canonicalCandidateDigest(pending!.binding), } } const first = receiptFor('http://127.0.0.1:31001', 'attempt-1') const second = receiptFor('http://127.0.0.1:31002', 'attempt-2') - expect(first.materialization).toEqual(second.materialization) - expect(first.binding.materializationReceiptDigest).toBe( - second.binding.materializationReceiptDigest, - ) - expect(first.binding.bindingDigest).not.toBe(second.binding.bindingDigest) + expect(first.declarationDigest).toBe(second.declarationDigest) + expect(first.bindingDigest).not.toBe(second.bindingDigest) }) }) diff --git a/tests/kernel/nested-coordination-durability.test.ts b/tests/kernel/nested-coordination-durability.test.ts index 0b4bd67e..42d62b82 100644 --- a/tests/kernel/nested-coordination-durability.test.ts +++ b/tests/kernel/nested-coordination-durability.test.ts @@ -11,6 +11,7 @@ import type { } from '../../src/runtime/supervise/supervisor-agent' import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' async function callTool( url: string, @@ -41,11 +42,10 @@ async function callTool( } function rootBrain() { - const manager = { - name: 'identical-manager', + const manager = testAgentProfile('identical-manager', { harness: 'codex', metadata: { role: 'driver' }, - } + }) return scriptedBrain([ { toolCalls: [ @@ -106,11 +106,10 @@ describe('nested supervisor coordination durability', () => { driveHarnessMaterialization: fullProfileMaterialization, maxTurns: 8, } - const profile = { - name: 'root', + const profile = testAgentProfile('root', { harness: 'cli-base', prompt: { systemPrompt: 'Run both managers.' }, - } as const + }) await supervise(profile, 'root task', { ...options, brain: rootBrain() }) await supervise(profile, 'root task', { ...options, brain: rootBrain() }) @@ -171,11 +170,10 @@ describe('nested supervisor coordination durability', () => { const brain: ToolLoopChat = async () => { turn += 1 if (turn === 1) { - const manager = { - name: 'identical-manager', + const manager = testAgentProfile('identical-manager', { harness: 'codex', metadata: { role: 'driver' }, - } + }) return { toolCalls: [ { @@ -223,7 +221,10 @@ describe('nested supervisor coordination durability', () => { } await supervise( - { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run both managers.' } }, + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'Run both managers.' }, + }), 'root task', { backend: { @@ -267,7 +268,10 @@ describe('nested supervisor coordination durability', () => { const seen: Array>> = [] await supervise( - { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run both managers.' } }, + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'Run both managers.' }, + }), 'root task', { backend: { @@ -288,11 +292,10 @@ describe('nested supervisor coordination durability', () => { { name: 'spawn_agent', arguments: { - profile: { - name: 'identical-manager', + profile: testAgentProfile('identical-manager', { harness: 'codex', metadata: { role: 'driver' }, - }, + }), task: 'same task', key: 'manager-a', }, @@ -300,11 +303,10 @@ describe('nested supervisor coordination durability', () => { { name: 'spawn_agent', arguments: { - profile: { - name: 'identical-manager', + profile: testAgentProfile('identical-manager', { harness: 'codex', metadata: { role: 'driver' }, - }, + }), task: 'same task', key: 'manager-b', }, diff --git a/tests/kernel/sandbox-leaf-settle.test.ts b/tests/kernel/sandbox-leaf-settle.test.ts index 50e07ebe..07ec71fc 100644 --- a/tests/kernel/sandbox-leaf-settle.test.ts +++ b/tests/kernel/sandbox-leaf-settle.test.ts @@ -13,10 +13,14 @@ import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangl import { describe, expect, it } from 'vitest' import { createExecutor } from '../../src/runtime/supervise/runtime' import type { AgentSpec, ExecutorContext, UsageEvent } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' const spec: AgentSpec = { - profile: { name: 'leaf', prompt: { systemPrompt: 'do the thing' } }, - harness: null, + profile: testAgentProfile('leaf', { + harness: 'opencode', + prompt: { systemPrompt: 'do the thing' }, + }), + harness: 'opencode', } function ctx(): ExecutorContext { diff --git a/tests/kernel/strategy-evolution.test.ts b/tests/kernel/strategy-evolution.test.ts index 281ebab8..f1a4cbcf 100644 --- a/tests/kernel/strategy-evolution.test.ts +++ b/tests/kernel/strategy-evolution.test.ts @@ -17,6 +17,7 @@ import type { BenchmarkReport } from '../../src/runtime/run-benchmark' import type { AgenticSurface, AgenticTask } from '../../src/runtime/strategy' import { sample } from '../../src/runtime/strategy' import { runStrategyEvolution, selectChampion } from '../../src/runtime/strategy-evolution' +import { testAgentProfile } from './test-agent-profile' // ── Fixtures ────────────────────────────────────────────────────────────────────── @@ -87,15 +88,30 @@ export default defineStrategy('one-shot', async ({ shot }) => { function scriptedChat(replies: string[]) { const seen: string[] = [] let i = 0 + const profile = testAgentProfile('strategy-author', { + harness: 'cli-base', + model: { provider: 'offline', default: 'author-model' }, + }) const chat = { - chat: async (req: { messages: Array<{ content: string }> }) => { - seen.push(req.messages.map((m) => m.content).join('\n')) - const reply = replies[Math.min(i, replies.length - 1)] as string - i += 1 - return { content: reply } + profile, + executor: { + backend: 'router' as const, + routerBaseUrl: 'http://router.test/v1', + routerKey: 'test-key', + complete: async (body: Record) => { + const messages = body.messages as Array<{ content: string }> + seen.push(messages.map((m) => m.content).join('\n')) + const reply = replies[Math.min(i, replies.length - 1)] as string + i += 1 + return { + choices: [{ message: { content: reply } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + model: 'author-model', + } + }, }, } - return { chat: chat as never, seen } + return { chat, seen } } const fenced = (code: string) => `\`\`\`ts\n${code}\n\`\`\`` @@ -103,20 +119,27 @@ const fenced = (code: string) => `\`\`\`ts\n${code}\n\`\`\`` function stubWorkerRouter(): void { vi.stubGlobal( 'fetch', - vi.fn(async () => ({ - ok: true, - json: async () => ({ - choices: [{ message: { content: 'DONE' } }], - usage: { prompt_tokens: 10, completion_tokens: 5 }, - }), - })), + vi.fn(async (_url: string, init?: { body?: string }) => { + const request = JSON.parse(init?.body ?? '{}') as { model?: string } + return { + ok: true, + json: async () => ({ + choices: [{ message: { content: 'DONE' } }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + model: request.model, + }), + } + }), ) } const worker = { routerBaseUrl: 'http://router.test/v1', routerKey: 'test-key', - model: 'test-model', + workerProfile: testAgentProfile('strategy-worker', { + harness: 'cli-base', + model: { provider: 'offline', default: 'test-model' }, + }), } const sliceTasks = (calls: Array<{ offset: number; n: number }>) => { @@ -185,7 +208,7 @@ describe('runStrategyEvolution', () => { modelPreflight: async (model) => { checkedModels.push(model) }, - author: { chat, model: 'author-model' }, + author: chat, budget: 3, concurrency: 2, generations: 1, @@ -221,7 +244,7 @@ describe('runStrategyEvolution', () => { trainN: 8, holdoutN: 8, worker, - author: { chat }, + author: chat, budget: 3, generations: 1, populationSize: 2, @@ -244,7 +267,7 @@ describe('runStrategyEvolution', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -265,7 +288,7 @@ describe('runStrategyEvolution', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 2, @@ -290,7 +313,7 @@ describe('runStrategyEvolution', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -395,7 +418,7 @@ describe('band-aware scoring', () => { trainN: 6, holdoutN: minimumPairedTasks, worker, - author: { chat }, + author: chat, budget: 3, generations: 1, populationSize: 1, @@ -428,7 +451,7 @@ describe('band-aware scoring', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -467,7 +490,7 @@ describe('tool catalog', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -491,7 +514,7 @@ describe('lossesDetail binary', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -522,7 +545,7 @@ describe('reproducer certification', () => { trainN: 6, holdoutN: 6, worker, - author: { chat }, + author: chat, budget: 3, generations: 1, populationSize: 1, @@ -549,7 +572,7 @@ describe('reproducer certification', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, @@ -567,13 +590,17 @@ describe('reproducer certification', () => { import { readFileSync, writeFileSync } from 'node:fs' describe('checkpoint and resume', () => { - const baseCfg = (chat: never, ckptPath: string, extra: Record = {}) => ({ + const baseCfg = ( + chat: ReturnType['chat'], + ckptPath: string, + extra: Record = {}, + ) => ({ environment: shotCountingSurface(), tasks: sliceTasks([]), trainN: 6, holdoutN: 6, worker, - author: { chat }, + author: chat, budget: 3, generations: 1, populationSize: 1, @@ -640,7 +667,7 @@ describe('checkpoint and resume', () => { trainN: 4, holdoutN: 4, worker, - author: { chat }, + author: chat, budget: 2, generations: 1, populationSize: 1, diff --git a/tests/kernel/strategy-suite.test.ts b/tests/kernel/strategy-suite.test.ts index 63da6ffb..bab8079a 100644 --- a/tests/kernel/strategy-suite.test.ts +++ b/tests/kernel/strategy-suite.test.ts @@ -35,6 +35,7 @@ import { authorStrategy, strategyAuthorContract, } from '../../src/runtime/strategy-author' +import { testAgentProfile } from './test-agent-profile' // ── Fixtures ────────────────────────────────────────────────────────────────────── @@ -79,10 +80,12 @@ function stubRouter(): CapturedChatRequest[] { vi.stubGlobal( 'fetch', vi.fn(async (_url: string, init?: { body?: string }) => { - captured.push(JSON.parse(init?.body ?? '{}') as CapturedChatRequest) + const request = JSON.parse(init?.body ?? '{}') as CapturedChatRequest & { model?: string } + captured.push(request) const body = { choices: [{ message: { content: 'DONE' } }], usage: { prompt_tokens: 10, completion_tokens: 5 }, + model: request.model, } // Both response-reading styles: runShot uses json(); agent-eval's llm-client // reads text() — a stub missing either silently downs the analyst leaf. @@ -104,7 +107,7 @@ function memoryComplete( return async (body) => { const req = body as CapturedChatRequest & { model?: string } const text = req.messages.map((m) => m.content).join('\n') - if (text.includes('third-person OBSERVER')) { + if (text.includes('TRACE (in order;')) { return { choices: [ { @@ -125,12 +128,14 @@ function memoryComplete( }, ], usage: { prompt_tokens: 7, completion_tokens: 3 }, + model: req.model, } } capturedWorkers.push(req) return { choices: [{ message: { content: 'DONE' } }], usage: { prompt_tokens: 10, completion_tokens: 5 }, + model: req.model, } } } @@ -138,7 +143,11 @@ function memoryComplete( const worker = { routerBaseUrl: 'http://router.test/v1', routerKey: 'test-key', - model: 'test-model', + workerProfile: testAgentProfile('strategy-worker', { + harness: 'cli-base', + model: { provider: 'offline', default: 'test-model' }, + prompt: { systemPrompt: task.systemPrompt }, + }), } afterEach(() => { @@ -431,7 +440,7 @@ describe('addressable optimization coordinates', () => { expect(strategyAuthorContract).toContain('systemPrompt') }) - it('analystModel routes the critique call to the critic model, not the worker', async () => { + it('analystProfile routes the critique call to the critic model, not the worker', async () => { const captured = stubRouter() const surface = fixtureSurface(() => ({ passes: 0, total: 1 })) const critiqued = defineStrategy('critiqued', async ({ shot, critique }) => { @@ -443,7 +452,10 @@ describe('addressable optimization coordinates', () => { surface, task, ...worker, - analystModel: 'critic-model', + analystProfile: testAgentProfile('strategy-critic', { + harness: 'cli-base', + model: { provider: 'offline', default: 'critic-model' }, + }), strategy: critiqued, budget: 2, }) @@ -478,14 +490,27 @@ describe('addressable optimization coordinates', () => { const module = [ "export default { name: 'noop', driver: () => ({ name: 'noop', act: async () => ({ kind: 'done', deliverable: {} }) }) }", ].join('\n') - const chat = { - chat: async (req: { messages: Array<{ content: string }> }) => { - seen.push(req.messages.map((m) => m.content).join('\n')) - return { content: `\`\`\`ts\n${module}\n\`\`\`` } + const profile = testAgentProfile('strategy-author', { + harness: 'cli-base', + model: { provider: 'offline', default: 'author-model' }, + }) + const executor = { + backend: 'router' as const, + routerBaseUrl: 'http://router.test/v1', + routerKey: 'test-key', + complete: async (body: Record) => { + const messages = body.messages as Array<{ content: string }> + seen.push(messages.map((m) => m.content).join('\n')) + return { + choices: [{ message: { content: `\`\`\`ts\n${module}\n\`\`\`` } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + model: 'author-model', + } }, - } as unknown as Parameters[0]['chat'] + } const { strategy } = await authorStrategy({ - chat, + profile, + executor, contract: 'CUSTOM CONTRACT vNEXT', environmentName: 'fixture', lossesJson: '[]', @@ -531,7 +556,19 @@ describe('shot tool selection', () => { await shot({ tools: ['read_thing'] }) return { score: 0, resolved: false, completions: 1, progression: [0], shots: 1 } }) - await runAgentic({ surface, task, ...worker, strategy: focused, budget: 1 }) + await runAgentic({ + surface, + task, + ...worker, + workerProfile: testAgentProfile('focused-worker', { + harness: 'cli-base', + model: { provider: 'offline', default: 'test-model' }, + prompt: { systemPrompt: task.systemPrompt }, + tools: { read_thing: true }, + }), + strategy: focused, + budget: 1, + }) const body = captured[0] as { tools?: Array<{ function: { name: string } }> } expect(body.tools?.map((t) => t.function.name)).toEqual(['read_thing']) }) @@ -790,8 +827,10 @@ describe('consult', () => { await runAgentic({ surface, task, ...worker, strategy: controller, budget: 2 }) // The consult call is the SECOND router request; its system prompt is the raw instruction. const consultReq = captured[1] as { messages?: Array<{ role: string; content: string }> } - expect(consultReq?.messages?.[0]?.role).toBe('system') - expect(consultReq?.messages?.[0]?.content).toContain('VERDICT: STOP') + const instruction = consultReq?.messages?.find((message) => + message.content.includes('VERDICT: STOP'), + ) + expect(instruction?.content).toContain('VERDICT: STOP') // The stubbed model replies 'DONE'; consult returns it verbatim (no findings filter). expect(reply).toBe('DONE') }) diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index dbb91d92..f2c581b6 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -24,9 +24,18 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' const budget: Budget = { maxIterations: 100, maxTokens: 100_000 } +function rootProfile(overrides: Parameters[1] = {}): AgentProfile { + return testAgentProfile('root', { harness: 'cli-base', ...overrides }) +} + +function workerProfile(name = 'worker'): AgentProfile { + return testAgentProfile(name) +} + /** A registry table over a plain record — the resolver port every `opts.registry` entry speaks. */ function table(entries: Record): SuperviseRegistryTable { const map = new Map(Object.entries(entries)) @@ -40,6 +49,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { return (async function* () { yield { kind: 'iteration' } as UsageEvent yield { kind: 'tokens', input: 5, output: 5 } as UsageEvent + yield { kind: 'cost', usd: 0 } as UsageEvent })() }, teardown: () => Promise.resolve({ destroyed: true }), @@ -50,7 +60,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 }, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: workerProfile(name), harness: null, executor: ex } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -71,7 +81,7 @@ function failingLeaf(name: string, reason: string): Agent { throw new Error('a failed leaf has no terminal artifact') }, } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: workerProfile(name), harness: null, executor: ex } return { name, act: async () => undefined, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -81,19 +91,13 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('runs a supervisor to delivery from just profile + task + worker seam + brain + budget', async () => { const brain = scriptedBrain([ { - toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, - ], + toolCalls: [{ name: 'spawn_agent', arguments: { profile: workerProfile(), task: 'go' } }], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'done' }, ]) const result = await supervise( - { - name: 'root', - harness: 'cli-base', - prompt: { systemPrompt: 'drive the worker' }, - }, + rootProfile({ prompt: { systemPrompt: 'drive the worker' } }), 'solve it', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 42 }), brain }, ) @@ -127,7 +131,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }, } const spec: AgentSpec = { - profile: { name: 'blocked-worker' } as AgentProfile, + profile: workerProfile('blocked-worker'), harness: null, executor, } @@ -137,15 +141,13 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ executorSpec: spec, } as Agent & { executorSpec: AgentSpec } } - const running = supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const running = supervise(rootProfile(), 'solve it', { budget, signal: controller.signal, makeWorkerAgent: blockedLeaf, brain: scriptedBrain([ { - toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, - ], + toolCalls: [{ name: 'spawn_agent', arguments: { profile: workerProfile(), task: 'go' } }], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, ]), @@ -171,7 +173,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }) const seen: Array>> = [] let turn = 0 - const running = supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const running = supervise(rootProfile(), 'solve it', { budget, rootHandle: handle, makeWorkerAgent: () => deliveringLeaf('unused', {}), @@ -213,7 +215,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ { content: 'must not need another turn' }, ]) const result = await supervise( - { name: 'root', harness: null, systemPrompt: 'solve or delegate' }, + rootProfile({ prompt: { systemPrompt: 'solve or delegate' } }), 'solve it directly', { budget, @@ -237,7 +239,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + { name: 'spawn_agent', arguments: { profile: workerProfile(), task: 'go' } }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -250,7 +252,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ runDir: dir, } - const first = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const first = await supervise(rootProfile(), 'solve it', { ...opts, brain: script(), }) @@ -268,7 +270,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ // A second `supervise()` against the SAME runDir + runId takes the resume path. Without the // `resume` flag threaded through, this would fail loud in `beginTree` ("already begun at …, // refusing to overwrite") because the wall-clock `at` differs between the two calls. - const second = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const second = await supervise(rootProfile(), 'solve it', { ...opts, brain: script(), }) @@ -317,12 +319,12 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ runDir: dir, onCoordinationEvent, } - const first = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const first = await supervise(rootProfile(), 'solve it', { ...common, brain: scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + { name: 'spawn_agent', arguments: { profile: workerProfile(), task: 'go' } }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -343,7 +345,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(replayCommitted).toBe(true) return replayScript(...args) } - const second = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + const second = await supervise(rootProfile(), 'solve it', { ...common, brain: replayBrain, }) @@ -360,7 +362,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('gives two attempts of one keyed assignment distinct worker and event identities', async () => { let attempt = 0 const events: Array<{ eventId: string; workerId: string; assignmentId?: string }> = [] - const result = await supervise({ name: 'root', harness: 'cli-base' }, 'retry once', { + const result = await supervise(rootProfile(), 'retry once', { budget, makeWorkerAgent: () => attempt++ === 0 @@ -371,7 +373,11 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ toolCalls: [ { name: 'spawn_agent', - arguments: { profile: { name: 'same-worker' }, task: 'go', key: 'same-assignment' }, + arguments: { + profile: workerProfile('same-worker'), + task: 'go', + key: 'same-assignment', + }, }, ], }, @@ -380,7 +386,11 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ toolCalls: [ { name: 'spawn_agent', - arguments: { profile: { name: 'same-worker' }, task: 'go', key: 'same-assignment' }, + arguments: { + profile: workerProfile('same-worker'), + task: 'go', + key: 'same-assignment', + }, }, ], }, @@ -412,9 +422,8 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ backend: 'router-tools', routerBaseUrl: 'http://localhost', routerKey: 'k', - model: 'm', } as ExecutorConfig) - const w = make({ name: 'w' }) as Agent & { executorSpec: AgentSpec } + const w = make(workerProfile('w')) as Agent & { executorSpec: AgentSpec } expect(w.name).toBe('w') expect(w.executorSpec.executorFactory).toBeDefined() expect(w.executorSpec.executor).toBeUndefined() @@ -425,12 +434,14 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ backend: 'router', routerBaseUrl: 'http://router.test', routerKey: 'key', - model: 'safe-model', } const make = workerFromBackend(backend) const mutableBackend = backend as { backend: string } mutableBackend.backend = 'cli' - const worker = make({ name: 'worker' }) as Agent & { + const worker = make(testAgentProfile('worker', { harness: 'cli-base' })) as Agent< + unknown, + unknown + > & { executorSpec: AgentSpec } const executor = worker.executorSpec.executorFactory?.(worker.executorSpec, { @@ -447,7 +458,6 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', sessionId: 'SHARED', }, { @@ -462,7 +472,6 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ bridge: { bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'model', sessionId: 'SHARED', }, }, @@ -478,19 +487,24 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ backend: 'router', routerBaseUrl: 'http://localhost', routerKey: 'k', - model: 'm', }) expect(() => - routerWorker({ - name: 'rich-worker', - model: { default: 'm', reasoningEffort: 'high' }, - tools: { shell: true }, - }), - ).toThrow(/modelReasoningEffort, tools/) + routerWorker( + testAgentProfile('rich-worker', { + model: { provider: 'offline', default: 'm', reasoningEffort: 'high' }, + tools: { shell: true }, + }), + ), + ).toThrow(/would drop axis changes.*tools/s) const rawCliWorker = workerFromBackend({ backend: 'cli', bin: '/bin/true' }) expect(() => - rawCliWorker({ name: 'raw-cli', prompt: { systemPrompt: 'This used to be ignored.' } }), + rawCliWorker( + testAgentProfile('raw-cli', { + harness: 'cli-base', + prompt: { systemPrompt: 'This used to be ignored.' }, + }), + ), ).toThrow(/systemPrompt/) const localWorktreeWorker = workerFromBackend({ @@ -499,22 +513,26 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ harness: 'claude-code', }) expect(() => - localWorktreeWorker({ - name: 'local-worktree', - connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], - }), + localWorktreeWorker( + testAgentProfile('local-worktree', { + harness: 'claude-code', + connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], + }), + ), ).toThrow(/connections/) const bridgedWorktreeWorker = workerFromBackend({ backend: 'cli-worktree', repoRoot: '/workspace', - bridge: { bridgeUrl: 'http://localhost', bridgeBearer: 'secret', model: 'm' }, + bridge: { bridgeUrl: 'http://localhost', bridgeBearer: 'secret' }, }) expect(() => - bridgedWorktreeWorker({ - name: 'bridged-worktree', - connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], - }), + bridgedWorktreeWorker( + testAgentProfile('bridged-worktree', { + harness: 'codex', + connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], + }), + ), ).not.toThrow() }) @@ -522,7 +540,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ const makeWorkerAgent = () => deliveringLeaf('w', {}) expect(() => supervise( - { name: 'root', harness: 'cli-base' }, + rootProfile(), { value: 1n }, { budget, @@ -532,7 +550,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ ), ).toThrow(/canonical JSON/) expect(() => - supervise({ name: 'root', harness: 'cli-base' }, 'task', { + supervise(rootProfile(), 'task', { budget, makeWorkerAgent, brain: scriptedBrain([{ content: 'unused' }]), @@ -549,11 +567,10 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ { name: 'spawn_agent', arguments: { - profile: { - name: 'unsafe-worker', + profile: testAgentProfile('unsafe-worker', { prompt: { systemPrompt: 'run the task' }, hooks: { beforeTool: [{ command: 'curl https://example.test' }] }, - }, + }), task: 'go', }, }, @@ -561,13 +578,12 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }, { content: 'profile was refused' }, ]) - const result = await supervise({ name: 'root', harness: 'cli-base' }, 't', { + const result = await supervise(rootProfile(), 't', { budget, backend: { backend: 'bridge', bridgeUrl: 'http://127.0.0.1:1', bridgeBearer: 'unused', - model: 'codex/test', }, brain, journal, @@ -584,19 +600,17 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it.each([ { capability: 'remote MCP', - profile: { - name: 'remote-mcp-worker', + profile: testAgentProfile('remote-mcp-worker', { mcp: { metadata: { transport: 'http' as const, url: 'http://169.254.169.254/latest/meta-data' }, }, - }, + }), }, { capability: 'hub connection', - profile: { - name: 'connected-worker', + profile: testAgentProfile('connected-worker', { connections: [{ connectionId: 'private-mail', capabilities: ['read'] }], - }, + }), }, ])('fails closed on an authored $capability unless the caller grants it', async ({ profile }) => { const journal = new InMemorySpawnJournal() @@ -607,13 +621,12 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ { content: 'profile was refused' }, ]) - const result = await supervise({ name: 'root', harness: 'cli-base' }, 't', { + const result = await supervise(rootProfile(), 't', { budget, backend: { backend: 'bridge', bridgeUrl: 'http://127.0.0.1:1', bridgeBearer: 'unused', - model: 'codex/test', }, brain, journal, @@ -627,34 +640,32 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ ]) }) - // The seam is where the skill's flat vocabulary meets the leaves' canonical one: the profile the - // executor is built against must already be the shape it reads. - it('workerFromBackend hands the leaf the CANONICAL profile the supervisor authored flat', () => { + it('workerFromBackend preserves the exact canonical profile it receives', () => { const make = workerFromBackend({ backend: 'router-tools', routerBaseUrl: 'http://localhost', routerKey: 'k', - model: 'm', } as ExecutorConfig) - const w = make({ - name: 'w', - systemPrompt: 'authored instructions', - model: 'authored-model', - }) as Agent & { executorSpec: AgentSpec } + const authored = testAgentProfile('w', { + harness: 'cli-base', + prompt: { systemPrompt: 'authored instructions' }, + model: { provider: 'offline', default: 'authored-model' }, + }) + const w = make(authored) as Agent & { executorSpec: AgentSpec } const profile = w.executorSpec.profile as { prompt?: { systemPrompt?: string } model?: unknown systemPrompt?: unknown } expect(profile.prompt?.systemPrompt).toBe('authored instructions') - expect(profile.model).toEqual({ default: 'authored-model' }) + expect(profile.model).toEqual({ provider: 'offline', default: 'authored-model' }) expect(profile.systemPrompt).toBeUndefined() }) it('fails loud with neither backend nor makeWorkerAgent', () => { - expect(() => supervise({ name: 'r', harness: 'cli-base' }, 't', { budget })).toThrow( - /backend|makeWorkerAgent/, - ) + expect(() => + supervise(testAgentProfile('r', { harness: 'cli-base' }), 't', { budget }), + ).toThrow(/backend|makeWorkerAgent/) }) it('refuses spawn authorization with a caller-owned worker factory before anything starts', async () => { @@ -664,7 +675,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ let authorizationCalls = 0 expect(() => - supervise({ name: 'r', harness: 'cli-base' }, 't', { + supervise(testAgentProfile('r', { harness: 'cli-base' }), 't', { budget, journal, runId: 'invalid-custom-authority', @@ -693,11 +704,18 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('allowedModels rejects a profile model outside the allowed set', () => { expect(() => - supervise({ name: 'r', harness: 'cli-base', model: { default: 'gpt-4.1' } }, 't', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - allowedModels: ['deepseek-v4-flash'], - }), + supervise( + testAgentProfile('r', { + harness: 'cli-base', + model: { provider: 'openai', default: 'gpt-4.1' }, + }), + 't', + { + budget, + makeWorkerAgent: () => deliveringLeaf('w', {}), + allowedModels: ['deepseek-v4-flash'], + }, + ), ).toThrow(/gpt-4\.1.*not in the allowed set/) }) @@ -719,29 +737,35 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }, ])('allowedModels rejects a hidden $field', ({ profile, rejected }) => { expect(() => - supervise({ name: 'r', harness: 'cli-base', ...profile }, 't', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - allowedModels: ['safe'], - }), + supervise( + testAgentProfile('r', { + harness: 'cli-base', + model: { provider: 'offline', default: 'safe' }, + ...profile, + }), + 't', + { + budget, + makeWorkerAgent: () => deliveringLeaf('w', {}), + allowedModels: ['safe'], + }, + ), ).toThrow(new RegExp(`${rejected}.*not in the allowed set`)) }) it('refuses a fixed session id on the reusable driver backend', () => { expect(() => - supervise({ name: 'r', harness: 'codex' }, 't', { + supervise(testAgentProfile('r', { harness: 'codex' }), 't', { budget, backend: { backend: 'bridge', bridgeUrl: 'http://127.0.0.1:1', bridgeBearer: 'unused', - model: 'worker-model', }, driverBackend: { backend: 'bridge', bridgeUrl: 'http://127.0.0.1:1', bridgeBearer: 'unused', - model: 'driver-model', sessionId: 'SHARED', }, }), @@ -750,51 +774,23 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('refuses an automatic external supervisor on a backend that cannot receive coordination tools', () => { expect(() => - supervise({ name: 'r', harness: 'codex' }, 't', { + supervise(testAgentProfile('r', { harness: 'codex' }), 't', { budget, backend: { backend: 'router-tools', routerBaseUrl: 'http://127.0.0.1:1', routerKey: 'unused', - model: 'safe', }, }), ).toThrow(/requires a local bridge driverBackend.*explicit driveHarness.*resolveDriveHarness/) }) - it('allowedModels rejects a router model outside the allowed set', () => { - expect(() => - supervise({ name: 'r', harness: 'cli-base' }, 't', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - router: { routerBaseUrl: 'http://localhost', routerKey: 'k', model: 'gpt-4.1' }, - allowedModels: ['deepseek-v4-flash'], - }), - ).toThrow(/gpt-4\.1.*not in the allowed set/) - }) - - it('allowedModels rejects a backend model outside the allowed set', () => { - expect(() => - supervise({ name: 'r', harness: 'cli-base' }, 't', { - budget, - backend: { - backend: 'router-tools', - routerBaseUrl: 'http://localhost', - routerKey: 'k', - model: 'gpt-4.1', - } as ExecutorConfig, - allowedModels: ['deepseek-v4-flash'], - }), - ).toThrow(/gpt-4\.1.*not in the allowed set/) - }) - it('allowedModels passes when every configured model is in the set', async () => { const result = await supervise( - { - name: 'root', + testAgentProfile('root', { harness: 'cli-base', - model: { default: 'deepseek-v4-flash' }, - }, + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }), 't', { budget, @@ -802,8 +798,10 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ router: { routerBaseUrl: 'http://unused.test', routerKey: 'test', - model: 'deepseek-v4-flash', - complete: async () => ({ choices: [{ message: { content: 'done' } }] }), + complete: async () => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'done' } }], + }), }, allowedModels: ['deepseek-v4-flash'], }, @@ -813,7 +811,10 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('allowedModels unset is unrestricted (any model passes)', async () => { const result = await supervise( - { name: 'root', harness: 'cli-base', model: { default: 'anything' } }, + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'offline', default: 'anything' }, + }), 't', { budget, @@ -821,8 +822,10 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ router: { routerBaseUrl: 'http://unused.test', routerKey: 'test', - model: 'anything', - complete: async () => ({ choices: [{ message: { content: 'done' } }] }), + complete: async () => ({ + model: 'anything', + choices: [{ message: { content: 'done' } }], + }), }, }, ) @@ -831,11 +834,18 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('allowedModels reads a canonical AgentProfile model through its resolved default id', () => { expect(() => - supervise({ name: 'r', harness: null, model: { default: 'gpt-4.1' } }, 't', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - allowedModels: ['deepseek-v4-flash'], - }), + supervise( + testAgentProfile('r', { + harness: 'cli-base', + model: { provider: 'openai', default: 'gpt-4.1' }, + }), + 't', + { + budget, + makeWorkerAgent: () => deliveringLeaf('w', {}), + allowedModels: ['deepseek-v4-flash'], + }, + ), ).toThrow(/gpt-4\.1.*not in the allowed set/) }) }) @@ -845,12 +855,15 @@ describe('supervise — a canonical AgentProfile root reaches the router as a mo const sentModels: unknown[] = [] // The offline seam (RouterConfig.complete): the real routerBrain path runs, no network. const result = await supervise( - { - name: 'root', - harness: null, - model: { default: 'anthropic/claude-opus-5', reasoningEffort: 'high' }, + testAgentProfile('root', { + harness: 'cli-base', + model: { + provider: 'anthropic', + default: 'anthropic/claude-opus-5', + reasoningEffort: 'high', + }, prompt: { systemPrompt: 'delegate, do not solve' }, - }, + }), 'solve it', { budget, @@ -858,10 +871,9 @@ describe('supervise — a canonical AgentProfile root reaches the router as a mo router: { routerBaseUrl: 'http://router.invalid', routerKey: 'k', - model: 'config-fallback-model', complete: async (body) => { sentModels.push(body.model) - return { choices: [{ message: { content: 'done' } }] } + return { model: body.model, choices: [{ message: { content: 'done' } }] } }, }, }, @@ -873,12 +885,11 @@ describe('supervise — a canonical AgentProfile root reaches the router as a mo it('sends the instruction lines to the router inside the system message', async () => { const systemMessages: unknown[] = [] await supervise( - { - name: 'root', - harness: null, + testAgentProfile('root', { + harness: 'cli-base', prompt: { systemPrompt: 'delegate, do not solve', instructions: ['keep it small'] }, resources: { instructions: 'prefer the fewest workers' }, - }, + }), 'solve it', { budget, @@ -886,11 +897,10 @@ describe('supervise — a canonical AgentProfile root reaches the router as a mo router: { routerBaseUrl: 'http://router.invalid', routerKey: 'k', - model: 'm', complete: async (body) => { const messages = body.messages as Array<{ role: string; content: unknown }> systemMessages.push(messages.find((m) => m.role === 'system')?.content) - return { choices: [{ message: { content: 'done' } }] } + return { model: body.model, choices: [{ message: { content: 'done' } }] } }, }, }, @@ -900,54 +910,31 @@ describe('supervise — a canonical AgentProfile root reaches the router as a mo ) }) - it("keeps the router config's model when the profile's hints resolve to no id", async () => { - const sentModels: unknown[] = [] - // `AgentProfileModelHints.default` is optional upstream — this profile passes - // `agentProfileSchema.safeParse`, so it must RUN, not be rejected. - const result = await supervise( - { name: 'root', harness: null, model: { provider: 'anthropic', small: 'cheap' } }, - 'solve it', - { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - router: { - routerBaseUrl: 'http://router.invalid', - routerKey: 'k', - model: 'config-fallback-model', - complete: async (body) => { - sentModels.push(body.model) - return { choices: [{ message: { content: 'done' } }] } - }, - }, - }, - ) - expect(result.kind).toBeDefined() - expect(sentModels).toEqual(['config-fallback-model']) - }) - - it("keeps the router config's own model when the profile names none", async () => { - const sentModels: unknown[] = [] - await supervise({ name: 'root', harness: null }, 'solve it', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', {}), - router: { - routerBaseUrl: 'http://router.invalid', - routerKey: 'k', - model: 'config-fallback-model', - complete: async (body) => { - sentModels.push(body.model) - return { choices: [{ message: { content: 'done' } }] } + it('refuses an incomplete model identity instead of filling it from backend config', () => { + expect(() => + supervise( + { + name: 'root', + harness: 'cli-base', + model: { provider: 'anthropic' }, + } as AgentProfile, + 'solve it', + { + budget, + makeWorkerAgent: () => deliveringLeaf('w', {}), + router: { routerBaseUrl: 'http://router.invalid', routerKey: 'k' }, }, - }, - }) - expect(sentModels).toEqual(['config-fallback-model']) + ), + ).toThrow(/AgentProfile\.model\.default is missing/) }) }) describe('supervise — the code-valued options are nameable, so a run configuration can carry them', () => { const spawnAwaitStop = () => scriptedBrain([ - { toolCalls: [{ name: 'spawn_agent', arguments: { profile: {}, task: 'go' } }] }, + { + toolCalls: [{ name: 'spawn_agent', arguments: { profile: workerProfile(), task: 'go' } }], + }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'done' }, ]) @@ -957,7 +944,7 @@ describe('supervise — the code-valued options are nameable, so a run configura { toolCalls: [{ name: 'submit_result', arguments: { result: { answer: 42 } } }] }, { content: 'must not need another turn' }, ]) - const result = await supervise({ name: 'root', harness: null }, 'solve it directly', { + const result = await supervise(rootProfile(), 'solve it directly', { budget, makeWorkerAgent: () => deliveringLeaf('unused', {}), brain, @@ -973,7 +960,7 @@ describe('supervise — the code-valued options are nameable, so a run configura }) it('a NAMED finalizer decides the run output', async () => { - const result = await supervise({ name: 'root', harness: null }, 'solve it', { + const result = await supervise(rootProfile(), 'solve it', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 42 }), brain: spawnAwaitStop(), @@ -990,7 +977,7 @@ describe('supervise — the code-valued options are nameable, so a run configura it('a registry TYPO is a ConfigError naming the option, the name, and the table', () => { expect(() => - supervise({ name: 'root', harness: null }, 't', { + supervise(rootProfile(), 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), deliverable: 'answer-is-43', @@ -998,7 +985,7 @@ describe('supervise — the code-valued options are nameable, so a run configura }), ).toThrow(ConfigError) expect(() => - supervise({ name: 'root', harness: null }, 't', { + supervise(rootProfile(), 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), deliverable: 'answer-is-43', @@ -1009,7 +996,7 @@ describe('supervise — the code-valued options are nameable, so a run configura it('a name with no registry for that option fails loud saying so', () => { expect(() => - supervise({ name: 'root', harness: null }, 't', { + supervise(rootProfile(), 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), finalizer: 'count-delivered', @@ -1019,7 +1006,7 @@ describe('supervise — the code-valued options are nameable, so a run configura it('names the probes option in its own resolution failure', () => { expect(() => - supervise({ name: 'root', harness: null }, 't', { + supervise(rootProfile(), 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), probes: 'file-exists', @@ -1044,7 +1031,7 @@ describe('supervise — the code-valued options are nameable, so a run configura return { check: () => true } }, } - const result = await supervise({ name: 'root', harness: null }, 'solve it', { + const result = await supervise(rootProfile(), 'solve it', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 42 }), brain: spawnAwaitStop(), @@ -1058,7 +1045,7 @@ describe('supervise — the code-valued options are nameable, so a run configura }) it('a non-string option value is untouched (existing callers keep passing values)', async () => { - const result = await supervise({ name: 'root', harness: null }, 'solve it', { + const result = await supervise(rootProfile(), 'solve it', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 42 }), brain: spawnAwaitStop(), @@ -1077,7 +1064,7 @@ describe('supervise — the coordination bind is opt-in and fails closed off loo it('refuses a non-loopback coordination host with no acknowledgment', () => { expect(() => - supervise({ name: 'root', harness: 'opencode' }, 't', { + supervise(testAgentProfile('root', { harness: 'opencode' }), 't', { ...harnessOpts, coordination: { host: '10.0.0.7', port: 8931 }, }), @@ -1086,7 +1073,7 @@ describe('supervise — the coordination bind is opt-in and fails closed off loo it('passes an acknowledged non-loopback bind through to the coordination server', async () => { let url = '' - await supervise({ name: 'root', harness: 'opencode' }, 't', { + await supervise(testAgentProfile('root', { harness: 'opencode' }), 't', { ...harnessOpts, driveHarness: async ({ coordinationMcpUrl }) => { url = coordinationMcpUrl @@ -1098,7 +1085,7 @@ describe('supervise — the coordination bind is opt-in and fails closed off loo it('binds the requested loopback host with no acknowledgment needed', async () => { let url = '' - await supervise({ name: 'root', harness: 'opencode' }, 't', { + await supervise(testAgentProfile('root', { harness: 'opencode' }), 't', { ...harnessOpts, driveHarness: async ({ coordinationMcpUrl }) => { url = coordinationMcpUrl diff --git a/tests/kernel/supervise-global-concurrency.test.ts b/tests/kernel/supervise-global-concurrency.test.ts index 797ecab3..f061c239 100644 --- a/tests/kernel/supervise-global-concurrency.test.ts +++ b/tests/kernel/supervise-global-concurrency.test.ts @@ -15,6 +15,7 @@ import type { Scope, } from '../../src/runtime/supervise/types' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' const zeroCost = { iterations: 1, tokens: { input: 1, output: 1 }, usd: 0, ms: 0 } const knownZero = { @@ -59,7 +60,11 @@ function trackedLeaf(name: string, activity?: Activity, holdMs = 0): Agent Promise.resolve({ destroyed: true }), resultArtifact: () => result, } - const spec: AgentSpec = { profile: { name }, harness: null, executor } + const spec: AgentSpec = { + profile: testAgentProfile(name, { harness: 'cli-base' }), + harness: null, + executor, + } return { name, act: async () => result.out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -77,7 +82,11 @@ function failingLeaf(name: string): Agent { throw new Error('failed executor has no result') }, } - const spec: AgentSpec = { profile: { name }, harness: null, executor } + const spec: AgentSpec = { + profile: testAgentProfile(name, { harness: 'cli-base' }), + harness: null, + executor, + } return { name, act: async () => undefined, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -103,7 +112,11 @@ function abortableLeaf(name: string): Agent { throw new Error('aborted executor has no result') }, } - const spec: AgentSpec = { profile: { name }, harness: null, executor } + const spec: AgentSpec = { + profile: testAgentProfile(name, { harness: 'cli-base' }), + harness: null, + executor, + } return { name, act: async () => undefined, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -123,7 +136,7 @@ describe('supervise tree-wide worker capacity', () => { executorSpec: AgentSpec } unkillable.executorSpec = { - profile: { name: 'unkillable' }, + profile: testAgentProfile('unkillable', { harness: 'cli-base' }), harness: null, executor: { runtime: 'router', @@ -182,17 +195,17 @@ describe('supervise tree-wide worker capacity', () => { const childProfiles: AgentProfile[] = depth === 1 ? [ - { - name: `${profile.name}-sub-manager`, + testAgentProfile(`${profile.name}-sub-manager`, { harness: 'cli-base', metadata: { role: 'driver', depth: 2 }, - }, + }), ] - : [0, 1].map((index) => ({ - name: `${profile.name}-leaf-${index}`, - harness: 'cli-base', - metadata: { role: 'worker', depth: 3 }, - })) + : [0, 1].map((index) => + testAgentProfile(`${profile.name}-leaf-${index}`, { + harness: 'cli-base', + metadata: { role: 'worker', depth: 3 }, + }), + ) const brain = scriptedBrain([ { toolCalls: childProfiles.map((child) => ({ @@ -232,11 +245,10 @@ describe('supervise tree-wide worker capacity', () => { toolCalls: [0, 1].map((index) => ({ name: 'spawn_agent', arguments: { - profile: { - name: `manager-${index}`, + profile: testAgentProfile(`manager-${index}`, { harness: 'cli-base', metadata: { role: 'driver', depth: 1 }, - }, + }), task: `run branch ${index}`, }, })), @@ -247,7 +259,7 @@ describe('supervise tree-wide worker capacity', () => { ]) const result = await supervise( - { name: 'root', harness: 'cli-base' }, + testAgentProfile('root', { harness: 'cli-base' }), 'run a three-level tree', { budget: { maxIterations: 500, maxTokens: 500_000 }, diff --git a/tests/kernel/supervise-otel-spans.test.ts b/tests/kernel/supervise-otel-spans.test.ts index e3cbd7b0..552d8e5d 100644 --- a/tests/kernel/supervise-otel-spans.test.ts +++ b/tests/kernel/supervise-otel-spans.test.ts @@ -8,7 +8,6 @@ * * Fully offline — scripted leaf executors and a scripted driver, no network, sandbox, or subprocess. */ -import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { OtelExporter, OtelSpan } from '../../src/otel-export' @@ -30,6 +29,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // ── Offline fixtures ────────────────────────────────────────────────────────── @@ -57,7 +57,7 @@ function workerLeaf( spent, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -491,24 +491,31 @@ describe('fixture integrity', () => { /** The whole `supervise()` call, minus the telemetry choice under test. */ function superviseOnce(otel?: SuperviseOptions['otel']) { - return supervise({ name: 'root', harness: null, systemPrompt: 'drive the worker' }, 'solve it', { - budget: { maxIterations: 100, maxTokens: 100_000 }, - runId: 'front-door', - // Injected clock: the two arms of the identical-result comparison must not diverge on a - // real-millisecond `settledAt` boundary. - now: () => 1_000, - makeWorkerAgent: () => workerLeaf('w', { answer: 42 }, { input: 5, output: 5 }, 1), - brain: scriptedBrain([ - { - toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, - ], - }, - { toolCalls: [{ name: 'await_event', arguments: {} }] }, - { content: 'done' }, - ]), - ...(otel ? { otel } : {}), - }) + return supervise( + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'drive the worker' }, + }), + 'solve it', + { + budget: { maxIterations: 100, maxTokens: 100_000 }, + runId: 'front-door', + // Injected clock: the two arms of the identical-result comparison must not diverge on a + // real-millisecond `settledAt` boundary. + now: () => 1_000, + makeWorkerAgent: () => workerLeaf('w', { answer: 42 }, { input: 5, output: 5 }, 1), + brain: scriptedBrain([ + { + toolCalls: [ + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'done' }, + ]), + ...(otel ? { otel } : {}), + }, + ) } describe('supervise(): telemetry is opt-in at the front door', () => { diff --git a/tests/kernel/supervise-worker-trace.test.ts b/tests/kernel/supervise-worker-trace.test.ts index b232c677..ca88c4cb 100644 --- a/tests/kernel/supervise-worker-trace.test.ts +++ b/tests/kernel/supervise-worker-trace.test.ts @@ -16,12 +16,9 @@ * the environment that process actually observed — not on the options object we built for it. */ -import { mkdtemp, readFile, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' import { deriveHexId } from '@tangle-network/agent-trace-contract' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { OtelExporter, OtelSpan } from '../../src/otel-export' import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' @@ -41,6 +38,7 @@ import type { SupervisorOpts, } from '../../src/runtime/supervise/types' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // ── Fixtures ────────────────────────────────────────────────────────────────── @@ -83,8 +81,8 @@ function registryOf(factory: ExecutorFactory): ExecutorRegistry { }) } -const OUTER_CHILD_BUDGET: Budget = { maxIterations: 8, maxTokens: 5_000, maxUsd: 5 } -const INNER_CHILD_BUDGET: Budget = { maxIterations: 4, maxTokens: 1_000, maxUsd: 1 } +const OUTER_CHILD_BUDGET: Budget = { maxIterations: 8, maxTokens: 5_000 } +const INNER_CHILD_BUDGET: Budget = { maxIterations: 4, maxTokens: 1_000 } /** Spawns its declared children, drains them, and returns the first `done` child's output. */ function scriptedDriver( @@ -112,14 +110,14 @@ function scriptedDriver( function resolvedLeaf(name: string): Agent { return { name, - executorSpec: { profile: { name }, harness: null }, + executorSpec: { profile: testAgentProfile(name), harness: null }, act: () => Promise.resolve(undefined), } as unknown as Agent } function supervisorOpts(over: Partial): SupervisorOpts { return { - budget: over.budget ?? { maxIterations: 100, maxTokens: 100_000, maxUsd: 10 }, + budget: over.budget ?? { maxIterations: 100, maxTokens: 100_000 }, runId: over.runId ?? 'run', journal: over.journal ?? new InMemorySpawnJournal(), blobs: over.blobs ?? new InMemoryResultBlobStore(), @@ -476,39 +474,28 @@ describe('the sandbox arm carries the context onto the box itself', () => { * this whole change exists to remove. This case is the guard on that. */ describe('supervise({ backend, otel }) stamps its workers too', () => { - let dir: string - beforeAll(async () => { - dir = await mkdtemp(join(tmpdir(), 'worker-trace-')) - }) - afterAll(async () => { - await rm(dir, { recursive: true, force: true }) - }) - - /** The worker writes its inherited environment to a file, so the assertion does not depend on how - * the finalizer treats an ungated CLI artifact. */ - function probeArgs(outPath: string): string[] { - return [ - '-e', - 'require("node:fs").writeFileSync(process.argv[1], JSON.stringify({' + - 'TRACE_ID: process.env.TRACE_ID ?? null,' + - 'PARENT_SPAN_ID: process.env.PARENT_SPAN_ID ?? null,' + - 'TRACEPARENT: process.env.TRACEPARENT ?? null}))', - outPath, - ] - } - - function superviseOnce(outPath: string, exporter?: OtelExporter) { - return supervise( - { name: 'root', harness: null, systemPrompt: 'drive the worker' }, + async function superviseOnce(exporter?: OtelExporter) { + const fake = fakeSandboxClient() + const result = await supervise( + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'drive the worker' }, + }), 'solve it', { budget: { maxIterations: 100, maxTokens: 100_000 }, runId: 'front-door', - backend: { backend: 'cli', bin: process.execPath, args: probeArgs(outPath) }, + backend: { backend: 'sandbox', harness: 'opencode', sandboxClient: fake.client }, brain: scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { + profile: testAgentProfile('worker'), + task: 'go', + }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -517,14 +504,15 @@ describe('supervise({ backend, otel }) stamps its workers too', () => { ...(exporter ? { otel: { exporter } } : {}), }, ) + return { result, created: fake.created } } it('hands the front door worker the run trace id and the root span id', async () => { const { exporter, spans } = recordingExporter() - const outPath = join(dir, 'traced.json') - await superviseOnce(outPath, exporter) + const { result, created } = await superviseOnce(exporter) - const env = JSON.parse(await readFile(outPath, 'utf8')) as InheritedEnv + expect(result.kind).toBe('winner') + const env = created[0]?.env const root = spans.find((s) => s.name === 'supervisor.run') expect(root).toBeDefined() expect(env.TRACE_ID).toBe(root?.traceId) @@ -535,12 +523,8 @@ describe('supervise({ backend, otel }) stamps its workers too', () => { }) it('stamps nothing when the front door configures no telemetry', async () => { - const outPath = join(dir, 'untraced.json') - await superviseOnce(outPath) - expect(JSON.parse(await readFile(outPath, 'utf8'))).toEqual({ - TRACE_ID: null, - PARENT_SPAN_ID: null, - TRACEPARENT: null, - }) + const { result, created } = await superviseOnce() + expect(result.kind).toBe('winner') + expect(created[0]?.env).toBeUndefined() }) }) diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index d7546cc4..1d800aff 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -1,4 +1,3 @@ -import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' @@ -24,6 +23,7 @@ import type { } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' const perWorker: Budget = { maxIterations: 4, maxTokens: 1000 } @@ -35,6 +35,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { return (async function* () { yield { kind: 'iteration' } as UsageEvent yield { kind: 'tokens', input: 5, output: 5 } as UsageEvent + yield { kind: 'cost', usd: 0 } as UsageEvent })() }, teardown: () => Promise.resolve({ destroyed: true }), @@ -45,7 +46,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 }, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor: ex } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -85,18 +86,20 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen const brain = scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: testAgentProfile('worker'), task: 'go' }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'done' }, ]) const root = supervisorAgent( - { - name: 'root', + testAgentProfile('root', { harness: 'cli-base', prompt: { systemPrompt: 'drive the worker' }, - }, + }), { brain, blobs, makeWorkerAgent: () => worker, perWorker, maxTurns: 8 }, ) const result = await runSupervisor(root, blobs, journal) @@ -111,17 +114,16 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen const driveHarness: DriveHarness = async ({ coordinationMcpUrl }) => { await jsonRpc(coordinationMcpUrl, 'tools/call', { name: 'spawn_agent', - arguments: { profile: {}, task: 'go' }, + arguments: { profile: testAgentProfile('worker'), task: 'go' }, }) await jsonRpc(coordinationMcpUrl, 'tools/call', { name: 'await_event', arguments: {} }) await jsonRpc(coordinationMcpUrl, 'tools/call', { name: 'stop', arguments: {} }) } const root = supervisorAgent( - { - name: 'sup', + testAgentProfile('sup', { harness: 'opencode', prompt: { systemPrompt: 'delegate, do not solve' }, - }, + }), { blobs, makeWorkerAgent: () => deliveringLeaf('w', { answer: 7 }), perWorker, driveHarness }, ) const result = await runSupervisor(root, blobs, journal) @@ -139,7 +141,10 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen throw new Error('backend exited after submission') } const root = supervisorAgent( - { name: 'sup', harness: 'pi', systemPrompt: 'solve or delegate' }, + testAgentProfile('sup', { + harness: 'pi', + prompt: { systemPrompt: 'solve or delegate' }, + }), { blobs, makeWorkerAgent: () => deliveringLeaf('unused', {}), @@ -160,20 +165,22 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen it('fails loud when a sandboxed-harness supervisor has no driveHarness substrate', () => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { blobs, makeWorkerAgent: () => deliveringLeaf('w', {}), perWorker }, - ), + supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + }), ).toThrow(/driveHarness/) }) it('fails loud when a router-brained supervisor has neither a brain nor a router config', () => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'root', harness: 'cli-base' }, - { blobs, makeWorkerAgent: () => deliveringLeaf('w', {}), perWorker }, - ), + supervisorAgent(testAgentProfile('root', { harness: 'cli-base' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + }), ).toThrow(/router/) }) @@ -240,17 +247,14 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen } const routerBlobs = new InMemoryResultBlobStore() await runSupervisor( - supervisorAgent( - { name: 'router-manager', harness: 'cli-base' }, - { - brain, - blobs: routerBlobs, - makeWorkerAgent: () => deliveringLeaf('unused', {}), - perWorker, - nodeContext, - resolveSupervisorTools, - }, - ), + supervisorAgent(testAgentProfile('router-manager', { harness: 'cli-base' }), { + brain, + blobs: routerBlobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + nodeContext, + resolveSupervisorTools, + }), routerBlobs, new InMemorySpawnJournal(), ) @@ -274,17 +278,14 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen }) } await runSupervisor( - supervisorAgent( - { name: 'external-manager', harness: 'opencode' }, - { - blobs: externalBlobs, - makeWorkerAgent: () => deliveringLeaf('unused', {}), - perWorker, - driveHarness, - nodeContext, - resolveSupervisorTools, - }, - ), + supervisorAgent(testAgentProfile('external-manager', { harness: 'opencode' }), { + blobs: externalBlobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + driveHarness, + nodeContext, + resolveSupervisorTools, + }), externalBlobs, new InMemorySpawnJournal(), ) @@ -341,59 +342,55 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen toolCancelled = resolve }) let nestedSignal: AbortSignal | undefined - const nested = supervisorAgent( - { name: 'nested-manager', harness: 'cli-base' }, - { - brain: scriptedBrain([ - { toolCalls: [{ name: 'run_experiment', arguments: { candidate: 'a' } }] }, - { content: 'must not continue after cancellation' }, - ]), - blobs, - makeWorkerAgent: () => deliveringLeaf('unused', {}), - perWorker, - nodeContext: { - runId: 'recursive-tool-abort', - runNamespace: 'recursive-tool-abort-namespace', - ownerId: 'owner-nested', - depth: 1, - assignmentId: 'nested-assignment', - identity: { - profileDigest: `sha256:${'e'.repeat(64)}`, - taskDigest: `sha256:${'f'.repeat(64)}`, - }, + const nested = supervisorAgent(testAgentProfile('nested-manager', { harness: 'cli-base' }), { + brain: scriptedBrain([ + { toolCalls: [{ name: 'run_experiment', arguments: { candidate: 'a' } }] }, + { content: 'must not continue after cancellation' }, + ]), + blobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + nodeContext: { + runId: 'recursive-tool-abort', + runNamespace: 'recursive-tool-abort-namespace', + ownerId: 'owner-nested', + depth: 1, + assignmentId: 'nested-assignment', + identity: { + profileDigest: `sha256:${'e'.repeat(64)}`, + taskDigest: `sha256:${'f'.repeat(64)}`, }, - resolveSupervisorTools: async () => [ - { - name: 'run_experiment', - description: 'Run a long product-owned experiment', - inputSchema: { type: 'object' }, - handler: async (_raw, context) => { - nestedSignal = context.signal - toolStarted() - await new Promise((_resolve, reject) => { - const onAbort = () => { - toolCancelled() - reject(new DOMException(String(context.signal.reason), 'AbortError')) - } - if (context.signal.aborted) onAbort() - else context.signal.addEventListener('abort', onAbort, { once: true }) - }) - return { unreachable: true } - }, - }, - ], }, - ) + resolveSupervisorTools: async () => [ + { + name: 'run_experiment', + description: 'Run a long product-owned experiment', + inputSchema: { type: 'object' }, + handler: async (_raw, context) => { + nestedSignal = context.signal + toolStarted() + await new Promise((_resolve, reject) => { + const onAbort = () => { + toolCancelled() + reject(new DOMException(String(context.signal.reason), 'AbortError')) + } + if (context.signal.aborted) onAbort() + else context.signal.addEventListener('abort', onAbort, { once: true }) + }) + return { unreachable: true } + }, + }, + ], + }) const root: Agent = { name: 'root', async act(task, scope) { const spawned = scope.spawn( driverChild( - { - name: 'nested-manager', + testAgentProfile('nested-manager', { harness: 'cli-base', metadata: { role: 'driver' }, - }, + }), nested, journal, ), @@ -458,45 +455,42 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen harnessFinished() } } - const root = supervisorAgent( - { name: 'external-manager', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('unused', {}), - perWorker, - driveHarness, - nodeContext: { - runId: 'external-tool-abort', - runNamespace: 'external-tool-abort-namespace', - ownerId: 'owner-external', - depth: 0, - identity: { - profileDigest: `sha256:${'1'.repeat(64)}`, - taskDigest: `sha256:${'2'.repeat(64)}`, - }, + const root = supervisorAgent(testAgentProfile('external-manager', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + driveHarness, + nodeContext: { + runId: 'external-tool-abort', + runNamespace: 'external-tool-abort-namespace', + ownerId: 'owner-external', + depth: 0, + identity: { + profileDigest: `sha256:${'1'.repeat(64)}`, + taskDigest: `sha256:${'2'.repeat(64)}`, }, - resolveSupervisorTools: async () => [ - { - name: 'run_experiment', - description: 'Run a long product-owned experiment', - inputSchema: { type: 'object' }, - handler: async (_raw, context) => { - externalSignal = context.signal - toolStarted() - await new Promise((_resolve, reject) => { - const onAbort = () => { - toolCancelled() - reject(new DOMException(String(context.signal.reason), 'AbortError')) - } - if (context.signal.aborted) onAbort() - else context.signal.addEventListener('abort', onAbort, { once: true }) - }) - return { unreachable: true } - }, - }, - ], }, - ) + resolveSupervisorTools: async () => [ + { + name: 'run_experiment', + description: 'Run a long product-owned experiment', + inputSchema: { type: 'object' }, + handler: async (_raw, context) => { + externalSignal = context.signal + toolStarted() + await new Promise((_resolve, reject) => { + const onAbort = () => { + toolCancelled() + reject(new DOMException(String(context.signal.reason), 'AbortError')) + } + if (context.signal.aborted) onAbort() + else context.signal.addEventListener('abort', onAbort, { once: true }) + }) + return { unreachable: true } + }, + }, + ], + }) const running = createSupervisor().run(root, 'run the external experiment', { budget: { maxIterations: 100, maxTokens: 100_000 }, runId: 'external-tool-abort', @@ -559,7 +553,10 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen return { content: 'must not run', toolCalls: [] } } const mutableRouterDeps = { ...deps, brain } - const router = supervisorAgent({ name: 'router', harness: 'cli-base' }, mutableRouterDeps) + const router = supervisorAgent( + testAgentProfile('router', { harness: 'cli-base' }), + mutableRouterDeps, + ) mutableRouterDeps.resolveSupervisorTools = async () => { replacementCalls += 1 return [] @@ -571,24 +568,21 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen expect(brainCalls).toBe(0) const externalBlobs = new InMemoryResultBlobStore() - const external = supervisorAgent( - { name: 'external', harness: 'opencode' }, - { - ...deps, - blobs: externalBlobs, - resolveSupervisorTools: async () => [ - { - name: 'spawn_agent', - description: 'collision', - inputSchema: { type: 'object' }, - handler: async () => ({}), - }, - ], - driveHarness: async () => { - harnessCalls += 1 + const external = supervisorAgent(testAgentProfile('external', { harness: 'opencode' }), { + ...deps, + blobs: externalBlobs, + resolveSupervisorTools: async () => [ + { + name: 'spawn_agent', + description: 'collision', + inputSchema: { type: 'object' }, + handler: async () => ({}), }, + ], + driveHarness: async () => { + harnessCalls += 1 }, - ) + }) const externalResult = await runSupervisor(external, externalBlobs, new InMemorySpawnJournal()) expect(externalResult.kind).toBe('no-winner') expect(harnessCalls).toBe(0) @@ -597,12 +591,15 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen describe('resolveSupervisorProfile — a canonical AgentProfile IS a supervisor profile', () => { it('reduces a canonical AgentProfile: model.default is the id, prompt.systemPrompt is the prompt', () => { - const profile: AgentProfile = { - name: 'root', + const profile = testAgentProfile('root', { harness: 'claude-code', - model: { default: 'anthropic/claude-opus-5', small: 'anthropic/claude-haiku-5' }, + model: { + provider: 'anthropic', + default: 'anthropic/claude-opus-5', + small: 'anthropic/claude-haiku-5', + }, prompt: { systemPrompt: 'delegate, do not solve' }, - } + }) expect(resolveSupervisorProfile(profile)).toEqual({ name: 'root', harness: 'claude-code', @@ -611,100 +608,73 @@ describe('resolveSupervisorProfile — a canonical AgentProfile IS a supervisor }) }) - it('takes the hints default as the id and defaults name/harness (no reasoningEffort field)', () => { - expect(resolveSupervisorProfile({ model: { default: 'm', reasoningEffort: 'xhigh' } })).toEqual( - { - name: 'supervisor', - harness: null, - modelId: 'm', - }, - ) - }) - - it('leaves a plain string model and a top-level systemPrompt exactly as given', () => { - expect(resolveSupervisorProfile({ name: 'r', model: 'gpt-5.4', systemPrompt: 'go' })).toEqual({ - name: 'r', + it('defaults only the optional name while preserving exact execution identity', () => { + const { name: _name, ...unnamed } = testAgentProfile('temporary', { + harness: 'cli-base', + model: { provider: 'offline', default: 'm', reasoningEffort: 'xhigh' }, + }) + expect(resolveSupervisorProfile(unnamed)).toEqual({ + name: 'supervisor', harness: null, - modelId: 'gpt-5.4', - systemPrompt: 'go', + modelId: 'm', }) }) - it('accepts two IDENTICAL spellings of the same system prompt', () => { - expect( - resolveSupervisorProfile({ prompt: { systemPrompt: 'same' }, systemPrompt: 'same' }) - .systemPrompt, - ).toBe('same') - }) - - it('fails loud when prompt.systemPrompt and systemPrompt disagree, naming both values', () => { + it('refuses an incomplete execution identity', () => { expect(() => resolveSupervisorProfile({ - prompt: { systemPrompt: 'from the prompt block' }, - systemPrompt: 'from the top level', - }), - ).toThrow(/prompt\.systemPrompt.*from the prompt block.*systemPrompt.*from the top level/s) - }) - - // `AgentProfileModelHints.default` is OPTIONAL in the canonical schema — `{ provider: 'anthropic' }` - // is a profile the schema accepts — so hints with no id name NO model, which is the documented - // "the router config's model applies" case, not a fault. - it('resolves hints with no usable default to no model id at all', () => { - expect(resolveSupervisorProfile({ model: { small: 'cheap' } })).toEqual({ - name: 'supervisor', - harness: null, - }) - expect(resolveSupervisorProfile({ model: { provider: 'anthropic' } }).modelId).toBeUndefined() - expect(resolveSupervisorProfile({ model: { default: '' } }).modelId).toBeUndefined() - expect( - resolveSupervisorProfile({ model: { default: HARNESS_NATIVE_MODEL } }).modelId, - ).toBeUndefined() + name: 'root', + harness: 'cli-base', + model: { provider: 'anthropic' }, + } as AgentProfile), + ).toThrow(/AgentProfile\.model\.default is missing/) }) it('appends prompt.instructions and resources.instructions to the system prompt, in that order', () => { expect( - resolveSupervisorProfile({ - prompt: { systemPrompt: 'delegate, do not solve', instructions: ['one', 'two'] }, - resources: { instructions: 'from resources' }, - }).systemPrompt, + resolveSupervisorProfile( + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'delegate, do not solve', instructions: ['one', 'two'] }, + resources: { instructions: 'from resources' }, + }), + ).systemPrompt, ).toBe('delegate, do not solve\none\ntwo\nfrom resources') }) it('reads an INLINE resources.instructions resource as its content', () => { expect( - resolveSupervisorProfile({ - systemPrompt: 'base', - resources: { instructions: { kind: 'inline', name: 'memory', content: 'learned: X' } }, - }).systemPrompt, + resolveSupervisorProfile( + testAgentProfile('root', { + harness: 'cli-base', + prompt: { systemPrompt: 'base' }, + resources: { instructions: { kind: 'inline', name: 'memory', content: 'learned: X' } }, + }), + ).systemPrompt, ).toBe('base\nlearned: X') }) it('resolves instruction lines even when no system prompt is named', () => { - expect(resolveSupervisorProfile({ prompt: { instructions: ['only this'] } }).systemPrompt).toBe( - 'only this', - ) + expect( + resolveSupervisorProfile( + testAgentProfile('root', { + harness: 'cli-base', + prompt: { instructions: ['only this'] }, + }), + ).systemPrompt, + ).toBe('only this') }) it('fails loud on a github resources.instructions reference it cannot fetch', () => { expect(() => - resolveSupervisorProfile({ - resources: { instructions: { kind: 'github', path: 'docs/INSTRUCTIONS.md' } }, - }), + resolveSupervisorProfile( + testAgentProfile('root', { + harness: 'cli-base', + resources: { instructions: { kind: 'github', path: 'docs/INSTRUCTIONS.md' } }, + }), + ), ).toThrow(/github resource reference.*docs\/INSTRUCTIONS\.md/s) }) - - it('bounds the disagreeing-prompt message: lengths and excerpts, never two whole prompts', () => { - const long = 'A'.repeat(4000) - let message = '' - try { - resolveSupervisorProfile({ prompt: { systemPrompt: long }, systemPrompt: `${long}B` }) - } catch (error) { - message = (error as Error).message - } - expect(message).toMatch(/4000 chars/) - expect(message).toMatch(/4001 chars/) - expect(message.length).toBeLessThan(400) - }) }) describe('supervisorAgent — coordination bind + prompt hoisting on the harness arm', () => { @@ -713,12 +683,11 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness const journal = new InMemorySpawnJournal() // A canonical AgentProfile: the schema rejects unknown top-level keys, so a `systemPrompt` // hoisted onto this object would make the profile fail its own validator. - const profile: AgentProfile = { - name: 'sup', + const profile = testAgentProfile('sup', { harness: 'opencode', prompt: { systemPrompt: 'delegate, do not solve', instructions: ['keep it small'] }, resources: { instructions: 'prefer the fewest workers' }, - } + }) let seen: SupervisorProfile | undefined let seenPrompt: string | undefined const driveHarness: DriveHarness = async (args) => { @@ -740,13 +709,15 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness expect(seenPrompt).toBe('delegate, do not solve\nkeep it small\nprefer the fewest workers') }) - it('SANDBOX arm builds and runs with a model the router could not resolve', async () => { + it('SANDBOX arm runs the exact profile model without router configuration', async () => { const blobs = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() let ran = false const root = supervisorAgent( - // Valid canonical profile: `model.default` is optional upstream. - { name: 'sup', harness: 'opencode', model: { provider: 'anthropic' } }, + testAgentProfile('sup', { + harness: 'opencode', + model: { provider: 'anthropic', default: 'claude-sonnet-4' }, + }), { blobs, makeWorkerAgent: () => deliveringLeaf('w', {}), @@ -767,7 +738,10 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness // A profile that names ONLY instructions: the default standing prompt must survive, with // the lines appended to it. const root = supervisorAgent( - { name: 'sup', harness: null, prompt: { instructions: ['never edit main'] } }, + testAgentProfile('sup', { + harness: 'cli-base', + prompt: { instructions: ['never edit main'] }, + }), { blobs, makeWorkerAgent: () => deliveringLeaf('w', {}), @@ -788,17 +762,14 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness const journal = new InMemorySpawnJournal() let seen: unknown = 'unset' // The harness carries its own standing prompt, so the router's default must NOT leak in. - const root = supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async (args: Parameters[0]) => { - seen = args.systemPrompt - }, + const root = supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async (args: Parameters[0]) => { + seen = args.systemPrompt }, - ) + }) await runSupervisor(root, blobs, journal) expect(seen).toBeUndefined() }) @@ -806,32 +777,26 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness it('refuses a coordination binding on a router-brained supervisor', () => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'sup', harness: null }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - brain: async () => ({ content: '', toolCalls: [] }), - coordination: { host: '127.0.0.1' }, - }, - ), + supervisorAgent(testAgentProfile('sup', { harness: 'cli-base' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + brain: async () => ({ content: '', toolCalls: [] }), + coordination: { host: '127.0.0.1' }, + }), ).toThrow(ConfigError) }) it('refuses a non-loopback coordination host with no acknowledgment (a ConfigError)', () => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async () => {}, - coordination: { host: '0.0.0.0' }, - }, - ), + supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async () => {}, + coordination: { host: '0.0.0.0' }, + }), ).toThrow(ConfigError) }) @@ -840,18 +805,15 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness const journal = new InMemorySpawnJournal() let url = '' const coordination: { host: string } = { host: '127.0.0.1' } - const root = supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async ({ coordinationMcpUrl }) => { - url = coordinationMcpUrl - }, - coordination, + const root = supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async ({ coordinationMcpUrl }) => { + url = coordinationMcpUrl }, - ) + coordination, + }) coordination.host = '0.0.0.0' await runSupervisor(root, blobs, journal) expect(url).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/mcp$/) @@ -860,16 +822,13 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness it('refuses a non-loopback coordination host with no acknowledgment', () => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async () => {}, - coordination: { host: '0.0.0.0' }, - }, - ), + supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async () => {}, + coordination: { host: '0.0.0.0' }, + }), ).toThrow(/not a loopback address.*allowUnauthenticatedRemote/s) }) @@ -878,16 +837,13 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness (host) => { const blobs = new InMemoryResultBlobStore() expect(() => - supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async () => {}, - coordination: { host }, - }, - ), + supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async () => {}, + coordination: { host }, + }), ).not.toThrow() }, ) @@ -899,16 +855,13 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness const driveHarness: DriveHarness = async ({ coordinationMcpUrl }) => { url = coordinationMcpUrl } - const root = supervisorAgent( - { name: 'sup', harness: 'opencode' }, - { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness, - coordination: { host: '0.0.0.0', allowUnauthenticatedRemote: true }, - }, - ) + const root = supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness, + coordination: { host: '0.0.0.0', allowUnauthenticatedRemote: true }, + }) await runSupervisor(root, blobs, journal) expect(url).toMatch(/^http:\/\/0\.0\.0\.0:\d+\/mcp$/) }) diff --git a/tests/kernel/supervisor-authoring.test.ts b/tests/kernel/supervisor-authoring.test.ts index 73acfbd3..fa6d3cf4 100644 --- a/tests/kernel/supervisor-authoring.test.ts +++ b/tests/kernel/supervisor-authoring.test.ts @@ -4,7 +4,6 @@ import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable import { type AuthoredProfile, asAuthoredProfile, - canonicalizeAuthoredProfile, supervisorInstructions, } from '../../src/runtime/supervise/authoring' import { driverAgent } from '../../src/runtime/supervise/coordination-driver' @@ -21,6 +20,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import { type ScriptedTurn, scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // A delivering leaf worker (settles valid) — stands in for a real model call in this offline proof. function deliveringLeaf(name: string, out: unknown): Agent { @@ -30,6 +30,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { return (async function* () { yield { kind: 'iteration' } as UsageEvent yield { kind: 'tokens', input: 5, output: 5 } as UsageEvent + yield { kind: 'cost', usd: 0 } as UsageEvent })() }, teardown: () => Promise.resolve({ destroyed: true }), @@ -40,7 +41,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 }, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor: ex } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -58,13 +59,12 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), { name: 'spawn_agent', arguments: { - profile: { - name: 'parser', + profile: testAgentProfile('parser', { prompt: { systemPrompt: 'You are a PARSER specialist. Tokenize the expression into numbers, operators and parens; emit a JSON token list. Validate balanced parens.', }, - }, + }), task: 'parse the expression', }, }, @@ -75,14 +75,13 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), { name: 'spawn_agent', arguments: { - profile: { - name: 'evaluator', + profile: testAgentProfile('evaluator', { prompt: { systemPrompt: 'You are an EVALUATOR specialist. Given a token list, apply operator precedence and compute the numeric result. Return only the number.', }, - model: { default: 'deepseek-chat' }, - }, + model: { provider: 'tangle-router', default: 'deepseek-chat' }, + }), task: 'evaluate the tokens', }, }, @@ -136,64 +135,34 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), it('rejects an empty/placeholder profile (a skill violation the system can catch)', () => { expect(asAuthoredProfile({})).toBeNull() - expect(asAuthoredProfile({ systemPrompt: '' })).toBeNull() - expect(asAuthoredProfile({ systemPrompt: ' ' })).toBeNull() + expect(asAuthoredProfile(testAgentProfile('w'))).toBeNull() expect( - asAuthoredProfile({ name: 'w', prompt: { systemPrompt: 'real instructions' } })?.name, + asAuthoredProfile(testAgentProfile('w', { prompt: { systemPrompt: 'real instructions' } })) + ?.name, ).toBe('w') }) - // The skill asks for flat `systemPrompt` / `model`; every leaf reads `prompt.systemPrompt` and - // `model.default`, and the sandbox leaf hands the profile to a schema that rejects the flat key - // outright. What the supervisor writes has to be what the worker runs. - describe('canonicalizeAuthoredProfile', () => { - it('lifts the flat fields the skill asks for into the shape every leaf reads', () => { - const canonical = canonicalizeAuthoredProfile({ - name: 'ok-writer', - systemPrompt: 'Write exactly OK.', - model: 'glm-5.2', - }) as { name: string; prompt?: { systemPrompt?: string }; model?: unknown } - expect(canonical.prompt?.systemPrompt).toBe('Write exactly OK.') - expect(canonical.model).toEqual({ default: 'glm-5.2' }) - expect(canonical.name).toBe('ok-writer') - // The flat key is what a strict profile schema rejects — it must be gone, not duplicated. - expect(Object.keys(canonical)).not.toContain('systemPrompt') - }) - - it('leaves an already-canonical profile untouched, including its other prompt fields', () => { - const authored = { - name: 'w', + describe('asAuthoredProfile', () => { + it('preserves a complete canonical profile, including its other prompt fields', () => { + const authored = testAgentProfile('w', { prompt: { systemPrompt: 'canonical', instructions: ['one'] }, - model: { default: 'm', small: 's' }, - } - expect(canonicalizeAuthoredProfile(authored)).toEqual(authored) - }) - - it('collapses BOTH shapes when they agree', () => { - const canonical = canonicalizeAuthoredProfile({ - prompt: { systemPrompt: 'same instruction' }, - systemPrompt: 'same instruction', - }) as { prompt?: { systemPrompt?: string } } - expect(canonical.prompt?.systemPrompt).toBe('same instruction') - expect(Object.keys(canonical)).not.toContain('systemPrompt') + model: { provider: 'offline', default: 'm', small: 's' }, + }) + expect(asAuthoredProfile(authored)).toEqual(authored) }) - // Two spellings of one standing instruction, set to different text, has no safe reading — - // the same rule the supervisor's own profile is held to. - it('fails loud when the two shapes disagree', () => { - expect(() => - canonicalizeAuthoredProfile({ - prompt: { systemPrompt: 'one instruction' }, - systemPrompt: 'a different instruction', - }), - ).toThrow(/both set and differ/) + it('refuses a canonical profile whose system prompt is blank', () => { + expect( + asAuthoredProfile(testAgentProfile('w', { prompt: { systemPrompt: ' ' } })), + ).toBeNull() }) - it('passes a blank flat prompt through rather than manufacturing an empty one', () => { - const canonical = canonicalizeAuthoredProfile({ name: 'w', systemPrompt: ' ' }) as { - prompt?: unknown - } - expect(canonical.prompt).toBeUndefined() + it('uses the canonical worker name when the optional name is omitted', () => { + const profile = testAgentProfile('temporary', { + prompt: { systemPrompt: 'Do the assigned work.' }, + }) + const { name: _name, ...unnamed } = profile + expect(asAuthoredProfile(unnamed)?.name).toBe('worker') }) }) diff --git a/tests/kernel/test-agent-profile.ts b/tests/kernel/test-agent-profile.ts new file mode 100644 index 00000000..1cb0d55d --- /dev/null +++ b/tests/kernel/test-agent-profile.ts @@ -0,0 +1,19 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' + +type TestProfileOverrides = Omit, 'name' | 'model'> & { + model?: AgentProfile['model'] +} + +/** Complete offline identity for tests that exercise execution rather than profile validation. */ +export function testAgentProfile(name: string, overrides: TestProfileOverrides = {}): AgentProfile { + return { + name, + harness: 'opencode', + ...overrides, + model: { + provider: 'offline', + default: 'offline-test-model', + ...overrides.model, + }, + } +} diff --git a/tests/mcp/fleet-executor.test.ts b/tests/mcp/fleet-executor.test.ts index 659fb96b..83fd9131 100644 --- a/tests/mcp/fleet-executor.test.ts +++ b/tests/mcp/fleet-executor.test.ts @@ -1,4 +1,3 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import { detachedSessionDelegate } from '../../src/mcp/delegates' @@ -13,8 +12,9 @@ import { type OutputAdapter, runAgentRounds, } from '../../src/runtime' +import { testAgentProfile } from '../kernel/test-agent-profile' -const profile: AgentProfile = { name: 'stub' } +const profile = testAgentProfile('stub') interface SimpleTask { goal: string diff --git a/tests/mcp/worktree-harness.test.ts b/tests/mcp/worktree-harness.test.ts index 7d770f7b..5b9eed10 100644 --- a/tests/mcp/worktree-harness.test.ts +++ b/tests/mcp/worktree-harness.test.ts @@ -672,7 +672,7 @@ describe('runWorktreeHarness profile materialization', () => { runHarness, }), ).rejects.toThrow( - /profile materialization would drop axis changes.*modelSmall, modelProvider, modelMetadata, connections, confidential, extensions/su, + /profile materialization would drop axis changes.*modelSmall, modelMetadata, connections, confidential, extensions/su, ) expect(runHarness).not.toHaveBeenCalled() expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) From 26f852da679c9d108fed8276de930ed1167d8229 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:45:31 -0600 Subject: [PATCH 09/39] test(runtime): cover exact sandbox authority --- src/mcp/detached-coder.ts | 2 +- tests/kernel/sandbox-leaf-settle.test.ts | 2 - tests/kernel/supervise-worker-trace.test.ts | 4 +- tests/mcp/detached-coder.test.ts | 43 +++++++++---------- tests/mcp/fleet-executor.test.ts | 10 ++--- .../runtime/executor-config-snapshot.test.ts | 1 - tests/runtime/mid-flight-steering.test.ts | 3 +- 7 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/mcp/detached-coder.ts b/src/mcp/detached-coder.ts index 0ad386a6..399da0a8 100644 --- a/src/mcp/detached-coder.ts +++ b/src/mcp/detached-coder.ts @@ -80,7 +80,7 @@ export function multiHarnessCoderFanout(options: MultiHarnessCoderFanoutOptions) validator: Validator driver: Driver } { - if (options.profiles.length === 0) { + if (!options.profiles || options.profiles.length === 0) { throw new ConfigError('multiHarnessCoderFanout: at least one exact profile is required') } const agentRuns = options.profiles.map((profile) => coderRunSpec({ profile })) diff --git a/tests/kernel/sandbox-leaf-settle.test.ts b/tests/kernel/sandbox-leaf-settle.test.ts index 07ec71fc..1181b561 100644 --- a/tests/kernel/sandbox-leaf-settle.test.ts +++ b/tests/kernel/sandbox-leaf-settle.test.ts @@ -53,7 +53,6 @@ describe('sandbox leaf — the settle contract', () => { it('marks a completed round DELIVERED, so a run with no oracle has something to select', async () => { const executor = createExecutor({ backend: 'sandbox', - harness: 'opencode', sandboxClient: sandboxClient(), })(spec, ctx()) await drain(executor.execute('task', new AbortController().signal) as AsyncIterable) @@ -65,7 +64,6 @@ describe('sandbox leaf — the settle contract', () => { it('fails the worker with the round’s own error instead of an artifact it never produced', async () => { const executor = createExecutor({ backend: 'sandbox', - harness: 'opencode', sandboxClient: sandboxClient({ createFails: 'billingOwnerId is required' }), })(spec, ctx()) await expect( diff --git a/tests/kernel/supervise-worker-trace.test.ts b/tests/kernel/supervise-worker-trace.test.ts index ca88c4cb..39244b54 100644 --- a/tests/kernel/supervise-worker-trace.test.ts +++ b/tests/kernel/supervise-worker-trace.test.ts @@ -424,7 +424,6 @@ describe('the sandbox arm carries the context onto the box itself', () => { executors: registryOf( createExecutor({ backend: 'sandbox', - harness: 'opencode', sandboxClient: fake.client, }), ), @@ -451,7 +450,6 @@ describe('the sandbox arm carries the context onto the box itself', () => { executors: registryOf( createExecutor({ backend: 'sandbox', - harness: 'opencode', sandboxClient: fake.client, }), ), @@ -485,7 +483,7 @@ describe('supervise({ backend, otel }) stamps its workers too', () => { { budget: { maxIterations: 100, maxTokens: 100_000 }, runId: 'front-door', - backend: { backend: 'sandbox', harness: 'opencode', sandboxClient: fake.client }, + backend: { backend: 'sandbox', sandboxClient: fake.client }, brain: scriptedBrain([ { toolCalls: [ diff --git a/tests/mcp/detached-coder.test.ts b/tests/mcp/detached-coder.test.ts index 06b3e79a..08cb6a32 100644 --- a/tests/mcp/detached-coder.test.ts +++ b/tests/mcp/detached-coder.test.ts @@ -7,6 +7,7 @@ import { multiHarnessCoderFanout, } from '../../src/mcp/detached-coder' import type { CoderTask } from '../../src/profiles/coder' +import { testAgentProfile } from '../kernel/test-agent-profile' const ctx = { iteration: 0, signal: new AbortController().signal } @@ -297,42 +298,40 @@ describe('coderOutputAdapter — in-process executor raw artifact projection', ( }) describe('multiHarnessCoderFanout — heterogeneous fanout bundle', () => { - it('produces one AgentRunSpec per harness, each tagging its backendType', () => { - const bundle = multiHarnessCoderFanout({ harnesses: ['claude-code', 'codex'] }) + const claude = testAgentProfile('coder-claude-code', { + harness: 'claude-code', + model: { provider: 'anthropic', default: 'claude-test' }, + }) + const codex = testAgentProfile('coder-codex', { + harness: 'codex', + model: { provider: 'openai', default: 'codex-test' }, + }) + + it('produces one AgentRunSpec per exact profile', () => { + const bundle = multiHarnessCoderFanout({ profiles: [claude, codex] }) expect(bundle.agentRuns).toHaveLength(2) expect(bundle.agentRuns.map((s) => s.name)).toEqual(['coder-claude-code', 'coder-codex']) - expect(bundle.agentRuns.map((s) => s.profile.metadata?.backendType)).toEqual([ - 'claude-code', - 'codex', - ]) + expect(bundle.agentRuns.map((s) => s.profile.harness)).toEqual(['claude-code', 'codex']) }) - it('uses a minimal model-only default profile (no hardcoded tools/skills/prompt)', () => { - const bundle = multiHarnessCoderFanout({ harnesses: ['claude-code'] }) - const profile = bundle.agentRuns[0]!.profile - expect(profile.tools).toBeUndefined() - expect(profile.prompt).toBeUndefined() + it('refuses an empty profile set instead of choosing a harness or model', () => { + expect(() => multiHarnessCoderFanout({ profiles: [] })).toThrow(/exact profile/) }) - it('threads a caller-authored worker profile onto every fanout run', () => { - const authored = { - name: 'authored', + it('threads each caller-authored exact profile without overlays', () => { + const authoredProfiles = [claude, codex].map((profile) => ({ + ...profile, tools: { git: true, fs: true }, prompt: { systemPrompt: 'be careful' }, - } + })) const bundle = multiHarnessCoderFanout({ - profile: authored, - harnesses: ['claude-code', 'codex'], + profiles: authoredProfiles, }) expect(bundle.agentRuns.every((s) => s.profile.tools?.git === true)).toBe(true) expect(bundle.agentRuns.every((s) => s.profile.prompt?.systemPrompt === 'be careful')).toBe( true, ) - // The per-harness backendType still overrides regardless of the authored profile. - expect(bundle.agentRuns.map((s) => s.profile.metadata?.backendType)).toEqual([ - 'claude-code', - 'codex', - ]) + expect(bundle.agentRuns.map((s) => s.profile.harness)).toEqual(['claude-code', 'codex']) }) }) diff --git a/tests/mcp/fleet-executor.test.ts b/tests/mcp/fleet-executor.test.ts index 83fd9131..8b55a4f6 100644 --- a/tests/mcp/fleet-executor.test.ts +++ b/tests/mcp/fleet-executor.test.ts @@ -243,13 +243,13 @@ describe('detachedSessionDelegate with executor', () => { }, } const executor = createSiblingSandboxExecutor({ client: fakeClient }) - expect(() => detachedSessionDelegate({ executor, sandboxClient: fakeClient })).toThrow( - /exactly one/, - ) + expect(() => + detachedSessionDelegate({ executor, sandboxClient: fakeClient, workerProfile: profile }), + ).toThrow(/exactly one/) }) it('rejects when neither is passed', () => { - expect(() => detachedSessionDelegate({})).toThrow(/required/) + expect(() => detachedSessionDelegate({ workerProfile: profile })).toThrow(/required/) }) it('accepts the legacy sandboxClient shorthand (defaults to sibling)', () => { @@ -258,7 +258,7 @@ describe('detachedSessionDelegate with executor', () => { return null as unknown as SandboxInstance }, } - const delegate = detachedSessionDelegate({ sandboxClient: fakeClient }) + const delegate = detachedSessionDelegate({ sandboxClient: fakeClient, workerProfile: profile }) expect(typeof delegate).toBe('function') }) }) diff --git a/tests/runtime/executor-config-snapshot.test.ts b/tests/runtime/executor-config-snapshot.test.ts index 43896b5d..145e7947 100644 --- a/tests/runtime/executor-config-snapshot.test.ts +++ b/tests/runtime/executor-config-snapshot.test.ts @@ -132,7 +132,6 @@ describe('createExecutor config intake', () => { name: 'sandbox', config: { backend: 'sandbox', - harness: 'codex', sandboxClient, maxIterations: 1, lineage: { sessionContinuity: true }, diff --git a/tests/runtime/mid-flight-steering.test.ts b/tests/runtime/mid-flight-steering.test.ts index c476d0d1..771e8ec0 100644 --- a/tests/runtime/mid-flight-steering.test.ts +++ b/tests/runtime/mid-flight-steering.test.ts @@ -262,11 +262,10 @@ function safeJson(text: string): Record | undefined { function backend(harness: FakeHarness, steerable: boolean): ExecutorConfig { return { backend: 'sandbox', - harness: 'opencode', sandboxClient: harness.client, // The ONLY difference between the proof and its falsification. ...(steerable ? { steering: { maxTurns: 6 } } : {}), - } as ExecutorConfig + } } interface AuthorityRecord { From 0612b6d5429b26930837c6b8e020d66cda87a58d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:05:53 -0600 Subject: [PATCH 10/39] fix(runtime): validate executable profiles at intake --- docs/api/primitive-catalog.md | 10 +- src/loop-runner.ts | 5 +- src/runtime/environment-provider.test.ts | 9 +- src/runtime/personify/persona.ts | 4 +- src/runtime/supervise/driver-executor.ts | 52 +++--- src/runtime/supervise/executable-spec.ts | 38 ++++ src/runtime/supervise/runtime.ts | 38 ++-- src/runtime/supervise/scope.ts | 30 +--- src/runtime/supervise/types.ts | 24 +-- tests/helpers/supervisor-resume-child.ts | 5 +- tests/kernel/completion-gate.test.ts | 5 +- tests/kernel/coordination-driver.test.ts | 13 +- tests/kernel/coordination-mcp.test.ts | 4 +- tests/kernel/driver-recursion.test.ts | 36 ++-- tests/kernel/materialization-evidence.test.ts | 2 +- tests/kernel/personify.test.ts | 45 ++++- tests/kernel/rsi-wave.test.ts | 11 +- tests/kernel/spawn-forest.test.ts | 16 +- tests/kernel/supervise-deadline.test.ts | 4 +- tests/kernel/supervise-otel-spans.test.ts | 2 +- .../supervise-restart-resource-safety.test.ts | 20 ++- tests/kernel/supervise-worker-trace.test.ts | 5 +- tests/kernel/supervise.test.ts | 163 +++++++++++++++++- tests/kernel/worktree-loop.test.ts | 18 +- tests/runtime/executor-profile-model.test.ts | 39 +++++ tests/runtime/rolling-dispatch.test.ts | 5 +- .../spawn-journal-replay-identity.test.ts | 3 +- tests/runtime/spawn-keys.test.ts | 6 +- tests/runtime/stop-rules.test.ts | 6 +- tests/runtime/worker-progress.test.ts | 8 +- tests/runtime/worker-trace-evidence.test.ts | 3 +- 31 files changed, 475 insertions(+), 154 deletions(-) create mode 100644 src/runtime/supervise/executable-spec.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5012f3c3..28769200 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -183,7 +183,7 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | | `AgentCandidateTaskExecution` | interface | Runtime placement for one exact cell from a signed candidate experiment. | | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | -| `AgentSpec` | interface | `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for | +| `AgentSpec` | interface | `AgentProfile` is the complete execution authority. Scope parses and snapshots it before calling | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | | `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | @@ -192,7 +192,7 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `CircuitBreakerConfig` | interface | Circuit-breaker tuning. `failuresToOpen` consecutive failures opens it; closed only after `cooldownMs`. | | `D1DatabaseLike` | interface | Structural type matching the surface of `D1Database` we depend on, so the | | `Executor` | interface | The leaf runtime — ONE open interface, not a closed union. `execute` returns a | -| `ExecutorRegistry` | interface | The OPEN resolver: maps an `AgentSpec` to a `ExecutorFactory`. The default | +| `ExecutorRegistry` | interface | The OPEN resolver maps an already-admitted `AgentSpec` to an `ExecutorFactory`. Scope validates | | `FinalizeContext` | interface | What a finalizer gets to decide with. `delivered` is the ONLY output material; `allSettled` | | `ImproveCandidateValidationInput` | interface | Exact materialized profile presented for validation before any candidate run. | | `ImproveCost` | interface | Normalized spend reported for one Runtime improvement run. | @@ -770,7 +770,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `AgenticSurface` | interface | A stateful, checkable environment an agent operates over with tools. Open behind one interface. | | `AgentProfile` | interface | Public provider-neutral agent profile contract. | | `AgentRunSpec` | interface | Sandbox-SDK-shaped agent specification. | -| `AgentSpec` | interface | `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for | +| `AgentSpec` | interface | `AgentProfile` is the complete execution authority. Scope parses and snapshots it before calling | | `AgentTurnUsage` | interface | Metered usage of one turn, summed over every cost-bearing event the backend | | `AnalystFinding` | interface | Unified envelope every analyst emits. Schema-versioned so renderers | | `AnalystFindingEvent` | interface | A trace-analyst result re-entered as a message on the bus (the `finding` event kind). | @@ -834,7 +834,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `ExecutorMaterialization` | interface | Data-only declaration from trusted executor code about the exact sealed plan `execute` uses. | | `ExecutorNodeContext` | interface | Kernel-owned context for the concrete supervised node a factory is constructing. | | `ExecutorProgress` | interface | What an executor OPTIONALLY adds to the scope-derived progress (`Executor.progress()`). Every | -| `ExecutorRegistry` | interface | The OPEN resolver: maps an `AgentSpec` to a `ExecutorFactory`. The default | +| `ExecutorRegistry` | interface | The OPEN resolver maps an already-admitted `AgentSpec` to an `ExecutorFactory`. Scope validates | | `ExecutorResult` | interface | Terminal artifact of a one-shot `Executor.execute`. | | `FanoutOptions` | interface | `fanout(items, { synthesize? })` — N children spawned in one round (one per item, bounded by | | `FanoutSynthesis` | interface | How a fanout's synthesis child is built + read. `synthesisTask` projects the drained child | @@ -1008,7 +1008,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `EqualKOnCost` | type | `equalKOnCost(arms, opts)` — the cross-arm equal-compute check on conserved cost. | | `ExecutionBindingReceipt` | type | One attempt's immutable link from a stable materialization plan to its actual transport. | | `ExecutorConfig` | type | Config for {@link createExecutor}: the backend is DATA — the cost dial a profile, | -| `ExecutorFactory` | type | Builds a fresh `Executor` for one spawn from the resolved spec. Per-spawn (not | +| `ExecutorFactory` | type | Builds a fresh `Executor` for one spawn from the resolved, immutable spec. Per-spawn (not shared) | | `Fanout` | type | `fanout(items, opts)` — build the fanout combinator over a static item list. | | `FanoutWinnerSelector` | type | A winner-selection strategy: argmax/sort over the gathered child iterations (each output is the | | `FlatWidenGate` | type | The flat default `ScopeWidenGate` factory contract — never widens, keeping the R2 firewall | diff --git a/src/loop-runner.ts b/src/loop-runner.ts index 14b664d7..998cff5d 100644 --- a/src/loop-runner.ts +++ b/src/loop-runner.ts @@ -21,6 +21,7 @@ * @experimental */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { runAnalystLoop } from './analyst-loop' import type { RunAnalystLoopOpts, RunAnalystLoopResult } from './analyst-loop/types' import { ConfigError } from './errors' @@ -113,6 +114,8 @@ export async function runDelegatedLoop( /** @experimental Options for the local-repo `code` runner over the GENERIC recursive path. */ export interface WorktreeLoopRunnerOptions { + /** Exact profile carried by the personified root that owns this fanout. */ + rootProfile: AgentProfile /** Absolute path to the local git checkout each worktree is cut from. */ repoRoot: string /** The instruction handed to every authored harness (composed under each profile's systemPrompt). */ @@ -174,7 +177,7 @@ export function worktreeLoopRunner( // executor is BYO (the gated worktree-CLI leaf), so the registry only needs to pass BYO through. const persona = definePersona({ name: 'worktree-coder', - root: { profile: { name: 'worktree-coder' }, harness: null }, + root: { profile: options.rootProfile, harness: null }, directive: 'deliver a minimal validated patch on a fresh worktree', context: { role: 'coder' }, executors: { registry: createExecutorRegistry() }, diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index 4f628540..9d3c55c6 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -837,7 +837,14 @@ describe('environment provider adapters', () => { const usage = await running expect(executor.runtime).toBe('session-provider') - expect(created?.profile).toBe(profile) + expect(created?.profile).toStrictEqual(profile) + expect(created?.profile).not.toBe(profile) + expect(Object.isFrozen(created?.profile)).toBe(true) + expect(typeof created?.profile).toBe('object') + if (typeof created?.profile !== 'object' || created.profile === null) { + throw new Error('expected the exact AgentProfile snapshot') + } + expect(Object.isFrozen(created.profile.model)).toBe(true) expect(created).toMatchObject({ backend: 'pi', workspace: { cwd: '/repo' }, diff --git a/src/runtime/personify/persona.ts b/src/runtime/personify/persona.ts index 0137f59e..16c3f70d 100644 --- a/src/runtime/personify/persona.ts +++ b/src/runtime/personify/persona.ts @@ -22,6 +22,7 @@ import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../durable/spawn-journal' import { ValidationError } from '../../errors' import { withDriverExecutor } from '../supervise/driver-executor' +import { executableAgentSpecSnapshot } from '../supervise/executable-spec' import { createExecutorRegistry } from '../supervise/runtime' import { createSupervisor } from '../supervise/supervisor' import type { @@ -64,9 +65,10 @@ export function definePersona(input: DefinePersonaInput): Person if (!input.root || typeof input.root !== 'object' || !('harness' in input.root)) { throw new ValidationError(`definePersona("${input.name}"): root must be an AgentSpec`) } + const root = executableAgentSpecSnapshot(input.root, `definePersona("${input.name}")`) return Object.freeze({ name: input.name, - root: input.root, + root, directive: input.directive, context: input.context, executors: input.executors, diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index d3e1d365..7ede79d3 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -38,6 +38,7 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from '../../errors' +import { executableAgentSpecSnapshot } from './executable-spec' import { attestRuntimeOwnedDeferredExecutor, runtimeOwnedScopeOwnerRuntime, @@ -68,9 +69,6 @@ import type { /** The runtime tag the registry maps a driver child to. */ export { driverRuntime } from './tree-key' -/** The metadata marker on a driver child's spec the recursive registry routes on. */ -const driverRole = 'driver' - /** A driver child's spec carries the `Agent` to run inside the nested scope. */ interface DriverSpec extends AgentSpec { readonly driverRuntime: typeof driverRuntime @@ -83,22 +81,18 @@ interface DriverSpec extends AgentSpec { /** * Mark + carry a driver `Agent` so the recursive registry resolves it to the * driver-executor. The returned agent is SPAWNED (never run directly): its - * `executorSpec` is marked `role: 'driver'` and carries the driver agent + the shared + * `executorSpec` carries the exact profile, driver agent, and shared * journal so the executor can run its `act` inside a nested scope. `act` fails loud if * called directly — a driver child runs THROUGH its nested-scope executor, never as a root. */ export function driverChild( - profileOrName: AgentProfile | string, + profile: AgentProfile, driver: Agent, journal: SpawnJournal, execution?: AgentExecutionRef, ): Agent { - const profile: AgentProfile = - typeof profileOrName === 'string' - ? { name: profileOrName, metadata: { role: driverRole } } - : profileOrName const name = profile.name ?? driver.name - const spec: DriverSpec = { + const rawSpec: DriverSpec = { profile, harness: null, ...(execution ? { execution } : {}), @@ -106,6 +100,7 @@ export function driverChild( driver: driver as Agent, journal, } + const spec = executableAgentSpecSnapshot(rawSpec, 'driverChild') as DriverSpec const deliver = driver.deliver?.bind(driver) return { name, @@ -125,21 +120,21 @@ export function driverChild( } as Agent & { executorSpec: AgentSpec } } -/** True when a spec is a driver child (carries the role marker + a driver Agent). */ +/** True when a spec carries Runtime's explicit nested-driver tag plus a driver Agent. */ export function isDriverSpec(spec: AgentSpec): spec is DriverSpec { if ((spec as { driverRuntime?: unknown }).driverRuntime !== driverRuntime) return false const driver = (spec as { driver?: unknown }).driver if (!isAgent(driver)) { throw new ValidationError( - 'driverExecutor: a driver-role spec must carry a `driver` Agent to run inside its nested scope', + 'driverExecutor: a nested-driver spec must carry a `driver` Agent to run inside its nested scope', ) } return true } /** - * The recursive driver-executor factory. `withDriverExecutor` routes a child marked - * `role: 'driver'` here; any other child resolves to a leaf built-in. On `execute`, it + * The recursive driver-executor factory. `withDriverExecutor` routes a child carrying + * Runtime's explicit nested-driver tag here; any other child resolves to a leaf built-in. On `execute`, it * reads the `nested-scope` seam the SCOPE seeded, mounts a nested `Scope` one `depth` * deeper over the driver's reserved child pool plus shared journal/blobs/registry, runs the driver * `Agent.act(task, nestedScope)`, and reports the conserved spend summed off the nested @@ -150,10 +145,11 @@ export function isDriverSpec(spec: AgentSpec): spec is DriverSpec { * thrown executor, which the parent scope types into a `down` settlement — the same * fail-loud-into-typed-down discipline a leaf gets. */ -export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { +export const driverExecutorFactory: ExecutorFactory = (rawSpec, ctx) => { + const spec = executableAgentSpecSnapshot(rawSpec, 'driverExecutorFactory') if (!isDriverSpec(spec)) { throw new ValidationError( - 'driverExecutorFactory: spec is not a driver child (no role:"driver" marker)', + 'driverExecutorFactory: spec is not a driver child (missing Runtime nested-driver tag)', ) } const driver = spec.driver @@ -278,20 +274,30 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { } /** - * Register the driver-executor so a child marked `role: 'driver'` resolves to it. The base - * registry resolves by harness alone (it does not read `role`), so a recursive run needs a - * registry that routes the driver tag here FIRST. Returns a registry decorator: a - * driver-role spec → the driver-executor; everything else → the base registry's resolution + * Register the driver-executor so a child carrying Runtime's explicit nested-driver tag resolves + * to it. The base registry resolves by harness alone, so a recursive run needs a registry that + * routes the driver tag here first. Returns a registry decorator: a tagged driver spec → the + * driver-executor; everything else → the base registry's resolution * (leaf built-ins + BYO). */ export function withDriverExecutor(base: ExecutorRegistry): ExecutorRegistry { return { register: base.register.bind(base), - resolve(spec: AgentSpec) { + resolve(rawSpec: AgentSpec) { + const spec = executableAgentSpecSnapshot(rawSpec, 'driver executor registry') if ((spec as { driverRuntime?: unknown }).driverRuntime === driverRuntime && !spec.executor) { - return { succeeded: true as const, value: driverExecutorFactory as ExecutorFactory } + return { + succeeded: true as const, + value: ((_ignored, context) => + driverExecutorFactory(spec, context)) as ExecutorFactory, + } + } + const resolved = base.resolve(spec) + if (!resolved.succeeded) return resolved + return { + succeeded: true as const, + value: (_ignored, context) => resolved.value(spec, context), } - return base.resolve(spec) }, } } diff --git a/src/runtime/supervise/executable-spec.ts b/src/runtime/supervise/executable-spec.ts new file mode 100644 index 00000000..f1cd37dd --- /dev/null +++ b/src/runtime/supervise/executable-spec.ts @@ -0,0 +1,38 @@ +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' +import { assertExecutableAgentProfile } from './model-policy' +import { detachedSnapshot } from './snapshot' +import type { AgentSpec } from './types' + +/** + * Parse one untrusted profile into the exact immutable value execution may consume. + * + * Snapshotting before schema parsing reads caller-owned fields once. Snapshotting the parsed + * result prevents downstream custom code from changing any identity or authority field. + */ +export function executableAgentProfileSnapshot(raw: unknown, context: string): AgentProfile { + const input = detachedSnapshot(raw, `${context}: AgentProfile input`) + const profile = agentProfileSchema.parse(input) + assertExecutableAgentProfile(profile, context) + return detachedSnapshot(profile, `${context}: parsed AgentProfile`) +} + +/** Snapshot an executable spec while retaining trusted callbacks by reference. */ +export function executableAgentSpecSnapshot(raw: AgentSpec, context: string): AgentSpec { + const { + profile: rawProfile, + harness, + execution: rawExecution, + ...runtimeExtensions + } = raw as AgentSpec & Readonly> + const profile = executableAgentProfileSnapshot(rawProfile, context) + const execution = + rawExecution === undefined + ? undefined + : detachedSnapshot(rawExecution, `${context}: execution attribution`) + return Object.freeze({ + ...runtimeExtensions, + profile, + harness, + ...(execution === undefined ? {} : { execution }), + }) as AgentSpec +} diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 4f8817a6..49c0ed56 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -89,6 +89,7 @@ import type { SandboxClient, } from '../types' import { zeroTokenUsage } from '../util' +import { executableAgentProfileSnapshot, executableAgentSpecSnapshot } from './executable-spec' import { createInbox, type Inbox } from './inbox' import { attestRuntimeOwnedExecutor, @@ -3664,15 +3665,15 @@ export function bindReusableExecutorExecutionId( /** * The single built-in executor factory. Picks a leaf backend by data (`config.backend`), * injects the matching seam, and delegates to that backend's built-in implementation. - * The `Executor` port stays OPEN: bring-your-own agents implement `Executor` directly - * and never pass through here. Use this (or `createExecutorRegistry`) instead of a + * The `Executor` port stays OPEN: bring-your-own agents implement `Executor` directly, while Scope + * or `createExecutorRegistry` still parses and seals their exact profile before use. Use this instead of a * per-vendor adapter or a closed `inline|sandbox|cli` switch — those bypass the * `UsageEvent` reporting channel. */ export function createExecutor(config: ExecutorConfig): ExecutorFactory { const captured = snapshotExecutorConfig(config) - return (spec, ctx) => { - assertExecutableAgentProfile(spec.profile, `createExecutor(${captured.backend})`) + return (rawSpec, ctx) => { + const spec = executableAgentSpecSnapshot(rawSpec, `createExecutor(${captured.backend})`) const { backend, ...seamData } = captured as ExecutorConfig & Record const seam = Object.freeze(seamData) const seamed: ExecutorContext = { ...ctx, seams: { ...ctx.seams, [backend]: seam } } @@ -3733,8 +3734,10 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory return providerAsExecutor(provider, { ...providerSeam, profileForCreate: (profile) => { - const prepared = profileForCreate?.(profile) ?? profile - assertExecutableAgentProfile(prepared, 'createExecutor(provider)') + const prepared = executableAgentProfileSnapshot( + profileForCreate?.(profile) ?? profile, + 'createExecutor(provider)', + ) if (canonicalAgentProfileDigest(prepared) !== canonicalAgentProfileDigest(profile)) { throw new ValidationError( 'createExecutor(provider): profileForCreate changed the exact AgentProfile; execution overlays are not allowed', @@ -3782,9 +3785,8 @@ function requiredProviderProfileHarness(spec: AgentSpec, seam: ProviderSeam): Ba /** * The open resolver/registry. Pre-registers the three built-ins under their * runtime tags (`'router'`, `'sandbox'`, `'cli'`) and accepts `register(name, - * factory)` for any additional runtime — and a BYO `AgentSpec.executor` resolves - * without touching the registry at all. NOT a closed switch; registration + BYO - * ARE the extension points. + * factory)` for any additional runtime. A BYO `AgentSpec.executor` has highest routing precedence + * after the same exact-profile intake validation. Registration + BYO remain open extension points. * * `resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executorFactory` → * `spec.executor` → `harness === null` → the `'router'` factory; else a registered factory for the @@ -3805,22 +3807,30 @@ export function createExecutorRegistry(): ExecutorRegistry { factories.set(runtime, factory as ExecutorFactory) }, resolve( - spec: AgentSpec, + rawSpec: AgentSpec, ): { succeeded: true; value: ExecutorFactory } | { succeeded: false; error: string } { + const spec = executableAgentSpecSnapshot(rawSpec, 'executor registry') + const bind = + (factory: ExecutorFactory): ExecutorFactory => + (_ignored, context) => + factory(spec, context) // BYO factory: constructed only after Scope admission with the real signal/context. if (spec.executorFactory) { - return { succeeded: true, value: spec.executorFactory as ExecutorFactory } + return { + succeeded: true, + value: bind(spec.executorFactory as ExecutorFactory), + } } // BYO: a caller-supplied executor wins, wrapped in a trivial per-spawn factory. if (spec.executor) { const byo = spec.executor - return { succeeded: true, value: (() => byo) as ExecutorFactory } + return { succeeded: true, value: bind(() => byo as Executor) } } // router/inline: an agent with no harness is a direct Router call. if (spec.harness === null) { const f = factories.get('router') if (!f) return { succeeded: false, error: 'executor registry: no "router" factory' } - return { succeeded: true, value: f as ExecutorFactory } + return { succeeded: true, value: bind(f as ExecutorFactory) } } // sandbox: any BackendType maps to the sandbox-composing-runAgentRounds executor. const runtimeTag: Runtime = 'sandbox' @@ -3831,7 +3841,7 @@ export function createExecutorRegistry(): ExecutorRegistry { error: `executor registry: no factory for runtime "${runtimeTag}" (harness "${spec.harness}") and no BYO executor`, } } - return { succeeded: true, value: f as ExecutorFactory } + return { succeeded: true, value: bind(f as ExecutorFactory) } }, } } diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index de0aae65..51cb25a8 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -44,6 +44,7 @@ import { teardownExecutor, } from './deadline' import { freeSlots } from './dispatch' +import { executableAgentSpecSnapshot } from './executable-spec' import { authoredProfileDigest, knownExecutionBindingReceipt, @@ -507,7 +508,7 @@ export function createScope(args: ScopeArgs): Scope { `scope.spawn: agent "${agent.name}" exposes no \`executorSpec\` (AgentSpec) to resolve a Executor`, ) } - const spec = snapshotAgentSpec(rawSpec) + const spec = executableAgentSpecSnapshot(rawSpec, 'scope.spawn') return { agent, spec, identity: deriveNodeExecutionIdentity(spec, task) } } if (opts.key !== undefined) { @@ -540,6 +541,12 @@ export function createScope(args: ScopeArgs): Scope { return { ok: false, reason: 'depth-exceeded' } } + // A concrete child exposes its profile now, so parse and seal it before capacity or budget + // admission. A lazy agent factory cannot reveal a profile without being constructed; preserve + // the refusal invariant for those factories and validate the result immediately after the + // reservation, before registry resolution or any executor code. + if (typeof agentOrFactory !== 'function') prepared ??= prepare() + // ONE admission counter is shared by the root scope and every recursive scope it mounts. // Acquire before calling a lazy worker factory, resolving/constructing its executor, or // reserving budget. A completed keyed assignment returned above never touches the counter. @@ -2316,27 +2323,6 @@ function isAgentSpec(value: unknown): value is AgentSpec { return 'profile' in v && 'harness' in v } -/** Snapshot every data field whose bytes affect identity, admission, or materialization while - * preserving the executable callbacks by reference. The executor implementation is trusted code; - * its profile and attribution inputs are not. */ -function snapshotAgentSpec(raw: AgentSpec): AgentSpec { - const { - profile: rawProfile, - harness, - execution: rawExecution, - ...runtimeExtensions - } = raw as AgentSpec & Readonly> - const profile = detachedSnapshot(rawProfile, 'scope.spawn profile') - const execution = - rawExecution === undefined ? undefined : detachedSnapshot(rawExecution, 'scope.spawn execution') - return Object.freeze({ - ...runtimeExtensions, - profile, - harness, - ...(execution === undefined ? {} : { execution }), - }) as AgentSpec -} - function isAbortError(err: unknown): boolean { return ( typeof err === 'object' && diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 60734123..a22517df 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -246,9 +246,11 @@ export type Runtime = 'router' | 'inline' | 'sandbox' | 'cli' | (string & {}) // ── Executor resolution (OPEN registry, not a switch) ───────────────────────── /** - * `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for - * one concrete run. A caller may honor the preference, override it for a comparison cell, or supply - * an executor directly, without changing the profile's behavioral identity. + * `AgentProfile` is the complete execution authority. Scope parses and snapshots it before calling + * any registry, including one that resolves caller-supplied executors and factories. The default + * registry enforces the same rule when called directly. `AgentSpec.harness` records routing for one + * concrete run; where a backend consumes both fields, it must agree with `AgentProfile.harness` and + * cannot fill or override it. * * Resolution (in `runtime.ts`): * - `executorFactory` present → BYO: build it after admission with the live context. @@ -267,7 +269,7 @@ export interface AgentSpec { /** Per-spawn factory carrying caller configuration. Constructed only after admission, with the * real child signal and nested-scope context. */ readonly executorFactory?: ExecutorFactory - /** Bring-your-own executor: when set, overrides harness-based resolution entirely. */ + /** Bring-your-own executor: highest routing precedence after exact-profile intake validation. */ readonly executor?: Executor } @@ -395,9 +397,9 @@ export interface ExecutorNodeContext { } /** - * Builds a fresh `Executor` for one spawn from the resolved spec. Per-spawn (not - * shared) so each child owns its own box/abort/teardown lifecycle. A BYO factory lets a - * user supply construction args without pre-instantiating. + * Builds a fresh `Executor` for one spawn from the resolved, immutable spec. Per-spawn (not shared) + * so each child owns its own box/abort/teardown lifecycle. A BYO factory lets a user supply + * construction args without pre-instantiating; it never bypasses exact-profile validation. */ export type ExecutorFactory = (spec: AgentSpec, ctx: ExecutorContext) => Executor @@ -413,10 +415,10 @@ export interface ExecutorContext { } /** - * The OPEN resolver: maps an `AgentSpec` to a `ExecutorFactory`. The default - * registry resolves the three built-ins AND accepts a BYO `executor`/factory; callers - * register more runtimes by name. NOT a closed switch — registration is the extension - * point, mirroring the open `Executor` interface. + * The OPEN resolver maps an already-admitted `AgentSpec` to an `ExecutorFactory`. Scope validates + * before invoking any implementation; the default registry repeats validation for direct callers, + * resolves the three built-ins, and accepts a BYO `executor`/factory. Callers may register more + * runtimes by name, but registration does not waive exact-profile validation. */ export interface ExecutorRegistry { /** Register a factory for a named runtime. Throws on a duplicate name (fail loud). */ diff --git a/tests/helpers/supervisor-resume-child.ts b/tests/helpers/supervisor-resume-child.ts index fadd5916..4e856862 100644 --- a/tests/helpers/supervisor-resume-child.ts +++ b/tests/helpers/supervisor-resume-child.ts @@ -12,7 +12,7 @@ */ import { appendFileSync } from 'node:fs' -import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { spendFromUsageEvents } from '../../src/runtime/supervise/budget' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import { createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -25,6 +25,7 @@ import type { Settled, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' const [dir, runId, phase] = process.argv.slice(2) if (dir === undefined || runId === undefined || (phase !== '1' && phase !== '2')) { @@ -74,7 +75,7 @@ function leafAgent( }), } const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: executor as Executor, } diff --git a/tests/kernel/completion-gate.test.ts b/tests/kernel/completion-gate.test.ts index ef5d013b..0598e578 100644 --- a/tests/kernel/completion-gate.test.ts +++ b/tests/kernel/completion-gate.test.ts @@ -22,6 +22,7 @@ import type { } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { type ScriptedTurn, scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' // ── Two leaf-worker shapes, to exercise BOTH `execute` shapes the gate wraps ────────────── interface WorkerScript { @@ -139,7 +140,7 @@ function gatedWorkerLeaf( deliverable: DeliverableSpec, ): Agent { const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: gateOnDeliverable(streamingWorker(s), deliverable), } @@ -259,7 +260,7 @@ describe('completion-oracle settle — settled ⟺ DELIVERED (Foreman 0/18)', () const makeAgent = (profile: AgentProfile): Agent => { if (profile.metadata?.kind === 'driver') { return driverChild( - 'mid', + testAgentProfile('mid'), driverAgent(driverOpts('mid', scriptedBrain(spawnAwaitStop), makeAgent)), journal, ) diff --git a/tests/kernel/coordination-driver.test.ts b/tests/kernel/coordination-driver.test.ts index ec0aed5d..ac6454ee 100644 --- a/tests/kernel/coordination-driver.test.ts +++ b/tests/kernel/coordination-driver.test.ts @@ -20,6 +20,7 @@ import type { import type { ToolLoopChat } from '../../src/runtime/tool-loop' import type { RuntimeHookEvent } from '../../src/runtime-hooks' import { type ScriptedTurn, scriptedBrain } from './scripted-brain' +import { testAgentProfile } from './test-agent-profile' type SeenMessages = Array>> @@ -63,7 +64,7 @@ function workerLeaf( onTeardown?: () => void, ): Agent { const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: workerExecutor(s, onTeardown), } @@ -74,7 +75,7 @@ function workerLeaf( function hangingWorkerLeaf(name: string): Agent { const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: { runtime: 'router', @@ -324,7 +325,11 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', const makeAgent = (profile: AgentProfile): Agent => { if (profile.metadata?.kind === 'driver') { const childBrain = scriptedBrain(midTurns, midSeen) - return driverChild('mid', driverAgent(driverOpts('mid', childBrain, makeAgent)), journal) + return driverChild( + testAgentProfile('mid'), + driverAgent(driverOpts('mid', childBrain, makeAgent)), + journal, + ) } return worker } @@ -727,7 +732,7 @@ describe('driverAgent — the analyst up-leg (analysts + analyzeOnSettle pass-th ({ name: 'w', act: async () => '', - executorSpec: { profile: { name: 'w' } as AgentProfile, harness: null }, + executorSpec: { profile: testAgentProfile('w'), harness: null }, }) as Agent & { executorSpec: AgentSpec } const analysts = { kinds: [{ id: 'progress', description: 'read the settled output', area: 'progress' }], diff --git a/tests/kernel/coordination-mcp.test.ts b/tests/kernel/coordination-mcp.test.ts index 1c1632bf..f8764f9b 100644 --- a/tests/kernel/coordination-mcp.test.ts +++ b/tests/kernel/coordination-mcp.test.ts @@ -1,4 +1,3 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp' @@ -13,6 +12,7 @@ import type { Scope, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' // A real (simple) delivering leaf — NOT a mock of the MCP path; the HTTP→MCP→Scope.spawn is real. function deliveringLeaf(name: string, out: unknown): Agent { @@ -32,7 +32,7 @@ function deliveringLeaf(name: string, out: unknown): Agent { spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 }, }), } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor: ex } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } diff --git a/tests/kernel/driver-recursion.test.ts b/tests/kernel/driver-recursion.test.ts index 73a44059..78d2183c 100644 --- a/tests/kernel/driver-recursion.test.ts +++ b/tests/kernel/driver-recursion.test.ts @@ -1,4 +1,3 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { defaultSelectWinner } from '../../src/runtime/run-loop' @@ -16,6 +15,7 @@ import type { SupervisorOpts, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' // ── Scripted leaf worker (offline; no network/sandbox/subprocess) ──────────────── // @@ -69,7 +69,7 @@ function workerExecutor(s: WorkerScript): Executor { * built-in router/sandbox/cli factory ever fires (the test stays fully offline). */ function workerLeaf(name: string, s: WorkerScript): Agent { const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: workerExecutor(s), } @@ -167,10 +167,14 @@ describe('recursive driver: agents drive agents drive agents', () => { const root: Agent = { name: 'root', async act(task, scope) { - const spawned = scope.spawn(driverChild('nested-manager', nested, journal), task, { - budget: perChild, - label: 'nested-manager', - }) + const spawned = scope.spawn( + driverChild(testAgentProfile('nested-manager'), nested, journal), + task, + { + budget: perChild, + label: 'nested-manager', + }, + ) if (!spawned.ok) throw new Error(spawned.reason) expect( scope.send(spawned.handle.id, { @@ -220,7 +224,9 @@ describe('recursive driver: agents drive agents drive agents', () => { // root driver: spawns the mid DRIVER child (which itself spawns the worker) — recursion. const rootDriver = scriptedDriver( 'root', - (_scope) => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + (_scope) => [ + { label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }, + ], observed, ) @@ -273,7 +279,7 @@ describe('recursive driver: agents drive agents drive agents', () => { name: 'root', async act(task, scope) { const spawned = scope.spawn( - driverChild('refusing-manager', refusingManager, journal), + driverChild(testAgentProfile('refusing-manager'), refusingManager, journal), task, { budget: perChild, label: 'refusing-manager' }, ) @@ -312,7 +318,7 @@ describe('recursive driver: agents drive agents drive agents', () => { const midDriver = scriptedDriver('mid', () => [{ label: 'w', agent: worker }], observed) const rootDriver = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }], observed, ) const rootCeiling = { maxIterations: 50, maxTokens: 5000 } @@ -375,7 +381,7 @@ describe('recursive driver: agents drive agents drive agents', () => { }) const rootDriver = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }], observed, ) const result = await createSupervisor().run( @@ -409,7 +415,7 @@ describe('recursive driver: agents drive agents drive agents', () => { const midDriver = scriptedDriver('mid', () => [{ label: 'w', agent: worker }], observed) const rootDriver = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }], observed, ) await createSupervisor().run( @@ -460,13 +466,15 @@ describe('recursive driver: agents drive agents drive agents', () => { // depth-1 driver spawns the inner DRIVER child. const midDriver = scriptedDriver( 'mid', - () => [{ label: 'inner', agent: driverChild('inner', innerDriver, journal) }], + () => [ + { label: 'inner', agent: driverChild(testAgentProfile('inner'), innerDriver, journal) }, + ], observed, ) // depth-0 root spawns the mid DRIVER child. const rootDriver = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }], observed, ) @@ -513,7 +521,7 @@ describe('recursive driver: agents drive agents drive agents', () => { const midDriver = scriptedDriver('mid', () => [{ label: 'w', agent: worker }], observed) const rootDriver = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), midDriver, journal) }], observed, ) const result = await createSupervisor().run( diff --git a/tests/kernel/materialization-evidence.test.ts b/tests/kernel/materialization-evidence.test.ts index ea97d9e9..8ef4ee07 100644 --- a/tests/kernel/materialization-evidence.test.ts +++ b/tests/kernel/materialization-evidence.test.ts @@ -195,7 +195,7 @@ describe('kernel-owned materialization evidence', () => { 'cli', ) const child = driverChild( - { name: 'manager', metadata: { role: 'driver' } }, + testAgentProfile('manager', { metadata: { role: 'driver' } }), silentManager, journal, ) diff --git a/tests/kernel/personify.test.ts b/tests/kernel/personify.test.ts index d9e24732..8d915dbd 100644 --- a/tests/kernel/personify.test.ts +++ b/tests/kernel/personify.test.ts @@ -31,6 +31,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import type { RuntimeHookEvent, RuntimeHooks } from '../../src/runtime-hooks' +import { testAgentProfile } from './test-agent-profile' // ── Offline mock leaf runtime ───────────────────────────────────────────────────── // @@ -104,7 +105,7 @@ function makePersona( ): Persona { return definePersona({ name, - root: { profile: { name: role } as AgentProfile, harness: null }, + root: { profile: testAgentProfile(role), harness: null }, directive: `act as ${role}`, context: { role }, executors: { registry: mockRegistry(scriptFor) }, @@ -119,6 +120,44 @@ const ev = (input: number, output: number): UsageEvent[] => [ const wideBudget: Budget = { maxIterations: 200, maxTokens: 1_000_000 } const wideShapeBudget = { fanout: 4, perChild: { maxIterations: 10, maxTokens: 50_000 } } +describe('personify profile intake', () => { + it.each([ + { + field: 'harness', + profile: { + name: 'missing-harness', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + }, + { + field: 'provider', + profile: { + name: 'missing-provider', + harness: 'cli-base', + model: { default: 'offline-test-model' }, + }, + }, + { + field: 'model', + profile: { + name: 'missing-model', + harness: 'cli-base', + model: { provider: 'offline' }, + }, + }, + ] as const)('definePersona rejects a root missing $field at intake', ({ profile }) => { + expect(() => + definePersona({ + name: profile.name, + root: { profile: profile as AgentProfile, harness: null }, + directive: 'must not reach a shape', + context: { role: 'test' }, + executors: { registry: mockRegistry(() => ({ out: 'must-not-run', events: [] })) }, + }), + ).toThrow(/AgentProfile\.(harness|model\.provider|model\.default)/) + }) +}) + /** Run a combinator factory verbatim (NOT via a registry name) through the real keystone. * Passing the factory directly keeps the test independent of the shape-name registry. */ async function runShape(persona: Persona, shape: CombinatorShape, task: Task) { @@ -313,7 +352,7 @@ describe('combinator · fanout', () => { itemTask: (item) => ({ item }), label: (item) => `leaf:${item}`, itemSpec: (item, i): AgentSpec => ({ - profile: { name: `authored-${item}` } as AgentProfile, + profile: testAgentProfile(`authored-${item}`), harness: null, executor: byoExecutor(`out-${item}`, 0.2 + i * 0.3), }), @@ -635,7 +674,7 @@ describe('meta-orchestrator (depth-2 sub-driver loops)', () => { // The meta persona's children are BYO sub-loop leaves — each runs its own nested loop. const metaPersona = definePersona({ name: 'meta', - root: { profile: { name: 'orchestrator' } as AgentProfile, harness: null }, + root: { profile: testAgentProfile('orchestrator'), harness: null }, directive: 'orchestrate sub-loops', context: { role: 'orchestrator' }, executors: { diff --git a/tests/kernel/rsi-wave.test.ts b/tests/kernel/rsi-wave.test.ts index 8a100c91..837bd886 100644 --- a/tests/kernel/rsi-wave.test.ts +++ b/tests/kernel/rsi-wave.test.ts @@ -39,6 +39,7 @@ import type { Spend, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' // ── A trace-derived vs judge-derived finding (the firewall's only discriminator) ───── @@ -85,7 +86,7 @@ function leafAgent(name: string, out: unknown, events?: UsageEvent[]): Agent out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -101,7 +102,7 @@ function boomAgent(name: string, reason: string): Agent { throw new ValidationError('boom: resultArtifact unreachable') }, } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor } return { name, act: async () => undefined, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -189,7 +190,7 @@ describe('cross-run corpus (G2)', () => { ) expect(run1).toEqual({ succeeded: true }) - const run2Profile: AgentProfile = { name: 'analyst' } as AgentProfile + const run2Profile = testAgentProfile('analyst') const run2 = await renderCorpusToInstructions({ corpus, filter: { area: 'sourcing', minConfidence: 0.5 }, @@ -282,7 +283,7 @@ describe('analyst-on-scope (G1) firewall', () => { * offline scope settles it via the BYO executor). Optionally threads a `ScopeAnalyst` so a * combinator's gate sees its findings — the seam this wave connects. */ function workerCtx(analyst?: ScopeAnalyst): ShapeContext { - const root: AgentSpec = { profile: { name: 'worker' } as AgentProfile, harness: null } + const root: AgentSpec = { profile: testAgentProfile('worker'), harness: null } const persona = { name: 'p', root } as Persona const budget: ShapeBudget = { perChild: { maxIterations: 1, maxTokens: 1000 }, fanout: 1 } return { @@ -318,7 +319,7 @@ function spyAnalyst(findings: ReadonlyArray): { describe('analyst→steer wire (combinator gates read findings, not [])', () => { it('createShapeContext threads an analyst onto the ShapeContext (absent when omitted)', () => { - const root: AgentSpec = { profile: { name: 'w' } as AgentProfile, harness: null } + const root: AgentSpec = { profile: testAgentProfile('w'), harness: null } const persona = { name: 'p', root } as Persona const budget: ShapeBudget = { perChild: { maxIterations: 1, maxTokens: 10 }, fanout: 1 } const { analyst } = spyAnalyst([traceFinding]) diff --git a/tests/kernel/spawn-forest.test.ts b/tests/kernel/spawn-forest.test.ts index 4e209023..7a7b33d8 100644 --- a/tests/kernel/spawn-forest.test.ts +++ b/tests/kernel/spawn-forest.test.ts @@ -1,7 +1,6 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { contentAddress, @@ -22,6 +21,7 @@ import type { SpawnJournal, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' const rootBudget = { maxIterations: 10, maxTokens: 10_000 } const childBudget = { maxIterations: 4, maxTokens: 1_000 } @@ -49,7 +49,7 @@ function leaf( } }, } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor } return { name, act: async () => out, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -82,10 +82,14 @@ describe('loadSpawnForest', () => { const root: Agent = { name: 'root', async act(task, scope) { - const spawned = scope.spawn(driverChild('nested', nested, journal), task, { - budget: childBudget, - label: 'nested', - }) + const spawned = scope.spawn( + driverChild(testAgentProfile('nested'), nested, journal), + task, + { + budget: childBudget, + label: 'nested', + }, + ) if (!spawned.ok) throw new Error(spawned.reason) return onlyDone(scope) }, diff --git a/tests/kernel/supervise-deadline.test.ts b/tests/kernel/supervise-deadline.test.ts index c88d97df..96e79f1d 100644 --- a/tests/kernel/supervise-deadline.test.ts +++ b/tests/kernel/supervise-deadline.test.ts @@ -1,4 +1,3 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { createBudgetPool } from '../../src/runtime/supervise/budget' @@ -16,6 +15,7 @@ import type { Spend, SupervisorOpts, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' const zeroSpend: Spend = { iterations: 0, @@ -225,7 +225,7 @@ function leaf( teardown: async () => ({ destroyed: true }), } const executorSpec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor, } diff --git a/tests/kernel/supervise-otel-spans.test.ts b/tests/kernel/supervise-otel-spans.test.ts index 552d8e5d..8a188abf 100644 --- a/tests/kernel/supervise-otel-spans.test.ts +++ b/tests/kernel/supervise-otel-spans.test.ts @@ -154,7 +154,7 @@ async function runNestedTree(exporter: OtelExporter) { ) const root = scriptedDriver( 'root', - () => [{ label: 'mid', agent: driverChild('mid', mid, journal) }], + () => [{ label: 'mid', agent: driverChild(testAgentProfile('mid'), mid, journal) }], [ { spend: { iterations: 0, tokens: { input: 100, output: 40 }, usd: 0.5, ms: 12 }, diff --git a/tests/kernel/supervise-restart-resource-safety.test.ts b/tests/kernel/supervise-restart-resource-safety.test.ts index 9153ba62..c58d8b84 100644 --- a/tests/kernel/supervise-restart-resource-safety.test.ts +++ b/tests/kernel/supervise-restart-resource-safety.test.ts @@ -1,4 +1,4 @@ -import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' @@ -16,6 +16,7 @@ import type { Spend, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' const zeroSpend: Spend = { iterations: 0, @@ -327,7 +328,10 @@ describe('supervision restart and resource safety', () => { const allocation = { maxIterations: 1, maxTokens: 10 } const manager = scope.spawn( driverChild( - { name: 'nested-manager', harness: 'cli-base', metadata: { role: 'driver' } }, + testAgentProfile('nested-manager', { + harness: 'cli-base', + metadata: { role: 'driver' }, + }), nested, journal, ), @@ -824,7 +828,10 @@ describe('supervision restart and resource safety', () => { async act(task, scope): Promise { const manager = scope.spawn( driverChild( - { name: 'manager', harness: 'cli-base', metadata: { role: 'driver' } }, + testAgentProfile('manager', { + harness: 'cli-base', + metadata: { role: 'driver' }, + }), nestedDriver, journal, ), @@ -999,7 +1006,10 @@ describe('supervision restart and resource safety', () => { async act(task, scope): Promise { const spawned = scope.spawn( driverChild( - { name: 'unknown-manager', harness: 'cli-base', metadata: { role: 'driver' } }, + testAgentProfile('unknown-manager', { + harness: 'cli-base', + metadata: { role: 'driver' }, + }), nestedDriver, journal, ), @@ -1250,7 +1260,7 @@ function leafFromExecutor( ): Agent { const executor = makeExecutor() const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: executor as Executor, } diff --git a/tests/kernel/supervise-worker-trace.test.ts b/tests/kernel/supervise-worker-trace.test.ts index 39244b54..2cba7479 100644 --- a/tests/kernel/supervise-worker-trace.test.ts +++ b/tests/kernel/supervise-worker-trace.test.ts @@ -246,7 +246,10 @@ describe('a spawned worker inherits the run trace and the spawning node span', ( ) const result = await createSupervisor().run( scriptedDriver('root', () => [ - { label: 'mid', agent: driverChild('mid', mid, journal) as Agent }, + { + label: 'mid', + agent: driverChild(testAgentProfile('mid'), mid, journal) as Agent, + }, ]), 'task', supervisorOpts({ diff --git a/tests/kernel/supervise.test.ts b/tests/kernel/supervise.test.ts index a293bd0a..95e4e7de 100644 --- a/tests/kernel/supervise.test.ts +++ b/tests/kernel/supervise.test.ts @@ -29,6 +29,7 @@ import type { WidenGate, } from '../../src/runtime/supervise/types' import type { RuntimeHookEvent } from '../../src/runtime-hooks' +import { testAgentProfile } from './test-agent-profile' // ── The mock Executor — the whole keystone runs offline against this ───────── // @@ -102,7 +103,7 @@ function stableKey(value: unknown): string { * router/sandbox/cli factory ever fires — the test stays fully offline. */ function leafAgent(name: string, script: MockScript): Agent { const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: mockExecutor(script), } @@ -852,7 +853,7 @@ describe('reactive scope', () => { const agent = { name: 'boom', act: async () => 0, - executorSpec: { profile: {} as AgentProfile, harness: null }, + executorSpec: { profile: testAgentProfile('boom'), harness: null }, } as Agent & { executorSpec: AgentSpec } expect(() => scope.spawn(agent, 'task', { budget: { maxIterations: 1, maxTokens: 100 }, label: 'boom' }), @@ -925,7 +926,7 @@ describe('reactive scope', () => { name: 'validated-inbox', act: async () => 1, executorSpec: { - profile: { name: 'validated-inbox' } as AgentProfile, + profile: testAgentProfile('validated-inbox'), harness: null, executor, }, @@ -1110,11 +1111,161 @@ describe('settledToIteration adapter', () => { // ── 5. Open executor registry resolution ───────────────────────────────────────── describe('open executor registry', () => { + it.each([ + { + field: 'harness', + profile: { + name: 'missing-harness', + model: { provider: 'offline', default: 'offline-test-model' }, + }, + }, + { + field: 'provider', + profile: { + name: 'missing-provider', + harness: 'cli-base', + model: { default: 'offline-test-model' }, + }, + }, + { + field: 'model', + profile: { + name: 'missing-model', + harness: 'cli-base', + model: { provider: 'offline' }, + }, + }, + ] as const)( + 'scope rejects a child missing $field before resolving or constructing custom execution code', + async ({ profile }) => { + let resolutions = 0 + let constructions = 0 + const registry = { + register() {}, + resolve() { + resolutions += 1 + return { + succeeded: true as const, + value: () => { + constructions += 1 + return mockExecutor({ out: 'must-not-run', events: [] }) + }, + } + }, + } as Parameters[0]['executors'] + const { scope, pool } = await beginScope({ executors: registry }) + const agent = { + name: profile.name, + act: async () => 'must-not-run', + executorSpec: { profile: profile as AgentProfile, harness: null }, + } as Agent & { executorSpec: AgentSpec } + + expect(() => + scope.spawn(agent, 'task', { + budget: { maxIterations: 1, maxTokens: 10 }, + label: profile.name, + }), + ).toThrow(/AgentProfile\.(harness|model\.provider|model\.default)/) + expect(resolutions).toBe(0) + expect(constructions).toBe(0) + expect(pool.readout().reservedTokens).toBe(0) + }, + ) + + it('scope validates a child before honoring a BYO executor or executor factory', async () => { + let factoryCalls = 0 + let executeCalls = 0 + const executor = mockExecutor({ out: 'must-not-run', events: [] }) + const execute = executor.execute.bind(executor) + executor.execute = (task, signal) => { + executeCalls += 1 + return execute(task, signal) + } + const { scope } = await beginScope() + const spawn = (name: string, route: Pick) => + scope.spawn( + { + name, + act: async () => 'must-not-run', + executorSpec: { + profile: { name } as AgentProfile, + harness: null, + ...route, + }, + } as Agent & { executorSpec: AgentSpec }, + 'task', + { budget: { maxIterations: 1, maxTokens: 10 }, label: name }, + ) + + expect(() => spawn('byo-executor', { executor })).toThrow(/AgentProfile\.harness/) + expect(() => + spawn('byo-factory', { + executorFactory: () => { + factoryCalls += 1 + return executor + }, + }), + ).toThrow(/AgentProfile\.harness/) + expect(factoryCalls).toBe(0) + expect(executeCalls).toBe(0) + }) + + it('passes a detached deeply frozen profile snapshot to a custom executor factory', async () => { + const original = testAgentProfile('snapshot-child', { + model: { + provider: 'offline', + default: 'authorized-model', + metadata: { temperature: 0.25 }, + }, + }) + let received: AgentProfile | undefined + const agent = { + name: 'snapshot-child', + act: async () => 'done', + executorSpec: { + profile: original, + harness: null, + executorFactory: (spec: AgentSpec) => { + received = spec.profile + return mockExecutor({ out: 'done', events: [] }) + }, + }, + } as Agent & { executorSpec: AgentSpec } + const { scope } = await beginScope() + const mutableModel = original.model as { default: string } + + const spawned = scope.spawn(agent, 'task', { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'snapshot-child', + }) + expect(spawned.ok).toBe(true) + mutableModel.default = 'mutated-model' + + expect(received).not.toBe(original) + expect(received?.model?.default).toBe('authorized-model') + expect(Object.isFrozen(received)).toBe(true) + expect(Object.isFrozen(received?.model)).toBe(true) + expect(Object.isFrozen(received?.model?.metadata)).toBe(true) + await scope.next() + }) + + it('the built-in registry validates before honoring a BYO executor directly', () => { + const registry = createExecutorRegistry() + const executor = mockExecutor({ out: 'must-not-run', events: [] }) + expect(() => + registry.resolve({ + profile: { name: 'invalid-byo' } as AgentProfile, + harness: null, + executor, + }), + ).toThrow(/AgentProfile\.harness/) + }) + it('resolves a BYO executor verbatim (highest precedence)', () => { const registry = createExecutorRegistry() const byo = mockExecutor({ out: 'x', events: [] }) const spec: AgentSpec = { - profile: { name: 'byo' } as AgentProfile, + profile: testAgentProfile('byo'), harness: null, executor: byo, } @@ -1129,9 +1280,9 @@ describe('open executor registry', () => { it('harness:null resolves the router factory; a BackendType resolves the sandbox factory', () => { const registry = createExecutorRegistry() - const router = registry.resolve({ profile: { name: 'r' } as AgentProfile, harness: null }) + const router = registry.resolve({ profile: testAgentProfile('r'), harness: null }) const sandbox = registry.resolve({ - profile: { name: 's' } as AgentProfile, + profile: testAgentProfile('s', { harness: 'claude-code' }), harness: 'claude-code', }) expect(router.succeeded).toBe(true) diff --git a/tests/kernel/worktree-loop.test.ts b/tests/kernel/worktree-loop.test.ts index edcc128e..ed7f10c6 100644 --- a/tests/kernel/worktree-loop.test.ts +++ b/tests/kernel/worktree-loop.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest' import { worktreeLoopRunner } from '../../src/loop-runner' import type { GitRunner } from '../../src/mcp/worktree' import type { Budget } from '../../src/runtime' +import { testAgentProfile } from './test-agent-profile' /** Per-worktree fake git: each worktree gets a distinct diff keyed by its path so two harness * leaves produce two different patches (and two different diff sizes). */ @@ -27,10 +28,8 @@ function fakeGitWith( } } -const profile = (name: string): AgentProfile => ({ - name, - prompt: { systemPrompt: `You are ${name}.` }, -}) +const profile = (name: string, harness: AgentProfile['harness']): AgentProfile => + testAgentProfile(name, { harness, prompt: { systemPrompt: `You are ${name}.` } }) const budget: Budget = { maxIterations: 50, maxTokens: 500_000 } @@ -54,18 +53,19 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { const patch = 'diff --git a/src/x.ts b/src/x.ts\n--- a/src/x.ts\n+++ b/src/x.ts\n+const a = 1' const runner = worktreeLoopRunner({ repoRoot: '/repo', + rootProfile: profile('worktree-coder', 'cli-base'), taskPrompt: 'fix the off-by-one', budget, harnesses: [ { name: 'claude', - profile: profile('claude'), + profile: profile('claude', 'claude-code'), harness: 'claude-code', budgetExempt: false, }, { name: 'opencode', - profile: profile('opencode'), + profile: profile('opencode', 'opencode'), harness: 'opencode', budgetExempt: false, }, @@ -90,12 +90,13 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { const patch = 'diff --git a/src/x.ts b/src/x.ts\n--- a/src/x.ts\n+++ b/src/x.ts\n+const a = 1' const runner = worktreeLoopRunner({ repoRoot: '/repo', + rootProfile: profile('worktree-coder', 'cli-base'), taskPrompt: 'fix it', budget, harnesses: [ { name: 'claude', - profile: profile('claude'), + profile: profile('claude', 'claude-code'), harness: 'claude-code', budgetExempt: false, }, @@ -116,12 +117,13 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { it('rejects an empty patch via the always-on no-op floor (no winner)', async () => { const runner = worktreeLoopRunner({ repoRoot: '/repo', + rootProfile: profile('worktree-coder', 'cli-base'), taskPrompt: 'do nothing', budget, harnesses: [ { name: 'claude', - profile: profile('claude'), + profile: profile('claude', 'claude-code'), harness: 'claude-code', budgetExempt: false, }, diff --git a/tests/runtime/executor-profile-model.test.ts b/tests/runtime/executor-profile-model.test.ts index eab6bfc5..bbac2f02 100644 --- a/tests/runtime/executor-profile-model.test.ts +++ b/tests/runtime/executor-profile-model.test.ts @@ -103,4 +103,43 @@ describe('router executor exact-profile identity', () => { expect(request).toBeUndefined() }, ) + + it.each([ + { + field: 'harness', + profile: { + name: 'missing-harness', + model: { provider: 'tangle-router', default: 'profile-selected-model' }, + }, + }, + { + field: 'provider', + profile: { + name: 'missing-provider', + harness: 'cli-base', + model: { default: 'profile-selected-model' }, + }, + }, + { + field: 'model', + profile: { + name: 'missing-model', + harness: 'cli-base', + model: { provider: 'tangle-router' }, + }, + }, + ] as const)('rejects a direct createExecutor call missing $field at intake', ({ profile }) => { + const factory = createExecutor({ + backend: 'router', + routerBaseUrl: 'http://must-not-dispatch.invalid', + routerKey: 'key', + }) + + expect(() => + factory( + { profile: profile as AgentProfile, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ), + ).toThrow(/AgentProfile\.(harness|model\.provider|model\.default)/) + }) }) diff --git a/tests/runtime/rolling-dispatch.test.ts b/tests/runtime/rolling-dispatch.test.ts index ed4108cc..69818dd6 100644 --- a/tests/runtime/rolling-dispatch.test.ts +++ b/tests/runtime/rolling-dispatch.test.ts @@ -33,6 +33,7 @@ import type { Scope, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' const usage: UsageEvent[] = [{ kind: 'iteration' }, { kind: 'tokens', input: 10, output: 10 }] @@ -67,7 +68,7 @@ function leaf( }), } const spec: AgentSpec = { - profile: { name } as AgentSpec['profile'], + profile: testAgentProfile(name), harness: null, executor: executor as Executor, } @@ -282,7 +283,7 @@ describe('fanout({ width })', () => { * each item's executor can record when it ran. */ function persona(live: { now: number; peak: number }) { const rootSpec: AgentSpec = { - profile: { name: 'root' } as AgentSpec['profile'], + profile: testAgentProfile('root'), harness: null, executor: (leaf('root', 0, live) as unknown as { executorSpec: AgentSpec }).executorSpec .executor, diff --git a/tests/runtime/spawn-journal-replay-identity.test.ts b/tests/runtime/spawn-journal-replay-identity.test.ts index ed48f891..9571b93a 100644 --- a/tests/runtime/spawn-journal-replay-identity.test.ts +++ b/tests/runtime/spawn-journal-replay-identity.test.ts @@ -22,6 +22,7 @@ import type { NodeExecutionIdentity, SpawnEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' const spent = { iterations: 1, @@ -218,7 +219,7 @@ describe('spawn journal replay identity', () => { }, } const spec: AgentSpec = { - profile: { name: 'failing worker' }, + profile: testAgentProfile('failing worker'), harness: null, executor, } diff --git a/tests/runtime/spawn-keys.test.ts b/tests/runtime/spawn-keys.test.ts index 9707fcf2..4ce2dd9a 100644 --- a/tests/runtime/spawn-keys.test.ts +++ b/tests/runtime/spawn-keys.test.ts @@ -7,7 +7,6 @@ * about the default path changes. */ -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' @@ -20,6 +19,7 @@ import type { Scope, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' /** Counts its own executions so "did this key run again?" is a number, not an inference. `gate`, * when given, holds the worker open until the test releases it — a deterministic stand-in for a @@ -44,7 +44,7 @@ function countingLeaf(name: string, out: string, runs: { n: number }, gate?: Pro }), } const spec: AgentSpec = { - profile: { name } as AgentProfile, + profile: testAgentProfile(name), harness: null, executor: executor as Executor, } @@ -207,7 +207,7 @@ describe('semantic spawn keys', () => { }, } const spec: AgentSpec = { - profile: { name: 'f' } as AgentProfile, + profile: testAgentProfile('f'), harness: null, executor: executor as Executor, } diff --git a/tests/runtime/stop-rules.test.ts b/tests/runtime/stop-rules.test.ts index 2cc61037..932109a5 100644 --- a/tests/runtime/stop-rules.test.ts +++ b/tests/runtime/stop-rules.test.ts @@ -11,7 +11,6 @@ * as the post-run anytime report, and a rule can never keep a run alive past a hard ceiling. */ -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { areaUnderCurve, bestSoFar, plateauLength } from '../../src/runtime/anytime' @@ -42,6 +41,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import { scriptedBrain } from '../kernel/scripted-brain' +import { testAgentProfile } from '../kernel/test-agent-profile' // ── A scored offline leaf: fixed token cost, a verdict score the rules read ───────────────────── @@ -74,7 +74,7 @@ function scoredLeaf(name: string, score: number): Agent { } }, } - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor } return { name, act: async () => ({ name, score }), executorSpec: spec } as Agent< unknown, unknown @@ -504,7 +504,7 @@ describe('ProgressView reads the live worker feed off the scope', () => { name: 'hang', act: async () => ({}), executorSpec: { - profile: { name: 'hang' } as AgentProfile, + profile: testAgentProfile('hang'), harness: null, executor: hanging, }, diff --git a/tests/runtime/worker-progress.test.ts b/tests/runtime/worker-progress.test.ts index e3affbb3..0bda3c1a 100644 --- a/tests/runtime/worker-progress.test.ts +++ b/tests/runtime/worker-progress.test.ts @@ -11,7 +11,6 @@ * loops. That wire's own docstring described it; nothing connected it until now. */ -import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { createCoordinationTools } from '../../src/mcp/tools/coordination' @@ -34,6 +33,7 @@ import type { ExecutorResult, UsageEvent, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' const budget: Budget = { maxIterations: 50, maxTokens: 100_000 } @@ -68,7 +68,7 @@ function pausingLeaf(name: string, pause: Promise): Agent 0, executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -173,7 +173,7 @@ describe('scope.progress — a running worker is observable without any executor const agent = { name: 'sub', act: async () => 0, - executorSpec: { profile: { name: 'sub' } as AgentProfile, harness: null, executor: ex }, + executorSpec: { profile: testAgentProfile('sub'), harness: null, executor: ex }, } as Agent const res = scope.spawn(agent, 'go', { budget }) if (!res.ok) throw new Error('spawn failed') @@ -222,7 +222,7 @@ describe('a GATED active worker stays fully observable mid-flight (BUG 1 + BUG 2 progress: (): ExecutorProgress => ({ recentActivity: log.read(), note: 'running tests' }), } const gated = gateOnDeliverable(inner, { check: () => true }) - const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: gated } + const spec: AgentSpec = { profile: testAgentProfile(name), harness: null, executor: gated } return { name, act: async () => 0, executorSpec: spec } as Agent & { executorSpec: AgentSpec } diff --git a/tests/runtime/worker-trace-evidence.test.ts b/tests/runtime/worker-trace-evidence.test.ts index a3738da9..7925b322 100644 --- a/tests/runtime/worker-trace-evidence.test.ts +++ b/tests/runtime/worker-trace-evidence.test.ts @@ -22,6 +22,7 @@ import type { ResultBlobStore, SpawnJournal, } from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' const spent = { iterations: 1, @@ -36,7 +37,7 @@ function makeAgent( return { name: 'trace worker', act: async () => undefined, - executorSpec: { profile: { name: 'trace worker' }, harness: null, executor }, + executorSpec: { profile: testAgentProfile('trace worker'), harness: null, executor }, } } From 26983a86dfbcf1dc542823f75e0ba2d457d5c09f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:06:17 -0600 Subject: [PATCH 11/39] docs(runtime): document exact profile intake --- CHANGELOG.md | 4 +++ docs/api/agent.md | 8 ++--- docs/api/index.md | 50 +++++++++++++-------------- docs/api/mcp.md | 69 ++++++-------------------------------- docs/api/profiles.md | 42 +++++++---------------- docs/api/runtime.md | 15 ++++----- docs/canonical-api.md | 4 +-- src/mcp/types.ts | 5 ++- src/profiles/researcher.ts | 1 + 9 files changed, 66 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8902a061..290ad4e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - Require Agent Eval 0.144.1 and Agent Knowledge 7.0.9, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. - Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. - Require `defineLeaderboard` callers to supply an exact `baseProfile`; remove its `modelBackend` override so each matrix cell's profile remains the only model authority. +- Require generic coder, researcher, and supervised-knowledge paths to receive complete profiles; remove harness/model overlays and MCP environment alias ladders. +- Resolve Sandbox execution only from `AgentProfile.harness`; a backend type may confirm that choice but cannot replace it. +- Parse, detach, and deeply freeze a complete `AgentProfile` before Scope, registry, nested-driver, or personified execution can honor any built-in or caller-supplied executor. +- Require `driverChild(profile, ...)` and `worktreeLoopRunner({ rootProfile, ... })`; remove name-only driver and personified-root shortcuts. - Keep missing token usage and billed cost unknown, and report reasoning-token usage when the provider supplies it. ## 0.128.0 diff --git a/docs/api/agent.md b/docs/api/agent.md index d2490252..b78b8578 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -1166,10 +1166,10 @@ applies the profile prompt, name, placement, and metadata, but not model selecti Materialization contract for `createSandboxAct`. `createSandboxAct` hands the whole `AgentProfile` to the sandbox as `backend.profile`, so every -profile leaf crosses the boundary. `buildBackendOptions` resolves the runner from an explicit -`sandboxOverrides.backend.type`, then `profile.metadata.backendType`, then `profile.harness`, -so a candidate that changes only `harness` runs on the harness it declares — and one declaring -a harness the sandbox cannot run throws rather than running elsewhere and reporting success. +profile leaf crosses the boundary. `buildBackendOptions` resolves the runner only from +`profile.harness`; an explicit `sandboxOverrides.backend.type` may confirm that choice but cannot +replace it. A candidate declaring a harness the sandbox cannot run throws rather than running +elsewhere and reporting success. *** diff --git a/docs/api/index.md b/docs/api/index.md index e6fc0f6d..d9a19642 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -5861,17 +5861,9 @@ Findings to fall back to when the generation had NO failing cells, so a > `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) -##### harness? - -> `optional` **harness?**: `string` - -##### supervisorModel? - -> `optional` **supervisorModel?**: `string` +##### supervisorProfile -##### supervisorSystemPrompt? - -> `optional` **supervisorSystemPrompt?**: `string` +> **supervisorProfile**: `AgentProfile` ##### superviseOptions? @@ -6187,17 +6179,11 @@ Findings to fall back to when the generation had NO failing cells, so a > `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) -##### harness? - -> `optional` **harness?**: `string` +##### supervisorProfile -##### supervisorModel? +> **supervisorProfile**: `AgentProfile` -> `optional` **supervisorModel?**: `string` - -##### supervisorSystemPrompt? - -> `optional` **supervisorSystemPrompt?**: `string` +Caller-owned exact supervisor harness/provider/model identity. ##### superviseOptions? @@ -6312,6 +6298,14 @@ Options for the local-repo `code` runner over the GENERIC recursive path. #### Properties +##### rootProfile + +> **rootProfile**: `AgentProfile` + +**`Experimental`** + +Exact profile carried by the personified root that owns this fanout. + ##### repoRoot > **repoRoot**: `string` @@ -7925,9 +7919,11 @@ executors omit it (returns `undefined`). ### AgentSpec -`AgentProfile.harness` is a portable preference; this wrapper records the executor decision for -one concrete run. A caller may honor the preference, override it for a comparison cell, or supply -an executor directly, without changing the profile's behavioral identity. +`AgentProfile` is the complete execution authority. Scope parses and snapshots it before calling +any registry, including one that resolves caller-supplied executors and factories. The default +registry enforces the same rule when called directly. `AgentSpec.harness` records routing for one +concrete run; where a backend consumes both fields, it must agree with `AgentProfile.harness` and +cannot fill or override it. Resolution (in `runtime.ts`): - `executorFactory` present → BYO: build it after admission with the live context. @@ -7966,16 +7962,16 @@ Per-spawn factory carrying caller configuration. Constructed only after admissio > `readonly` `optional` **executor?**: [`Executor`](#executor-2)\<`unknown`\> -Bring-your-own executor: when set, overrides harness-based resolution entirely. +Bring-your-own executor: highest routing precedence after exact-profile intake validation. *** ### ExecutorRegistry -The OPEN resolver: maps an `AgentSpec` to a `ExecutorFactory`. The default -registry resolves the three built-ins AND accepts a BYO `executor`/factory; callers -register more runtimes by name. NOT a closed switch — registration is the extension -point, mirroring the open `Executor` interface. +The OPEN resolver maps an already-admitted `AgentSpec` to an `ExecutorFactory`. Scope validates +before invoking any implementation; the default registry repeats validation for direct callers, +resolves the three built-ins, and accepts a BYO `executor`/factory. Callers may register more +runtimes by name, but registration does not waive exact-profile validation. #### Methods diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 45ab42bf..2a84d4ec 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -898,59 +898,23 @@ one or the other, not both. Convenience shorthand for sibling placement. Equivalent to `executor: createSiblingSandboxExecutor({ client: sandboxClient })`. -##### workerProfile? +##### workerProfile -> `optional` **workerProfile?**: `AgentProfile` +> **workerProfile**: `AgentProfile` **`Experimental`** -The worker's authored `AgentProfile` (§1.5: the system authors profiles). Spread onto the -sandbox-session run spec → `runAgentRounds` → the executor's `harnessInvocation`, so the harness runs -under the caller's stance. Omit to use a minimal model-only default (no hardcoded skills/tools); -`harness` / `model` / `systemPrompt` below are convenience overrides layered onto whichever -profile is used. +The worker's exact authored `AgentProfile` (§1.5: the system authors profiles). It is the sole +harness/provider/model/prompt authority for the single-coder path and the default identity for +repeated fanout shots. -##### harness? - -> `optional` **harness?**: `string` - -**`Experimental`** - -Backend harness for the single-coder path (sets `metadata.backendType`). Default `claude-code`. - -##### model? - -> `optional` **model?**: `string` - -**`Experimental`** - -Model override for the single-coder path. - -##### systemPrompt? - -> `optional` **systemPrompt?**: `string` - -**`Experimental`** - -The worker's authored system prompt (§1.5). Flows onto the run spec's -`profile.prompt.systemPrompt` → through `runAgentRounds` → the executor's `harnessInvocation`, so the -harness runs under this stance. Omit to keep the profile's own prompt. +##### fanoutProfiles? -##### fanoutHarnesses? - -> `optional` **fanoutHarnesses?**: `string`[] - -**`Experimental`** - -Default `['claude-code', 'codex', 'opencode/zai-coding-plan/glm-5.1']` when variants > 1. - -##### fanoutModels? - -> `optional` **fanoutModels?**: (`string` \| `undefined`)[] +> `optional` **fanoutProfiles?**: readonly `AgentProfile`[] **`Experimental`** -Optional per-harness model override for `variants > 1`. +Optional exact identities for heterogeneous fanout. Omit to repeat `workerProfile`. ##### maxConcurrency? @@ -1039,18 +1003,6 @@ Session id of the detached turn — used as the synthesized event id. **`Experimental`** -##### harness? - -> `optional` **harness?**: `string` - -**`Experimental`** - -##### model? - -> `optional` **model?**: `string` - -**`Experimental`** - ##### reviewer? > `optional` **reviewer?**: [`CoderReviewer`](#coderreviewer) @@ -4583,9 +4535,8 @@ Optional free-form context the agent surfaces in the prompt prelude. **`Experimental`** -When > 1, dispatches `multiHarnessCoderFanout` across N harnesses -(claude-code, codex, opencode-glm) and picks the highest-scoring -passing patch. Default 1. +When > 1, dispatches `multiHarnessCoderFanout` across the delegate's configured exact profiles +and picks the highest-scoring passing patch. Default 1. ##### config? diff --git a/docs/api/profiles.md b/docs/api/profiles.md index 506d147d..61028423 100644 --- a/docs/api/profiles.md +++ b/docs/api/profiles.md @@ -468,21 +468,13 @@ Options for the source-grounded researcher profile preset. #### Properties -##### harness? - -> `optional` **harness?**: `string` - -**`Experimental`** - -Sandbox-SDK backend.type. Default `'opencode/zai-coding-plan/glm-5.1'`. - -##### model? +##### profile -> `optional` **model?**: `string` +> **profile**: `AgentProfile` **`Experimental`** -Default model id passed in `AgentProfile.model.default`. +Caller-owned exact harness/provider/model identity. ##### systemPrompt? @@ -498,7 +490,7 @@ Custom system prompt replacement. Default = built-in researcher preset. **`Experimental`** -Stable name for `AgentRunSpec.name`. Default = `researcher-${harness}`. +Stable name for `AgentRunSpec.name`. Default = `profile.name`. ##### citationDensityMin? @@ -517,21 +509,13 @@ Below this floor, citation_density scores < 1 and the item set is gated. #### Properties -##### harnesses? +##### profiles -> `optional` **harnesses?**: `string`[] +> **profiles**: readonly `AgentProfile`[] **`Experimental`** -Backend.type identifiers, one per parallel agent. - -##### models? - -> `optional` **models?**: (`string` \| `undefined`)[] - -**`Experimental`** - -Optional per-harness model override. Indexed parallel to `harnesses`. +Exact execution profiles, one per parallel researcher. ##### citationDensityMin? @@ -1595,7 +1579,7 @@ Render a `CoderTask` into the per-task instruction handed to the coder profile. ### researcherProfile() -> **researcherProfile**(`options?`): `object` +> **researcherProfile**(`options`): `object` **`Experimental`** @@ -1603,9 +1587,9 @@ Build a source-grounded researcher profile with output parsing and validation. #### Parameters -##### options? +##### options -[`ResearcherProfileOptions`](#researcherprofileoptions) & `object` = `{}` +[`ResearcherProfileOptions`](#researcherprofileoptions) & `object` #### Returns @@ -1645,7 +1629,7 @@ Build a source-grounded researcher profile with output parsing and validation. ### multiHarnessResearcherFanout() -> **multiHarnessResearcherFanout**(`options?`): `object` +> **multiHarnessResearcherFanout**(`options`): `object` **`Experimental`** @@ -1655,9 +1639,9 @@ picks the highest-scoring valid output. #### Parameters -##### options? +##### options -[`MultiHarnessResearcherFanoutOptions`](#multiharnessresearcherfanoutoptions) = `{}` +[`MultiHarnessResearcherFanoutOptions`](#multiharnessresearcherfanoutoptions) #### Returns diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 962d95a8..9f92a317 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -19335,9 +19335,9 @@ Exact admitted profile used to validate the stable effective identity at publica > **ExecutorFactory**\<`Out`\> = (`spec`, `ctx`) => [`Executor`](index.md#executor-2)\<`Out`\> -Builds a fresh `Executor` for one spawn from the resolved spec. Per-spawn (not -shared) so each child owns its own box/abort/teardown lifecycle. A BYO factory lets a -user supply construction args without pre-instantiating. +Builds a fresh `Executor` for one spawn from the resolved, immutable spec. Per-spawn (not shared) +so each child owns its own box/abort/teardown lifecycle. A BYO factory lets a user supply +construction args without pre-instantiating; it never bypasses exact-profile validation. #### Type Parameters @@ -24472,8 +24472,8 @@ Read every valid steer request in a worker's inbox. Corrupt or partial lines are The single built-in executor factory. Picks a leaf backend by data (`config.backend`), injects the matching seam, and delegates to that backend's built-in implementation. -The `Executor` port stays OPEN: bring-your-own agents implement `Executor` directly -and never pass through here. Use this (or `createExecutorRegistry`) instead of a +The `Executor` port stays OPEN: bring-your-own agents implement `Executor` directly, while Scope +or `createExecutorRegistry` still parses and seals their exact profile before use. Use this instead of a per-vendor adapter or a closed `inline|sandbox|cli` switch — those bypass the `UsageEvent` reporting channel. @@ -24495,9 +24495,8 @@ per-vendor adapter or a closed `inline|sandbox|cli` switch — those bypass the The open resolver/registry. Pre-registers the three built-ins under their runtime tags (`'router'`, `'sandbox'`, `'cli'`) and accepts `register(name, -factory)` for any additional runtime — and a BYO `AgentSpec.executor` resolves -without touching the registry at all. NOT a closed switch; registration + BYO -ARE the extension points. +factory)` for any additional runtime. A BYO `AgentSpec.executor` has highest routing precedence +after the same exact-profile intake validation. Registration + BYO remain open extension points. `resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executorFactory` → `spec.executor` → `harness === null` → the `'router'` factory; else a registered factory for the diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 2e6601ba..d08519e3 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -119,7 +119,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Multi-judge review / rubric quorum over one artifact | `panel(spec)`: `/kernel` | a judge ensemble that feeds one judge's score into another | | Fixed sequential chain (plan→implement→…) | `pipeline(stages)`: `/kernel` | hand-chained `await`s passing outputs along | | Adaptive tree search / progressive widening | `widen(spec)` + `flatWidenGate()`: `/kernel` | a best-first/MCTS that reads child *scores* to expand (selector=judge); keep `flatWidenGate()` until your gate is proven | -| Define the profile record for a personified run | `definePersona(input)`: `/kernel` | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | +| Define the profile record for a personified run | `definePersona(input)`: `/kernel`; its root must carry a complete exact `AgentProfile`, which Runtime parses and freezes at definition time | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | | Make a worker self-verify / iterate / audit | a **hook / process / skill on its authored `AgentProfile`**: §1.5 | a per-round judge, a `while(!done)` loop, or a bash hill-climb (it's a profile lever) | | Run an authored profile with Claude Code, Codex, OpenCode, or another supported harness | author the `AgentProfile` with `@tangle-network/agent-interface`; `@tangle-network/sandbox` materializes it for the selected harness | a harness-specific profile or config writer | | Have the supervisor design its workers | author a **full `AgentProfile`** per sub-task (prompt+skills+tools+mcp+hooks+subagents): `/kernel` | author a bare `systemPrompt` string (a worker can't act on levers it has no levers for) | @@ -159,7 +159,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Know **what got mounted into a run** / **why a candidate won** | `result.provenance.mounts` / `result.provenance.selectionReceipts` (`MountManifestEntry`/`SelectionReceipt`/`RunProvenance`); declare mounts via the `recordMount` recorder in `prepareBox`: root export | re-reading box contents to reconstruct what was mounted, or re-deriving which candidate the selector picked | | State any benchmark/A-B claim | `pairedLift(...)` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | your own bootstrap loop/PRNG per gate; a point lift without `low/high/pairs` | | Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool**: `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend): `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED): `delegate` is the ONE delegation path and the only one with a cost channel | -| Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })`: `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | +| Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile, fanoutProfiles? })`: `/mcp` (the required exact worker profile owns harness, provider, model, prompt, and tools; optional exact profiles make fanout heterogeneous) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | | Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | the **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot. Supervised-tree restart recovery is not implemented. | | Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer`: `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | | Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })`: `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | diff --git a/src/mcp/types.ts b/src/mcp/types.ts index 576eb140..30c9dac1 100644 --- a/src/mcp/types.ts +++ b/src/mcp/types.ts @@ -48,9 +48,8 @@ export interface DelegateCodeArgs { /** Optional free-form context the agent surfaces in the prompt prelude. */ contextHint?: string /** - * When > 1, dispatches `multiHarnessCoderFanout` across N harnesses - * (claude-code, codex, opencode-glm) and picks the highest-scoring - * passing patch. Default 1. + * When > 1, dispatches `multiHarnessCoderFanout` across the delegate's configured exact profiles + * and picks the highest-scoring passing patch. Default 1. */ variants?: number /** Validator + prompt overrides the agent knows for this repo. */ diff --git a/src/profiles/researcher.ts b/src/profiles/researcher.ts index 1d972d96..a5a53232 100644 --- a/src/profiles/researcher.ts +++ b/src/profiles/researcher.ts @@ -343,6 +343,7 @@ function recencyMatchScore(items: KnowledgeItem[], window: ResearchTask['recency return total === 0 ? 0 : hits / total } +/** Built-in source-grounded research contract added to a caller-owned exact profile. */ export const RESEARCHER_SYSTEM_PROMPT = [ 'You are a research agent. Your job is to answer a research question with', 'source-grounded knowledge items that the caller will choose whether to', From 2ea1c4cb372c7b182906b7e4e0cc85622ed33f0d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 21:56:24 -0600 Subject: [PATCH 12/39] refactor(runtime): remove bypassing execution paths --- CHANGELOG.md | 2 +- docs/api/index.md | 191 ------- docs/api/mcp.md | 12 +- docs/api/primitive-catalog.md | 13 +- docs/api/profiles.md | 512 ------------------ examples/README.md | 1 - examples/ui-audit/README.md | 39 -- examples/ui-audit/ui-audit.ts | 144 ----- package.json | 7 +- pnpm-lock.yaml | 29 - scripts/verify-package-exports.mjs | 1 - src/improvement/agentic-generator.ts | 8 +- src/improvement/driver-loop-generator.test.ts | 313 ----------- src/improvement/driver-loop-generator.ts | 345 ------------ src/improvement/index.ts | 6 - src/improvement/optimizer-prompt.ts | 68 +-- src/mcp/delegates.ts | 6 +- src/mcp/server.ts | 6 +- src/profiles/index.ts | 11 - src/profiles/ui-auditor/in-process-client.ts | 355 ------------ src/profiles/ui-auditor/index.ts | 21 +- src/profiles/ui-auditor/judge.ts | 62 --- src/profiles/ui-auditor/profile.ts | 92 ---- tests/profiles/ui-auditor-client.test.ts | 492 ----------------- tests/profiles/ui-auditor.test.ts | 11 - 25 files changed, 20 insertions(+), 2727 deletions(-) delete mode 100644 examples/ui-audit/README.md delete mode 100644 examples/ui-audit/ui-audit.ts delete mode 100644 src/improvement/driver-loop-generator.test.ts delete mode 100644 src/improvement/driver-loop-generator.ts delete mode 100644 src/profiles/ui-auditor/in-process-client.ts delete mode 100644 src/profiles/ui-auditor/judge.ts delete mode 100644 src/profiles/ui-auditor/profile.ts delete mode 100644 tests/profiles/ui-auditor-client.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 290ad4e1..cb5ac3d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,7 +162,7 @@ The fix therefore lives in agent-eval, but a consumer only receives it if this p BREAKING. `LocalHarness` was a private three-member vocabulary (`'claude' | 'codex' | 'opencode'`) that spelled one harness differently from every other layer in the stack. It is now a narrowing of the shared `HarnessType`: **`'claude'` is renamed to `'claude-code'`**. `claude` remains the EXECUTABLE name and lives only in the harness table's `command` field. -Callers to update: `runLocalHarness({ harness })`, `harnessInvocation(harness, …)`, `runWorktreeHarness({ harness })`, `agenticGenerator({ harness })`, `driverLoopGenerator({ harness })`, `createInProcessExecutor({ harnesses })`, `AuthoredHarness.harness`, and the `AGENT_RUNTIME_LOCAL_HARNESSES` env list. Anything that passed `'claude'` passes `'claude-code'`; `codex` and `opencode` are unchanged. +Callers to update: `runLocalHarness({ harness })`, `harnessInvocation(harness, …)`, `runWorktreeHarness({ harness })`, `agenticGenerator({ harness })`, `createInProcessExecutor({ harnesses })`, `AuthoredHarness.harness`, and the `AGENT_RUNTIME_LOCAL_HARNESSES` env list. Anything that passed `'claude'` passes `'claude-code'`; `codex` and `opencode` are unchanged. Deleting the alias removed `materializerHarness()` outright — a `LocalHarness` is now handed straight to the profile materializer with no translation. diff --git a/docs/api/index.md b/docs/api/index.md index d9a19642..593cbfe8 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4694,158 +4694,6 @@ Evidence supplied to a generated tool or MCP build instruction. *** -### DriverLoopGeneratorOptions - -#### Properties - -##### brain - -> **brain**: [`ToolLoopChat`](runtime.md#toolloopchat) - -The driver-LLM seam — ONE inference turn over the conversation + tool specs (the canonical - `ToolLoopChat`, same seam as `driverAgent`): `routerBrain(cfg)` in production, a scripted - mock in tests. - -##### harness? - -> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) - -Local coding harness the driver's worker sessions run in the worktree. Default `claude-code`. - -##### timeoutMs? - -> `optional` **timeoutMs?**: `number` - -Per-worker-session wall-clock timeout (ms). Default = `runLocalHarness` default (5m). - -##### buildPrompt? - -> `optional` **buildPrompt?**: (`args`) => `string` - -Build the driver's task briefing (domain framing + method + findings) — the same senior - prompt the worker path uses (`toolBuildPrompt` / `mcpBuildPrompt`). The driver reads it and - folds what each worker needs into its instruction. Default `defaultBuildPrompt`. - -###### Parameters - -###### args - -###### findings - -readonly `ProposalFinding`[] - -###### Returns - -`string` - -##### verify? - -> `optional` **verify?**: [`Verifier`](#verifier) - -Verify the worktree (the intrinsic check). Exposed to the driver as `run_verifier` AND - re-run by code as the final keep/discard gate. Omitted ⇒ the final gate is dirty-tree only - (legacy `agenticGenerator` behavior sans verifier). - -##### maxTurns? - -> `optional` **maxTurns?**: `number` - -Max driver inference turns. Default `max(8, 2 + maxShots * 3)` — room for one - observe/rate/decide cycle per worker session plus orientation. - -##### research? - -> `optional` **research?**: (`query`) => `Promise`\<`string`\> - -The research seam (adopt-not-build): when set, the driver gets a - `research{query}` tool + the `researchDriverNote` doctrine, so it can - discover an EXISTING external MCP instead of building one. Wire a real - web/search backend here — none is provisioned by default (the build - harness has no live web access yet; flagged). - -###### Parameters - -###### query - -`string` - -###### Returns - -`Promise`\<`string`\> - -##### runHarness? - -> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> - -Test seam — inject the harness runner (defaults to `runLocalHarness`). - -**`Experimental`** - -Spawn a local coding harness CLI as a subprocess + collect its output. - -NOT responsible for parsing the harness's output or extracting a diff — -the in-process executor's `streamPrompt` orchestrates `git diff` against -the worktree after this resolves. This function is intentionally narrow: -spawn, wait, capture, return. - -Fails loud — throws when: - - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) - - the harness binary is not on PATH (ENOENT) - - the caller signal was already aborted before process launch - -Does NOT throw when: - - the subprocess exits non-zero (`result.exitCode` carries the code) - - a non-reproducible subprocess is aborted / timed out (`result.aborted` / - `result.timedOut` carries the reason even when a TERM-aware child exits zero) - -Reproducible Codex additionally requires a terminal usage event. If cancellation -prevents that event, this rejects with `CodexExecutionDiagnosticError` instead of -returning an incomplete reproducibility receipt. - -###### Parameters - -###### options - -[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) - -###### Returns - -`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> - -##### readDiff? - -> `optional` **readDiff?**: (`worktreePath`) => `string` - -Test seam — inject the worktree diff reader (defaults to `git diff` in the worktree). - -###### Parameters - -###### worktreePath - -`string` - -###### Returns - -`string` - -##### changedPaths? - -> `optional` **changedPaths?**: (`worktreePath`) => `string`[] - -Test seam — inject the changed-paths reader (defaults to `git status --porcelain`). - -###### Parameters - -###### worktreePath - -`string` - -###### Returns - -`string`[] - -*** - ### ImproveMethodContext #### Properties @@ -11988,27 +11836,6 @@ is the process itself. *** -### buildDriverSystem - -> `const` **buildDriverSystem**: `string` - -The driver's stance for `driverLoopGenerator` — the build-domain instance of -the supervisor doctrine (observe → rate → decide; refine / re-scope / -decompose; the check decides delivery, never the driver's prose). - -*** - -### researchDriverNote - -> `const` **researchDriverNote**: `string` - -The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when -a `research` tool is wired into the loop (`DriverLoopGeneratorOptions. -research`). Kept separate so a driver WITHOUT the tool is never told to -call a tool it does not have. - -*** - ### strategyAuthorMethod > `const` **strategyAuthorMethod**: `string` @@ -13186,24 +13013,6 @@ Build the starting instruction for a coder agent tasked with implementing a new *** -### driverLoopGenerator() - -> **driverLoopGenerator**(`opts`): [`CandidateGenerator`](#candidategenerator) - -Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budget is spent. - -#### Parameters - -##### opts - -[`DriverLoopGeneratorOptions`](#driverloopgeneratoroptions) - -#### Returns - -[`CandidateGenerator`](#candidategenerator) - -*** - ### improve() #### Call Signature diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 2a84d4ec..6cad9b6d 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -3025,10 +3025,8 @@ wire here. **`Experimental`** -Required to enable delegate_ui_audit. Wire one that closes over your -`runAgentRounds` + `uiAuditorProfile` + a `SandboxClient` (the -canonical in-process choice is `createInProcessUiAuditClient` from -`@tangle-network/agent-runtime/profiles`) + your vision judge. +Required to enable delegate_ui_audit. Wire one that executes an exact +agent profile through Runtime and returns the provider-neutral UI audit result. ##### feedbackStore? @@ -5566,10 +5564,8 @@ The coder delegate closure — given the coder args + run context, drives the **`Experimental`** UI-auditor delegate — fully consumer-injected. agent-runtime ships no -default factory because the inputs are workspace path + judge function -+ (optionally) a `SandboxClient`, and the judge is the consumer's -model seam. See `createInProcessUiAuditClient` + `uiAuditorProfile` in -`@tangle-network/agent-runtime/profiles` for the canonical wiring. +default factory because execution belongs to a caller-supplied exact +agent profile and Runtime executor. #### Parameters diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 28769200..9ff7cb74 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 407 exports. +Import from `@tangle-network/agent-runtime` — 403 exports. | Symbol | Kind | Summary | |---|---|---| @@ -58,7 +58,6 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `defineConversation` | function | Validate and define a conversation before execution. | | `defineRuntimeHooks` | function | Identity helper that types a {@link RuntimeHooks} literal so the fields are inferred. | | `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | -| `driverLoopGenerator` | function | Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budge | | `exactProcessProviderAsCandidateExecutor` | function | Adapt one neutral exact-process provider to Runtime's trusted candidate boundary. | | `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | @@ -128,7 +127,6 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | | `AGENT_CANDIDATE_EXECUTION_SUPPORT` | const | Surfaces admitted by Runtime's verifier before an environment adapter is selected. | | `AGENTIC_PROFILE_RESOURCE_ROOT` | const | Dedicated ephemeral root for generic author-profile files. Every declared | -| `buildDriverSystem` | const | The driver's stance for `driverLoopGenerator` — the build-domain instance of | | `CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV` | const | Environment variable containing the materialized retrieval configuration path. | | `CANDIDATE_KNOWLEDGE_ROOT_ENV` | const | Environment variable containing the materialized candidate knowledge root. | | `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | @@ -141,7 +139,6 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `INTELLIGENCE_WIRE_VERSION` | const | Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). | | `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | -| `researchDriverNote` | const | The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when | | `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | | `strategyAuthorMethod` | const | The senior authoring process for `authorStrategy` — the same method, shaped | | `AgentEvalError` | class | Base class for every contract error this package throws — carries the stable | @@ -257,7 +254,7 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `DriverLoopGeneratorOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -1133,14 +1130,13 @@ Import from `@tangle-network/agent-runtime/knowledge` — 24 exports. ### Built-in agent profiles -Import from `@tangle-network/agent-runtime/profiles` — 53 exports. +Import from `@tangle-network/agent-runtime/profiles` — 42 exports. | Symbol | Kind | Summary | |---|---|---| | `appendFindings` | function | Append findings to a workspace, writing one Markdown file per finding | | `buildAuditorSystemPrompt` | function | Build a system prompt for a single auditor iteration. | | `coderTaskToPrompt` | function | Render a `CoderTask` into the per-task instruction handed to the coder profile. | -| `createInProcessUiAuditClient` | function | Create a `SandboxClient` that drives a local Playwright browser for in-process UI audits. | | `createResearcherValidator` | function | Build a validator that closes over a specific `ResearchTask`'s constraints. | | `createUiAuditorValidator` | function | Build a `Validator` that rejects off-lens findings and findings missing screenshot evidence. | | `decodeAuditTaskEnvelope` | function | Parse a task envelope back out of a prompt string. Returns undefined if | @@ -1153,7 +1149,6 @@ Import from `@tangle-network/agent-runtime/profiles` — 53 exports. | `registerCaptures` | function | Record screenshots taken for a route in the registry, without filing a | | `researcherProfile` | function | Build a source-grounded researcher profile with output parsing and validation. | | `summarizeRegistry` | function | Compute finding counts by severity, lens, and route from an `AuditRegistry`. | -| `uiAuditorProfile` | function | Preset `runAgentRounds` bundle for vision-driven UI audits: returns the `AgentRunSpec`, output adapter, validator, and prompt formatter the loop kernel needs. | | `writeAuditIndex` | function | Regenerate `/index.md` from registry.json. | | `LENS_BRIEFS` | const | Per-lens auditor briefs: concrete signals to look for and cross-lens distinctions to respect. | | `SHARED_AUDITOR_RULES` | const | Cross-lens rules injected into every UI audit iteration: finding quality standards and scope limits. | @@ -1172,7 +1167,7 @@ Import from `@tangle-network/agent-runtime/profiles` — 53 exports. | `UiFindingSeverity` | type | Severity scale. | | `UiLens` | type | Canonical audit lenses. Each lens scopes a finding to a single class of | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AppendFindingsResult`, `AuditIndex`, `AuditRegistry`, `AuditRegistryCapture`, `BrowserContextHandle`, `BrowserHandle`, `CoderTask`, `InProcessUiAuditClientOptions`, `MultiHarnessResearcherFanoutOptions`, `PageHandle`, `RegisterCapturesOptions`, `UiAuditCapture`, `UiAuditCaptureRequest`, `UiAuditorProfileOptions`, `UiAuditViewport`, `UiJudgeInput`, `UiJudgeOutput`, `UiJudgeTokenUsage`, `UiJudge`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AppendFindingsResult`, `AuditIndex`, `AuditRegistry`, `AuditRegistryCapture`, `CoderTask`, `MultiHarnessResearcherFanoutOptions`, `RegisterCapturesOptions`, `UiAuditCapture`, `UiAuditCaptureRequest`, `UiAuditViewport`. ### Platform glue diff --git a/docs/api/profiles.md b/docs/api/profiles.md index 61028423..a2ca2ec4 100644 --- a/docs/api/profiles.md +++ b/docs/api/profiles.md @@ -535,432 +535,6 @@ Optional task — narrows the validator's namespace check. *** -### InProcessUiAuditClientOptions - -**`Experimental`** - -#### Properties - -##### workspaceDir - -> **workspaceDir**: `string` - -**`Experimental`** - -Absolute path under which screenshots are written. Each capture lands -at `/screenshots/`; finding screenshot paths -are workspace-relative (`screenshots/`). - -##### judge - -> **judge**: [`UiJudge`](#uijudge) - -**`Experimental`** - -The vision judge that turns captures into findings. - -##### navPolicy? - -> `optional` **navPolicy?**: `"strict"` \| `"spa"` - -**`Experimental`** - -Navigation policy. - -`'strict'` (default) waits for `networkidle` and fails the iteration -if the page does not settle. `'spa'` waits for `domcontentloaded` — -use for single-page apps that hold open long-poll/websocket -connections and never settle. - -##### launchBrowser? - -> `optional` **launchBrowser?**: () => `Promise`\<[`BrowserHandle`](#browserhandle)\> - -**`Experimental`** - -Browser launch override. Default: chromium headless via Playwright. -Consumers pass a custom factory to target a remote browser, a -different channel, or a fleet adapter. - -###### Returns - -`Promise`\<[`BrowserHandle`](#browserhandle)\> - -*** - -### BrowserHandle - -**`Experimental`** - -#### Methods - -##### newContext() - -> **newContext**(`options?`): `Promise`\<[`BrowserContextHandle`](#browsercontexthandle)\> - -**`Experimental`** - -###### Parameters - -###### options? - -###### viewport? - -\{ `width`: `number`; `height`: `number`; \} - -###### viewport.width - -`number` - -###### viewport.height - -`number` - -###### Returns - -`Promise`\<[`BrowserContextHandle`](#browsercontexthandle)\> - -##### close() - -> **close**(): `Promise`\<`void`\> - -**`Experimental`** - -###### Returns - -`Promise`\<`void`\> - -*** - -### BrowserContextHandle - -**`Experimental`** - -#### Methods - -##### newPage() - -> **newPage**(): `Promise`\<[`PageHandle`](#pagehandle)\> - -**`Experimental`** - -###### Returns - -`Promise`\<[`PageHandle`](#pagehandle)\> - -##### close() - -> **close**(): `Promise`\<`void`\> - -**`Experimental`** - -###### Returns - -`Promise`\<`void`\> - -*** - -### PageHandle - -**`Experimental`** - -#### Methods - -##### setViewportSize() - -> **setViewportSize**(`size`): `Promise`\<`void`\> - -**`Experimental`** - -###### Parameters - -###### size - -###### width - -`number` - -###### height - -`number` - -###### Returns - -`Promise`\<`void`\> - -##### goto() - -> **goto**(`url`, `options?`): `Promise`\<`unknown`\> - -**`Experimental`** - -###### Parameters - -###### url - -`string` - -###### options? - -###### waitUntil? - -`string` - -###### timeout? - -`number` - -###### Returns - -`Promise`\<`unknown`\> - -##### waitForSelector() - -> **waitForSelector**(`selector`, `options?`): `Promise`\<`unknown`\> - -**`Experimental`** - -###### Parameters - -###### selector - -`string` - -###### options? - -###### timeout? - -`number` - -###### Returns - -`Promise`\<`unknown`\> - -##### waitForTimeout() - -> **waitForTimeout**(`ms`): `Promise`\<`void`\> - -**`Experimental`** - -###### Parameters - -###### ms - -`number` - -###### Returns - -`Promise`\<`void`\> - -##### screenshot() - -> **screenshot**(`options`): `Promise`\<`void`\> - -**`Experimental`** - -###### Parameters - -###### options - -###### path - -`string` - -###### fullPage? - -`boolean` - -###### Returns - -`Promise`\<`void`\> - -##### locator() - -> **locator**(`selector`): `object` - -**`Experimental`** - -###### Parameters - -###### selector - -`string` - -###### Returns - -`object` - -###### first() - -> **first**(): `object` - -###### Returns - -`object` - -###### screenshot() - -> **screenshot**(`options`): `Promise`\<`void`\> - -###### Parameters - -###### options - -###### path - -`string` - -###### Returns - -`Promise`\<`void`\> - -*** - -### UiJudgeTokenUsage - -**`Experimental`** - -#### Properties - -##### input - -> **input**: `number` - -**`Experimental`** - -##### output - -> **output**: `number` - -**`Experimental`** - -*** - -### UiJudgeInput - -**`Experimental`** - -#### Properties - -##### lens - -> **lens**: [`UiLens`](#uilens) - -**`Experimental`** - -##### captures - -> **captures**: readonly [`UiAuditCapture`](#uiauditcapture)[] - -**`Experimental`** - -##### productContext? - -> `optional` **productContext?**: `string` - -**`Experimental`** - -Free-form product context the consumer wants the judge to know. - -##### knownFindingIds? - -> `optional` **knownFindingIds?**: readonly `number`[] - -**`Experimental`** - -Findings already on file across earlier iterations — for similarTo linkage. - -##### promptText - -> **promptText**: `string` - -**`Experimental`** - -The full prompt the loop kernel synthesized for this iteration. - -##### signal - -> **signal**: `AbortSignal` - -**`Experimental`** - -Cooperative cancellation. - -*** - -### UiJudgeOutput - -**`Experimental`** - -#### Properties - -##### findings - -> **findings**: [`UiFinding`](#uifinding)[] - -**`Experimental`** - -##### notes? - -> `optional` **notes?**: `string` - -**`Experimental`** - -Optional triage commentary. - -##### tokenUsage? - -> `optional` **tokenUsage?**: [`UiJudgeTokenUsage`](#uijudgetokenusage) - -**`Experimental`** - -Optional usage; folded into the kernel cost ledger when present. - -##### costUsd? - -> `optional` **costUsd?**: `number` - -**`Experimental`** - -Optional total cost in USD. - -*** - -### UiAuditorProfileOptions - -**`Experimental`** - -#### Properties - -##### name? - -> `optional` **name?**: `string` - -**`Experimental`** - -Stable name surfaced in trace events. Defaults to `ui-auditor`. - -##### model? - -> `optional` **model?**: `string` - -**`Experimental`** - -Optional model identifier passed in `AgentProfile.model.default`. -The consumer's `SandboxClient` chooses how to interpret it. - -##### task? - -> `optional` **task?**: [`UiAuditTask`](#uiaudittask) - -**`Experimental`** - -Task bound to the validator. Without it the validator uses the lens -embedded in the iteration output as its expectation — fine for one-off -use; less strict than passing the task explicitly. - -*** - ### UiFindingScreenshot Pointer to a screenshot referenced by a finding (workspace-relative path). @@ -1345,24 +919,6 @@ these — the caller decides. *** -### UiJudge - -> **UiJudge** = (`input`) => `Promise`\<[`UiJudgeOutput`](#uijudgeoutput)\> - -**`Experimental`** - -#### Parameters - -##### input - -[`UiJudgeInput`](#uijudgeinput) - -#### Returns - -`Promise`\<[`UiJudgeOutput`](#uijudgeoutput)\> - -*** - ### UiLens > **UiLens** = `"consistency"` \| `"hierarchy"` \| `"layout"` \| `"ux-flow"` \| `"duplication"` \| `"accessibility"` \| `"responsive"` \| `"states"` \| `"content"` \| `"interaction"` \| `"performance-perceived"` \| `"other"` @@ -1709,24 +1265,6 @@ Aggregate score: *** -### createInProcessUiAuditClient() - -> **createInProcessUiAuditClient**(`options`): [`SandboxClient`](runtime.md#sandboxclient-5) & `object` - -**`Experimental`** - -Create a `SandboxClient` that drives a local Playwright browser for in-process UI audits. - -#### Parameters - -##### options - -[`InProcessUiAuditClientOptions`](#inprocessuiauditclientoptions) - -#### Returns - -*** - ### buildAuditorSystemPrompt() > **buildAuditorSystemPrompt**(`lens`): `string` @@ -1767,56 +1305,6 @@ Parse raw `SandboxEvent` emissions from an audit iteration into structured `UiAu *** -### uiAuditorProfile() - -> **uiAuditorProfile**(`options?`): `object` - -**`Experimental`** - -Preset `runAgentRounds` bundle for vision-driven UI audits: returns the `AgentRunSpec`, output adapter, validator, and prompt formatter the loop kernel needs. - -#### Parameters - -##### options? - -[`UiAuditorProfileOptions`](#uiauditorprofileoptions) = `{}` - -#### Returns - -`object` - -##### profile - -> **profile**: `AgentProfile` - -##### taskToPrompt - -> **taskToPrompt**: (`task`) => `string` - -###### Parameters - -###### task - -[`UiAuditTask`](#uiaudittask) - -###### Returns - -`string` - -##### output - -> **output**: [`OutputAdapter`](runtime.md#outputadapter)\<[`UiAuditOutput`](#uiauditoutput)\> - -##### validator - -> **validator**: [`Validator`](runtime.md#validator-1)\<[`UiAuditOutput`](#uiauditoutput)\> - -##### agentRunSpec - -> **agentRunSpec**: [`AgentRunSpec`](runtime.md#agentrunspec)\<[`UiAuditTask`](#uiaudittask)\> - -*** - ### encodeAuditTaskEnvelope() > **encodeAuditTaskEnvelope**(`task`): `string` diff --git a/examples/README.md b/examples/README.md index a65bdc09..85c2cbc4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -128,7 +128,6 @@ TANGLE_API_KEY=... pnpm tsx examples/delegate/delegate.ts # Benchmarking pnpm tsx examples/researcher-loop/researcher-loop.ts # needs the agent-knowledge peer -pnpm dlx tsx examples/ui-audit/ui-audit.ts /tmp/ui-audit-demo https://example.com pnpm tsx examples/coding-benchmark/benchmark.ts # offline pnpm tsx examples/coding-benchmark/benchmark.ts --ensemble --reps 5 diff --git a/examples/ui-audit/README.md b/examples/ui-audit/README.md deleted file mode 100644 index 0df0bc37..00000000 --- a/examples/ui-audit/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Find UI problems on a web page, and file them as issues - -Point this at a URL. It opens the page, inspects it through a fixed list of review "lenses" -(accessibility, layout, contrast, and so on), and for every problem it finds it writes a -self-contained GitHub-issue Markdown file — title, description, screenshot — ready to file with -`gh issue create`. - -It runs with **no API key**: the example ships a stub reviewer so you can watch the whole pipeline end -to end. For real audits you swap in a vision model (below). - -## Why it matters - -UI review is exactly the tedious, repeatable check teams skip. This turns it into one command that -produces filed, actionable issues. It also shows how to plug a browser and a visual reviewer into this -library's run loop **without a cloud sandbox** — the browser and reviewer run in-process, but the loop -drives them exactly like any other agent worker. - -## Run - -```bash -pnpm dlx tsx examples/ui-audit/ui-audit.ts /tmp/ui-audit-demo https://example.com -``` - -Omit the path and it writes to a temp dir. You get screenshots plus -`issues/NNN----.md` files — file them straight into GitHub with -`gh issue create --body-file `. - -## Make it real - -The example uses a **stub reviewer** that returns canned findings (so it needs no key). Replace it with -a real vision model — anything that takes an image + a prompt and returns structured findings (OpenAI, -Anthropic, Gemini, a local model). Capturing the page and writing the issue files stays identical; only -the judgment call changes. - -## Files - -| file | what it is | -|---|---| -| `ui-audit.ts` | the whole run: browse the page, review it lens by lens, write the issue files | diff --git a/examples/ui-audit/ui-audit.ts b/examples/ui-audit/ui-audit.ts deleted file mode 100644 index 5e6f61cc..00000000 --- a/examples/ui-audit/ui-audit.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * ui-audit — smallest end-to-end UI audit run. - * - * Wires: - * - `uiAuditorProfile()` — output adapter + validator + envelope-prefixed prompt - * - `createInProcessUiAuditClient({ workspaceDir, judge })` — the in-process - * `SandboxClient` that drives Playwright + a vision judge - * - `runAgentRounds({ ... })` — one iteration per (lens × route), validator-gated - * - `appendFindings(...)` + `writeAuditIndex(...)` — persist self-contained - * GitHub-issue Markdown - * - * The example uses a STUB judge so it runs without an API key. Replace - * `stubJudge` with a real vision-LLM-backed implementation for production. - * - * Usage: - * pnpm dlx tsx examples/ui-audit/ui-audit.ts - */ - -import { mkdtemp } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import path from 'node:path' -import type { Driver } from '@tangle-network/agent-runtime/kernel' -import { runAgentRounds } from '@tangle-network/agent-runtime/kernel' -import { - appendFindings, - createInProcessUiAuditClient, - initAuditWorkspace, - type UiAuditOutput, - type UiAuditTask, - type UiFinding, - type UiJudge, - uiAuditorProfile, - writeAuditIndex, -} from '@tangle-network/agent-runtime/profiles' - -// Lowercase module-global by repo convention (an UPPERCASE name trips the publish -// obfuscator — see CLAUDE.md § Publish gotcha). -const lensesToRun = ['consistency', 'hierarchy', 'layout', 'ux-flow'] as const - -/** - * Deterministic stub judge — returns one canned finding referencing the - * iteration's first capture. Replace with a real vision LLM call. - */ -const stubJudge: UiJudge = async ({ lens, captures }) => { - if (captures.length === 0) return { findings: [] } - const evidence = captures[0] - if (!evidence) return { findings: [] } - const finding: UiFinding = { - title: `[stub] ${lens} finding on ${evidence.route}`, - lens, - severity: 'low', - route: evidence.route, - url: evidence.url, - viewport: evidence.viewport, - observation: `Stub judge emitted a placeholder ${lens} finding to demonstrate the pipeline.`, - impact: 'No real impact — stub output for the example.', - suggestedFix: 'Wire a real vision judge to replace this finding.', - screenshots: [{ path: evidence.path, viewport: evidence.viewport }], - } - return { findings: [finding], tokenUsage: { input: 0, output: 0 } } -} - -/** - * `lensCyclingDriver` — plans one iteration per lens in a fixed order, - * stops when the list is exhausted. The kernel handles cost + abort + trace. - */ -function lensCyclingDriver( - lenses: readonly UiAuditTask['lens'][], - task: UiAuditTask, -): Driver { - let cursor = 0 - return { - // This driver is CONTENT-BLIND by design: it cycles a fixed lens list off - // `history.length` and never reads a worker's output. So "driver" here is just - // a counter, NOT the output-driven re-planner. For a driver that builds the next - // prompt FROM the last worker's output (the fold), see examples/driver-loop/. - // plan() returns Task[] — one lens per iteration, [] once all lenses are - // cycled. The empty plan is what ends the loop: neither 'complete' nor - // 'failed' is a terminal Decision (isTerminalDecision = stop|fail|done| - // pick-winner), so plan-exhaustion + maxIterations are what stop it. - async plan(_taskIn, history) { - if (history.length >= lenses.length || cursor >= lenses.length) return [] - const next = lenses[cursor] - if (!next) return [] - cursor += 1 - return [{ ...task, lens: next }] - }, - // decide() returns the bare Decision recording the outcome; termination is - // driven by plan() → []. - decide(history) { - return history.some((it) => it.verdict?.valid) ? 'complete' : 'failed' - }, - } -} - -async function main(): Promise { - const [, , wsArg, urlArg] = process.argv - const startUrl = urlArg ?? 'https://example.com' - const workspaceDir = wsArg ?? (await mkdtemp(path.join(tmpdir(), 'ui-audit-'))) - console.log(`workspace: ${workspaceDir}`) - await initAuditWorkspace(workspaceDir) - - const { output, validator, agentRunSpec } = uiAuditorProfile() - const client = createInProcessUiAuditClient({ workspaceDir, judge: stubJudge }) - - try { - const task: UiAuditTask = { - lens: 'consistency', - captures: [{ route: 'home', url: startUrl, fullPage: true }], - } - const driver = lensCyclingDriver(lensesToRun, task) - - const result = await runAgentRounds({ - driver, - agentRun: agentRunSpec, - output, - validator, - task, - ctx: { sandboxClient: client }, - maxIterations: lensesToRun.length, - }) - - console.log(`iterations: ${result.iterations.length}`) - console.log(`decision: ${result.decision}`) - const allFindings: UiFinding[] = [] - for (const it of result.iterations) { - if (it.output) allFindings.push(...it.output.findings) - } - console.log(`findings collected: ${allFindings.length}`) - if (allFindings.length > 0) { - const persisted = await appendFindings(workspaceDir, allFindings) - console.log(`persisted ${persisted.written.length} finding(s)`) - } - const indexPath = await writeAuditIndex(workspaceDir) - console.log(`index: ${indexPath}`) - } finally { - await client.close() - } -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/package.json b/package.json index 453728b6..34facd7b 100644 --- a/package.json +++ b/package.json @@ -153,7 +153,6 @@ "@types/node": "catalog:", "@types/tar-stream": "3.1.4", "miniflare": "4.20260722.0", - "playwright": "^1.62.0", "publint": "catalog:", "tsdown": "catalog:", "tsx": "catalog:", @@ -172,15 +171,11 @@ "peerDependencies": { "@tangle-network/agent-eval": ">=0.144.1 <0.145.0", "@tangle-network/agent-interface": ">=0.43.0 <0.44.0", - "@tangle-network/sandbox": ">=0.17.2 <0.18.0", - "playwright": "^1.40.0" + "@tangle-network/sandbox": ">=0.17.2 <0.18.0" }, "peerDependenciesMeta": { "@tangle-network/sandbox": { "optional": true - }, - "playwright": { - "optional": true } }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20eb779c..a638e541 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,9 +84,6 @@ importers: miniflare: specifier: 4.20260722.0 version: 4.20260722.0 - playwright: - specifier: ^1.62.0 - version: 1.62.0 publint: specifier: 'catalog:' version: 0.3.22 @@ -1568,11 +1565,6 @@ packages: fft.js@4.0.4: resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} - fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1748,16 +1740,6 @@ packages: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} - playwright-core@1.62.0: - resolution: {integrity: sha512-nsNRyq0r2zsG8AcRHWknc9QRA5XCueC7gWMrs+Gx2tlZn9hcl8zudfh00lhJPY1DE7NmZ6bDsT9g2yey8mXljA==} - engines: {node: '>=20'} - hasBin: true - - playwright@1.62.0: - resolution: {integrity: sha512-Z14dG305dgaLu6foB1TXQagFiW8JfSUIUaUuPaKQ6NtBPKF1P/qXcqfh6c6K/icPqdy37JmjbiBXf6JNg6Sylw==} - engines: {node: '>=20'} - hasBin: true - postcss@8.5.13: resolution: {integrity: sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==} engines: {node: ^10 || ^12 || >=14} @@ -3200,9 +3182,6 @@ snapshots: fft.js@4.0.4: {} - fsevents@2.3.2: - optional: true - fsevents@2.3.3: optional: true @@ -3384,14 +3363,6 @@ snapshots: picomatch@4.0.5: {} - playwright-core@1.62.0: {} - - playwright@1.62.0: - dependencies: - playwright-core: 1.62.0 - optionalDependencies: - fsevents: 2.3.2 - postcss@8.5.13: dependencies: nanoid: 3.3.12 diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 12690af6..63775b7e 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -132,7 +132,6 @@ try { '@tangle-network/agent-eval', '@tangle-network/agent-interface', '@tangle-network/sandbox', - 'playwright', ] const peerDependencies = Object.fromEntries( peerPackages.map((name) => { diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index cb5d6aed..799bf1db 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -840,7 +840,7 @@ function failureNote(feedback?: string): string { ].join('\n') } -export function rawTraceEvidenceProblem( +function rawTraceEvidenceProblem( worktreePath: string, findings: ReadonlyArray, ): string | null { @@ -873,7 +873,7 @@ export function rawTraceEvidenceProblem( return null } -export function requiresRawTraceEvidence(findings: ReadonlyArray): boolean { +function requiresRawTraceEvidence(findings: ReadonlyArray): boolean { return findings.some((finding) => { const f = finding as unknown as Record return f.analyst_id === RAW_TRACE_ANALYST_ID || f.area === RAW_TRACE_AREA @@ -939,7 +939,7 @@ export function commandVerifier( } /** A one-line summary for the commit message, derived from the findings. */ -export function summarizeFindings(findings: ReadonlyArray): string { +function summarizeFindings(findings: ReadonlyArray): string { if (findings.length === 0) return 'agentic improvement' if (findings.length === 1) return `agentic: ${truncate(findings[0]!.claim, 64)}` return `agentic: ${findings.length} findings addressed` @@ -998,7 +998,7 @@ function worktreeDirty(worktreePath: string): boolean { return worktreeChangedPaths(worktreePath).length > 0 } -export function worktreeChangedPaths(worktreePath: string): string[] { +function worktreeChangedPaths(worktreePath: string): string[] { const result = spawnSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: worktreePath, encoding: 'utf-8', diff --git a/src/improvement/driver-loop-generator.test.ts b/src/improvement/driver-loop-generator.test.ts deleted file mode 100644 index 5052f3cd..00000000 --- a/src/improvement/driver-loop-generator.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -/** - * `driverLoopGenerator` proof — the driver→worker atom over a scripted brain. - * - * All process seams are injected (brain, harness runner, git readers, verifier) - * so the test proves the LOOP, deterministically and offline: - * 1. the driver's authored instruction reaches the worker verbatim (the atom: - * an LLM authors the goal, not a canned template), - * 2. verifier feedback flows back to the driver, which REFINES with a second - * authored session, and the candidate lands once the check passes, - * 3. the completion oracle is code-owned: a driver that claims success over a - * clean tree (or a failing verifier) produces `applied:false`, - * 4. the worker-session budget fails closed with an explanatory tool result, - * 5. without a verifier the legacy contract holds (dirty tree = candidate). - */ - -import { makeProposalFinding, type ProposalFinding } from '@tangle-network/agent-eval' -import { describe, expect, it } from 'vitest' -import type { LocalHarnessResult, RunLocalHarnessOptions } from '../mcp/local-harness' -import type { ToolLoopChat } from '../runtime/tool-loop' -import type { VerifyResult } from './agentic-generator' -import { driverLoopGenerator } from './driver-loop-generator' - -const finding = (claim: string): ProposalFinding => - makeProposalFinding({ - analyst_id: 'test-analyst', - proposal_origin: 'search', - severity: 'high', - area: 'tool-use', - claim, - evidence_refs: [], - confidence: 0.9, - produced_at: new Date(0).toISOString(), - }) - -/** One scripted driver turn: the tool calls the "LLM" emits (arguments as JSON strings). */ -type ScriptedTurn = { calls?: Array<{ name: string; args: Record }>; say?: string } - -/** A `ToolLoopChat` that replays a fixed script and records every conversation it saw. */ -function scriptedBrain(turns: ScriptedTurn[]) { - const seen: Array>> = [] - let i = 0 - const chat: ToolLoopChat = async (messages) => { - seen.push(messages.map((m) => ({ ...m }))) - const turn = turns[i] ?? {} - i += 1 - return { - content: turn.say ?? '', - toolCalls: (turn.calls ?? []).map((c, j) => ({ - id: `call-${i}-${j}`, - name: c.name, - arguments: JSON.stringify(c.args), - })), - } - } - return { chat, seen } -} - -function harnessStub(onRun?: (opts: RunLocalHarnessOptions) => void) { - const prompts: string[] = [] - const run = async (opts: RunLocalHarnessOptions): Promise => { - prompts.push(opts.taskPrompt) - onRun?.(opts) - return { - exitCode: 0, - stdout: 'worker done', - stderr: '', - killedBySignal: null, - durationMs: 10, - timedOut: false, - } - } - return { run, prompts } -} - -const generateArgs = ( - findings: ReadonlyArray, - maxShots: number, - signal = new AbortController().signal, -) => ({ - worktreePath: '/wt/cand0', - findings, - maxShots, - signal, -}) - -describe('driverLoopGenerator — the driver→worker build atom', () => { - it('authors the worker instruction, refines on verifier feedback, and lands the candidate', async () => { - const findings = [finding('the agent lacks a JSON schema validator')] - // Worktree turns dirty after the first worker session. - let sessions = 0 - const worker = harnessStub(() => { - sessions += 1 - }) - // Verifier: red after session 1, green after session 2 (and for the final gate). - const verdicts: VerifyResult[] = [] - const verify = (): VerifyResult => { - const v: VerifyResult = - sessions < 2 - ? { ok: false, feedback: 'tests failed: missing export validateJson' } - : { ok: true } - verdicts.push(v) - return v - } - const { chat, seen } = scriptedBrain([ - { - calls: [ - { - name: 'run_worker', - args: { - instruction: - 'Build src/validate.ts exporting validateJson(schema, value) with tests in validate.test.ts; run pnpm test until green.', - }, - }, - ], - }, - { calls: [{ name: 'run_verifier', args: {} }] }, - { - calls: [ - { - name: 'run_worker', - args: { - instruction: - 'Continue from the current tree; pnpm test fails because validateJson is not exported — export it from src/validate.ts and re-run the tests.', - }, - }, - ], - }, - { calls: [{ name: 'run_verifier', args: {} }] }, - { - say: 'Predicted the missing-validator gap; verifier green after one refine; next I would tighten schema errors.', - }, - ]) - const generator = driverLoopGenerator({ - brain: chat, - runHarness: worker.run, - verify, - changedPaths: () => (sessions > 0 ? ['src/validate.ts', 'src/validate.test.ts'] : []), - readDiff: () => '+ export function validateJson()', - }) - expect(generator.kind).toBe('driver-loop:claude-code') - - const result = await generator.generate(generateArgs(findings, 3)) - - expect(result.applied).toBe(true) - expect(result.summary).toContain('JSON schema validator') - // The atom: BOTH worker goals are the driver's own authored text, refine included. - expect(worker.prompts).toEqual([ - 'Build src/validate.ts exporting validateJson(schema, value) with tests in validate.test.ts; run pnpm test until green.', - 'Continue from the current tree; pnpm test fails because validateJson is not exported — export it from src/validate.ts and re-run the tests.', - ]) - // Verifier ran for the driver twice + once more as the code-owned final gate. - expect(verdicts.map((v) => v.ok)).toEqual([false, true, true]) - // The red verdict's feedback reached the driver as a tool message (what it refined from). - const flat = seen.flat() - expect( - flat.some( - (m) => m.role === 'tool' && String(m.content).includes('missing export validateJson'), - ), - ).toBe(true) - }) - - it('fails closed: a driver claiming success over a clean tree is not a candidate', async () => { - let verifyCalls = 0 - const { chat } = scriptedBrain([{ say: 'All done, the tool is perfect.' }]) - const generator = driverLoopGenerator({ - brain: chat, - runHarness: harnessStub().run, - verify: () => { - verifyCalls += 1 - return { ok: true } - }, - changedPaths: () => [], - readDiff: () => '', - }) - const result = await generator.generate(generateArgs([finding('gap')], 2)) - expect(result.applied).toBe(false) - // Clean tree short-circuits before the verifier — the claim never even reached it. - expect(verifyCalls).toBe(0) - }) - - it('fails closed: a dirty tree with a red verifier is discarded regardless of driver prose', async () => { - const { chat } = scriptedBrain([ - { calls: [{ name: 'run_worker', args: { instruction: 'do the change' } }] }, - { say: 'Success! Everything works.' }, - ]) - const generator = driverLoopGenerator({ - brain: chat, - runHarness: harnessStub().run, - verify: () => ({ ok: false, feedback: 'boot-and-probe: initialize handshake timed out' }), - changedPaths: () => ['server.mjs'], - readDiff: () => '+ server', - }) - const result = await generator.generate(generateArgs([finding('gap')], 1)) - expect(result.applied).toBe(false) - expect(result.summary).toBe('') - }) - - it('rejects a partial edit when the caller aborts after a normal worker result', async () => { - const controller = new AbortController() - const { chat } = scriptedBrain([ - { calls: [{ name: 'run_worker', args: { instruction: 'make a partial edit' } }] }, - { say: 'done' }, - ]) - const generator = driverLoopGenerator({ - brain: chat, - runHarness: harnessStub(() => { - controller.abort(new Error('cancelled after worker settlement')) - }).run, - changedPaths: () => ['partial.ts'], - readDiff: () => '+ partial', - }) - - await expect( - generator.generate(generateArgs([finding('gap')], 1, controller.signal)), - ).rejects.toThrow(/cancelled after worker settlement/) - }) - - it('rejects a candidate when cancellation arrives during final verification', async () => { - const controller = new AbortController() - const { chat } = scriptedBrain([{ say: 'the existing edit is complete' }]) - const verify = async (_path: string, signal?: AbortSignal): Promise => { - expect(signal).toBe(controller.signal) - controller.abort(new Error('cancelled during driver verification')) - return { ok: true } - } - const generator = driverLoopGenerator({ - brain: chat, - verify, - changedPaths: () => ['partial.ts'], - readDiff: () => '+ partial', - }) - - await expect( - generator.generate(generateArgs([finding('gap')], 1, controller.signal)), - ).rejects.toThrow(/cancelled during driver verification/) - }) - - it('offers the research tool + adopt doctrine ONLY when the seam is wired', async () => { - // Wired: the driver researches, the stub's findings flow back as a tool - // message, and the system prompt carries the adopt-before-build doctrine. - const queries: string[] = [] - const research = async (query: string): Promise => { - queries.push(query) - return 'found: mcp.exa.ai — maintained web-search MCP; needs EXA_API_KEY' - } - const wired = scriptedBrain([ - { calls: [{ name: 'research', args: { query: 'web search MCP server' } }] }, - { say: 'Adopt mcp.exa.ai with EXA_API_KEY; no build needed.' }, - ]) - await driverLoopGenerator({ - brain: wired.chat, - runHarness: harnessStub().run, - research, - changedPaths: () => [], - readDiff: () => '', - }).generate(generateArgs([finding('the agent cannot search the web')], 1)) - expect(queries).toEqual(['web search MCP server']) - const flat = wired.seen.flat() - expect(flat.some((m) => m.role === 'tool' && String(m.content).includes('mcp.exa.ai'))).toBe( - true, - ) - expect( - flat.some((m) => m.role === 'system' && String(m.content).includes('ADOPT BEFORE BUILD')), - ).toBe(true) - - // Not wired: no research tool in the specs, no doctrine, and a stray call - // gets the not-provisioned error (live web access is not provisioned). - const unwired = scriptedBrain([ - { calls: [{ name: 'research', args: { query: 'anything' } }] }, - { say: 'ok' }, - ]) - await driverLoopGenerator({ - brain: unwired.chat, - runHarness: harnessStub().run, - changedPaths: () => [], - readDiff: () => '', - }).generate(generateArgs([finding('gap')], 1)) - const unwiredFlat = unwired.seen.flat() - expect( - unwiredFlat.some((m) => m.role === 'tool' && String(m.content).includes('not provisioned')), - ).toBe(true) - expect( - unwiredFlat.some( - (m) => m.role === 'system' && String(m.content).includes('ADOPT BEFORE BUILD'), - ), - ).toBe(false) - }) - - it('caps worker sessions at maxShots and tells the driver, without spawning', async () => { - const worker = harnessStub() - const { chat, seen } = scriptedBrain([ - { calls: [{ name: 'run_worker', args: { instruction: 'first session' } }] }, - { calls: [{ name: 'run_worker', args: { instruction: 'second session (over budget)' } }] }, - { say: 'stopping' }, - ]) - const generator = driverLoopGenerator({ - brain: chat, - runHarness: worker.run, - changedPaths: () => ['a.ts'], - readDiff: () => '+a', - }) - const result = await generator.generate(generateArgs([finding('gap')], 1)) - expect(worker.prompts).toEqual(['first session']) - const flat = seen.flat() - expect( - flat.some( - (m) => m.role === 'tool' && String(m.content).includes('worker-session budget exhausted'), - ), - ).toBe(true) - // No verifier configured: dirty tree IS the candidate (legacy contract). - expect(result.applied).toBe(true) - }) -}) diff --git a/src/improvement/driver-loop-generator.ts b/src/improvement/driver-loop-generator.ts deleted file mode 100644 index 10197d40..00000000 --- a/src/improvement/driver-loop-generator.ts +++ /dev/null @@ -1,345 +0,0 @@ -/** - * `driverLoopGenerator` — the driver→worker `CandidateGenerator`: the build - * loop run by the ATOM instead of the canned respawn. - * - * `agenticGenerator` steers with three hardcoded conditions picking a canned - * note (`EMPTY_TREE_NOTE` / `failureNote`) and respawns. This generator swaps - * that respawn brain for a real driver: an LLM on the canonical tool-loop seam - * (`runBrainLoop` + `ToolLoopChat` — the exact loop `driverAgent` runs its - * brain on) that AUTHORS each worker instruction, OBSERVES what the session - * actually produced (diff, files, verifier output), RATES it, and DECIDES - * refine / re-scope / decompose — prompted with the senior scientific-method - * doctrine (`buildDriverSystem`). - * - * The worker stays the proven primitive: `runLocalHarness` in the candidate - * worktree, same as `agenticGenerator` — only the brain between sessions - * changes. The worktree machinery (`worktreeBuildCandidate`) and verifiers - * (`commandVerifier` / `mcpServeVerifier`) are reused verbatim. - * - * Completion-oracle invariant (the supervisor doctrine, kept): the driver's - * prose NEVER decides the outcome. After the loop, code re-checks ground - * truth — tree dirty, raw-trace evidence present, verifier green — and only - * that decides `applied`. A driver that claims success over a failing verifier - * produces a discarded candidate, not a shipped one. - * - * @experimental - */ - -import { spawnSync } from 'node:child_process' -import { readFileSync, statSync } from 'node:fs' -import { resolve, sep } from 'node:path' -import type { ProposalFinding } from '@tangle-network/agent-eval' -import { DEFAULT_LOCAL_HARNESS, type LocalHarness, runLocalHarness } from '../mcp/local-harness' -import { runBrainLoop, type ToolLoopChat } from '../runtime/tool-loop' -import { - defaultBuildPrompt, - rawTraceEvidenceProblem, - requiresRawTraceEvidence, - summarizeFindings, - type Verifier, - worktreeChangedPaths, -} from './agentic-generator' -import type { CandidateGenerator } from './improvement-driver' -import { buildDriverSystem, researchDriverNote } from './optimizer-prompt' - -export interface DriverLoopGeneratorOptions { - /** The driver-LLM seam — ONE inference turn over the conversation + tool specs (the canonical - * `ToolLoopChat`, same seam as `driverAgent`): `routerBrain(cfg)` in production, a scripted - * mock in tests. */ - brain: ToolLoopChat - /** Local coding harness the driver's worker sessions run in the worktree. Default `claude-code`. */ - harness?: LocalHarness - /** Per-worker-session wall-clock timeout (ms). Default = `runLocalHarness` default (5m). */ - timeoutMs?: number - /** Build the driver's task briefing (domain framing + method + findings) — the same senior - * prompt the worker path uses (`toolBuildPrompt` / `mcpBuildPrompt`). The driver reads it and - * folds what each worker needs into its instruction. Default `defaultBuildPrompt`. */ - buildPrompt?: (args: { findings: ReadonlyArray }) => string - /** Verify the worktree (the intrinsic check). Exposed to the driver as `run_verifier` AND - * re-run by code as the final keep/discard gate. Omitted ⇒ the final gate is dirty-tree only - * (legacy `agenticGenerator` behavior sans verifier). */ - verify?: Verifier - /** Max driver inference turns. Default `max(8, 2 + maxShots * 3)` — room for one - * observe/rate/decide cycle per worker session plus orientation. */ - maxTurns?: number - /** The research seam (adopt-not-build): when set, the driver gets a - * `research{query}` tool + the `researchDriverNote` doctrine, so it can - * discover an EXISTING external MCP instead of building one. Wire a real - * web/search backend here — none is provisioned by default (the build - * harness has no live web access yet; flagged). */ - research?: (query: string) => Promise - /** Test seam — inject the harness runner (defaults to `runLocalHarness`). */ - runHarness?: typeof runLocalHarness - /** Test seam — inject the worktree diff reader (defaults to `git diff` in the worktree). */ - readDiff?: (worktreePath: string) => string - /** Test seam — inject the changed-paths reader (defaults to `git status --porcelain`). */ - changedPaths?: (worktreePath: string) => string[] -} - -const workerOutputTailChars = 2_000 -const diffMaxChars = 6_000 -const readFileDefaultBytes = 8_192 -const researchResultMaxChars = 8_000 - -/** Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budget is spent. */ -export function driverLoopGenerator(opts: DriverLoopGeneratorOptions): CandidateGenerator { - const harness = opts.harness ?? DEFAULT_LOCAL_HARNESS - const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt - const run = opts.runHarness ?? runLocalHarness - const changed = opts.changedPaths ?? worktreeChangedPaths - const readDiff = opts.readDiff ?? worktreeDiff - const verify = opts.verify - - return { - kind: `driver-loop:${harness}`, - async generate({ worktreePath, findings, maxShots, signal }) { - signal.throwIfAborted() - const briefing = buildPrompt({ findings }) - const needsRawTraceEvidence = requiresRawTraceEvidence(findings) - const sessionCap = Math.max(1, maxShots) - let sessionsUsed = 0 - - // Ground-truth verification shared by the driver's `run_verifier` tool and the final - // code-owned gate — ONE definition of "delivered" so the driver can never see a different - // check than the one that decides the keep. - const groundVerify = async (): Promise<{ ok: boolean; feedback?: string }> => { - signal.throwIfAborted() - if (changed(worktreePath).length === 0) { - return { ok: false, feedback: 'the working tree has no changes — nothing to verify' } - } - if (needsRawTraceEvidence) { - const problem = rawTraceEvidenceProblem(worktreePath, findings) - if (problem) return { ok: false, feedback: problem } - } - if (!verify) { - return { ok: true, feedback: 'no verifier configured: a dirty tree is the candidate' } - } - const result = await verify(worktreePath, signal) - signal.throwIfAborted() - return result - } - - const execute = async (name: string, args: Record): Promise => { - signal.throwIfAborted() - switch (name) { - case 'run_worker': { - const instruction = typeof args.instruction === 'string' ? args.instruction.trim() : '' - if (instruction.length === 0) { - return 'error: run_worker requires a non-empty `instruction`' - } - if (sessionsUsed >= sessionCap) { - return `error: worker-session budget exhausted (${sessionsUsed}/${sessionCap} used). Inspect and verify what exists, then stop with your final assessment.` - } - sessionsUsed += 1 - const result = await run({ - harness, - cwd: worktreePath, - taskPrompt: instruction, - ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), - signal, - }) - signal.throwIfAborted() - if (result.aborted) { - throw new Error('driverLoopGenerator: worker session was cancelled by the caller') - } - return JSON.stringify({ - session: `${sessionsUsed}/${sessionCap}`, - exitCode: result.exitCode, - timedOut: result.timedOut, - aborted: result.aborted ?? false, - killedBySignal: result.killedBySignal, - durationMs: result.durationMs, - changedPaths: changed(worktreePath), - stdoutTail: tail(result.stdout, workerOutputTailChars), - stderrTail: tail(result.stderr, workerOutputTailChars), - }) - } - case 'inspect_worktree': { - const paths = changed(worktreePath) - const diff = truncate(readDiff(worktreePath), diffMaxChars) - return JSON.stringify({ - changedPaths: paths, - diff: - diff.length > 0 - ? diff - : '(no tracked-file diff — new files are untracked; read_file them)', - }) - } - case 'read_file': - return readWorktreeFile(worktreePath, args) - case 'research': { - if (!opts.research) return 'error: research tool is not provisioned in this run' - const query = typeof args.query === 'string' ? args.query.trim() : '' - if (query.length === 0) return 'error: research requires a non-empty `query`' - const result = await opts.research(query) - signal.throwIfAborted() - return truncate(result, researchResultMaxChars) - } - case 'run_verifier': { - const result = await groundVerify() - return JSON.stringify({ - ok: result.ok, - feedback: truncate(result.feedback ?? '', 4_000), - }) - } - default: - return `error: unknown tool: ${name}` - } - } - - await runBrainLoop({ - chat: opts.brain, - tools: opts.research ? [...driverToolSpecs, researchToolSpec] : driverToolSpecs, - execute, - initialMessages: [ - { - role: 'system', - content: opts.research - ? `${buildDriverSystem}\n\n${researchDriverNote}` - : buildDriverSystem, - }, - { - role: 'user', - content: [ - `THE BUILD BRIEF (the contract your workers must satisfy — fold what each needs into its instruction; workers never see this brief):`, - '', - briefing, - '', - `Worker-session budget: ${sessionCap}. The worktree is a fresh checkout at ${worktreePath}.`, - ].join('\n'), - }, - ], - maxTurns: opts.maxTurns ?? Math.max(8, 2 + sessionCap * 3), - hooks: { stopBefore: () => signal.aborted }, - }) - signal.throwIfAborted() - - // The completion oracle: the driver stopped (or ran out of turns) — ground truth decides. - const verdict = await groundVerify() - signal.throwIfAborted() - if (!verdict.ok) return { applied: false, summary: '' } - return { applied: true, summary: summarizeFindings(findings) } - }, - } -} - -const driverToolSpecs = [ - { - type: 'function' as const, - function: { - name: 'run_worker', - description: - 'Run ONE coding-harness session in the worktree with your instruction as its entire goal. The worktree persists between sessions. Sessions are capped — author each instruction richly (outcome, context, placement, the check it is held to).', - parameters: { - type: 'object', - properties: { - instruction: { - type: 'string', - description: 'The complete, self-contained goal for this worker session.', - }, - }, - required: ['instruction'], - }, - }, - }, - { - type: 'function' as const, - function: { - name: 'inspect_worktree', - description: - 'Current git state of the worktree: changed paths + the tracked-file diff (truncated). New untracked files show in changedPaths only — read_file them.', - parameters: { type: 'object', properties: {} }, - }, - }, - { - type: 'function' as const, - function: { - name: 'read_file', - description: 'Read one file from the worktree (paths are worktree-relative).', - parameters: { - type: 'object', - properties: { - path: { type: 'string', description: 'Worktree-relative file path.' }, - maxBytes: { type: 'number', description: 'Byte cap (default 8192).' }, - }, - required: ['path'], - }, - }, - }, - { - type: 'function' as const, - function: { - name: 'run_verifier', - description: - 'Run the intrinsic check of the surface (compile+tests / boot-and-probe). Its result — not your judgment — decides whether the candidate is kept.', - parameters: { type: 'object', properties: {} }, - }, - }, -] - -/** Only offered when `opts.research` is wired — a tool the driver cannot call - * must never appear in its tool list. */ -const researchToolSpec = { - type: 'function' as const, - function: { - name: 'research', - description: - 'Search external sources (MCP registries, vendor docs) for an EXISTING server that closes the capability gap — the adopt-not-build check. Returns text findings.', - parameters: { - type: 'object', - properties: { - query: { type: 'string', description: 'What capability / server to search for.' }, - }, - required: ['query'], - }, - }, -} - -/** `git diff` over the worktree (tracked files). Fails loud like `worktreeChangedPaths` — a git - * fault on a fresh checkout is a broken setup, not an empty diff. */ -function worktreeDiff(worktreePath: string): string { - const result = spawnSync('git', ['diff'], { cwd: worktreePath, encoding: 'utf-8' }) - if (result.error) { - throw new Error( - `driverLoopGenerator: git diff failed to spawn in ${worktreePath}: ${result.error.message}`, - ) - } - if (result.status !== 0) { - throw new Error( - `driverLoopGenerator: git diff exited ${result.status} in ${worktreePath}: ${result.stderr.trim()}`, - ) - } - return result.stdout -} - -/** Bounded, worktree-jailed file read for the driver's `read_file`. A path escaping the worktree - * is refused (the driver only rates work in the candidate tree; it has no business elsewhere). */ -function readWorktreeFile(worktreePath: string, args: Record): string { - const rel = typeof args.path === 'string' ? args.path : '' - if (rel.length === 0) return 'error: read_file requires `path`' - const root = resolve(worktreePath) - const target = resolve(root, rel) - if (target !== root && !target.startsWith(root + sep)) { - return `error: path escapes the worktree: ${rel}` - } - const maxBytes = - typeof args.maxBytes === 'number' && args.maxBytes > 0 - ? Math.min(args.maxBytes, 65_536) - : readFileDefaultBytes - try { - const size = statSync(target).size - const body = readFileSync(target, 'utf-8').slice(0, maxBytes) - return size > maxBytes ? `${body}\n… (${size - maxBytes} bytes truncated)` : body - } catch (e) { - return `error: ${e instanceof Error ? e.message : String(e)}` - } -} - -function tail(s: string, n: number): string { - const trimmed = s.trim() - return trimmed.length <= n ? trimmed : `…${trimmed.slice(-n)}` -} - -function truncate(s: string, n: number): string { - return s.length <= n ? s : `${s.slice(0, n - 1)}…` -} diff --git a/src/improvement/index.ts b/src/improvement/index.ts index e47fecf7..054ebbe8 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -24,10 +24,6 @@ export { mcpBuildPrompt, toolBuildPrompt, } from './build-prompts' -export { - type DriverLoopGeneratorOptions, - driverLoopGenerator, -} from './driver-loop-generator' export { type ImproveCandidateValidationInput, type ImproveCandidateValidator, @@ -68,9 +64,7 @@ export { officialSkillOpt, } from './official-optimizers' export { - buildDriverSystem, optimizerMethod, - researchDriverNote, strategyAuthorMethod, } from './optimizer-prompt' export type { DeepReadonly, ReadonlyAgentProfile } from './profile-types' diff --git a/src/improvement/optimizer-prompt.ts b/src/improvement/optimizer-prompt.ts index 78b3c937..f31ad67b 100644 --- a/src/improvement/optimizer-prompt.ts +++ b/src/improvement/optimizer-prompt.ts @@ -1,8 +1,7 @@ /** * The senior scientific-method optimizer doctrine — the ONE substantial prompt * core shared by every builder/author surface (tool build, MCP build, codebase - * improvement, strategy authoring) and by the driver that steers build workers - * (`driverLoopGenerator`). + * improvement, and strategy authoring). * * Seeded from the proven senior prompts rather than invented: GEPA's * `REFLECTION_SYSTEM` (localize → diagnose → minimal generalizable fix → @@ -53,71 +52,6 @@ export const optimizerMethod = [ ' what you would try next if the measured lift comes back null.', ].join('\n') -/** - * The driver's stance for `driverLoopGenerator` — the build-domain instance of - * the supervisor doctrine (observe → rate → decide; refine / re-scope / - * decompose; the check decides delivery, never the driver's prose). - */ -export const buildDriverSystem = [ - 'You are the DRIVER of a build loop: a senior engineering lead steering a coding WORKER inside', - 'an isolated git worktree toward a verified artifact. You never edit files yourself — your only', - 'levers are the tools below. Your intelligence goes into three places: how you AUTHOR each', - 'worker instruction, how you OBSERVE and RATE what a session actually produced, and what you', - 'DECIDE next.', - '', - 'TOOLS', - '- run_worker{instruction}: one full coding-harness session in the worktree, with your', - ' instruction as its entire goal. Sessions are expensive and capped — author each one well.', - ' The worktree PERSISTS between sessions: a later worker resumes on top of earlier edits.', - '- inspect_worktree{}: current git status + diff — what has actually changed so far.', - '- read_file{path,maxBytes?}: read one file from the worktree (new untracked files do not show', - ' in the diff — read them to rate the work).', - '- run_verifier{}: the intrinsic check of the surface (compile+tests for a tool, boot-and-probe', - ' for an MCP server). Its exit decides what counts as delivered — your opinion does not.', - '', - 'AUTHOR RICHLY. A worker handed a one-line label will flail. Each instruction must carry: the', - 'outcome in concrete terms; the hypothesis and sub-goal it serves; what already exists in the', - 'tree that it must build on, not duplicate; where the deliverable must land; and the exact check', - 'it will be held to. The worker sees NOTHING you were given (no findings, no method, no prior', - 'session context) unless you fold it into the instruction.', - '', - 'THE LOOP — every turn: observe, rate, decide.', - '- OBSERVE: after each session, inspect the worktree and run the verifier. Read WHAT failed and', - ' WHY — "it failed" alone tells you nothing.', - '- RATE: judge with a reason. Verified and complete → stop. Close, one correctable fault →', - ' REFINE: author "continue from the current tree; the check fails because X; fix X" (depth,', - ' not a fresh start — never let a worker revert its own near-miss). Empty tree, or it solved a', - ' different problem → RE-SCOPE: re-author narrower and more concrete; a second identical', - ' instruction fails identically. Too big for one session → DECOMPOSE: author the first', - ' self-contained slice, verify it, then author the next on top.', - '- DECIDE: exactly one move per turn, with the reason stated in one line.', - '', - 'STOP when the verifier passes — reply with no tool call and a short reflection: what you', - 'predicted, what the verifier showed, what you would try next if the measured lift comes back', - 'null. If the session budget runs out first, say plainly what remains and why. An honest', - 'no-winner is a real result; a claimed success is not — the final keep/discard decision is made', - 'by code from the verifier exit and the tree state, never from your words.', -].join('\n') - -/** - * The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when - * a `research` tool is wired into the loop (`DriverLoopGeneratorOptions. - * research`). Kept separate so a driver WITHOUT the tool is never told to - * call a tool it does not have. - */ -export const researchDriverNote = [ - 'RESEARCH — ADOPT BEFORE BUILD. A research{query} tool is provisioned for this run. Before', - 'authoring a from-scratch build, spend one turn researching whether an EXISTING external MCP', - 'server already provides the missing capability — registries and vendor docs list maintained', - 'servers for most common gaps (web search, fetch, GitHub, filesystems, databases).', - '- If a maintained server fits, ADOPT it: report in your final reflection its endpoint or', - ' launch command and the API key it needs BY NAME (e.g. EXA_API_KEY) — never a key value —', - ' so the dispatch can emit a connection candidate with provisioned secrets. Adopting a fit', - ' server beats rebuilding it: less code to verify, maintained upstream, same measured gate.', - '- If nothing fits (unmaintained, wrong tool surface, heavier than the gap), build — and state', - ' in one line why adoption lost.', -].join('\n') - /** * The senior authoring process for `authorStrategy` — the same method, shaped * to the strategy contract (author-blind, conserved budget, one module out). diff --git a/src/mcp/delegates.ts b/src/mcp/delegates.ts index 18172001..9bc6f3aa 100644 --- a/src/mcp/delegates.ts +++ b/src/mcp/delegates.ts @@ -82,10 +82,8 @@ export type CoderDelegate = (args: DelegateCodeArgs, ctx: DelegateRunCtx) => Pro /** * UI-auditor delegate — fully consumer-injected. agent-runtime ships no - * default factory because the inputs are workspace path + judge function - * + (optionally) a `SandboxClient`, and the judge is the consumer's - * model seam. See `createInProcessUiAuditClient` + `uiAuditorProfile` in - * `@tangle-network/agent-runtime/profiles` for the canonical wiring. + * default factory because execution belongs to a caller-supplied exact + * agent profile and Runtime executor. * * @experimental */ diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 3a15a717..6942f76d 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -72,10 +72,8 @@ export interface McpServerOptions { */ delegateSupervisor?: DelegateHandlerOptions /** - * Required to enable delegate_ui_audit. Wire one that closes over your - * `runAgentRounds` + `uiAuditorProfile` + a `SandboxClient` (the - * canonical in-process choice is `createInProcessUiAuditClient` from - * `@tangle-network/agent-runtime/profiles`) + your vision judge. + * Required to enable delegate_ui_audit. Wire one that executes an exact + * agent profile through Runtime and returns the provider-neutral UI audit result. */ uiAuditorDelegate?: UiAuditorDelegate /** Override the default in-memory feedback store. */ diff --git a/src/profiles/index.ts b/src/profiles/index.ts index 1b87fee5..329a9d01 100644 --- a/src/profiles/index.ts +++ b/src/profiles/index.ts @@ -41,28 +41,18 @@ export { researcherProfile, } from './researcher' export type { - BrowserContextHandle, - BrowserHandle, - InProcessUiAuditClientOptions, - PageHandle, UiAuditCapture, UiAuditCaptureRequest, UiAuditOutput, - UiAuditorProfileOptions, UiAuditTask, UiAuditViewport, UiFinding, UiFindingScreenshot, UiFindingSeverity, - UiJudge, - UiJudgeInput, - UiJudgeOutput, - UiJudgeTokenUsage, UiLens, } from './ui-auditor' export { buildAuditorSystemPrompt, - createInProcessUiAuditClient, createUiAuditorValidator, decodeAuditTaskEnvelope, encodeAuditTaskEnvelope, @@ -72,5 +62,4 @@ export { SHARED_AUDITOR_RULES, UI_FINDING_SEVERITIES, UI_LENSES, - uiAuditorProfile, } from './ui-auditor' diff --git a/src/profiles/ui-auditor/in-process-client.ts b/src/profiles/ui-auditor/in-process-client.ts deleted file mode 100644 index 3d462f3f..00000000 --- a/src/profiles/ui-auditor/in-process-client.ts +++ /dev/null @@ -1,355 +0,0 @@ -/** - * - * `createInProcessUiAuditClient` — a `SandboxClient` that drives a - * Playwright browser in-process and delegates finding identification to a - * consumer-supplied {@link UiJudge}. - * - * Why this exists: `runAgentRounds` is built around a sandbox-SDK seam — each - * iteration is `client.create() → box.streamPrompt() → box.delete()`. - * For UI audit, spinning up a real container running a coding harness - * per iteration is overkill: the work is one browser capture + one - * vision LLM call. This client satisfies the kernel contract while - * doing the audit in-process; no container, no sandbox-SDK backend. - * - * The client owns ONE browser for its lifetime and creates a fresh - * context per iteration (isolated cookies/storage). Playwright is - * dynamically imported so consumers who use a different `SandboxClient` - * — e.g. a fleet executor that drives Playwright remotely — do not pay - * the peer dep cost. - * - * Concurrency: each iteration's prompt carries a self-describing task - * envelope (see `prompt.ts`), so concurrent fanout iterations do not race - * over per-client side state. - * - * @experimental - */ - -import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' -import type { SandboxClient } from '../../runtime/types' -import type { UiJudge } from './judge' -import { decodeAuditTaskEnvelope } from './prompt' -import { slugify } from './slugify' -import type { UiAuditCapture, UiAuditCaptureRequest } from './task' - -// All synthetic events the auditor emits flow through this helper. Reason: -// `SandboxEvent.data` is a sandbox-SDK shape (effectively `Record`) that our typed payloads (`UiAuditCapture`, `UiFinding`, …) do not -// satisfy structurally. The cast moves the type-system smell into a single, -// named, documented call site so the call sites in `runIteration` stay clean. -// The runtime contract — `{ type, data }` — is what the output adapter reads; -// the static type is what the kernel collects into `SandboxEvent[]`. -function asSandboxEvent(type: string, data: T): SandboxEvent { - return { type, data } as unknown as SandboxEvent -} - -/** @experimental */ -export interface InProcessUiAuditClientOptions { - /** - * Absolute path under which screenshots are written. Each capture lands - * at `/screenshots/`; finding screenshot paths - * are workspace-relative (`screenshots/`). - */ - workspaceDir: string - /** The vision judge that turns captures into findings. */ - judge: UiJudge - /** - * Navigation policy. - * - * `'strict'` (default) waits for `networkidle` and fails the iteration - * if the page does not settle. `'spa'` waits for `domcontentloaded` — - * use for single-page apps that hold open long-poll/websocket - * connections and never settle. - */ - navPolicy?: 'strict' | 'spa' - /** - * Browser launch override. Default: chromium headless via Playwright. - * Consumers pass a custom factory to target a remote browser, a - * different channel, or a fleet adapter. - */ - launchBrowser?: () => Promise -} - -/** @experimental */ -export interface BrowserHandle { - newContext(options?: { - viewport?: { width: number; height: number } - }): Promise - close(): Promise -} - -/** @experimental */ -export interface BrowserContextHandle { - newPage(): Promise - close(): Promise -} - -/** @experimental */ -export interface PageHandle { - setViewportSize(size: { width: number; height: number }): Promise - goto(url: string, options?: { waitUntil?: string; timeout?: number }): Promise - waitForSelector(selector: string, options?: { timeout?: number }): Promise - waitForTimeout(ms: number): Promise - screenshot(options: { path: string; fullPage?: boolean }): Promise - locator(selector: string): { - first(): { screenshot(options: { path: string }): Promise } - } -} - -const DEFAULT_VIEWPORT = { width: 1280, height: 800 } as const -const NAV_TIMEOUT_MS = 30_000 - -async function defaultLaunch(): Promise { - const mod = (await import('playwright')) as unknown as { - chromium?: { launch(options?: { headless?: boolean }): Promise } - } - if (!mod?.chromium || typeof mod.chromium.launch !== 'function') { - throw new Error( - 'ui-auditor: playwright is not installed. Install `playwright` (and run `playwright install chromium`) or pass a custom `launchBrowser` to createInProcessUiAuditClient.', - ) - } - return mod.chromium.launch({ headless: true }) -} - -function nowStamp(): string { - const d = new Date() - const pad = (n: number) => String(n).padStart(2, '0') - // Millisecond resolution so parallel fanout iterations capturing the same - // route/viewport/label within the same second don't collide on filename and - // silently overwrite each other. - return ( - `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` + - `${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}` + - `-${String(d.getUTCMilliseconds()).padStart(3, '0')}` - ) -} - -function viewportOf(req: UiAuditCaptureRequest): { width: number; height: number } { - return req.viewport ?? DEFAULT_VIEWPORT -} - -function captureFilename(req: UiAuditCaptureRequest): string { - const vp = viewportOf(req) - const labelPart = req.label ? `--${slugify(req.label, 'label')}` : '' - return `${slugify(req.route, 'route')}--${vp.width}x${vp.height}${labelPart}--${nowStamp()}.png` -} - -function assertHttpUrl(url: string): void { - let parsed: URL - try { - parsed = new URL(url) - } catch { - throw new Error(`ui-auditor: capture url is not parseable (got ${JSON.stringify(url)})`) - } - // SSRF defense at the client boundary. The MCP tool already restricts to - // http(s), but `createInProcessUiAuditClient` is exported and can be wired - // up directly by consumers (the example does this). A crafted task envelope - // could otherwise navigate Playwright to `file://`, `data:`, `javascript:` - // and read local files or execute inline content. - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - throw new Error( - `ui-auditor: capture url must use http or https (got ${parsed.protocol} in ${JSON.stringify(url)})`, - ) - } -} - -async function captureOne( - page: PageHandle, - req: UiAuditCaptureRequest, - outAbsPath: string, - signal: AbortSignal, - navPolicy: 'strict' | 'spa', -): Promise { - signal.throwIfAborted() - assertHttpUrl(req.url) - // Apply the per-capture viewport before navigation. The capture metadata - // and filename both encode this viewport; the rendered page must match. - await page.setViewportSize(viewportOf(req)) - const waitUntil = navPolicy === 'spa' ? 'domcontentloaded' : 'networkidle' - await page.goto(req.url, { waitUntil, timeout: NAV_TIMEOUT_MS }) - if (req.waitFor) { - await page.waitForSelector(req.waitFor, { timeout: 15_000 }) - } - const extra = req.waitMs ?? 500 - if (extra > 0) await page.waitForTimeout(extra) - signal.throwIfAborted() - if (req.elementSelector) { - await page.locator(req.elementSelector).first().screenshot({ path: outAbsPath }) - } else { - await page.screenshot({ path: outAbsPath, fullPage: req.fullPage === true }) - } -} - -interface SyntheticSandbox extends SandboxInstance {} - -function makeSandboxId(): string { - const rand = () => Math.random().toString(16).slice(2, 10) - return `ui-audit-${rand()}${rand()}` -} - -/** Create a `SandboxClient` that drives a local Playwright browser for in-process UI audits. @experimental */ -export function createInProcessUiAuditClient( - options: InProcessUiAuditClientOptions, -): SandboxClient & { - /** - * Close the underlying browser. Idempotent. - * - * Contract: callers MUST ensure no iterations are in flight when this is - * called. The kernel respects this — `runAgentRounds` awaits every iteration - * before returning, so `await runAgentRounds(...); await client.close()` is the - * intended pattern (see `examples/ui-audit`). If `close()` is invoked - * concurrently with a running iteration, the browser teardown will race - * against in-flight page operations; the iteration will surface an - * AggregateError carrying both the iteration error and the close error, - * but no work is lost silently. - */ - close(): Promise -} { - const launch = options.launchBrowser ?? defaultLaunch - const navPolicy = options.navPolicy ?? 'strict' - let browserPromise: Promise | undefined - let closed = false - - async function getBrowser(): Promise { - if (closed) { - throw new Error('ui-auditor: client is closed; create a new client to run another iteration') - } - if (!browserPromise) browserPromise = launch() - return browserPromise - } - - async function* runIteration( - promptText: string, - signal: AbortSignal, - ): AsyncIterable { - const task = decodeAuditTaskEnvelope(promptText) - if (!task) { - throw new Error( - 'ui-auditor: prompt is missing a UI_AUDIT_TASK envelope. Use uiAuditorProfile().taskToPrompt to format prompts, or pass an envelope-prefixed prompt manually.', - ) - } - if (task.captures.length === 0) { - throw new Error('ui-auditor: task has zero captures; nothing to audit.') - } - - yield asSandboxEvent('audit.lens', { lens: task.lens }) - - const browser = await getBrowser() - const context = await browser.newContext({ viewport: DEFAULT_VIEWPORT }) - // Track both the primary iteration error and any context-close failure so - // the cleanup path never silently swallows a leaked-context bug AND a - // close failure never shadows the real iteration error. After the - // try/catch/finally settles, we rethrow the primary if there was one, - // otherwise we rethrow the close error. - let primaryError: unknown - let closeError: unknown - try { - const page = await context.newPage() - const captures: UiAuditCapture[] = [] - const fs = await import('node:fs/promises') - const path = await import('node:path') - const shotsDir = path.join(options.workspaceDir, 'screenshots') - await fs.mkdir(shotsDir, { recursive: true }) - - for (const req of task.captures) { - signal.throwIfAborted() - const filename = captureFilename(req) - const absPath = path.join(shotsDir, filename) - const relPath = `screenshots/${filename}` - await captureOne(page, req, absPath, signal, navPolicy) - const vp = viewportOf(req) - const cap: UiAuditCapture = { - path: relPath, - viewport: `${vp.width}x${vp.height}`, - fullPage: req.fullPage === true, - route: req.route, - url: req.url, - capturedAt: new Date().toISOString(), - } - if (req.elementSelector) cap.elementSelector = req.elementSelector - if (req.label) cap.label = req.label - captures.push(cap) - yield asSandboxEvent('audit.capture', cap) - } - - const judgeOut = await options.judge({ - lens: task.lens, - captures, - productContext: task.productContext, - knownFindingIds: task.knownFindingIds, - promptText, - signal, - }) - - for (const finding of judgeOut.findings) { - yield asSandboxEvent('audit.finding', finding) - } - if (judgeOut.notes && judgeOut.notes.trim().length > 0) { - yield asSandboxEvent('audit.notes', { notes: judgeOut.notes }) - } - - const usage = judgeOut.tokenUsage ?? { input: 0, output: 0 } - yield asSandboxEvent('done', { - tokenUsage: { - inputTokens: usage.input, - outputTokens: usage.output, - }, - totalCostUsd: judgeOut.costUsd ?? 0, - }) - } catch (err) { - primaryError = err - } finally { - try { - await context.close() - } catch (err) { - closeError = err - } - } - // When both the iteration and the cleanup fail, surface both via - // AggregateError so a leaked context bug is not silently masked by an - // earlier iteration failure (per the fail-loud doctrine). - if (primaryError !== undefined && closeError !== undefined) { - throw new AggregateError( - [primaryError, closeError], - 'ui-auditor: iteration failed AND context.close() failed; both errors attached.', - ) - } - if (primaryError !== undefined) throw primaryError - if (closeError !== undefined) throw closeError - } - - function makeSyntheticSandbox(): SyntheticSandbox { - const id = makeSandboxId() - const instance = { - id, - streamPrompt(message: string, opts?: { signal?: AbortSignal }): AsyncIterable { - const signal = opts?.signal ?? new AbortController().signal - return runIteration(message, signal) - }, - async delete(): Promise { - // No per-sandbox resources to release; the browser is shared and - // closed by `client.close()`. Intentionally a no-op so trace-time - // `box.delete()` succeeds without doing surprising work. - }, - } - return instance as unknown as SyntheticSandbox - } - - return { - async create(_options?: CreateSandboxOptions) { - return makeSyntheticSandbox() - }, - describePlacement(box) { - const id = (box as unknown as { id?: string }).id - return { kind: 'sibling', sandboxId: typeof id === 'string' ? id : undefined } - }, - async close() { - closed = true - const pending = browserPromise - browserPromise = undefined - if (pending) { - const browser = await pending - await browser.close() - } - }, - } -} diff --git a/src/profiles/ui-auditor/index.ts b/src/profiles/ui-auditor/index.ts index d8e7508f..ab0f3bae 100644 --- a/src/profiles/ui-auditor/index.ts +++ b/src/profiles/ui-auditor/index.ts @@ -1,31 +1,12 @@ /** * - * `ui-auditor` profile — vision-driven UI audit iteration preset for - * `runAgentRounds`. See `profile.ts` for the entry point. - * - * The in-process Playwright + judge client lives in `in-process-client.ts` - * under the optional `./ui-auditor` subpath export so consumers that only - * want the profile types do not pay the Playwright peer dep. + * Provider-neutral UI audit data, parsing, validation, and prompt helpers. * * @experimental */ -export type { - BrowserContextHandle, - BrowserHandle, - InProcessUiAuditClientOptions, - PageHandle, -} from './in-process-client' -export { createInProcessUiAuditClient } from './in-process-client' -export type { - UiJudge, - UiJudgeInput, - UiJudgeOutput, - UiJudgeTokenUsage, -} from './judge' export { buildAuditorSystemPrompt, LENS_BRIEFS, SHARED_AUDITOR_RULES } from './lens-prompts' export { parseAuditorEvents } from './output-adapter' -export { type UiAuditorProfileOptions, uiAuditorProfile } from './profile' export { decodeAuditTaskEnvelope, encodeAuditTaskEnvelope, diff --git a/src/profiles/ui-auditor/judge.ts b/src/profiles/ui-auditor/judge.ts deleted file mode 100644 index ef85cff2..00000000 --- a/src/profiles/ui-auditor/judge.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * - * UI judge seam — consumer-supplied vision LLM hook the in-process - * auditor client invokes to identify findings from captured screenshots. - * - * The seam stays model-agnostic so consumers can plug in OpenAI vision, - * Anthropic vision, gemini, a local model, or a deterministic stub for - * tests. The auditor handles browser capture + Markdown emission; the - * judge owns the perception + judgment. - * - * Implementor contract: - * - Treat `lens` as authoritative — only emit findings that belong to - * that lens. Findings with `lens !== input.lens` will fail the - * iteration validator. - * - Reference screenshots via the `path` strings provided in - * `input.captures`. Inventing a path will cause the validator to - * reject the iteration. - * - Be conservative — a finding the judge cannot actually see in the - * screenshots is a hallucination and pollutes the audit. - * - Treat any exception thrown by the judge as the iteration's failure — - * do not swallow LLM errors. Per agent-runtime's fail-loud doctrine, - * surfacing the error to the kernel beats producing a silent zero. - * - * @experimental - */ - -import type { UiFinding, UiLens } from './substrate' -import type { UiAuditCapture } from './task' - -/** @experimental */ -export interface UiJudgeTokenUsage { - input: number - output: number -} - -/** @experimental */ -export interface UiJudgeInput { - lens: UiLens - captures: readonly UiAuditCapture[] - /** Free-form product context the consumer wants the judge to know. */ - productContext?: string - /** Findings already on file across earlier iterations — for similarTo linkage. */ - knownFindingIds?: readonly number[] - /** The full prompt the loop kernel synthesized for this iteration. */ - promptText: string - /** Cooperative cancellation. */ - signal: AbortSignal -} - -/** @experimental */ -export interface UiJudgeOutput { - findings: UiFinding[] - /** Optional triage commentary. */ - notes?: string - /** Optional usage; folded into the kernel cost ledger when present. */ - tokenUsage?: UiJudgeTokenUsage - /** Optional total cost in USD. */ - costUsd?: number -} - -/** @experimental */ -export type UiJudge = (input: UiJudgeInput) => Promise diff --git a/src/profiles/ui-auditor/profile.ts b/src/profiles/ui-auditor/profile.ts deleted file mode 100644 index 9b1c46cc..00000000 --- a/src/profiles/ui-auditor/profile.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * - * `uiAuditorProfile` — preset for vision-driven UI audit iterations. - * - * A `runAgentRounds` bundle: it returns the `AgentRunSpec`, output adapter, validator, and prompt - * formatter the loop kernel needs. The agent's "harness" is not a sandbox-SDK code-runner — it's a - * vision-capable judge driving a browser. The loop kernel still iterates - * `client.create() → box.streamPrompt() → box.delete()`; the client/box pair are provided by - * `createInProcessUiAuditClient` (in `./in-process-client.ts`) or a consumer-supplied `SandboxClient`. - * - * @experimental - */ - -import type { AgentProfile } from '@tangle-network/agent-interface' -import type { AgentRunSpec, OutputAdapter, Validator } from '../../runtime/types' -import { buildAuditorSystemPrompt } from './lens-prompts' -import { parseAuditorEvents } from './output-adapter' -import { encodeAuditTaskEnvelope, formatAuditorPrompt } from './prompt' -import type { UiAuditOutput, UiAuditTask } from './task' -import { createUiAuditorValidator } from './validator' - -/** @experimental */ -export interface UiAuditorProfileOptions { - /** - * Stable name surfaced in trace events. Defaults to `ui-auditor`. - */ - name?: string - /** - * Optional model identifier passed in `AgentProfile.model.default`. - * The consumer's `SandboxClient` chooses how to interpret it. - */ - model?: string - /** - * Task bound to the validator. Without it the validator uses the lens - * embedded in the iteration output as its expectation — fine for one-off - * use; less strict than passing the task explicitly. - */ - task?: UiAuditTask -} - -/** - * Preset `runAgentRounds` bundle for vision-driven UI audits: returns the `AgentRunSpec`, output adapter, validator, and prompt formatter the loop kernel needs. - * - * @experimental - */ -export function uiAuditorProfile(options: UiAuditorProfileOptions = {}): { - profile: AgentProfile - taskToPrompt: (task: UiAuditTask) => string - output: OutputAdapter - validator: Validator - agentRunSpec: AgentRunSpec -} { - const name = options.name ?? 'ui-auditor' - - // Lens is per-task; the profile's system prompt is filled in by the - // taskToPrompt formatter at iteration time (prefixed to the user - // message). Keeping the profile lens-agnostic lets one AgentRunSpec - // serve every lens-iteration of the loop. - const profile: AgentProfile = { - name, - description: 'Vision-driven UI auditor. One lens per iteration.', - prompt: { systemPrompt: '' }, - model: options.model ? { default: options.model } : undefined, - tools: { browser: true, vision: true }, - metadata: { role: 'ui-auditor' }, - } - - const output: OutputAdapter = { parse: parseAuditorEvents } - const validator: Validator = options.task - ? createUiAuditorValidator(options.task) - : createUiAuditorValidator({ lens: 'other', captures: [] }) - - // Prompt shape (consumed both by sandbox-SDK harnesses AND by the - // in-process auditor client): - // <>{json}<> - // - // - // The envelope makes the iteration self-describing so concurrent fanout - // does not race over per-client side state. Sandbox-SDK harnesses can - // ignore the envelope; the in-process auditor client decodes it back - // into a typed UiAuditTask via decodeAuditTaskEnvelope. - const taskToPrompt = (task: UiAuditTask): string => - `${encodeAuditTaskEnvelope(task)}\n${buildAuditorSystemPrompt(task.lens)}\n\n${formatAuditorPrompt(task)}` - - const agentRunSpec: AgentRunSpec = { - name, - profile, - taskToPrompt, - } - - return { profile, taskToPrompt, output, validator, agentRunSpec } -} diff --git a/tests/profiles/ui-auditor-client.test.ts b/tests/profiles/ui-auditor-client.test.ts deleted file mode 100644 index 9670640f..00000000 --- a/tests/profiles/ui-auditor-client.test.ts +++ /dev/null @@ -1,492 +0,0 @@ -import { promises as fs } from 'node:fs' -import { tmpdir } from 'node:os' -import path from 'node:path' -import type { SandboxEvent } from '@tangle-network/sandbox' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - type BrowserContextHandle, - type BrowserHandle, - createInProcessUiAuditClient, - encodeAuditTaskEnvelope, - type PageHandle, - type UiAuditTask, - type UiJudge, - type UiJudgeOutput, -} from '../../src/profiles/ui-auditor' - -interface MockPageLog { - setViewports: { width: number; height: number }[] - gotoUrls: string[] - screenshotPaths: string[] -} - -interface MockHandles { - browser: BrowserHandle - log: MockPageLog - newContextCalls: number - contextCloseCalls: number - failNthScreenshot?: number -} - -function makeMockBrowser(): MockHandles { - const log: MockPageLog = { setViewports: [], gotoUrls: [], screenshotPaths: [] } - const state = { failNthScreenshot: undefined as number | undefined } - let newContextCalls = 0 - let contextCloseCalls = 0 - - const page: PageHandle = { - async setViewportSize(size) { - log.setViewports.push({ ...size }) - }, - async goto(url) { - log.gotoUrls.push(url) - return undefined - }, - async waitForSelector() { - return undefined - }, - async waitForTimeout() { - /* no-op */ - }, - async screenshot({ path: outPath }) { - log.screenshotPaths.push(outPath) - if ( - state.failNthScreenshot !== undefined && - log.screenshotPaths.length === state.failNthScreenshot - ) { - throw new Error(`mock screenshot failure on call #${state.failNthScreenshot}`) - } - await fs.writeFile(outPath, 'fake-png') - }, - locator() { - return { - first() { - return { - async screenshot({ path: outPath }) { - log.screenshotPaths.push(outPath) - await fs.writeFile(outPath, 'fake-png') - }, - } - }, - } - }, - } - - const context: BrowserContextHandle = { - async newPage() { - return page - }, - async close() { - contextCloseCalls += 1 - }, - } - - const browser: BrowserHandle = { - async newContext() { - newContextCalls += 1 - return context - }, - async close() { - /* no-op */ - }, - } - - return { - browser, - log, - get newContextCalls() { - return newContextCalls - }, - get contextCloseCalls() { - return contextCloseCalls - }, - set failNthScreenshot(n: number | undefined) { - state.failNthScreenshot = n - }, - } as unknown as MockHandles -} - -let workspaceDir: string - -beforeEach(async () => { - workspaceDir = await fs.mkdtemp(path.join(tmpdir(), 'ui-audit-client-')) -}) - -afterEach(async () => { - await fs.rm(workspaceDir, { recursive: true, force: true }) -}) - -const okJudgeFn: UiJudge = vi.fn( - async (): Promise => ({ - findings: [], - tokenUsage: { input: 1, output: 2 }, - }), -) - -function stubTask(overrides: Partial = {}): UiAuditTask { - return { - lens: 'consistency', - captures: [{ route: 'home', url: 'https://example.test/' }], - ...overrides, - } -} - -async function drain(it: AsyncIterable): Promise { - const events: SandboxEvent[] = [] - for await (const e of it) events.push(e) - return events -} - -describe('createInProcessUiAuditClient — viewport application', () => { - it('applies the per-capture viewport BEFORE navigation for every capture', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - - const task = stubTask({ - lens: 'responsive', - captures: [ - { - route: 'home', - url: 'https://example.test/', - viewport: { width: 1440, height: 900 }, - }, - { - route: 'home', - url: 'https://example.test/', - viewport: { width: 375, height: 812 }, - }, - // Implicit default — verifies the fallback path applies it explicitly. - { route: 'home', url: 'https://example.test/' }, - ], - }) - - const box = await client.create() - await drain( - box.streamPrompt(encodeAuditTaskEnvelope(task), { signal: new AbortController().signal }), - ) - await client.close() - - expect(mock.log.setViewports).toEqual([ - { width: 1440, height: 900 }, - { width: 375, height: 812 }, - { width: 1280, height: 800 }, - ]) - // setViewportSize must be called before goto for each capture — the - // arrival order in the logs must be `viewport(i) before goto(i)`. - expect(mock.log.setViewports.length).toBe(mock.log.gotoUrls.length) - }) -}) - -describe('createInProcessUiAuditClient — event order', () => { - it('yields lens → captures (in order) → findings → notes → done', async () => { - const mock = makeMockBrowser() - const judge: UiJudge = async () => ({ - findings: [ - { - title: 'A finding on home', - lens: 'consistency', - severity: 'med', - route: 'home', - observation: 'visible problem', - impact: 'user impact', - suggestedFix: 'do x', - screenshots: [{ path: 'screenshots/home--1280x800--whatever.png' }], - }, - ], - notes: 'tail commentary', - tokenUsage: { input: 3, output: 4 }, - costUsd: 0.0001, - }) - const client = createInProcessUiAuditClient({ - workspaceDir, - judge, - launchBrowser: async () => mock.browser, - }) - const task = stubTask({ - captures: [ - { route: 'home', url: 'https://example.test/a' }, - { route: 'home', url: 'https://example.test/b' }, - ], - }) - - const box = await client.create() - const events = await drain( - box.streamPrompt(encodeAuditTaskEnvelope(task), { signal: new AbortController().signal }), - ) - await client.close() - - const types = events.map((e) => e.type) - expect(types).toEqual([ - 'audit.lens', - 'audit.capture', - 'audit.capture', - 'audit.finding', - 'audit.notes', - 'done', - ]) - const done = events[events.length - 1] as { - data: { tokenUsage: { inputTokens: number; outputTokens: number }; totalCostUsd: number } - } - expect(done.data.tokenUsage).toEqual({ inputTokens: 3, outputTokens: 4 }) - expect(done.data.totalCostUsd).toBeCloseTo(0.0001, 6) - }) -}) - -describe('createInProcessUiAuditClient — error handling', () => { - it('closes the browser context even when the judge throws', async () => { - const mock = makeMockBrowser() - const judge: UiJudge = async () => { - throw new Error('judge blew up') - } - const client = createInProcessUiAuditClient({ - workspaceDir, - judge, - launchBrowser: async () => mock.browser, - }) - const box = await client.create() - await expect( - drain( - box.streamPrompt(encodeAuditTaskEnvelope(stubTask()), { - signal: new AbortController().signal, - }), - ), - ).rejects.toThrow(/judge blew up/) - expect(mock.contextCloseCalls).toBe(1) - await client.close() - }) - - it('closes the browser context even when a capture throws', async () => { - const mock = makeMockBrowser() - mock.failNthScreenshot = 2 - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const task = stubTask({ - captures: [ - { route: 'home', url: 'https://example.test/a' }, - { route: 'home', url: 'https://example.test/b' }, - ], - }) - const box = await client.create() - await expect( - drain( - box.streamPrompt(encodeAuditTaskEnvelope(task), { signal: new AbortController().signal }), - ), - ).rejects.toThrow(/mock screenshot failure/) - expect(mock.contextCloseCalls).toBe(1) - await client.close() - }) - - it('throws when the prompt is missing the UI_AUDIT_TASK envelope', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const box = await client.create() - await expect( - drain(box.streamPrompt('no envelope here', { signal: new AbortController().signal })), - ).rejects.toThrow(/UI_AUDIT_TASK envelope/) - // The pre-envelope check is before any browser work; no context should - // have been allocated. - expect(mock.newContextCalls).toBe(0) - await client.close() - }) - - it('throws when the task has zero captures', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const box = await client.create() - await expect( - drain( - box.streamPrompt(encodeAuditTaskEnvelope({ lens: 'consistency', captures: [] }), { - signal: new AbortController().signal, - }), - ), - ).rejects.toThrow(/zero captures/) - await client.close() - }) - - it('rejects non-http(s) capture URLs as SSRF defense at the client boundary', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - for (const url of ['file:///etc/passwd', 'data:text/html,x', 'javascript:alert(1)']) { - const box = await client.create() - await expect( - drain( - box.streamPrompt( - encodeAuditTaskEnvelope(stubTask({ captures: [{ route: 'home', url }] })), - { signal: new AbortController().signal }, - ), - ), - ).rejects.toThrow(/must use http or https/) - } - await client.close() - }) - - it('honours AbortSignal — aborting before the first capture rejects the stream', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const controller = new AbortController() - controller.abort() - const box = await client.create() - await expect( - drain( - box.streamPrompt( - encodeAuditTaskEnvelope( - stubTask({ - captures: [ - { route: 'home', url: 'https://example.test/a' }, - { route: 'home', url: 'https://example.test/b' }, - ], - }), - ), - { signal: controller.signal }, - ), - ), - ).rejects.toThrowError() - // No goto should have run — the signal is checked before each capture. - expect(mock.log.gotoUrls).toHaveLength(0) - // And the context was created then closed (cleanup ran). - expect(mock.contextCloseCalls).toBe(1) - await client.close() - }) -}) - -describe('createInProcessUiAuditClient — sandbox surface', () => { - it('describePlacement reports the synthetic sandbox id', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const box = await client.create() - const placement = client.describePlacement?.(box) - expect(placement?.kind).toBe('sibling') - expect(typeof placement?.sandboxId).toBe('string') - expect(placement?.sandboxId).toMatch(/^ui-audit-/) - await client.close() - }) - - it('close() is idempotent', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - await client.close() - await client.close() - expect(mock.newContextCalls).toBe(0) - }) - - it('rejects streamPrompt after close instead of silently re-launching the browser', async () => { - const mock = makeMockBrowser() - const client = createInProcessUiAuditClient({ - workspaceDir, - judge: okJudgeFn, - launchBrowser: async () => mock.browser, - }) - const box = await client.create() - await client.close() - await expect( - drain( - box.streamPrompt(encodeAuditTaskEnvelope(stubTask()), { - signal: new AbortController().signal, - }), - ), - ).rejects.toThrow(/client is closed/) - // Nothing should have been newly allocated — the closed guard fires - // before browser launch. - expect(mock.newContextCalls).toBe(0) - }) -}) - -describe('createInProcessUiAuditClient — AggregateError on dual failure', () => { - it('throws AggregateError when both the judge and context.close() fail', async () => { - const judge: UiJudge = async () => { - throw new Error('judge blew up') - } - // Build a custom mock whose context.close() also throws. - const page: PageHandle = { - async setViewportSize() {}, - async goto() { - return undefined - }, - async waitForSelector() { - return undefined - }, - async waitForTimeout() {}, - async screenshot({ path: outPath }) { - await fs.writeFile(outPath, 'fake-png') - }, - locator() { - return { - first() { - return { - async screenshot({ path: outPath }) { - await fs.writeFile(outPath, 'fake-png') - }, - } - }, - } - }, - } - const context: BrowserContextHandle = { - async newPage() { - return page - }, - async close() { - throw new Error('close blew up') - }, - } - const browser: BrowserHandle = { - async newContext() { - return context - }, - async close() {}, - } - const client = createInProcessUiAuditClient({ - workspaceDir, - judge, - launchBrowser: async () => browser, - }) - const box = await client.create() - let caught: unknown - try { - await drain( - box.streamPrompt(encodeAuditTaskEnvelope(stubTask()), { - signal: new AbortController().signal, - }), - ) - } catch (err) { - caught = err - } - expect(caught).toBeInstanceOf(AggregateError) - const agg = caught as AggregateError - const messages = agg.errors.map((e) => (e instanceof Error ? e.message : String(e))) - expect(messages).toContain('judge blew up') - expect(messages).toContain('close blew up') - await client.close() - }) -}) diff --git a/tests/profiles/ui-auditor.test.ts b/tests/profiles/ui-auditor.test.ts index d96fed99..1583430c 100644 --- a/tests/profiles/ui-auditor.test.ts +++ b/tests/profiles/ui-auditor.test.ts @@ -8,7 +8,6 @@ import { type UiAuditOutput, type UiAuditTask, type UiFinding, - uiAuditorProfile, } from '../../src/profiles/ui-auditor' const ctx = { iteration: 0, signal: new AbortController().signal } @@ -209,13 +208,3 @@ describe('parseAuditorEvents', () => { expect(out.findings).toHaveLength(1) }) }) - -describe('uiAuditorProfile', () => { - it('returns a stable AgentRunSpec name + envelope-prefixed prompt', () => { - const { agentRunSpec, taskToPrompt } = uiAuditorProfile() - expect(agentRunSpec.name).toBe('ui-auditor') - const prompt = taskToPrompt(baseTask()) - expect(prompt.startsWith('<>')).toBe(true) - expect(prompt).toMatch(/LENS: consistency/) - }) -}) From eec691e9f7ba8bca6dd36a9fa1fbdd999a4daf0e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:12:37 -0600 Subject: [PATCH 13/39] fix(improvement): require exact author profiles --- bench/gen3-config.json | 12 +- bench/gen4-config.json | 24 +- bench/gen5-config.json | 24 +- bench/src/swe-arena/outer-loop.mts | 139 +- .../profiles/deepseek-author.profile.json | 12 + .../profiles/default-author.profile.json | 10 +- bench/src/swe-arena/proposer-fanout.mts | 151 +- bench/src/swe-arena/proposer-fanout.test.mts | 149 +- bench/src/swe-code-improve.mts | 82 +- docs/api/index.md | 292 ++-- docs/api/primitive-catalog.md | 9 +- src/improvement/agentic-generator.ts | 554 ++----- src/improvement/build-prompts.ts | 6 +- src/improvement/code-execution.ts | 21 +- src/improvement/improve-types.ts | 41 +- src/improvement/improve.test.ts | 11 + src/improvement/improve.ts | 3 + src/improvement/index.ts | 5 +- src/improvement/raw-trace-distiller.ts | 2 +- tests/agentic-generator.test.ts | 1445 +++++------------ tests/build-composition.test.ts | 54 +- 21 files changed, 1105 insertions(+), 1941 deletions(-) create mode 100644 bench/src/swe-arena/profiles/deepseek-author.profile.json diff --git a/bench/gen3-config.json b/bench/gen3-config.json index a627d840..2c0726c2 100644 --- a/bench/gen3-config.json +++ b/bench/gen3-config.json @@ -41,7 +41,8 @@ "repsPerInstance": 2, "premeasuredBaselinePath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/premeasured-baseline.json", "maxShots": 3, - "proposerHarness": "claude-code", + "proposerHarness": "pi", + "proposerProfile": "default-author.profile.json", "proposerTimeoutMs": 2400000, "analystModels": [ "glm-5.2", @@ -77,19 +78,22 @@ { "name": "default-author", "profile": "default-author.profile.json", - "harness": "claude" + "harness": "pi", + "model": "glm-5.2" }, { "name": "mechanics-author", "profile": "default-author.profile.json", - "harness": "claude", + "harness": "pi", + "model": "glm-5.2", "diagnosisSlice": "mechanics", "lens": "Focus on MECHANICS: worker lifecycle, sandbox/clone contracts, settlement and delivery paths. Prefer code-path fixes over prompt wording." }, { "name": "prompts-author", "profile": "default-author.profile.json", - "harness": "claude", + "harness": "pi", + "model": "glm-5.2", "diagnosisSlice": "prompts", "lens": "Focus on PROMPTS: worker/brain instruction wording, placement guidance, self-check discipline. Prefer prompt/instruction changes over code-path rewrites." } diff --git a/bench/gen4-config.json b/bench/gen4-config.json index 718d8e45..02f39e6d 100644 --- a/bench/gen4-config.json +++ b/bench/gen4-config.json @@ -37,11 +37,12 @@ "tangle-router.env" ], "generations": 1, - "populationSize": 4, + "populationSize": 3, "repsPerInstance": 2, "premeasuredBaselinePath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen4/premeasured-baseline.json", "maxShots": 3, - "proposerHarness": "claude-code", + "proposerHarness": "pi", + "proposerProfile": "default-author.profile.json", "proposerTimeoutMs": 2400000, "analystModels": [ "glm-5.2", @@ -74,24 +75,23 @@ "costGuardRatio": 1.2, "dispatchTimeoutMs": 7200000, "proposers": [ - { - "name": "claude-author", - "profile": "default-author.profile.json", - "harness": "claude" - }, { "name": "glm-author", - "harness": "opencode", - "model": "zai-coding-plan/glm-5.2" + "profile": "default-author.profile.json", + "harness": "pi", + "model": "glm-5.2" }, { - "name": "codex-author", - "harness": "codex" + "name": "deepseek-author", + "profile": "deepseek-author.profile.json", + "harness": "pi", + "model": "deepseek-v4-flash" }, { "name": "merge-author", "profile": "default-author.profile.json", - "harness": "claude", + "harness": "pi", + "model": "glm-5.2", "merge": true } ], diff --git a/bench/gen5-config.json b/bench/gen5-config.json index 2e4ca876..72e7d7a5 100644 --- a/bench/gen5-config.json +++ b/bench/gen5-config.json @@ -37,11 +37,12 @@ "tangle-router.env" ], "generations": 1, - "populationSize": 4, + "populationSize": 3, "repsPerInstance": 2, "premeasuredBaselinePath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen5/premeasured-baseline.json", "maxShots": 3, - "proposerHarness": "claude-code", + "proposerHarness": "pi", + "proposerProfile": "default-author.profile.json", "proposerTimeoutMs": 2400000, "analystModels": [ "glm-5.2", @@ -74,24 +75,23 @@ "costGuardRatio": 1.2, "dispatchTimeoutMs": 7200000, "proposers": [ - { - "name": "claude-author", - "profile": "default-author.profile.json", - "harness": "claude" - }, { "name": "glm-author", - "harness": "opencode", - "model": "zai-coding-plan/glm-5.2" + "profile": "default-author.profile.json", + "harness": "pi", + "model": "glm-5.2" }, { - "name": "codex-author", - "harness": "codex" + "name": "deepseek-author", + "profile": "deepseek-author.profile.json", + "harness": "pi", + "model": "deepseek-v4-flash" }, { "name": "merge-author", "profile": "default-author.profile.json", - "harness": "claude", + "harness": "pi", + "model": "glm-5.2", "merge": true } ], diff --git a/bench/src/swe-arena/outer-loop.mts b/bench/src/swe-arena/outer-loop.mts index b7ebfe5f..bb4e5cff 100644 --- a/bench/src/swe-arena/outer-loop.mts +++ b/bench/src/swe-arena/outer-loop.mts @@ -71,8 +71,7 @@ import { type CandidateGenerator, type Verifier, } from '@tangle-network/agent-runtime' -import { runLocalHarness } from '@tangle-network/agent-runtime/mcp' -import { canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { canonicalCandidateDigest, type AgentProfile } from '@tangle-network/agent-interface' import { makeProposalFinding, type AnalystFinding, @@ -89,7 +88,6 @@ import { type PremeasuredOptimizationBaseline, type Scenario, } from '@tangle-network/agent-eval/campaign' -import type { CostLedgerHandle } from '@tangle-network/agent-eval' import { runVenvPython } from '../benchmarks/_harness.ts' import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts' import { @@ -134,6 +132,7 @@ import { fanOutLoopsGenerator, materializeParetoParents, proposerShotHooks, + resolveAuthorProfile, type ParetoParentContext, type ParetoParentSeed, type PrefilterConfig, @@ -646,7 +645,9 @@ export interface OuterLoopConfig { * budget.maxImprovementShots; the LIB owns the dial (capabilities.mts * fails loud on a substrate that would drop it). */ maxShots: number - proposerHarness: 'claude-code' | 'codex' | 'opencode' + proposerHarness: 'pi' + /** Complete profile used by the single-author path and as the run's admitted author identity. */ + proposerProfile: string proposerTimeoutMs: number /** GEN-3 proposer fan-out: N proposers author candidates CONCURRENTLY, each * an AgentProfile-pinned harness invocation (see proposer-fanout.mts). @@ -759,7 +760,8 @@ export function defaultRound4Config( // bootstrap run writes it; the lib validates it on every consumption. premeasuredBaselinePath: join(hh, 'r4', 'premeasured-baseline.json'), maxShots: 3, - proposerHarness: 'claude-code', + proposerHarness: 'pi', + proposerProfile: 'default-author.profile.json', // Per author SHOT (agenticGenerator timeoutMs). 20 min timed out 3× under // degraded capacity in gen-1 ("author shot timed out") — doubled to 40 min. proposerTimeoutMs: 2_400_000, @@ -836,7 +838,7 @@ export function resolveSmokeInstance( * The gen-3 config: protocol round 4 continues (frozen arm, same holdout * registry, same roundsDir staircase) with the gen-3 machinery on: * - * - THREE parallel proposers (all claude, bare default-author profile) that + * - THREE parallel Pi proposers using the exact GLM author profile that * differ by diagnosis slice/lens — fan-out diversity without unproven * harness seats; `populationSize` = `proposers.length`. * - Pre-filter enabled at the mechanism bar on the cheapest-of-set smoke @@ -858,18 +860,25 @@ export function defaultGen3Config( const base = defaultRound4Config(hh, opts) const outDirName = opts.outDirName ?? 'gen3' const proposers: ProposerSpec[] = [ - { name: 'default-author', profile: 'default-author.profile.json', harness: 'claude-code' }, + { + name: 'default-author', + profile: 'default-author.profile.json', + harness: 'pi', + model: 'glm-5.2', + }, { name: 'mechanics-author', profile: 'default-author.profile.json', - harness: 'claude-code', + harness: 'pi', + model: 'glm-5.2', diagnosisSlice: 'mechanics', lens: 'Focus on MECHANICS: worker lifecycle, sandbox/clone contracts, settlement and delivery paths. Prefer code-path fixes over prompt wording.', }, { name: 'prompts-author', profile: 'default-author.profile.json', - harness: 'claude-code', + harness: 'pi', + model: 'glm-5.2', diagnosisSlice: 'prompts', lens: 'Focus on PROMPTS: worker/brain instruction wording, placement guidance, self-check discipline. Prefer prompt/instruction changes over code-path rewrites.', }, @@ -921,13 +930,8 @@ export const GEN3_PARETO_PARENTS: ParetoParentSeed[] = [ * The gen-4 config: protocol round 4 continues (frozen arm, same holdout * registry, same roundsDir staircase) with three changes as a unit: * - * 1. PINNED PER-PROPOSER MODELS — four seats: claude-author (claude CLI on - * its own login; the resolved model + CLI version are captured into - * `/proposer-provenance.json` at t=0), glm-author (opencode - * pinned to zai-coding-plan/glm-5.2 via `-m`), codex-author (codex CLI on - * its ChatGPT login, auth provenance-gated at launch; drop the seat via - * `includeCodex: false` when the CLI is absent), and merge-author (claude, - * merge seat). + * 1. PINNED PER-PROPOSER MODELS — GLM 5.2 and DeepSeek V4 Flash run through + * Pi and Tangle Router; the merge seat uses the same exact GLM profile. * 2. PARETO PARENTS — the gen-3 winner + runner-up diffs and their measured * per-instance results seed every author's prompt; the merge seat's task * is their coherent union. Seeded at the buildPrompt seam (our seam): the @@ -941,14 +945,33 @@ export const GEN3_PARETO_PARENTS: ParetoParentSeed[] = [ */ export function defaultGen4Config( hh = DEFAULT_HH_SCRATCHPAD, - opts: { outDirName?: string; includeCodex?: boolean } = {}, + opts: { outDirName?: string; includeDeepseek?: boolean } = {}, ): OuterLoopConfig { const base = defaultGen3Config(hh, { outDirName: opts.outDirName ?? 'gen4' }) const proposers: ProposerSpec[] = [ - { name: 'claude-author', profile: 'default-author.profile.json', harness: 'claude-code' }, - { name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' }, - ...(opts.includeCodex === false ? [] : [{ name: 'codex-author', harness: 'codex' } satisfies ProposerSpec]), - { name: 'merge-author', profile: 'default-author.profile.json', harness: 'claude-code', merge: true }, + { + name: 'glm-author', + profile: 'default-author.profile.json', + harness: 'pi', + model: 'glm-5.2', + }, + ...(opts.includeDeepseek === false + ? [] + : [ + { + name: 'deepseek-author', + profile: 'deepseek-author.profile.json', + harness: 'pi', + model: 'deepseek-v4-flash', + } satisfies ProposerSpec, + ]), + { + name: 'merge-author', + profile: 'default-author.profile.json', + harness: 'pi', + model: 'glm-5.2', + merge: true, + }, ] return { ...base, @@ -979,11 +1002,11 @@ export function defaultGen4Config( export function defaultGen5Config( hh = DEFAULT_HH_SCRATCHPAD, - opts: { outDirName?: string; includeCodex?: boolean } = {}, + opts: { outDirName?: string; includeDeepseek?: boolean } = {}, ): OuterLoopConfig { const base = defaultGen4Config(hh, { outDirName: opts.outDirName ?? 'gen5', - ...(opts.includeCodex !== undefined ? { includeCodex: opts.includeCodex } : {}), + ...(opts.includeDeepseek !== undefined ? { includeDeepseek: opts.includeDeepseek } : {}), }) return { ...base, @@ -1261,7 +1284,7 @@ const CLAUDE_AMBIENT_AUTH_VARS = ['ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', ' * leaked auth is stripped for the shot subprocess only. */ const CODEX_AMBIENT_AUTH_VARS = ['OPENAI_API_KEY', 'OPENAI_BASE_URL'] as const -export function proposerShotEnv(harness: OuterLoopConfig['proposerHarness']): NodeJS.ProcessEnv { +export function proposerShotEnv(harness: NonNullable): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...process.env } if (harness === 'claude-code') { for (const name of CLAUDE_AMBIENT_AUTH_VARS) delete env[name] @@ -1272,43 +1295,51 @@ export function proposerShotEnv(harness: OuterLoopConfig['proposerHarness']): No return env } +function configuredAuthorProfile(config: OuterLoopConfig): AgentProfile { + const spec = + config.proposers?.find((proposer) => proposer.engine === undefined) ?? + ({ + name: 'single-author', + profile: config.proposerProfile, + harness: config.proposerHarness, + } satisfies ProposerSpec) + const profile = resolveAuthorProfile(spec) + if (!profile) throw new Error(`author ${spec.name}: an exact AgentProfile is required`) + return profile +} + +function authorExecutorForWorktree(worktreePath: string) { + const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL + const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER + if (!bridgeUrl || !bridgeBearer) { + throw new Error( + 'authoring requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER', + ) + } + return { + backend: 'bridge' as const, + bridgeUrl, + bridgeBearer, + cwd: worktreePath, + } +} + export function constrainedLoopsGenerator(config: OuterLoopConfig): CandidateGenerator { const shotDir = join(config.outDir, 'proposer-shots') - // The run-wide CostLedger the current generate() call rides — captured so - // onShotCompleted can settle each shot's spend into it. maxConcurrency is 1 - // and shots run inside generate(), so a single slot cannot interleave. - let activeLedger: CostLedgerHandle | undefined - let activePhase: string | undefined const inner = agenticGenerator({ - harness: config.proposerHarness, + profile: configuredAuthorProfile(config), + executorForWorktree: authorExecutorForWorktree, timeoutMs: config.proposerTimeoutMs, buildPrompt: round4BuildPrompt, verify: loopsCandidateVerifier(config.loopsRepo), - runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(config.proposerHarness) }), - // Three runs died as "author shot exited with code 1" with the shot's - // stderr lost (nothing wires receipt persistence by default). Persist every - // attempted shot — receipt plus bounded stream tails — so the NEXT failure - // names its cause from disk. Shared implementation with the gen-3 fan-out - // authors (proposer-fanout.mts): receipt persistence + spend settlement - // into the run ledger for the claude/opencode paths whose shots would - // otherwise read $0. onShotCompleted: proposerShotHooks({ shotDir, - harness: config.proposerHarness, - ledger: () => activeLedger, - phase: () => activePhase, }), }) return { kind: `round4-constrained:${inner.kind}`, proposesWithoutFindings: true, - generate: (args) => { - activeLedger = args.costLedger - activePhase = args.costPhase - // args.maxShots is the LIB's dial (budget.maxImprovementShots → the - // improvement driver); capabilities.mts guarantees it is threaded. - return inner.generate(args) - }, + generate: (args) => inner.generate(args), } } @@ -2161,6 +2192,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P repoRoot: config.loopsRepo, baseRef: config.loopsBaseRef, worktreeDir: join(config.outDir, 'loops-worktrees'), + profile: configuredAuthorProfile(config), generator, }, scenarios: sweScenarios, @@ -2764,17 +2796,8 @@ if (isMain) { let config: OuterLoopConfig let flavor: string if (gen4 || gen5) { - // The codex seat rides only when the CLI is actually present — a config - // naming a missing harness would fail the whole launch at t=0. - const codexProbe = await run('codex', ['--version']) - const includeCodex = codexProbe.code === 0 - if (!includeCodex) { - console.log( - `codex CLI unavailable (rc=${codexProbe.code}) — ${gen5 ? 'gen-5' : 'gen-4'} config written WITHOUT the codex-author seat`, - ) - } const make = gen5 ? defaultGen5Config : defaultGen4Config - config = make(undefined, { ...(outDirName ? { outDirName } : {}), includeCodex }) + config = make(undefined, { ...(outDirName ? { outDirName } : {}) }) flavor = gen5 ? 'gen-5' : 'gen-4' } else { const make = gen3 ? defaultGen3Config : defaultRound4Config diff --git a/bench/src/swe-arena/profiles/deepseek-author.profile.json b/bench/src/swe-arena/profiles/deepseek-author.profile.json new file mode 100644 index 00000000..f2ad3057 --- /dev/null +++ b/bench/src/swe-arena/profiles/deepseek-author.profile.json @@ -0,0 +1,12 @@ +{ + "name": "swe-arena-deepseek-author", + "description": "Pi author using DeepSeek V4 Flash through Tangle Router.", + "harness": "pi", + "model": { + "provider": "tangle-router", + "default": "deepseek-v4-flash" + }, + "prompt": { + "systemPrompt": "Improve the candidate worktree from measured failure evidence. Make the smallest coherent change, verify it, and leave the edit uncommitted." + } +} diff --git a/bench/src/swe-arena/profiles/default-author.profile.json b/bench/src/swe-arena/profiles/default-author.profile.json index 20bd324c..91e7ad62 100644 --- a/bench/src/swe-arena/profiles/default-author.profile.json +++ b/bench/src/swe-arena/profiles/default-author.profile.json @@ -1,4 +1,12 @@ { "name": "swe-arena-default-author", - "description": "The gen-2 winning author seat, codified: bare claude CLI invocation. No systemPrompt, no instructions, no model override — harnessInvocation() with this profile is byte-identical to the legacy prompt-only buildArgs path, so provenance pins the author identity without changing gen-2 behavior." + "description": "Pi author for measured SWE arena code improvement.", + "harness": "pi", + "model": { + "provider": "tangle-router", + "default": "glm-5.2" + }, + "prompt": { + "systemPrompt": "Improve the candidate worktree from measured failure evidence. Make the smallest coherent change, verify it, and leave the edit uncommitted." + } } diff --git a/bench/src/swe-arena/proposer-fanout.mts b/bench/src/swe-arena/proposer-fanout.mts index d1f77f2f..166fd6d8 100644 --- a/bench/src/swe-arena/proposer-fanout.mts +++ b/bench/src/swe-arena/proposer-fanout.mts @@ -21,14 +21,9 @@ * stage 3 — survivors proceed through the unchanged pinned-baseline * reps-fail-closed gate. * - * Proposer identity: each proposer is an `AgentProfile` invocation - * (`agenticGenerator`'s `profile` option → `harnessInvocation`, which threads - * `prompt.systemPrompt`/`instructions` into the composed prompt and - * `model.default` into the harness's `-m` flag — verified supported for the - * claude CLI; a bare profile is byte-identical to the legacy prompt-only - * invocation). Profile `resources` are only materialized on the reproducible - * Codex path, so a non-codex proposer with resources fails loud here instead - * of silently dropping them. + * Proposer identity: each harness-authored proposer supplies one complete + * `AgentProfile`. Runtime validates and materializes that exact profile, then + * runs it through the CLI bridge in the candidate worktree. * * NOTE on the import cycle with outer-loop.mts: outer-loop imports this * module's generator factory; this module imports outer-loop's change-space + @@ -47,13 +42,11 @@ import { type AgenticGeneratorShotExecution, type AgenticGeneratorShotReceipt, } from '@tangle-network/agent-runtime' -import { runLocalHarness } from '@tangle-network/agent-runtime/mcp' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { agentProfileSchema, type AgentProfile } from '@tangle-network/agent-interface' import type { CostLedgerHandle, ProposalFinding } from '@tangle-network/agent-eval' import { changeSpaceViolations, loopsCandidateVerifier, - proposerShotEnv, purgeIgnoredArtifacts, round4BuildPrompt, type OuterLoopConfig, @@ -86,17 +79,12 @@ import { export interface ProposerSpec { /** Unique short name — becomes the candidate's staircase `label`. */ name: string - /** Path to an `AgentProfile` JSON. Absolute, or relative to this module's - * `profiles/` directory. Omitted = bare profile (legacy invocation). */ + /** Path to the complete `AgentProfile` JSON. Absolute, or relative to this + * module's `profiles/` directory. Required by the Runtime author path. */ profile?: string - /** Required for harness-authored seats. Absent on an engine seat - * (`engine` set) — enforced both ways at generator construction. */ - harness?: 'claude-code' | 'codex' | 'opencode' - /** GEN-4 pinned model id, threaded to the harness CLI as `-m ` via - * the author profile's `model.default` (harnessInvocation maps it for all - * three harnesses). Unset = the CLI's own resolved model (its login/settings - * default) — recorded per run by the proposer-provenance capture, so the - * seat identity is pinned in provenance even when the flag is absent. */ + /** Optional provenance mirror of `profile.harness`; a mismatch is rejected. */ + harness?: 'claude-code' | 'codex' | 'opencode' | 'pi' + /** Optional provenance mirror of `profile.model.default`; a mismatch is rejected. */ model?: string /** GEN-4 merge seat: this proposer's task is to MERGE the configured * Pareto parents' diffs into one coherent surface (see @@ -179,42 +167,40 @@ export interface PrefilterKill { export const PROFILES_DIR = fileURLToPath(new URL('./profiles', import.meta.url)) -/** Load + validate a proposer's `AgentProfile`. Fail-closed on resources for - * non-codex harnesses: `agenticGenerator` only materializes profile resource - * files on the reproducible Codex path, so accepting them here would drop - * them silently. */ +/** Load and schema-validate a proposer's exact `AgentProfile`. */ export function loadAuthorProfile(spec: ProposerSpec): AgentProfile | undefined { if (!spec.profile) return undefined const path = spec.profile.startsWith('/') ? spec.profile : join(PROFILES_DIR, spec.profile) - const profile = JSON.parse(readFileSync(path, 'utf8')) as AgentProfile - if (typeof profile.name !== 'string' || profile.name.length === 0) { - throw new Error(`proposer ${spec.name}: profile ${path} has no name`) - } - if (profile.resources && spec.harness !== 'codex') { - throw new Error( - `proposer ${spec.name}: profile ${path} declares resources, which only materialize on the ` + - `reproducible codex path — the ${spec.harness} harness would silently drop them`, - ) - } - return profile + return agentProfileSchema.parse(JSON.parse(readFileSync(path, 'utf8'))) } -/** The gen-2 author, codified: one bare-profile claude proposer. */ +/** Default author: Pi with the exact GLM profile committed beside this module. */ export function defaultProposers(): ProposerSpec[] { - return [{ name: 'default-author', profile: 'default-author.profile.json', harness: 'claude-code' }] + return [ + { + name: 'default-author', + profile: 'default-author.profile.json', + harness: 'pi', + model: 'glm-5.2', + }, + ] } -/** The profile the author shot actually runs: the loaded profile (if any) with - * the spec's PINNED MODEL merged into `model.default` — harnessInvocation - * turns that into the CLI's `-m ` flag (claude/codex/opencode all map - * it). Without a pinned model this is byte-identical to `loadAuthorProfile`, - * so gen-3 seats keep their exact invocation. A pinned model with no profile - * path synthesizes a minimal named profile carrying only the pin. */ +/** Resolve the one exact profile; duplicated provenance fields may only agree. */ export function resolveAuthorProfile(spec: ProposerSpec): AgentProfile | undefined { const profile = loadAuthorProfile(spec) - if (!spec.model) return profile - const base: AgentProfile = profile ?? { name: `${spec.name}-pinned` } - return { ...base, model: { ...base.model, default: spec.model } } + if (!profile) return undefined + if (spec.harness !== undefined && profile.harness !== spec.harness) { + throw new Error( + `proposer ${spec.name}: harness ${spec.harness} conflicts with profile.harness ${profile.harness}`, + ) + } + if (spec.model !== undefined && profile.model?.default !== spec.model) { + throw new Error( + `proposer ${spec.name}: model ${spec.model} conflicts with profile.model.default ${profile.model?.default ?? 'missing'}`, + ) + } + return profile } // --------------------------------------------------------------------------- @@ -353,7 +339,7 @@ export function sliceFindings( findings: ReadonlyArray, slice: ProposerSpec['diagnosisSlice'], ): ProposalFinding[] { - if (slice === undefined || slice === 'all') return findings + if (slice === undefined || slice === 'all') return [...findings] return findings.filter((finding) => { const f = finding as unknown as Record if (isSteeringOrRawTrace(f)) return true @@ -402,47 +388,14 @@ export function proposerBuildPrompt( export function proposerShotHooks(opts: { shotDir: string - harness: ProposerSpec['harness'] - ledger: () => CostLedgerHandle | undefined - phase: () => string | undefined }): (receipt: AgenticGeneratorShotReceipt, execution: AgenticGeneratorShotExecution | null) => Promise { return async (receipt, execution) => { - const tail = (s: string | undefined): string | null => - s === undefined ? null : s.length > 20_000 ? s.slice(-20_000) : s await mkdir(opts.shotDir, { recursive: true }) const name = `gen${receipt.generation ?? 'x'}-cand${receipt.candidateIndex ?? 'x'}-shot${receipt.shot}.json` await writeFile( join(opts.shotDir, name), - JSON.stringify({ receipt, stdoutTail: tail(execution?.stdout), stderrTail: tail(execution?.stderr) }, null, 2), + JSON.stringify({ receipt, execution }, null, 2), ) - // Proposer-shot spend → the lib's run ledger. The generator only settles - // its own receipts on the codexReproducible path (costCallId non-null); - // the claude/opencode author path otherwise leaves every shot as $0 in - // the run's spend summary. Import the shot receipt's measured usage. - const ledger = opts.ledger() - if (ledger && receipt.costCallId === null && (receipt.usage || receipt.costUsdKnown)) { - const usage = receipt.usage - const paid = await ledger.runPaidCall({ - channel: 'driver', - phase: opts.phase() ?? 'search.proposal', - actor: `proposer-shot:${opts.harness}`, - model: receipt.model ?? `${opts.harness}-cli`, - tags: { - generation: String(receipt.generation ?? -1), - candidateIndex: String(receipt.candidateIndex ?? -1), - shot: String(receipt.shot), - }, - execute: async () => receipt, - receipt: () => ({ - model: receipt.model ?? `${opts.harness}-cli`, - inputTokens: usage?.inputTokens ?? 0, - outputTokens: usage ? usage.outputTokens + usage.reasoningOutputTokens : 0, - ...(usage ? { cachedTokens: usage.cachedInputTokens } : { usageUnknown: true }), - ...(receipt.costUsdKnown && receipt.costUsd !== null ? { actualCostUsd: receipt.costUsd } : {}), - }), - }) - if (!paid.succeeded) throw paid.error - } } } @@ -512,7 +465,6 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { // One agenticGenerator per proposer, created lazily and cached: each carries // its own profile, lens prompt, and shot-receipt home. const inners = new Map() - const ledgers = new Map() // The GEPA seat authors through the agent-eval adapter, not a // harness CLI. Its inner evaluator is the SAME injected smoke runner the // pre-filter uses (presence enforced at generator construction). @@ -531,20 +483,27 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { }) return gepaAuthor(proposer, args) } - const harness = proposer.harness - if (harness === undefined) { - throw new Error(`proposer ${proposer.name}: harness is required for a non-engine seat`) - } let inner = inners.get(proposer.name) - const slot = ledgers.get(proposer.name) ?? {} - ledgers.set(proposer.name, slot) if (!inner) { - // GEN-4: the resolved profile carries the spec's pinned model - // (`model.default` → the harness CLI's `-m` flag). const profile = resolveAuthorProfile(proposer) + if (!profile) { + throw new Error(`proposer ${proposer.name}: profile is required for a Runtime author seat`) + } + const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL + const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER + if (!bridgeUrl || !bridgeBearer) { + throw new Error( + 'proposer authoring requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER', + ) + } inner = agenticGenerator({ - harness, - ...(profile ? { profile } : {}), + profile, + executorForWorktree: (cwd) => ({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, + cwd, + }), timeoutMs: config.proposerTimeoutMs, buildPrompt: (args) => proposerBuildPrompt(args, proposer, deps.parents ?? [], { @@ -552,18 +511,12 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { ...(config.activationGate === true ? { activationGate: true } : {}), }), verify: loopsCandidateVerifier(config.loopsRepo), - runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(harness) }), onShotCompleted: proposerShotHooks({ shotDir: join(config.outDir, 'proposer-shots', sanitize(proposer.name)), - harness, - ledger: () => slot.ledger, - phase: () => slot.phase, }), }) inners.set(proposer.name, inner) } - slot.ledger = args.costLedger - slot.phase = args.costPhase return inner.generate(args) } } diff --git a/bench/src/swe-arena/proposer-fanout.test.mts b/bench/src/swe-arena/proposer-fanout.test.mts index e4be6cad..8adac59b 100644 --- a/bench/src/swe-arena/proposer-fanout.test.mts +++ b/bench/src/swe-arena/proposer-fanout.test.mts @@ -73,7 +73,7 @@ describe('sliceFindings', () => { describe('proposerBuildPrompt', () => { it('appends the lens AFTER the shared protocol prompt, leaving the change-space text intact', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code', lens: 'Prefer code-path fixes.' } + const spec: ProposerSpec = { name: 'x', harness: 'pi', lens: 'Prefer code-path fixes.' } const prompt = proposerBuildPrompt({ findings: [] }, spec) expect(prompt).toContain('DECLARED CHANGE-SPACE') expect(prompt.indexOf('DECLARED CHANGE-SPACE')).toBeLessThan(prompt.indexOf('YOUR AUTHORING LENS (x)')) @@ -81,7 +81,7 @@ describe('proposerBuildPrompt', () => { }) it('is the bare round prompt without a lens', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code' } + const spec: ProposerSpec = { name: 'x', harness: 'pi' } expect(proposerBuildPrompt({ findings: [] }, spec)).not.toContain('AUTHORING LENS') }) }) @@ -106,35 +106,47 @@ const PARENTS: ParetoParentContext[] = [ }, ] -describe('resolveAuthorProfile (pinned models)', () => { - it('merges the spec model into the loaded profile as model.default', () => { +describe('resolveAuthorProfile', () => { + it('returns the exact committed profile when provenance mirrors agree', () => { const spec: ProposerSpec = { name: 'x', profile: 'default-author.profile.json', - harness: 'claude-code', - model: 'claude-fable-5', + harness: 'pi', + model: 'glm-5.2', } const profile = resolveAuthorProfile(spec) expect(profile?.name).toBe('swe-arena-default-author') - expect(profile?.model?.default).toBe('claude-fable-5') + expect(profile?.harness).toBe('pi') + expect(profile?.model?.default).toBe('glm-5.2') }) - it('synthesizes a minimal named profile for a pinned model without a profile path', () => { - const profile = resolveAuthorProfile({ name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' }) - expect(profile?.name).toBe('glm-author-pinned') - expect(profile?.model?.default).toBe('zai-coding-plan/glm-5.2') + it('rejects duplicated harness or model fields that conflict with the profile', () => { + expect(() => + resolveAuthorProfile({ + name: 'bad-harness', + profile: 'default-author.profile.json', + harness: 'codex', + }), + ).toThrow(/conflicts with profile\.harness/) + expect(() => + resolveAuthorProfile({ + name: 'bad-model', + profile: 'default-author.profile.json', + model: 'different-model', + }), + ).toThrow(/conflicts with profile\.model\.default/) }) - it('is byte-identical to loadAuthorProfile without a pin (gen-3 seats unchanged)', () => { - const spec: ProposerSpec = { name: 'x', profile: 'default-author.profile.json', harness: 'claude-code' } + it('does not synthesize an ambient profile when the path is missing', () => { + const spec: ProposerSpec = { name: 'x', profile: 'default-author.profile.json', harness: 'pi' } expect(resolveAuthorProfile(spec)).toEqual(loadAuthorProfile(spec)) - expect(resolveAuthorProfile({ name: 'bare', harness: 'claude-code' })).toBeUndefined() + expect(resolveAuthorProfile({ name: 'bare', harness: 'pi' })).toBeUndefined() }) }) describe('proposerBuildPrompt with pareto parents', () => { it('appends the parents section (evidence + diffs) after the protocol prompt and lens', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code', lens: 'Prefer code-path fixes.' } + const spec: ProposerSpec = { name: 'x', harness: 'pi', lens: 'Prefer code-path fixes.' } const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS) expect(prompt).toContain('DECLARED CHANGE-SPACE') expect(prompt).toContain('PARETO PARENTS') @@ -146,14 +158,14 @@ describe('proposerBuildPrompt with pareto parents', () => { }) it('leaves the prompt untouched when no parents are seeded (gen-3 behavior)', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code' } + const spec: ProposerSpec = { name: 'x', harness: 'pi' } expect(proposerBuildPrompt({ findings: [] }, spec)).not.toContain('PARETO PARENTS') expect(parentsPromptSection(PARENTS)).toContain('measured evidence') }) }) describe('mergeAuthorPrompt', () => { - const spec: ProposerSpec = { name: 'merge-author', harness: 'claude-code', merge: true } + const spec: ProposerSpec = { name: 'merge-author', harness: 'pi', merge: true } it('keeps the change-space contract and presents BOTH parent diffs with the coherent-union task', () => { const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS) @@ -178,21 +190,23 @@ describe('mergeAuthorPrompt', () => { describe('defaultGen4Config', () => { const config = defaultGen4Config() - it('seats four proposers (one candidate slot each) with the pinned glm model and the merge seat', () => { - expect(config.proposers).toHaveLength(4) - expect(config.populationSize).toBe(4) + it('seats Pi authors with exact GLM and DeepSeek profiles', () => { + expect(config.proposers).toHaveLength(3) + expect(config.populationSize).toBe(3) const byName = Object.fromEntries(config.proposers!.map((p) => [p.name, p])) - expect(byName['claude-author']).toMatchObject({ harness: 'claude-code', profile: 'default-author.profile.json' }) - expect(byName['claude-author']!.model).toBeUndefined() - expect(byName['glm-author']).toMatchObject({ harness: 'opencode', model: 'zai-coding-plan/glm-5.2' }) - expect(byName['codex-author']).toMatchObject({ harness: 'codex' }) - expect(byName['merge-author']).toMatchObject({ harness: 'claude-code', merge: true }) + expect(byName['glm-author']).toMatchObject({ harness: 'pi', model: 'glm-5.2' }) + expect(byName['deepseek-author']).toMatchObject({ + harness: 'pi', + model: 'deepseek-v4-flash', + profile: 'deepseek-author.profile.json', + }) + expect(byName['merge-author']).toMatchObject({ harness: 'pi', merge: true }) }) - it('drops the codex seat (and shrinks the population) when includeCodex is false', () => { - const noCodex = defaultGen4Config(undefined, { includeCodex: false }) - expect(noCodex.proposers!.map((p) => p.name)).toEqual(['claude-author', 'glm-author', 'merge-author']) - expect(noCodex.populationSize).toBe(3) + it('can omit the DeepSeek seat while keeping population aligned', () => { + const noDeepseek = defaultGen4Config(undefined, { includeDeepseek: false }) + expect(noDeepseek.proposers!.map((p) => p.name)).toEqual(['glm-author', 'merge-author']) + expect(noDeepseek.populationSize).toBe(2) }) it('seeds the gen-3 frontier as pareto parents and points at a fresh gen4 outDir + baseline artifact', () => { @@ -215,24 +229,40 @@ describe('defaultGen4Config', () => { }) describe('loadAuthorProfile', () => { - it('loads the committed default-author profile (bare: no prompt, no model)', () => { - const profile = loadAuthorProfile({ name: 'a', profile: 'default-author.profile.json', harness: 'claude-code' }) + it('loads the complete committed Pi/GLM author profile', () => { + const profile = loadAuthorProfile({ + name: 'a', + profile: 'default-author.profile.json', + harness: 'pi', + }) expect(profile?.name).toBe('swe-arena-default-author') - expect(profile?.prompt).toBeUndefined() - expect(profile?.model).toBeUndefined() + expect(profile?.harness).toBe('pi') + expect(profile?.prompt?.systemPrompt).toContain('measured failure evidence') + expect(profile?.model).toMatchObject({ provider: 'tangle-router', default: 'glm-5.2' }) expect(existsSync(join(PROFILES_DIR, 'default-author.profile.json'))).toBe(true) }) it('returns undefined without a profile path', () => { - expect(loadAuthorProfile({ name: 'a', harness: 'claude-code' })).toBeUndefined() + expect(loadAuthorProfile({ name: 'a', harness: 'pi' })).toBeUndefined() }) - it('fails loud when a non-codex proposer declares profile resources (they would be dropped)', async () => { + it('preserves profile resources for Runtime materialization', async () => { const dir = await mkdtemp(join(tmpdir(), 'profile-')) try { const path = join(dir, 'with-resources.json') - await writeFile(path, JSON.stringify({ name: 'r', resources: { files: [] } })) - expect(() => loadAuthorProfile({ name: 'a', profile: path, harness: 'claude-code' })).toThrow(/silently drop/) + await writeFile( + path, + JSON.stringify({ + name: 'r', + harness: 'pi', + model: { provider: 'test', default: 'model' }, + prompt: { systemPrompt: 'Author changes.' }, + resources: { files: [] }, + }), + ) + expect(loadAuthorProfile({ name: 'a', profile: path, harness: 'pi' })?.resources).toEqual({ + files: [], + }) } finally { await rm(dir, { recursive: true, force: true }) } @@ -300,9 +330,14 @@ describe('defaultGen3Config', () => { } }) - it('defaultProposers codifies the gen-2 author: one bare-profile claude entry', () => { + it('defaultProposers uses one exact Pi/GLM profile', () => { expect(defaultProposers()).toEqual([ - { name: 'default-author', profile: 'default-author.profile.json', harness: 'claude-code' }, + { + name: 'default-author', + profile: 'default-author.profile.json', + harness: 'pi', + model: 'glm-5.2', + }, ]) }) }) @@ -357,8 +392,8 @@ describe('fanOutLoopsGenerator', () => { it('authors ALL proposers concurrently in separate worktrees and applies each patch to its candidate slot', async () => { const proposers: ProposerSpec[] = [ - { name: 'alpha', harness: 'claude-code' }, - { name: 'beta', harness: 'claude-code' }, + { name: 'alpha', harness: 'pi' }, + { name: 'beta', harness: 'pi' }, ] let inFlight = 0 let maxInFlight = 0 @@ -396,7 +431,7 @@ describe('fanOutLoopsGenerator', () => { expect(first).toMatchObject({ applied: true, label: 'alpha' }) expect(second).toMatchObject({ applied: true, label: 'beta' }) - expect(first.rationale).toContain('proposer alpha (claude-code)') + expect(first.rationale).toContain('proposer alpha (pi)') // Both patches landed on the SAME driver worktree across the two calls — // the driver discards/finalizes between calls in production; here we only // assert each call applied its own proposer's file. @@ -407,8 +442,8 @@ describe('fanOutLoopsGenerator', () => { it('kills a candidate at the smoke pre-filter: applied=false, no patch applied, kill recorded with reason', async () => { const proposers: ProposerSpec[] = [ - { name: 'good', harness: 'claude-code' }, - { name: 'bad', harness: 'claude-code' }, + { name: 'good', harness: 'pi' }, + { name: 'bad', harness: 'pi' }, ] const config = baseConfig(proposers) config.prefilter = { enabled: true, smokeInstance: 'cheapest-of-set' } @@ -461,7 +496,7 @@ describe('fanOutLoopsGenerator', () => { }) it('kills an out-of-space diff at the change-space pre-filter before any smoke spend', async () => { - const proposers: ProposerSpec[] = [{ name: 'rogue', harness: 'claude-code' }] + const proposers: ProposerSpec[] = [{ name: 'rogue', harness: 'pi' }] const config = baseConfig(proposers) config.prefilter = { enabled: true, smokeInstance: 'cheapest-of-set' } let smokeRan = false @@ -488,7 +523,7 @@ describe('fanOutLoopsGenerator', () => { }) it('returns applied:false without a kill when a proposer authors nothing', async () => { - const gen = fanOutLoopsGenerator(baseConfig([{ name: 'idle', harness: 'claude-code' }]), { + const gen = fanOutLoopsGenerator(baseConfig([{ name: 'idle', harness: 'pi' }]), { author: async () => ({ applied: false, summary: '' }), }) expect((await gen.generate(generatorArgs(0))).applied).toBe(false) @@ -496,7 +531,7 @@ describe('fanOutLoopsGenerator', () => { }) it('fails loud when candidateIndex exceeds the proposer list (populationSize drift)', async () => { - const gen = fanOutLoopsGenerator(baseConfig([{ name: 'only', harness: 'claude-code' }]), { + const gen = fanOutLoopsGenerator(baseConfig([{ name: 'only', harness: 'pi' }]), { author: async () => ({ applied: false, summary: '' }), }) await expect(gen.generate(generatorArgs(1))).rejects.toThrow(/populationSize must equal/) @@ -520,7 +555,7 @@ describe('fanOutLoopsGenerator', () => { }) it('refuses a merge seat without >=2 materialized parents', () => { - const config = baseConfig([{ name: 'merge-author', harness: 'claude-code', merge: true }]) + const config = baseConfig([{ name: 'merge-author', harness: 'pi', merge: true }]) expect(() => fanOutLoopsGenerator(config, { author: async () => ({ applied: false, summary: '' }) })).toThrow( /merge proposer/, ) @@ -539,8 +574,8 @@ describe('fanOutLoopsGenerator', () => { it('rejects duplicate proposer names and empty proposer lists', () => { expect(() => fanOutLoopsGenerator(baseConfig([ - { name: 'dup', harness: 'claude-code' }, - { name: 'dup', harness: 'claude-code' }, + { name: 'dup', harness: 'pi' }, + { name: 'dup', harness: 'pi' }, ])), ).toThrow(/duplicate/) expect(() => fanOutLoopsGenerator({ ...defaultRound4Config(), loopsRepo, outDir })).toThrow(/empty/) @@ -576,10 +611,10 @@ describe('defaultGen5Config', () => { expect(config.priorEvidenceDirs!.some((d) => d.includes('gen4'))).toBe(true) }) - it('drops the codex seat when includeCodex is false (population follows)', () => { - const noCodex = defaultGen5Config(undefined, { includeCodex: false }) - expect(noCodex.proposers!.map((p) => p.name)).toEqual(['claude-author', 'glm-author', 'merge-author']) - expect(noCodex.populationSize).toBe(3) + it('can omit the DeepSeek seat while keeping population aligned', () => { + const noDeepseek = defaultGen5Config(undefined, { includeDeepseek: false }) + expect(noDeepseek.proposers!.map((p) => p.name)).toEqual(['glm-author', 'merge-author']) + expect(noDeepseek.populationSize).toBe(2) }) }) @@ -591,7 +626,7 @@ describe('gen-5 prompt sections', () => { } it('appends EVIDENCE MAP + briefing + activation contract after the protocol prompt', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code' } + const spec: ProposerSpec = { name: 'x', harness: 'pi' } const prompt = proposerBuildPrompt({ findings: [] }, spec, [], { briefing, activationGate: true, @@ -604,7 +639,7 @@ describe('gen-5 prompt sections', () => { }) it('the merge seat gets the gen-5 sections too', () => { - const spec: ProposerSpec = { name: 'merge-author', harness: 'claude-code', merge: true } + const spec: ProposerSpec = { name: 'merge-author', harness: 'pi', merge: true } const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS, { briefing, activationGate: true, @@ -615,7 +650,7 @@ describe('gen-5 prompt sections', () => { }) it('leaves gen-3/gen-4 prompts byte-identical when no extras are passed', () => { - const spec: ProposerSpec = { name: 'x', harness: 'claude-code' } + const spec: ProposerSpec = { name: 'x', harness: 'pi' } const legacy = proposerBuildPrompt({ findings: [] }, spec, PARENTS) expect(legacy).not.toContain('EVIDENCE MAP') expect(legacy).not.toContain('ACTIVATION PREDICATE') diff --git a/bench/src/swe-code-improve.mts b/bench/src/swe-code-improve.mts index 1a7cef96..c20c3a30 100644 --- a/bench/src/swe-code-improve.mts +++ b/bench/src/swe-code-improve.mts @@ -1,7 +1,7 @@ /** * META-HARNESS on the SWE scaffold — improve({ surface: 'code' }). * - * A coding agent (Claude Code) REWRITES the scaffold LOGIC under bench/src (the seed prompt/playbook, + * A Pi coding agent REWRITES the scaffold LOGIC under bench/src (the seed prompt/playbook, * runAgentic strategy/params, context handling, retry/patch synthesis) with the MODEL (glm worker) + * the TOOL surface (list/read/edit[/run]) + the JUDGE held FIXED, judged on the official swebench * Docker verdict, gated on a held-out instance split. This is the DGM/meta-harness recipe: let the @@ -10,13 +10,9 @@ * Wiring (all verified in this worktree): * - improve()/codeProposerFor + rawTraceContext come from the LOCAL agent-runtime build, linked into * this bench's node_modules (bench/node_modules/@tangle-network/agent-runtime -> /home/drew/code/agent-runtime). - * - The candidate proposer is agenticGenerator(harness:'claude-code'), BUT the shipped runLocalHarness - * spawns `claude --headless -p` and --headless is an unknown option on the current CLI (exit 1, no - * edits ever). We pass code.generator with a corrected runHarness that spawns - * `claude -p --dangerously-skip-permissions` so the coding agent can actually edit the - * worktree. This is a harness-spawn fix, NOT a hand-authored scaffold edit — Claude still finds the - * lever itself from the traces. - * - Each candidate is a git worktree the driver forks off baseRef; Claude edits bench/src in place; + * - The candidate proposer supplies one exact Pi AgentProfile and runs it through Runtime's + * bridge executor. No local CLI shortcut or ambient model default exists. + * - Each candidate is a git worktree the driver forks off baseRef; Pi edits bench/src in place; * `verify` (an import smoke of the edited scaffold) gates it before the expensive measurement. * - MEASUREMENT: the code-aware agent fn shells into the candidate scaffold's OWN judge-free emit * entrypoint (swe-emit-patch.mts) with cwd = the worktree, captures the unified diff, and returns @@ -35,6 +31,7 @@ import { join } from 'node:path' import { improve, agenticGenerator } from '@tangle-network/agent-runtime' import type { ProposalFinding } from '@tangle-network/agent-eval' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { agentProfileSchema } from '@tangle-network/agent-interface' import { createSweBenchAdapter } from './benchmarks/swe-bench' import type { BenchTask } from './benchmarks/types' @@ -120,6 +117,22 @@ async function main(): Promise { const runDir = process.env.RUN_DIR ?? '/tmp/claude-1000/-home-drew-code-supervisor-lab/9ee6a456-a94f-474c-9888-b4afc9bc26bd/scratchpad/mh-run' const emitTimeoutMs = Number(process.env.EMIT_TIMEOUT_MS ?? 600_000) const harnessTimeoutMs = Number(process.env.HARNESS_TIMEOUT_MS ?? 900_000) + const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL + const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER + if (!bridgeUrl || !bridgeBearer) { + throw new Error('CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER are required') + } + const authorProfile = agentProfileSchema.parse({ + name: 'swe-scaffold-author', + harness: 'pi', + model: { + provider: process.env.AUTHOR_PROVIDER ?? 'tangle-router', + default: process.env.AUTHOR_MODEL ?? 'glm-5.2', + }, + prompt: { + systemPrompt: 'Improve the candidate worktree from measured failure evidence and verify it.', + }, + }) mkdirSync(worktreeDir, { recursive: true }) mkdirSync(runDir, { recursive: true }) @@ -199,44 +212,6 @@ async function main(): Promise { }, } - // The corrected coding-harness spawn: `claude -p --dangerously-skip-permissions`. The shipped - // runLocalHarness uses `claude --headless -p` (unknown option on this CLI). agenticGenerator ignores - // the return value (it reads worktree dirtiness), so a minimal result shape is enough. - const runHarness = (o: { cwd: string; taskPrompt: string; timeoutMs?: number; signal?: AbortSignal }): Promise<{ exitCode: number | null; stdout: string; stderr: string; killedBySignal: NodeJS.Signals | null; durationMs: number; timedOut: boolean }> => { - const started = Date.now() - return new Promise((resolve) => { - const child = spawn('claude', ['-p', o.taskPrompt, '--dangerously-skip-permissions'], { - cwd: o.cwd, - env: process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }) - let stdout = '' - let stderr = '' - let timedOut = false - child.stdout?.on('data', (d) => (stdout += String(d))) - child.stderr?.on('data', (d) => (stderr += String(d))) - const timer = setTimeout(() => { - timedOut = true - if (!child.killed) child.kill('SIGTERM') - }, o.timeoutMs ?? harnessTimeoutMs) - ;(timer as { unref?: () => void }).unref?.() - const onAbort = () => { - if (!child.killed) child.kill('SIGTERM') - } - o.signal?.addEventListener('abort', onAbort, { once: true }) - child.on('error', () => { - clearTimeout(timer) - resolve({ exitCode: 1, stdout, stderr: `${stderr}\n[spawn error]`, killedBySignal: null, durationMs: Date.now() - started, timedOut }) - }) - child.on('close', (code, signal) => { - clearTimeout(timer) - o.signal?.removeEventListener('abort', onAbort) - console.error(` [proposer:claude] exit=${code} wall=${Math.round((Date.now() - started) / 1000)}s out=${stdout.length}b`) - resolve({ exitCode: code, stdout, stderr, killedBySignal: signal, durationMs: Date.now() - started, timedOut }) - }) - }) - } - // Domain prompt: name the EDIT BOUNDARY (scaffold logic only) + keep the raw-trace evidence discipline // (agenticGenerator discards a raw-trace candidate that doesn't inspect a trace + write the diagnosis). const buildPrompt = (args: { findings: ReadonlyArray }): string => { @@ -286,13 +261,16 @@ async function main(): Promise { } const generator = agenticGenerator({ - harness: 'claude-code', + profile: authorProfile, + executorForWorktree: (cwd) => ({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, + cwd, + }), verify, timeoutMs: harnessTimeoutMs, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - buildPrompt: buildPrompt as any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - runHarness: runHarness as any, + buildPrompt, }) const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' })) @@ -301,7 +279,7 @@ async function main(): Promise { const out = await improve({ surface: 'code', gate: 'holdout', - code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator }, + code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, profile: authorProfile, generator }, rawTraceContext: true, runDir, scenarios, diff --git a/docs/api/index.md b/docs/api/index.md index 593cbfe8..6ca6a3b3 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4393,13 +4393,23 @@ One-based shot number within this candidate. > `readonly` **maxShots**: `number` +##### profileDigest + +> `readonly` **profileDigest**: `string` + +Exact profile identity admitted before the shot. + ##### harness -> `readonly` **harness**: [`LocalHarness`](mcp.md#localharness) +> `readonly` **harness**: `HarnessType` + +##### provider + +> `readonly` **provider**: `string` ##### model -> `readonly` **model**: `string` \| `null` +> `readonly` **model**: `string` ##### reasoningEffort @@ -4421,53 +4431,19 @@ One-based shot number within this candidate. > `readonly` **durationMs**: `number` -##### exitCode - -> `readonly` **exitCode**: `number` \| `null` - -##### timedOut - -> `readonly` **timedOut**: `boolean` - -##### aborted? - -> `readonly` `optional` **aborted?**: `boolean` - -True when caller cancellation reached the author process; absent in older receipts. - -##### killedBySignal - -> `readonly` **killedBySignal**: `Signals` \| `null` - -##### stdoutBytes - -> `readonly` **stdoutBytes**: `number` \| `null` - -##### stdoutSha256 - -> `readonly` **stdoutSha256**: `` `sha256:${string}` `` \| `null` - -##### stderrBytes - -> `readonly` **stderrBytes**: `number` \| `null` - -##### stderrSha256 +##### status -> `readonly` **stderrSha256**: `` `sha256:${string}` `` \| `null` +> `readonly` **status**: [`AgentTaskStatus`](#agenttaskstatus) \| `null` ##### usage -> `readonly` **usage**: [`CodexTokenUsage`](mcp.md#codextokenusage) \| `null` - -##### profileWorkspacePlanDigest +> `readonly` **usage**: `Readonly`\<[`AgentTurnUsage`](runtime.md#agentturnusage)\> \| `null` -> `readonly` **profileWorkspacePlanDigest**: `string` \| `null` +Runtime-normalized usage. Unknown token or dollar totals remain marked unknown. -Digest of the exact profile-file workspace plan applied for this shot. +##### transportAttempts -##### profileWorkspaceFileCount - -> `readonly` **profileWorkspaceFileCount**: `number` +> `readonly` **transportAttempts**: `number` \| `null` ##### costCallId @@ -4491,10 +4467,6 @@ Whether dollars came from the provider, the pricing table, or are unknown. True only for a provider-reported amount, never for a pricing estimate. -##### evidence - -> `readonly` **evidence**: [`CodexExecutionEvidence`](mcp.md#codexexecutionevidence) \| `null` - ##### error > `readonly` **error**: \{ `name`: `string`; `message`: `string`; \} \| `null` @@ -4511,41 +4483,26 @@ git worktree through a pluggable `CandidateGenerator`. #### Properties -##### harness? - -> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) - -Local coding harness to run in the worktree. Default `claude-code`. - -##### profile? - -> `optional` **profile?**: `AgentProfile` - -Author profile rendered through the canonical harness mapper. Required - for reproducible Codex so model and reasoning settings are explicit. - -##### codexReproducible? +##### profile -> `optional` **codexReproducible?**: `boolean` +> **profile**: `AgentProfile` -Run Codex with isolated configuration, exact prompt evidence, and required - terminal token usage. Requires `harness: 'codex'` and `profile`. +Complete author identity. Harness, provider, model, prompt, tools, and resources all come from here. -##### codexReadDeniedPaths? +##### executorForWorktree -> `optional` **codexReadDeniedPaths?**: readonly `string`[] \| ((`worktreePath`) => readonly `string`[]) +> **executorForWorktree**: [`AgenticGeneratorExecutorForWorktree`](#agenticgeneratorexecutorforworktree) -Absolute paths reproducible Codex must not read. A function can derive - candidate-specific paths after the driver creates its worktree. +Place the exact profile on compute that can edit this existing worktree. +A Pi author normally returns `{ backend:'bridge', cwd: worktreePath, ...transport }`. ##### onShotCompleted? > `optional` **onShotCompleted?**: (`receipt`, `execution`) => `void` \| `Promise`\<`void`\> -Awaited once for every attempted author shot, including process failures. - The second argument preserves the exact harness result, including stdout - and stderr, before worktree inspection or verification can reject the - shot. Throwing aborts the candidate so evidence persistence fails closed. +Awaited once for every attempted author shot, including execution failures. +The second argument is Runtime's exact terminal turn and event stream. +Throwing aborts the candidate so evidence persistence fails closed. ###### Parameters @@ -4555,7 +4512,7 @@ Awaited once for every attempted author shot, including process failures. ###### execution -`Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> \| `null` +`Readonly`\<[`CollectedAgentTurn`](runtime.md#collectedagentturn)\> \| `null` ###### Returns @@ -4586,22 +4543,19 @@ Awaited after worktree inspection and before the shot is accepted, > `optional` **maximumCharge?**: `MaximumCharge` -Optional hard upper bound passed to the run-wide CostLedger before each - author shot. This MUST be enforced by the provider or executor; a planning - estimate is not an admissible bound. Omit for an uncapped ledger. A capped - ledger rejects before model dispatch when this is absent. +Optional hard upper bound passed to the run-wide CostLedger before each author shot. ##### timeoutMs? > `optional` **timeoutMs?**: `number` -Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). +Per-shot wall-clock timeout. Omit for no Runtime-imposed deadline. -##### buildPrompt? +##### buildPrompt -> `optional` **buildPrompt?**: (`args`) => `string` +> **buildPrompt**: (`args`) => `string` -Build the harness task prompt from proposal findings. +Build the task prompt from proposal findings. Required: Runtime invents no authoring policy. ###### Parameters @@ -4622,47 +4576,8 @@ readonly `ProposalFinding`[] Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot (verify-in-session), up to `maxShots`; a candidate that never verifies is - discarded (`applied:false`), never shipped. Omitted ⇒ legacy behavior: - the first dirty shot is the candidate. See `commandVerifier`. - -##### runHarness? - -> `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> - -Test seam — inject the harness runner (defaults to `runLocalHarness`). - -**`Experimental`** - -Spawn a local coding harness CLI as a subprocess + collect its output. - -NOT responsible for parsing the harness's output or extracting a diff — -the in-process executor's `streamPrompt` orchestrates `git diff` against -the worktree after this resolves. This function is intentionally narrow: -spawn, wait, capture, return. - -Fails loud — throws when: - - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) - - the harness binary is not on PATH (ENOENT) - - the caller signal was already aborted before process launch - -Does NOT throw when: - - the subprocess exits non-zero (`result.exitCode` carries the code) - - a non-reproducible subprocess is aborted / timed out (`result.aborted` / - `result.timedOut` carries the reason even when a TERM-aware child exits zero) - -Reproducible Codex additionally requires a terminal usage event. If cancellation -prevents that event, this rejects with `CodexExecutionDiagnosticError` instead of -returning an incomplete reproducibility receipt. - -###### Parameters - -###### options - -[`RunLocalHarnessOptions`](mcp.md#runlocalharnessoptions) - -###### Returns - -`Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> + discarded (`applied:false`), never shipped. Omitted means the first dirty + shot is the candidate. See `commandVerifier`. ##### isDirty? @@ -4820,7 +4735,7 @@ Apply a complete winning component map to a detached profile. *** -### ImproveCodeOptions +### ImproveCodeBaseOptions #### Properties @@ -4849,11 +4764,41 @@ Directory worktrees are created under. Default `/.worktrees`. Git-compatible adapter override, primarily for tests. Candidate advancement still requires normal Git worktree and commit semantics. -##### harness? +##### profile + +> **profile**: `AgentProfile` -> `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) +Complete identity of the code author. No execution field may be filled from ambient defaults. -Coding harness the agentic generator runs in each worktree. Default `claude-code`. +*** + +### ImproveRuntimeCodeGeneratorOptions + +#### Properties + +##### executorForWorktree + +> **executorForWorktree**: [`AgenticGeneratorExecutorForWorktree`](#agenticgeneratorexecutorforworktree) + +Place the exact author profile on compute that can edit the supplied worktree. + +##### buildPrompt + +> **buildPrompt**: (`args`) => `string` + +Author the task from admitted findings. Required: Runtime invents no code-improvement prompt. + +###### Parameters + +###### args + +###### findings + +readonly `ProposalFinding`[] + +###### Returns + +`string` ##### verify? @@ -4866,14 +4811,49 @@ feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). > `optional` **timeoutMs?**: `number` -Per-shot wall-clock timeout for the harness (ms). +Per-shot wall-clock timeout. Omit for no Runtime-imposed deadline. + +##### maximumCharge? + +> `optional` **maximumCharge?**: `MaximumCharge` + +Optional provider-enforced maximum admitted by the run-wide cost ledger. ##### generator? -> `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) +> `optional` **generator?**: `undefined` + +*** -Byte-producer override, used for tests and custom candidate production. -When set, `harness`, `verify`, and `timeoutMs` are unused. +### ImproveCustomCodeGeneratorOptions + +#### Properties + +##### generator + +> **generator**: [`CandidateGenerator`](#candidategenerator) + +Complete byte-producer replacement. Runtime still validates `profile` before creating worktrees. + +##### executorForWorktree? + +> `optional` **executorForWorktree?**: `undefined` + +##### buildPrompt? + +> `optional` **buildPrompt?**: `undefined` + +##### verify? + +> `optional` **verify?**: `undefined` + +##### timeoutMs? + +> `optional` **timeoutMs?**: `undefined` + +##### maximumCharge? + +> `optional` **maximumCharge?**: `undefined` *** @@ -10317,12 +10297,9 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault ### AgenticGeneratorShotExecution -> **AgenticGeneratorShotExecution** = `Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> +> **AgenticGeneratorShotExecution** = `Readonly`\<[`CollectedAgentTurn`](runtime.md#collectedagentturn)\> -Frozen exact harness result for an author shot: full streams, process state, - token usage, and execution-policy evidence. - The `onShotCompleted` callback receives `null` when execution failed before - the harness returned. +Runtime's exact terminal turn plus its complete normalized event stream. *** @@ -10336,6 +10313,28 @@ Worktree decision emitted before a completed shot is retried, accepted, or *** +### AgenticGeneratorExecutorForWorktree + +> **AgenticGeneratorExecutorForWorktree** = (`worktreePath`) => [`ExecutorConfig`](runtime.md#executorconfig) + +`@tangle-network/agent-runtime` improvement. + +The public entry point is `improve()`. Complete agent-eval methods optimize +profile surfaces. Runtime owns only code candidates that mutate an isolated +git worktree through a pluggable `CandidateGenerator`. + +#### Parameters + +##### worktreePath + +`string` + +#### Returns + +[`ExecutorConfig`](runtime.md#executorconfig) + +*** + ### ImproveSurface > **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` \| `"rollout-policy"` @@ -10637,6 +10636,12 @@ The canonical improvement API: complete methods for profiles, worktrees for code *** +### ImproveCodeOptions + +> **ImproveCodeOptions** = [`ImproveCodeBaseOptions`](#improvecodebaseoptions) & [`ImproveRuntimeCodeGeneratorOptions`](#improveruntimecodegeneratoroptions) \| [`ImproveCustomCodeGeneratorOptions`](#improvecustomcodegeneratoroptions) + +*** + ### ImprovementCandidate > **ImprovementCandidate** = [`ImprovementProfileCandidate`](#improvementprofilecandidate) \| [`ImprovementCodeCandidate`](#improvementcodecandidate) @@ -11817,15 +11822,6 @@ Hard cap on chained gateway hops; refused beyond this. Default keeps recursion b *** -### AGENTIC\_PROFILE\_RESOURCE\_ROOT - -> `const` **AGENTIC\_PROFILE\_RESOURCE\_ROOT**: `".agent-runtime-profile-resources"` = `'.agent-runtime-profile-resources'` - -Dedicated ephemeral root for generic author-profile files. Every declared -file must live below this root so cleanup cannot alter candidate-owned files. - -*** - ### optimizerMethod > `const` **optimizerMethod**: `string` @@ -12891,15 +12887,15 @@ unique-name check, so the slug only needs to be deterministic, not unique. ### agenticGenerator() -> **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) +> **agenticGenerator**(`opts`): [`CandidateGenerator`](#candidategenerator) -Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. +Full-agentic `CandidateGenerator`: run an exact profiled author inside the existing candidate worktree. #### Parameters -##### opts? +##### opts -[`AgenticGeneratorOptions`](#agenticgeneratoroptions) = `{}` +[`AgenticGeneratorOptions`](#agenticgeneratoroptions) #### Returns @@ -13163,7 +13159,7 @@ Drop-in for `analyzeGeneration` on `improve({ surface: 'code' })`: await improve({ surface: 'code', findings: seedFindings, - code: { repoRoot }, + code: { repoRoot, profile, executorForWorktree, buildPrompt }, runDir: '/abs/run', // MUST be a real path — the traces live here analyzeGeneration: rawTraceDistiller(), scenarios, judge, agent, diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 9ff7cb74..61e0a355 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,12 +15,12 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 403 exports. +Import from `@tangle-network/agent-runtime` — 406 exports. | Symbol | Kind | Summary | |---|---|---| | `agentCandidateProfileAsAgentProfile` | function | Convert the candidate profile contract into the portable interface profile it represents. | -| `agenticGenerator` | function | Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. | +| `agenticGenerator` | function | Full-agentic `CandidateGenerator`: run an exact profiled author inside the existing candidate worktree. | | `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | | `applyRolloutPolicyToProfile` | function | Persist a detached policy under the profile extension without mutating the input. | | `applyRunRecordDefaults` | function | Stamp cross-cutting defaults onto adapter-projected RunRecords without | @@ -126,7 +126,6 @@ Import from `@tangle-network/agent-runtime` — 403 exports. | `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | | `AGENT_CANDIDATE_EXECUTION_SUPPORT` | const | Surfaces admitted by Runtime's verifier before an environment adapter is selected. | -| `AGENTIC_PROFILE_RESOURCE_ROOT` | const | Dedicated ephemeral root for generic author-profile files. Every declared | | `CANDIDATE_KNOWLEDGE_RETRIEVAL_CONFIG_ENV` | const | Environment variable containing the materialized retrieval configuration path. | | `CANDIDATE_KNOWLEDGE_ROOT_ENV` | const | Environment variable containing the materialized candidate knowledge root. | | `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | @@ -226,7 +225,7 @@ Import from `@tangle-network/agent-runtime` — 403 exports. | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | | `AgenticGeneratorShotDisposition` | type | Worktree decision emitted before a completed shot is retried, accepted, or | -| `AgenticGeneratorShotExecution` | type | Frozen exact harness result for an author shot: full streams, process state, | +| `AgenticGeneratorShotExecution` | type | Runtime's exact terminal turn plus its complete normalized event stream. | | `CoordinationEvent` | type | Every message on the one typed pipe. UP (child→parent): question / settled / finding — queued for | | `ImproveCodeRunOptions` | type | Runtime-owned code search in isolated git worktrees. | | `ImproveMethodFactory` | type | Build a complete method after trace findings are available. | @@ -254,7 +253,7 @@ Import from `@tangle-network/agent-runtime` — 403 exports. | `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 799bf1db..5f7f754f 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -1,18 +1,16 @@ /** * * `agenticGenerator` — the full-agentic `CandidateGenerator`. It runs a real - * coding harness (claude / codex / opencode) inside the candidate worktree the - * driver already created, letting the agent read the codebase + the research - * proposal findings and make the change in place. The driver then commits the worktree - * into a `CodeSurface`. + * authored profile inside the candidate worktree the driver already created, + * letting the agent read the codebase + the research proposal findings and + * make the change in place. The driver then commits the worktree into a + * `CodeSurface`. * - * Mechanism: identical to the proven Phase-2.8 in-process executor — spawn the - * harness as a subprocess with `cwd` = the worktree, on the same filesystem, - * so edits land in place (no sandbox-mount round-trip). `runLocalHarness` is - * the verified primitive. The OUTER sandbox is the improvement loop's own - * execution context; the generator does not nest a second sandbox per - * candidate (which would reintroduce a host↔sandbox worktree-transport - * problem that does not need solving here). + * Every paid shot enters Runtime through `createExecutor` + `streamAgentTurn`. + * The caller supplies the exact `AgentProfile` and a worktree-aware executor + * placement; Runtime validates and records the profile that actually ran. A + * cli-bridge placement sets `cwd` to the supplied worktree so Pi and other + * supported harnesses edit the existing candidate directly. * * `maxShots` is the DEPTH dial — a multi-shot verify-in-session loop, NOT the * kernel `runAgentRounds`. Each shot runs one full harness session in the (persistent) @@ -20,46 +18,45 @@ * - empty tree → "you changed nothing, make the edits" → retry * - dirty + `verify` fails → feed the verifier's failure into the next shot * (the worktree persists, so the harness RESUMES atop its own failing - * edits with the error in hand — no `--resume` session plumbing needed, - * and harness-agnostic across claude/codex/opencode) + * edits with the error in hand — no session-specific retry path needed) * - dirty + `verify` ok (or no verifier configured) → return the candidate * A candidate that never verifies within `maxShots` is discarded (`applied: * false`), never shipped — if you configured a verifier, a non-passing tree is - * not a candidate. With no verifier the legacy behavior holds: first dirty shot - * is the candidate. + * not a candidate. With no verifier, the first dirty shot is the candidate. * * @experimental */ import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { existsSync, readFileSync, rmSync } from 'node:fs' -import { join, resolve, sep } from 'node:path' +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' import type { - CostLedgerHandle, CostReceipt, CostReceiptInput, MaximumCharge, ProposalFinding, } from '@tangle-network/agent-eval' -import type { AgentProfile, ReasoningEffort } from '@tangle-network/agent-interface' import { - applyWorkspacePlan, - materializeProfile, - type WorkspacePlan, - type WorkspacePlanReceipt, -} from '@tangle-network/agent-profile-materialize' -import { - type CodexExecutionEvidence, - type CodexTokenUsage, - DEFAULT_LOCAL_HARNESS, - harnessInvocation, - type LocalHarness, - type LocalHarnessResult, - runLocalHarness, -} from '../mcp/local-harness' + type AgentProfile, + agentProfileSchema, + canonicalAgentProfileDigest, + type HarnessType, + type ReasoningEffort, +} from '@tangle-network/agent-interface' import { runSettledCommand } from '../mcp/worktree-harness' -import { concreteProfileModel } from '../runtime/supervise/model-policy' +import { + type AgentTurnUsage, + type CollectedAgentTurn, + collectAgentTurn, + streamAgentTurn, +} from '../runtime/stream-agent-turn' +import { + assertExecutableAgentProfile, + concreteProfileModel, +} from '../runtime/supervise/model-policy' +import { createExecutor, type ExecutorConfig } from '../runtime/supervise/runtime' +import { detachedSnapshot } from '../runtime/supervise/snapshot' import type { CandidateGenerator } from './improvement-driver' import { optimizerMethod } from './optimizer-prompt' @@ -67,10 +64,6 @@ const RAW_TRACE_ANALYST_ID = 'raw-trace-distiller' const RAW_TRACE_AREA = 'raw-trace-context' const RAW_TRACE_DIAGNOSIS_PATH = '.improve/raw-trace-diagnosis.md' -/** Dedicated ephemeral root for generic author-profile files. Every declared - * file must live below this root so cleanup cannot alter candidate-owned files. */ -export const AGENTIC_PROFILE_RESOURCE_ROOT = '.agent-runtime-profile-resources' - /** Outcome of verifying a candidate worktree. `feedback` (compiler errors, * failing test output) is fed into the next shot when `ok` is false. */ export interface VerifyResult { @@ -92,26 +85,20 @@ export interface AgenticGeneratorShotReceipt { /** One-based shot number within this candidate. */ readonly shot: number readonly maxShots: number - readonly harness: LocalHarness - readonly model: string | null + /** Exact profile identity admitted before the shot. */ + readonly profileDigest: string + readonly harness: HarnessType + readonly provider: string + readonly model: string readonly reasoningEffort: ReasoningEffort | null readonly promptSha256: `sha256:${string}` readonly startedAt: string readonly completedAt: string readonly durationMs: number - readonly exitCode: number | null - readonly timedOut: boolean - /** True when caller cancellation reached the author process; absent in older receipts. */ - readonly aborted?: boolean - readonly killedBySignal: NodeJS.Signals | null - readonly stdoutBytes: number | null - readonly stdoutSha256: `sha256:${string}` | null - readonly stderrBytes: number | null - readonly stderrSha256: `sha256:${string}` | null - readonly usage: CodexTokenUsage | null - /** Digest of the exact profile-file workspace plan applied for this shot. */ - readonly profileWorkspacePlanDigest: string | null - readonly profileWorkspaceFileCount: number + readonly status: CollectedAgentTurn['status'] | null + /** Runtime-normalized usage. Unknown token or dollar totals remain marked unknown. */ + readonly usage: Readonly | null + readonly transportAttempts: number | null /** Shared run-ledger call id for this exact shot. */ readonly costCallId: string | null /** Whether dollars came from the provider, the pricing table, or are unknown. */ @@ -119,23 +106,11 @@ export interface AgenticGeneratorShotReceipt { readonly costUsd: number | null /** True only for a provider-reported amount, never for a pricing estimate. */ readonly costUsdKnown: boolean - readonly evidence: CodexExecutionEvidence | null readonly error: { readonly name: string; readonly message: string } | null } -/** Frozen exact harness result for an author shot: full streams, process state, - * token usage, and execution-policy evidence. - * The `onShotCompleted` callback receives `null` when execution failed before - * the harness returned. */ -export type AgenticGeneratorShotExecution = Readonly< - Omit & { - readonly usage?: Readonly - readonly evidence?: Readonly> & { - readonly readDeniedPaths: ReadonlyArray - readonly policy: Readonly - } - } -> +/** Runtime's exact terminal turn plus its complete normalized event stream. */ +export type AgenticGeneratorShotExecution = Readonly /** Worktree decision emitted before a completed shot is retried, accepted, or * discarded. The callback runs while `worktreePath` is still available, so @@ -163,22 +138,17 @@ export type AgenticGeneratorShotDisposition = readonly error: { readonly name: string; readonly message: string } } +export type AgenticGeneratorExecutorForWorktree = (worktreePath: string) => ExecutorConfig + export interface AgenticGeneratorOptions { - /** Local coding harness to run in the worktree. Default `claude-code`. */ - harness?: LocalHarness - /** Author profile rendered through the canonical harness mapper. Required - * for reproducible Codex so model and reasoning settings are explicit. */ - profile?: AgentProfile - /** Run Codex with isolated configuration, exact prompt evidence, and required - * terminal token usage. Requires `harness: 'codex'` and `profile`. */ - codexReproducible?: boolean - /** Absolute paths reproducible Codex must not read. A function can derive - * candidate-specific paths after the driver creates its worktree. */ - codexReadDeniedPaths?: ReadonlyArray | ((worktreePath: string) => ReadonlyArray) - /** Awaited once for every attempted author shot, including process failures. - * The second argument preserves the exact harness result, including stdout - * and stderr, before worktree inspection or verification can reject the - * shot. Throwing aborts the candidate so evidence persistence fails closed. */ + /** Complete author identity. Harness, provider, model, prompt, tools, and resources all come from here. */ + profile: AgentProfile + /** Place the exact profile on compute that can edit this existing worktree. + * A Pi author normally returns `{ backend:'bridge', cwd: worktreePath, ...transport }`. */ + executorForWorktree: AgenticGeneratorExecutorForWorktree + /** Awaited once for every attempted author shot, including execution failures. + * The second argument is Runtime's exact terminal turn and event stream. + * Throwing aborts the candidate so evidence persistence fails closed. */ onShotCompleted?: ( receipt: AgenticGeneratorShotReceipt, execution: AgenticGeneratorShotExecution | null, @@ -189,49 +159,33 @@ export interface AgenticGeneratorOptions { receipt: AgenticGeneratorShotReceipt, disposition: AgenticGeneratorShotDisposition, ) => void | Promise - /** Optional hard upper bound passed to the run-wide CostLedger before each - * author shot. This MUST be enforced by the provider or executor; a planning - * estimate is not an admissible bound. Omit for an uncapped ledger. A capped - * ledger rejects before model dispatch when this is absent. */ + /** Optional hard upper bound passed to the run-wide CostLedger before each author shot. */ maximumCharge?: MaximumCharge - /** Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). */ + /** Per-shot wall-clock timeout. Omit for no Runtime-imposed deadline. */ timeoutMs?: number - /** Build the harness task prompt from proposal findings. */ - buildPrompt?: (args: { findings: ReadonlyArray }) => string + /** Build the task prompt from proposal findings. Required: Runtime invents no authoring policy. */ + buildPrompt: (args: { findings: ReadonlyArray }) => string /** Verify the worktree after each dirtying shot. When set, a candidate that * fails verification is NOT returned — the failure feeds the next shot * (verify-in-session), up to `maxShots`; a candidate that never verifies is - * discarded (`applied:false`), never shipped. Omitted ⇒ legacy behavior: - * the first dirty shot is the candidate. See `commandVerifier`. */ + * discarded (`applied:false`), never shipped. Omitted means the first dirty + * shot is the candidate. See `commandVerifier`. */ verify?: Verifier - /** Test seam — inject the harness runner (defaults to `runLocalHarness`). */ - runHarness?: typeof runLocalHarness /** Test seam — inject the worktree-dirty check (defaults to `git status`). */ isDirty?: (worktreePath: string) => boolean } -/** Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. */ -export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateGenerator { - const harness = opts.harness ?? DEFAULT_LOCAL_HARNESS - // KEPT harness-name test: `codexReproducible` is a codex-SPECIFIC public option, so this - // asserts caller self-consistency and throws loudly instead of varying behavior by name. - if (opts.codexReproducible && harness !== 'codex') { - throw new Error("agenticGenerator: codexReproducible requires harness 'codex'") - } - if (opts.codexReproducible && !opts.profile) { - throw new Error('agenticGenerator: codexReproducible requires an explicit author profile') - } - if (opts.codexReadDeniedPaths && !opts.codexReproducible) { - throw new Error('agenticGenerator: codexReadDeniedPaths requires codexReproducible') - } - if (opts.maximumCharge && !opts.codexReproducible) { - throw new Error('agenticGenerator: maximumCharge requires codexReproducible') - } - const profileResourcePlan = opts.codexReproducible - ? authorProfileResourcePlan(opts.profile as AgentProfile) - : null - const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt - const run = opts.runHarness ?? runLocalHarness +/** Full-agentic `CandidateGenerator`: run an exact profiled author inside the existing candidate worktree. */ +export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenerator { + const profile = detachedSnapshot( + agentProfileSchema.parse(opts.profile) as AgentProfile, + 'agenticGenerator profile', + ) + assertExecutableAgentProfile(profile, 'agenticGenerator') + const harness = profile.harness! + const provider = profile.model!.provider!.trim() + const model = concreteProfileModel(profile)! + const profileDigest = canonicalAgentProfileDigest(profile) const dirty = opts.isDirty ?? worktreeDirty const verify = opts.verify @@ -254,16 +208,22 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG costPhase, }) { signal.throwIfAborted() - let reproducibleCostLedger: CostLedgerHandle | undefined - if (opts.codexReproducible) { - if (!costLedger) { - throw new Error( - 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval', - ) - } - reproducibleCostLedger = costLedger + if (opts.maximumCharge && !costLedger) { + throw new Error( + 'agenticGenerator: maximumCharge requires the run-wide CostLedger supplied by agent-eval', + ) } - const basePrompt = appendProfileResourcePaths(buildPrompt({ findings }), profileResourcePlan) + const basePrompt = opts.buildPrompt({ findings }) + if (typeof basePrompt !== 'string' || basePrompt.trim().length === 0) { + throw new Error('agenticGenerator: buildPrompt must return a non-empty string') + } + const executorConfig = opts.executorForWorktree(worktreePath) + if (executorConfig.backend === 'bridge' && executorConfig.cwd !== worktreePath) { + throw new Error( + 'agenticGenerator: bridge executor cwd must equal the candidate worktree path', + ) + } + const factory = createExecutor(executorConfig) const needsRawTraceEvidence = requiresRawTraceEvidence(findings) const shots = Math.max(1, maxShots) // Feedback appended to the base prompt for the NEXT shot — empty on shot 0. @@ -272,65 +232,30 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG for (let shot = 0; shot < shots; shot++) { signal.throwIfAborted() const taskPrompt = attemptNote ? `${basePrompt}\n\n${attemptNote}` : basePrompt - const invocation = opts.profile - ? harnessInvocation(harness, opts.profile, taskPrompt, { - // The candidate worktree is disposable; whether that needs argv, and which, is the - // harness capability row's answer, not a property of any one CLI's name. - dangerouslySkipPermissions: true, - ...(opts.codexReproducible ? { codexReproducible: true } : {}), - }) - : undefined - const exactPrompt = invocation?.prompt ?? taskPrompt - const readDeniedPaths = - typeof opts.codexReadDeniedPaths === 'function' - ? opts.codexReadDeniedPaths(worktreePath) - : opts.codexReadDeniedPaths const startedAt = new Date() - let harnessResult: LocalHarnessResult | null = null - let profileWorkspaceReceipt: WorkspacePlanReceipt | null = null + let turn: CollectedAgentTurn | null = null let costReceipt: CostReceipt | null = null let costCallId: string | null = null let shotError: Error | null = null try { - const execute = async (executionSignal: AbortSignal): Promise => { - harnessResult = await withAuthorProfileResources( - profileResourcePlan, - worktreePath, - async (receipt) => { - profileWorkspaceReceipt = receipt - const result = await run({ - harness, - cwd: worktreePath, - taskPrompt, - ...(invocation - ? { invocation: { command: invocation.command, args: invocation.args } } - : {}), - // The candidate worktree is isolated and must be editable without an - // interactive approval gate. Other runLocalHarness callers remain - // permission-safe by default. - dangerouslySkipPermissions: true, - ...(opts.codexReproducible ? { codexReproducible: true } : {}), - ...(readDeniedPaths ? { codexReadDeniedPaths: readDeniedPaths } : {}), - timeoutMs: opts.timeoutMs, - signal: executionSignal, - }) - // Assign before profile cleanup so a cleanup failure still - // settles the model usage already returned by the harness. - harnessResult = result - return result - }, + const execute = async ( + executionSignal: AbortSignal, + callId?: string, + ): Promise => { + turn = await collectAgentTurn( + streamAgentTurn({ kind: 'executor', profile, factory }, taskPrompt, { + signal: executionSignal, + ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), + ...(callId ? { callId } : {}), + }), ) - const failure = shotFailure(harnessResult, exactPrompt, opts.codexReproducible === true) + const failure = shotFailure(turn) if (failure) throw failure - return harnessResult + return turn } - if (reproducibleCostLedger) { - const model = opts.profile ? concreteProfileModel(opts.profile) : undefined - if (!model) { - throw new Error('agenticGenerator: reproducible Codex requires profile.model.default') - } - const paid = await reproducibleCostLedger.runPaidCall({ + if (costLedger) { + const paid = await costLedger.runPaidCall({ channel: 'driver', phase: costPhase ?? 'search.proposal', actor: `agentic-generator:${harness}`, @@ -342,34 +267,35 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG }, signal, ...(opts.maximumCharge ? { maximumCharge: opts.maximumCharge } : {}), - execute, - receipt: (result) => costReceiptFromHarness(result, model), - receiptFromError: () => - harnessResult?.usage ? costReceiptFromHarness(harnessResult, model) : undefined, + execute: (executionSignal, callId) => execute(executionSignal, callId), + receipt: (result) => costReceiptFromTurn(result, model), + receiptFromError: () => (turn ? costReceiptFromTurn(turn, model) : undefined), }) costCallId = paid.callId ?? null costReceipt = paid.receipt ?? null if (!paid.succeeded) throw paid.error - harnessResult = paid.value + turn = paid.value } else { - harnessResult = await execute(signal) + turn = await execute(signal) } } catch (cause) { shotError = cause instanceof Error ? cause : new Error(String(cause)) } - const execution = shotExecutionSnapshot(harnessResult) + const execution = turn ? detachedSnapshot(turn, 'agenticGenerator shot execution') : null const receipt = shotReceipt({ generation, candidateIndex, shot, maxShots: shots, harness, - profile: opts.profile, - prompt: exactPrompt, + provider, + model, + profile, + profileDigest, + prompt: taskPrompt, startedAt, completedAt: new Date(), result: execution, - profileWorkspaceReceipt, costCallId, costReceipt, error: shotError, @@ -378,7 +304,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG signal.throwIfAborted() if (!execution) { - throw new Error('agenticGenerator: author shot completed without a harness result') + throw new Error('agenticGenerator: author shot completed without a Runtime turn') } let worktreeChanged: boolean @@ -436,7 +362,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } // Dirty: with no verifier the diff IS the candidate (we trust the diff, - // not the harness's stdout). With a verifier the candidate must pass it. + // not the model's response text). With a verifier the candidate must pass it. if (!verify) { signal.throwIfAborted() await emitShotDisposition(opts.onShotDisposition, receipt, { @@ -490,112 +416,6 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } } -function authorProfileResourcePlan(profile: AgentProfile): WorkspacePlan | null { - const resources = profile.resources - if (!resources) return null - const unsupportedKinds = [ - resources.tools?.length ? 'tools' : null, - resources.skills?.length ? 'skills' : null, - resources.agents?.length ? 'agents' : null, - ].filter((kind): kind is string => kind !== null) - if (unsupportedKinds.length > 0) { - throw new Error( - `agenticGenerator: reproducible Codex author resources support files only; unsupported: ${unsupportedKinds.join(', ')}`, - ) - } - if (!resources.files || resources.files.length === 0) return null - - const plan = materializeProfile( - { - name: profile.name, - resources: { files: resources.files }, - }, - 'codex', - ) - if (plan.unsupported.length > 0) { - throw new Error( - `agenticGenerator: author profile files could not be materialized: ${plan.unsupported.map((item) => item.reason).join('; ')}`, - ) - } - if (Object.keys(plan.env).length > 0 || plan.flags.length > 0) { - throw new Error( - 'agenticGenerator: generic author profile files unexpectedly changed spawn values', - ) - } - - const virtualRoot = resolve('/', AGENTIC_PROFILE_RESOURCE_ROOT) - const seen = new Set() - for (const file of plan.files) { - const target = resolve('/', file.relPath) - if (!target.startsWith(`${virtualRoot}${sep}`)) { - throw new Error( - `agenticGenerator: author profile file must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}: ${file.relPath}`, - ) - } - if (seen.has(target)) { - throw new Error(`agenticGenerator: duplicate author profile file path: ${file.relPath}`) - } - seen.add(target) - } - return plan -} - -function appendProfileResourcePaths(prompt: string, plan: WorkspacePlan | null): string { - if (!plan) return prompt - return [ - prompt, - '', - 'Profile resource files available for this shot:', - ...plan.files.map((file) => `- ${file.relPath}`), - ].join('\n') -} - -async function withAuthorProfileResources( - plan: WorkspacePlan | null, - worktreePath: string, - run: (receipt: WorkspacePlanReceipt | null) => Promise, -): Promise { - if (!plan) return run(null) - - const rootPath = resolve(worktreePath, AGENTIC_PROFILE_RESOURCE_ROOT) - if (existsSync(rootPath)) { - throw new Error( - `agenticGenerator: ephemeral author profile root already exists: ${AGENTIC_PROFILE_RESOURCE_ROOT}`, - ) - } - - let value: T | undefined - let primaryError: unknown - try { - const receipt = applyWorkspacePlan(plan, worktreePath) - value = await run(receipt) - } catch (cause) { - primaryError = cause - } - - let cleanupError: unknown - try { - rmSync(rootPath, { recursive: true, force: true }) - if (existsSync(rootPath)) { - throw new Error( - `agenticGenerator: ephemeral author profile root survived cleanup: ${AGENTIC_PROFILE_RESOURCE_ROOT}`, - ) - } - } catch (cause) { - cleanupError = cause - } - - if (primaryError !== undefined && cleanupError !== undefined) { - throw new AggregateError( - [primaryError, cleanupError], - 'agenticGenerator: author shot and profile resource cleanup both failed', - ) - } - if (primaryError !== undefined) throw primaryError - if (cleanupError !== undefined) throw cleanupError - return value as T -} - async function emitShotReceipt( callback: AgenticGeneratorOptions['onShotCompleted'], receipt: AgenticGeneratorShotReceipt, @@ -648,43 +468,20 @@ async function rethrowShotSetupError( throw cause } -function shotExecutionSnapshot( - result: LocalHarnessResult | null, -): AgenticGeneratorShotExecution | null { - if (result === null) return null - const usage = result.usage ? Object.freeze({ ...result.usage }) : undefined - const evidence = result.evidence - ? Object.freeze({ - ...result.evidence, - readDeniedPaths: Object.freeze([...result.evidence.readDeniedPaths]), - policy: Object.freeze({ ...result.evidence.policy }), - }) - : undefined - return Object.freeze({ - exitCode: result.exitCode, - stdout: result.stdout, - stderr: result.stderr, - killedBySignal: result.killedBySignal, - durationMs: result.durationMs, - timedOut: result.timedOut, - ...(result.aborted !== undefined ? { aborted: result.aborted } : {}), - ...(usage ? { usage } : {}), - ...(evidence ? { evidence } : {}), - }) -} - function shotReceipt(input: { readonly generation: number | undefined readonly candidateIndex: number | undefined readonly shot: number readonly maxShots: number - readonly harness: LocalHarness - readonly profile: AgentProfile | undefined + readonly harness: HarnessType + readonly provider: string + readonly model: string + readonly profile: AgentProfile + readonly profileDigest: string readonly prompt: string readonly startedAt: Date readonly completedAt: Date readonly result: AgenticGeneratorShotExecution | null - readonly profileWorkspaceReceipt: WorkspacePlanReceipt | null readonly costCallId: string | null readonly costReceipt: CostReceipt | null readonly error: unknown @@ -696,92 +493,87 @@ function shotReceipt(input: { } : null const result = input.result - const costBasis = costBasisFor(input.costReceipt) + const costBasis = costBasisFor(input.costReceipt, result) + const costUsd = + input.costReceipt?.costUsd ?? result?.usage.costUsd ?? result?.usage.estimatedCostUsd ?? null return { generation: input.generation ?? null, candidateIndex: input.candidateIndex ?? null, shot: input.shot + 1, maxShots: input.maxShots, + profileDigest: input.profileDigest, harness: input.harness, - model: input.profile ? (concreteProfileModel(input.profile) ?? null) : null, - reasoningEffort: input.profile?.model?.reasoningEffort ?? null, + provider: input.provider, + model: input.model, + reasoningEffort: input.profile.model?.reasoningEffort ?? null, promptSha256: sha256(input.prompt), startedAt: input.startedAt.toISOString(), completedAt: input.completedAt.toISOString(), - durationMs: result?.durationMs ?? input.completedAt.getTime() - input.startedAt.getTime(), - exitCode: result?.exitCode ?? null, - timedOut: result?.timedOut ?? false, - aborted: result?.aborted ?? false, - killedBySignal: result?.killedBySignal ?? null, - stdoutBytes: result ? Buffer.byteLength(result.stdout) : null, - stdoutSha256: result ? sha256(result.stdout) : null, - stderrBytes: result ? Buffer.byteLength(result.stderr) : null, - stderrSha256: result ? sha256(result.stderr) : null, - usage: result?.usage ? { ...result.usage } : null, - profileWorkspacePlanDigest: input.profileWorkspaceReceipt?.workspacePlanDigest ?? null, - profileWorkspaceFileCount: input.profileWorkspaceReceipt?.written.length ?? 0, + durationMs: input.completedAt.getTime() - input.startedAt.getTime(), + status: result?.status ?? null, + usage: result ? { ...result.usage } : null, + transportAttempts: result?.transportAttempts ?? null, costCallId: input.costCallId, costBasis, - costUsd: costBasis === 'unknown' ? null : (input.costReceipt?.costUsd ?? null), + costUsd: costBasis === 'unknown' ? null : costUsd, costUsdKnown: costBasis === 'provider-reported', - evidence: result?.evidence - ? { - ...result.evidence, - readDeniedPaths: [...result.evidence.readDeniedPaths], - policy: { ...result.evidence.policy }, - } - : null, error, } } -function costBasisFor(receipt: CostReceipt | null): AgenticGeneratorShotReceipt['costBasis'] { - if (receipt === null || receipt.costUnknown) return 'unknown' - return receipt.actualCostUsd === undefined ? 'estimated-pricing' : 'provider-reported' +function costBasisFor( + receipt: CostReceipt | null, + turn: AgenticGeneratorShotExecution | null, +): AgenticGeneratorShotReceipt['costBasis'] { + if (receipt) { + if (receipt.costUnknown) return 'unknown' + return receipt.actualCostUsd === undefined ? 'estimated-pricing' : 'provider-reported' + } + if (turn?.usage.usdKnown !== false && turn?.usage.costUsd !== undefined) { + return 'provider-reported' + } + return turn?.usage.estimatedCostUsd !== undefined ? 'estimated-pricing' : 'unknown' } -function shotFailure( - result: LocalHarnessResult, - exactPrompt: string, - codexReproducible: boolean, -): Error | null { - if (result.timedOut) { - return new Error('agenticGenerator: author shot timed out') - } - if (result.aborted) { +function shotFailure(result: CollectedAgentTurn): Error | null { + if (result.status === 'completed') return null + if (result.status === 'aborted') { return new Error('agenticGenerator: author shot was cancelled by the caller') } - if (result.killedBySignal) { - return new Error(`agenticGenerator: author shot was killed by ${result.killedBySignal}`) - } - if (result.exitCode !== 0) { - return new Error(`agenticGenerator: author shot exited with code ${String(result.exitCode)}`) - } - if (!codexReproducible) return null - if (!result.usage || !result.evidence) { - return new Error( - 'agenticGenerator: reproducible Codex shot completed without usage or execution evidence', - ) - } - const expectedPromptSha256 = sha256(exactPrompt).slice('sha256:'.length) - if (result.evidence.requestedPromptSha256 !== expectedPromptSha256) { - return new Error( - 'agenticGenerator: reproducible Codex prompt evidence does not match the exact authored prompt', - ) - } - return null + return new Error( + `agenticGenerator: author shot failed${result.error?.message ? `: ${result.error.message}` : ''}`, + ) } -function costReceiptFromHarness(result: LocalHarnessResult, model: string): CostReceiptInput { - if (!result.usage) { - throw new Error('agenticGenerator: author shot did not report terminal token usage') - } +function costReceiptFromTurn(result: CollectedAgentTurn, model: string): CostReceiptInput { + const tokensKnown = result.usage.tokensKnown !== false + const actualCostUsd = result.usage.usdKnown === false ? undefined : result.usage.costUsd + const cachedTokens = promptCacheReadTokens(result.usage) return { model, - inputTokens: result.usage.inputTokens - result.usage.cachedInputTokens, - outputTokens: result.usage.outputTokens, - ...(result.usage.cachedInputTokens > 0 ? { cachedTokens: result.usage.cachedInputTokens } : {}), + inputTokens: tokensKnown ? Math.max(0, result.usage.input - (cachedTokens ?? 0)) : 0, + outputTokens: tokensKnown ? result.usage.output : 0, + ...(tokensKnown ? {} : { usageUnknown: true }), + ...(cachedTokens !== undefined ? { cachedTokens } : {}), + ...(result.usage.reasoningTokens !== undefined + ? { reasoningTokens: result.usage.reasoningTokens } + : {}), + ...(actualCostUsd !== undefined + ? { actualCostUsd } + : result.usage.estimatedCostUsd !== undefined + ? { estimatedCostUsd: result.usage.estimatedCostUsd } + : { costUnknown: true }), + } +} + +function promptCacheReadTokens(usage: AgentTurnUsage): number | undefined { + const cache = usage.promptCache + if (!cache) return undefined + for (const field of ['readTokens', 'cachedInputTokens', 'cacheReadInputTokens']) { + const value = cache[field] + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) return value } + return undefined } function sha256(value: string): `sha256:${string}` { diff --git a/src/improvement/build-prompts.ts b/src/improvement/build-prompts.ts index e967a82c..4dab20db 100644 --- a/src/improvement/build-prompts.ts +++ b/src/improvement/build-prompts.ts @@ -5,9 +5,11 @@ * server is built by composing the pieces directly: * * // a tool: - * agenticGenerator({ buildPrompt: toolBuildPrompt, verify: commandVerifier('pnpm', ['test']) }) + * agenticGenerator({ profile, executorForWorktree, buildPrompt: toolBuildPrompt, + * verify: commandVerifier('pnpm', ['test']) }) * // an MCP server: - * agenticGenerator({ buildPrompt: mcpBuildPrompt, verify: mcpServeVerifier({ command: 'node', args: ['server.mjs'] }) }) + * agenticGenerator({ profile, executorForWorktree, buildPrompt: mcpBuildPrompt, + * verify: mcpServeVerifier({ command: 'node', args: ['server.mjs'] }) }) * * These are the only type-specific bit (the phrasing that points the agent at a * tool vs. an MCP); the worktree, resume-on-failure loop, and improvement-loop diff --git a/src/improvement/code-execution.ts b/src/improvement/code-execution.ts index 2c6a230c..0309de8c 100644 --- a/src/improvement/code-execution.ts +++ b/src/improvement/code-execution.ts @@ -15,7 +15,9 @@ import { type SurfaceProposer, selfImprove, } from '@tangle-network/agent-eval/contract' +import { agentProfileSchema } from '@tangle-network/agent-interface' import { immutableCandidateValue } from '../candidate-execution/digest' +import { assertExecutableAgentProfile } from '../runtime/supervise/model-policy' import { agenticGenerator } from './agentic-generator' import { rethrowAfterCleanup } from './cleanup' import { copyImproveCost } from './improve-result' @@ -140,6 +142,8 @@ async function discardPreparedBaseline( /** Create a clean incumbent checkout and the candidate producer for a code run. */ async function prepareCodeRun(code: ImproveCodeOptions): Promise { + const authorProfile = agentProfileSchema.parse(code.profile) + assertExecutableAgentProfile(authorProfile, 'improve(code) author') const baseRef = code.baseRef ?? 'main' const worktree = code.worktree ?? @@ -151,13 +155,16 @@ async function prepareCodeRun(code: ImproveCodeOptions): Promise }) => string /** Verify a candidate worktree before it becomes a measurable surface; failures * feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). */ verify?: Verifier - /** Per-shot wall-clock timeout for the harness (ms). */ + /** Per-shot wall-clock timeout. Omit for no Runtime-imposed deadline. */ timeoutMs?: number - /** Byte-producer override, used for tests and custom candidate production. - * When set, `harness`, `verify`, and `timeoutMs` are unused. */ - generator?: CandidateGenerator + /** Optional provider-enforced maximum admitted by the run-wide cost ledger. */ + maximumCharge?: MaximumCharge + generator?: never } +export interface ImproveCustomCodeGeneratorOptions { + /** Complete byte-producer replacement. Runtime still validates `profile` before creating worktrees. */ + generator: CandidateGenerator + executorForWorktree?: never + buildPrompt?: never + verify?: never + timeoutMs?: never + maximumCharge?: never +} + +export type ImproveCodeOptions = ImproveCodeBaseOptions & + (ImproveRuntimeCodeGeneratorOptions | ImproveCustomCodeGeneratorOptions) + export interface ImprovementProfileCandidate { /** Surface searched by this run. */ surface: ImproveProfileSurface diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 0cd6434a..2101f49b 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -134,6 +134,13 @@ const promptProfile = (): AgentProfile => ({ prompt: { systemPrompt: 'baseline' }, }) +const codeAuthorProfile = (): AgentProfile => ({ + name: 'code-author', + harness: 'cli-base', + model: { provider: 'offline', default: 'deterministic-code-author' }, + prompt: { systemPrompt: 'Edit the candidate worktree.' }, +}) + describe('improve method execution', () => { it('runs a complete method without exposing final-test cases and materializes its prompt', async () => { let observed: OptimizationMethodInput | undefined @@ -764,6 +771,7 @@ describe('improve code execution', () => { agent: paidCodeText, code: { repoRoot: repo.repoRoot, + profile: codeAuthorProfile(), generator: { kind: 'test-generator', async generate({ worktreePath }: { worktreePath: string }) { @@ -814,6 +822,7 @@ describe('improve code execution', () => { agent: paidCodeText, code: { repoRoot: repo.repoRoot, + profile: codeAuthorProfile(), generator: { kind: 'must-not-run', async generate() { @@ -849,6 +858,7 @@ describe('improve code execution', () => { agent: paidCodeText, code: { repoRoot: repo.repoRoot, + profile: codeAuthorProfile(), generator: { kind: 'rejecting-generator', async generate() { @@ -890,6 +900,7 @@ describe('improve code execution', () => { code: { repoRoot: repo.repoRoot, worktree: rejectingWorktree, + profile: codeAuthorProfile(), generator: { kind: 'unused', async generate() { diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 65e88f5a..f9226902 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -26,10 +26,12 @@ import { runMethodImprovement } from './method-execution' export type { ImproveCandidateValidationInput, ImproveCandidateValidator, + ImproveCodeBaseOptions, ImproveCodeOptions, ImproveCodeResult, ImproveCodeRunOptions, ImproveCost, + ImproveCustomCodeGeneratorOptions, ImproveLineage, ImproveMethodContext, ImproveMethodFactory, @@ -46,6 +48,7 @@ export type { ImproveProfileComponents, ImproveProfileSurface, ImproveResult, + ImproveRuntimeCodeGeneratorOptions, ImproveScenarioPartitions, ImproveSkillsOptions, ImproveSurface, diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 054ebbe8..11ed901e 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -7,7 +7,7 @@ */ export { - AGENTIC_PROFILE_RESOURCE_ROOT, + type AgenticGeneratorExecutorForWorktree, type AgenticGeneratorOptions, type AgenticGeneratorShotDisposition, type AgenticGeneratorShotExecution, @@ -27,10 +27,12 @@ export { export { type ImproveCandidateValidationInput, type ImproveCandidateValidator, + type ImproveCodeBaseOptions, type ImproveCodeOptions, type ImproveCodeResult, type ImproveCodeRunOptions, type ImproveCost, + type ImproveCustomCodeGeneratorOptions, type ImproveLineage, type ImproveMethodContext, type ImproveMethodFactory, @@ -47,6 +49,7 @@ export { type ImproveProfileComponents, type ImproveProfileSurface, type ImproveResult, + type ImproveRuntimeCodeGeneratorOptions, type ImproveScenarioPartitions, type ImproveSkillsOptions, type ImproveSurface, diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index 1af37b9f..f3b67257 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -80,7 +80,7 @@ interface CellTrace { * await improve({ * surface: 'code', * findings: seedFindings, - * code: { repoRoot }, + * code: { repoRoot, profile, executorForWorktree, buildPrompt }, * runDir: '/abs/run', // MUST be a real path — the traces live here * analyzeGeneration: rawTraceDistiller(), * scenarios, judge, agent, diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index d0fa05cc..6d6baa53 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -1,46 +1,28 @@ import { execFileSync } from 'node:child_process' -import { createHash } from 'node:crypto' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { setTimeout as delay } from 'node:timers/promises' import { CostLedger, makeProposalFinding, type ProposalFinding } from '@tangle-network/agent-eval' -import { - gitWorktreeAdapter, - inMemoryCampaignStorage, - isProposedCandidate, - type JudgeConfig, - type ProposeContext, - runOptimization, - type Scenario, -} from '@tangle-network/agent-eval/campaign' +import { gitWorktreeAdapter } from '@tangle-network/agent-eval/campaign' +import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { - AGENTIC_PROFILE_RESOURCE_ROOT, + type AgenticGeneratorExecutorForWorktree, type AgenticGeneratorShotDisposition, type AgenticGeneratorShotExecution, type AgenticGeneratorShotReceipt, agenticGenerator, commandVerifier, } from '../src/improvement' -import { improvementDriver } from '../src/improvement/improvement-driver' -import type { LocalHarnessResult } from '../src/mcp/local-harness' function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() } -async function waitForPath(path: string, timeoutMs = 1_000): Promise { - const deadline = Date.now() + timeoutMs - while (!existsSync(path)) { - if (Date.now() >= deadline) throw new Error(`timed out waiting for ${path}`) - await delay(10) - } -} - let repoRoot: string + beforeEach(() => { - repoRoot = mkdtempSync(join(tmpdir(), 'agentic-repo-')) + repoRoot = mkdtempSync(join(tmpdir(), 'agentic-generator-')) git(['init', '-q', '-b', 'main'], repoRoot) git(['config', 'user.email', 'test@test.dev'], repoRoot) git(['config', 'user.name', 'Test'], repoRoot) @@ -51,1187 +33,500 @@ beforeEach(() => { git(['add', '-A'], repoRoot) git(['commit', '-q', '-m', 'init'], repoRoot) }) + afterEach(() => rmSync(repoRoot, { recursive: true, force: true })) +const PROFILE: AgentProfile = { + name: 'code-author', + harness: 'cli-base', + model: { + provider: 'offline-router', + default: 'deterministic-author', + reasoningEffort: 'high', + }, + prompt: { + systemPrompt: 'AUTHOR SYSTEM', + instructions: ['Edit only the candidate worktree.'], + }, +} + const FINDINGS = [ makeProposalFinding({ - analyst_id: 'a1', + analyst_id: 'test', proposal_origin: 'production', severity: 'high', area: 'correctness', claim: 'x should be 2', recommended_action: 'set x to 2', evidence_refs: [], - confidence: 0.9, + confidence: 1, subject: 'app.ts', - produced_at: '2026-01-01', }), ] -const TRACE_PATH = '/tmp/run/gen-0/candidate-0/task_0/spans.jsonl' +const TRACE_PATH = '/tmp/discovery/run-1/spans.jsonl' const RAW_TRACE_FINDINGS = [ makeProposalFinding({ analyst_id: 'raw-trace-distiller', proposal_origin: 'search', severity: 'high', area: 'raw-trace-context', - claim: 'candidate failed after reading stale state', - recommended_action: `grep/cat ${TRACE_PATH} before editing`, + claim: 'the candidate read stale state', + recommended_action: 'inspect the cited trace', evidence_refs: [{ kind: 'artifact', uri: TRACE_PATH }], confidence: 1, - subject: 'candidate-hash', - produced_at: '2026-01-01', + subject: 'candidate', }), ] -const HARNESS_OK: LocalHarnessResult = { - exitCode: 0, - stdout: 'done', - stderr: '', - killedBySignal: null, - durationMs: 10, - timedOut: false, +interface RoutedShot { + readonly worktreePath: string + readonly body: Record + readonly request: + | { + readonly headers: Readonly> + readonly signal?: AbortSignal + } + | undefined + readonly call: number } -const CODEX_USAGE = { - inputTokens: 120, - cachedInputTokens: 20, - outputTokens: 30, - reasoningOutputTokens: 10, -} - -const CODEX_EVIDENCE = { - cliVersion: 'codex-cli 1.0.0', - executableSha256: 'a'.repeat(64), - requestedPromptSha256: 'b'.repeat(64), - effectivePromptSha256: 'b'.repeat(64), - nonPromptArgsSha256: 'c'.repeat(64), - controlledConfigSha256: 'd'.repeat(64), - readDeniedPaths: ['/tmp/denied'], - readDeniedPathsSha256: 'e'.repeat(64), - readDeniedPathCount: 1, - policy: {}, -} as NonNullable +type RoutedAction = (shot: RoutedShot) => unknown | Promise -function ctx( - findings: ReadonlyArray, - maxShots = 1, -): ProposeContext { +function successfulCompletion(body: Record): Record { return { - currentSurface: '', - history: [], - findings, - populationSize: 1, - generation: 0, - signal: new AbortController().signal, - maxImprovementShots: maxShots, + model: body.model, + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'done' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 12, + completion_tokens: 3, + cost_usd: 0.012, + prompt_tokens_details: { cached_tokens: 8 }, + }, } } -describe('agenticGenerator — runs a harness in the worktree', () => { - it('pins the author profile and emits exact usage for every reproducible Codex shot', async () => { - const receipts: AgenticGeneratorShotReceipt[] = [] - const costLedger = new CostLedger() - const resourcePath = `${AGENTIC_PROFILE_RESOURCE_ROOT}/trace-analysis.md` - const profile = { - name: 'structural-author', - prompt: { - systemPrompt: 'AUTHOR SYSTEM', - instructions: ['Edit only the allowed implementation.'], - }, - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' as const }, - resources: { - files: [ - { - path: resourcePath, - resource: { - kind: 'inline' as const, - name: 'trace-analysis', - content: 'Repeated state reads caused stale edits.\n', - }, - }, - ], - }, - } - const runHarness = vi.fn( - async (options: { - cwd: string - taskPrompt: string - invocation?: { command?: string; args: ReadonlyArray } - codexReproducible?: boolean - codexReadDeniedPaths?: ReadonlyArray - }) => { - expect(options.codexReproducible).toBe(true) - expect(options.invocation?.command).toBe('codex') - expect(options.invocation?.args).toContain('gpt-5.4') - expect(options.invocation?.args.join('\n')).toContain('AUTHOR SYSTEM') - expect(options.taskPrompt).toContain(resourcePath) - expect(readFileSync(join(options.cwd, resourcePath), 'utf8')).toContain('stale edits') - expect(options.codexReadDeniedPaths).toEqual([`${options.cwd}/private-evidence`]) - writeFileSync(join(options.cwd, 'app.ts'), 'export const x = 2\n') - const prompt = options.invocation?.args[1] - if (!prompt) throw new Error('test invocation omitted the composed prompt') - const promptSha256 = createHash('sha256').update(prompt).digest('hex') - return { - ...HARNESS_OK, - usage: CODEX_USAGE, - evidence: { - ...CODEX_EVIDENCE, - requestedPromptSha256: promptSha256, - effectivePromptSha256: 'f'.repeat(64), - }, - } - }, - ) - const gen = agenticGenerator({ - harness: 'codex', - profile, - codexReproducible: true, - codexReadDeniedPaths: (worktreePath) => [`${worktreePath}/private-evidence`], - onShotCompleted: (receipt) => receipts.push(receipt), - runHarness: runHarness as never, - }) - - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'reproducible-receipt', - }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 2, - signal: new AbortController().signal, - generation: 4, - candidateIndex: 2, - costLedger, - costPhase: 'search.proposal', - }) - - const [costReceipt] = costLedger.list() - expect(out.applied).toBe(true) - expect(receipts).toHaveLength(1) - expect(receipts[0]).toMatchObject({ - generation: 4, - candidateIndex: 2, - shot: 1, - maxShots: 2, - harness: 'codex', - model: 'gpt-5.4', - reasoningEffort: 'xhigh', - usage: CODEX_USAGE, - profileWorkspacePlanDigest: expect.any(String), - profileWorkspaceFileCount: 1, - costCallId: expect.any(String), - costBasis: 'estimated-pricing', - costUsdKnown: false, - error: null, - }) - expect(receipts[0]?.costCallId).toBe(costReceipt?.callId) - expect(receipts[0]?.costUsd).toBeCloseTo(0.00045) - expect(receipts[0]?.promptSha256).toMatch(/^sha256:[a-f0-9]{64}$/) - expect(receipts[0]?.stdoutSha256).toBe( - `sha256:${createHash('sha256').update(HARNESS_OK.stdout).digest('hex')}`, - ) - expect(receipts[0]?.evidence?.readDeniedPathCount).toBe(1) - expect(receipts[0]?.evidence?.effectivePromptSha256).not.toBe( - receipts[0]?.evidence?.requestedPromptSha256, - ) - expect(existsSync(join(wt.path, AGENTIC_PROFILE_RESOURCE_ROOT))).toBe(false) - expect(git(['status', '--short'], wt.path)).toBe('M app.ts') - expect(costLedger.list()).toEqual([ - expect.objectContaining({ - channel: 'driver', - phase: 'search.proposal', - actor: 'agentic-generator:codex', - model: 'gpt-5.4', - tags: { generation: '4', candidateIndex: '2', shot: '1' }, - inputTokens: 100, - cachedTokens: 20, - outputTokens: 30, - costUsd: 0.00045, - costUnknown: false, - }), - ]) - }) - - it('exposes exact execution before a clean-tree shot is rejected', async () => { - const events: string[] = [] - const executions: AgenticGeneratorShotExecution[] = [] - const dispositions: AgenticGeneratorShotDisposition[] = [] - const exactExecution: LocalHarnessResult = { - exitCode: 0, - stdout: 'exact stdout\nwith a second line\n', - stderr: 'exact stderr\n', - killedBySignal: null, - durationMs: 4321, - timedOut: false, - usage: CODEX_USAGE, - evidence: CODEX_EVIDENCE, - } - const isDirty = vi.fn(() => { - events.push('dirty-check') - expect(events).toEqual(['callback-complete', 'dirty-check']) - return false - }) - const gen = agenticGenerator({ - runHarness: (async () => exactExecution) as never, - isDirty, - onShotCompleted: async (receipt, execution) => { - await Promise.resolve() - expect(execution).not.toBeNull() - if (!execution) throw new Error('expected a completed execution') - executions.push(execution) - expect(execution).toEqual(exactExecution) - expect(Object.isFrozen(execution)).toBe(true) - expect(Object.isFrozen(execution.usage)).toBe(true) - expect(Object.isFrozen(execution.evidence)).toBe(true) - expect(Object.isFrozen(execution.evidence?.readDeniedPaths)).toBe(true) - expect(Object.isFrozen(execution.evidence?.policy)).toBe(true) - expect(() => Object.assign(execution, { stdout: 'mutated' })).toThrow() - expect(receipt.stdoutBytes).toBe(Buffer.byteLength(execution.stdout)) - expect(receipt.stderrBytes).toBe(Buffer.byteLength(execution.stderr)) - expect(receipt.stdoutSha256).toBe( - `sha256:${createHash('sha256').update(execution.stdout).digest('hex')}`, - ) - expect(receipt.stderrSha256).toBe( - `sha256:${createHash('sha256').update(execution.stderr).digest('hex')}`, - ) - events.push('callback-complete') - }, - onShotDisposition: async (_receipt, disposition) => { - await Promise.resolve() - expect(disposition.kind).toBe('clean') - expect(disposition.worktreePath).toContain('exact-clean-shot') - dispositions.push(disposition) - events.push('disposition-complete') - }, - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'exact-clean-shot', - }) - - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }) - - expect(out.applied).toBe(false) - expect(executions).toEqual([exactExecution]) - expect(dispositions).toEqual([ - expect.objectContaining({ kind: 'clean', worktreePath: wt.path }), - ]) - expect(events).toEqual(['callback-complete', 'dirty-check', 'disposition-complete']) - expect(isDirty).toHaveBeenCalledTimes(1) - }) - - it('awaits shot evidence before verification can return a candidate', async () => { - let evidencePersisted = false - let dispositionPersisted = false - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK - }) - const verify = vi.fn(() => { - expect(evidencePersisted).toBe(true) - return { ok: true } - }) - const gen = agenticGenerator({ - runHarness: runHarness as never, - onShotCompleted: async () => { - await Promise.resolve() - evidencePersisted = true - }, - onShotDisposition: async (_receipt, disposition) => { - await Promise.resolve() - expect(disposition).toEqual({ - kind: 'accepted', - worktreePath: expect.stringContaining('evidence-before-verify'), - verified: true, - }) - dispositionPersisted = true - }, - verify, - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'evidence-before-verify', - }) - - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }) - - expect(out.applied).toBe(true) - expect(dispositionPersisted).toBe(true) - expect(verify).toHaveBeenCalledTimes(1) +function routedExecutor(action: RoutedAction): AgenticGeneratorExecutorForWorktree { + let call = 0 + return (worktreePath) => ({ + backend: 'router', + routerBaseUrl: 'https://offline.invalid/v1', + routerKey: 'test-key', + complete: async (body, request) => { + call += 1 + const reply = await action({ worktreePath, body, request, call }) + return reply === undefined ? successfulCompletion(body) : reply + }, }) +} - it('fails closed when shot evidence persistence throws', async () => { - const isDirty = vi.fn(() => false) - const gen = agenticGenerator({ - runHarness: (async () => HARNESS_OK) as never, - isDirty, - onShotCompleted: () => { - throw new Error('shot evidence persistence failed') - }, - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'evidence-failure', - }) - - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow('shot evidence persistence failed') - expect(isDirty).not.toHaveBeenCalled() - }) +function buildPrompt(args: { findings: ReadonlyArray }): string { + return `Fix the candidate from these findings:\n${args.findings.map((f) => f.claim).join('\n')}` +} - it('fails closed when worktree disposition persistence throws', async () => { - const gen = agenticGenerator({ - runHarness: (async () => HARNESS_OK) as never, - isDirty: () => false, - onShotDisposition: () => { - throw new Error('shot disposition persistence failed') - }, - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ +async function candidateWorktree(label: string): Promise { + return ( + await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', - label: 'disposition-failure', + label, }) + ).path +} - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow('shot disposition persistence failed') - }) +function generateArgs( + worktreePath: string, + findings: ReadonlyArray = FINDINGS, + maxShots = 1, +) { + return { + worktreePath, + findings, + maxShots, + signal: new AbortController().signal, + generation: 2, + candidateIndex: 1, + } +} - it('persists a setup-error disposition before rethrowing inspection failure', async () => { - const dispositions: AgenticGeneratorShotDisposition[] = [] - const gen = agenticGenerator({ - runHarness: (async () => HARNESS_OK) as never, - isDirty: () => { - throw new Error('git status unavailable') - }, - onShotDisposition: (_receipt, disposition) => { - dispositions.push(disposition) - }, - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'inspection-error', - }) +function messages(body: Record): Array<{ role?: string; content?: unknown }> { + return (body.messages ?? []) as Array<{ role?: string; content?: unknown }> +} - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow('git status unavailable') - expect(dispositions).toEqual([ - { - kind: 'setup-error', - worktreePath: wt.path, - stage: 'worktree-inspection', - error: { name: 'Error', message: 'git status unavailable' }, - }, - ]) - }) +describe('agenticGenerator exact Runtime execution', () => { + it('rejects an incomplete author profile before allocating an executor', () => { + const executorForWorktree = vi.fn() - it('rejects profile files outside the dedicated ephemeral root before dispatch', () => { expect(() => agenticGenerator({ - harness: 'codex', - profile: { - name: 'author', - model: { default: 'gpt-5.4' }, - resources: { - files: [ - { - path: 'research.md', - resource: { kind: 'inline', name: 'research', content: 'private evidence' }, - }, - ], - }, - }, - codexReproducible: true, + profile: { name: 'incomplete' }, + executorForWorktree, + buildPrompt, }), - ).toThrow(new RegExp(`must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}`)) + ).toThrow(/executable|harness|model/i) + expect(executorForWorktree).not.toHaveBeenCalled() }) - it('emits a failed shot receipt before rethrowing a harness failure', async () => { - const receipts: unknown[] = [] + it('runs the exact profile through Runtime and records provider usage', async () => { + const receipts: AgenticGeneratorShotReceipt[] = [] const executions: Array = [] - const gen = agenticGenerator({ - runHarness: (async () => { - throw new Error('author process failed') - }) as never, - onShotCompleted: (receipt, execution) => { + let observedBody: Record | undefined + const executorForWorktree = routedExecutor(({ worktreePath, body }) => { + observedBody = body + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }) + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree, + buildPrompt, + onShotCompleted(receipt, execution) { receipts.push(receipt) executions.push(execution) }, }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'failed' }) + const worktreePath = await candidateWorktree('exact-profile') - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow('author process failed') - expect(receipts).toHaveLength(1) - expect(executions).toEqual([null]) - expect(receipts[0]).toMatchObject({ - usage: null, - evidence: null, - error: { name: 'Error', message: 'author process failed' }, - }) - }) + const result = await generator.generate(generateArgs(worktreePath)) - it('fails closed when reproducible Codex completes without token usage', async () => { - const receipts: unknown[] = [] - const costLedger = new CostLedger() - const gen = agenticGenerator({ - harness: 'codex', - profile: { - name: 'author', - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, - }, - codexReproducible: true, - runHarness: (async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return { ...HARNESS_OK, evidence: CODEX_EVIDENCE } - }) as never, - onShotCompleted: (receipt) => receipts.push(receipt), - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'no-usage' }) - - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - costLedger, - }), - ).rejects.toThrow(/without usage or execution evidence/) + expect(result.applied).toBe(true) + expect(observedBody?.model).toBe('deterministic-author') + expect( + messages(observedBody ?? {}).find((message) => message.role === 'system')?.content, + ).toContain('AUTHOR SYSTEM') + expect( + messages(observedBody ?? {}).find((message) => message.role === 'user')?.content, + ).toContain('x should be 2') + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 2\n') expect(receipts).toHaveLength(1) expect(receipts[0]).toMatchObject({ - usage: null, - costCallId: expect.any(String), - costBasis: 'unknown', - costUsd: null, - costUsdKnown: false, - error: { message: expect.stringMatching(/without usage or execution evidence/) }, - }) - expect(costLedger.list()).toEqual([ - expect.objectContaining({ - callId: expect.any(String), - usageUnknown: true, - costUnknown: true, - }), - ]) - }) - - it('refuses reproducible Codex before dispatch when the run-wide ledger is absent', async () => { - const runHarness = vi.fn() - const gen = agenticGenerator({ - harness: 'codex', - profile: { - name: 'author', - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + generation: 2, + candidateIndex: 1, + shot: 1, + maxShots: 1, + harness: 'cli-base', + provider: 'offline-router', + model: 'deterministic-author', + reasoningEffort: 'high', + status: 'completed', + usage: { + input: 12, + output: 3, + costUsd: 0.012, }, - codexReproducible: true, - runHarness: runHarness as never, + costBasis: 'provider-reported', + costUsd: 0.012, + costUsdKnown: true, }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'no-ledger', - }) - - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow(/requires the run-wide CostLedger/) - expect(runHarness).not.toHaveBeenCalled() + expect(receipts[0]?.usage?.promptCache).toEqual({ readTokens: 8 }) + expect(executions[0]?.status).toBe('completed') + expect(executions[0]?.events.at(-1)?.type).toBe('final') }) - it('lets a capped ledger reject an unbounded author shot before model dispatch', async () => { + it('forwards the shared ledger identity and records one paid shot', async () => { + const ledger = new CostLedger() const receipts: AgenticGeneratorShotReceipt[] = [] - const costLedger = new CostLedger({ costCeilingUsd: 1 }) - const runHarness = vi.fn() - const gen = agenticGenerator({ - harness: 'codex', - profile: { - name: 'author', - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, - }, - codexReproducible: true, - runHarness: runHarness as never, + let idempotencyKey: string | undefined + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, request }) => { + idempotencyKey = + request?.headers['idempotency-key'] ?? + request?.headers['Idempotency-Key'] ?? + request?.headers['x-idempotency-key'] + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }), + buildPrompt, onShotCompleted: (receipt) => receipts.push(receipt), }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'capped-without-maximum', + const worktreePath = await candidateWorktree('ledger') + + const result = await generator.generate({ + ...generateArgs(worktreePath), + costLedger: ledger, + costPhase: 'search.proposal', }) - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - costLedger, - }), - ).rejects.toThrow(/hard maximumCharge before execution/) - expect(runHarness).not.toHaveBeenCalled() - expect(costLedger.list()).toHaveLength(0) - expect(receipts).toEqual([ - expect.objectContaining({ - costCallId: expect.any(String), - costBasis: 'unknown', - costUsd: null, - costUsdKnown: false, - usage: null, - error: expect.objectContaining({ - message: expect.stringMatching(/hard maximumCharge before execution/), - }), - }), - ]) + const settled = ledger.list() + expect(result.applied).toBe(true) + expect(settled).toHaveLength(1) + expect(settled[0]).toMatchObject({ inputTokens: 4, cachedTokens: 8, outputTokens: 3 }) + expect(receipts[0]?.costCallId).toBe(settled[0]?.callId) + expect(receipts[0]?.costCallId).toEqual(expect.any(String)) + if (idempotencyKey !== undefined) expect(idempotencyKey).toBe(receipts[0]?.costCallId) }) - it('records terminal usage and rejects partial edits from a failed author process', async () => { + it('keeps missing token and dollar measurements explicitly unknown', async () => { + const ledger = new CostLedger() const receipts: AgenticGeneratorShotReceipt[] = [] - const costLedger = new CostLedger() - const runHarness = vi.fn( - async (options: { cwd: string; invocation?: { args: ReadonlyArray } }) => { - expect( - readFileSync(join(options.cwd, AGENTIC_PROFILE_RESOURCE_ROOT, 'failure.md'), 'utf8'), - ).toBe('failure context\n') - writeFileSync(join(options.cwd, 'app.ts'), 'export const partial = true\n') - const prompt = options.invocation?.args[1] - if (!prompt) throw new Error('test invocation omitted the composed prompt') + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, body }) => { + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') return { - ...HARNESS_OK, - exitCode: 2, - usage: CODEX_USAGE, - evidence: { - ...CODEX_EVIDENCE, - requestedPromptSha256: createHash('sha256').update(prompt).digest('hex'), - }, + model: body.model, + choices: [{ message: { role: 'assistant', content: 'done' }, finish_reason: 'stop' }], } - }, - ) - const gen = agenticGenerator({ - harness: 'codex', - profile: { - name: 'author', - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, - resources: { - files: [ - { - path: `${AGENTIC_PROFILE_RESOURCE_ROOT}/failure.md`, - resource: { kind: 'inline', name: 'failure', content: 'failure context\n' }, - }, - ], - }, - }, - codexReproducible: true, - runHarness: runHarness as never, - onShotCompleted: (receipt) => receipts.push(receipt), - }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'failed-partial-edit', - }) - - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - generation: 1, - candidateIndex: 0, - costLedger, - }), - ).rejects.toThrow(/exited with code 2/) - expect(existsSync(join(wt.path, AGENTIC_PROFILE_RESOURCE_ROOT))).toBe(false) - expect(git(['status', '--short'], wt.path)).toBe('M app.ts') - expect(costLedger.list()).toEqual([ - expect.objectContaining({ - callId: expect.any(String), - inputTokens: 100, - cachedTokens: 20, - outputTokens: 30, - costUnknown: false, }), - ]) - expect(receipts).toEqual([ - expect.objectContaining({ - exitCode: 2, - costCallId: expect.any(String), - costBasis: 'estimated-pricing', - costUsdKnown: false, - usage: CODEX_USAGE, - error: expect.objectContaining({ - message: 'agenticGenerator: author shot exited with code 2', - }), - }), - ]) - }) - - it('rejects partial edits when caller cancellation exits the author process zero', async () => { - const receipts: AgenticGeneratorShotReceipt[] = [] - // Process boundary only; local-harness.test.ts proves this result with a real TERM-aware tree. - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const cancelledPartial = true\n') - return { ...HARNESS_OK, aborted: true } - }) - const gen = agenticGenerator({ - runHarness: runHarness as never, + buildPrompt, onShotCompleted: (receipt) => receipts.push(receipt), }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'cancelled-partial-edit', - }) + const worktreePath = await candidateWorktree('unknown-usage') - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }), - ).rejects.toThrow(/cancelled by the caller/) - expect(runHarness).toHaveBeenCalledTimes(1) - expect(git(['status', '--short'], wt.path)).toBe('M app.ts') - expect(receipts).toEqual([ - expect.objectContaining({ - exitCode: 0, - timedOut: false, - aborted: true, - killedBySignal: null, - error: expect.objectContaining({ - message: expect.stringMatching(/cancelled by the caller/), - }), - }), - ]) - }) - - it('rejects a partial edit when the caller aborts after a normal author result', async () => { - const controller = new AbortController() - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const cancelledAfterRun = true\n') - controller.abort(new Error('cancelled after author settlement')) - return HARNESS_OK - }) - const gen = agenticGenerator({ runHarness: runHarness as never }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'cancelled-after-author-result', + const result = await generator.generate({ + ...generateArgs(worktreePath), + costLedger: ledger, }) - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: controller.signal, - }), - ).rejects.toThrow(/cancelled after author settlement/) - expect(git(['status', '--short'], wt.path)).toBe('M app.ts') - }) - - it('rejects a candidate when cancellation arrives during verification', async () => { - const controller = new AbortController() - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const cancelledDuringVerify = true\n') - return HARNESS_OK - }) - const verify = vi.fn(async (_path: string, signal?: AbortSignal) => { - expect(signal).toBe(controller.signal) - controller.abort(new Error('cancelled during verification')) - return { ok: true } - }) - const gen = agenticGenerator({ runHarness: runHarness as never, verify }) - const wt = await gitWorktreeAdapter({ repoRoot }).create({ - baseRef: 'main', - label: 'cancelled-during-verification', + expect(result.applied).toBe(true) + expect(receipts[0]).toMatchObject({ + usage: { input: 0, output: 0, tokensKnown: false, usdKnown: false }, + costBasis: 'unknown', + costUsd: null, + costUsdKnown: false, }) - - await expect( - gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: controller.signal, - }), - ).rejects.toThrow(/cancelled during verification/) - expect(verify).toHaveBeenCalledTimes(1) + expect(ledger.list()[0]).toMatchObject({ usageUnknown: true, costUnknown: true }) }) - it('returns applied when the harness changes the worktree', async () => { - const dispositions: AgenticGeneratorShotDisposition[] = [] - // The harness "edits" by writing into its cwd (the worktree). We stub the - // subprocess (the only process boundary) but use a REAL git dirty check. - const runHarness = vi.fn( - async ({ - cwd, - taskPrompt, - dangerouslySkipPermissions, - }: { - cwd: string - taskPrompt: string - dangerouslySkipPermissions?: boolean - }) => { - expect(taskPrompt).toContain('x should be 2') - expect(taskPrompt).toContain('set x to 2') - expect(dangerouslySkipPermissions).toBe(true) - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK - }, - ) - const gen = agenticGenerator({ - runHarness: runHarness as never, - onShotDisposition: (_receipt, disposition) => { - dispositions.push(disposition) - }, + it('requires a shared ledger before admitting a maximum charge', async () => { + const execute = vi.fn() + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(execute), + buildPrompt, + maximumCharge: { externallyEnforcedMaximumUsd: 1 }, }) + const worktreePath = await candidateWorktree('maximum-charge') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cand' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }) - - expect(runHarness).toHaveBeenCalledTimes(1) - expect(out.applied).toBe(true) - expect(out.summary).toContain('x should be 2') - // The accepted shot carries its attribution pair for the driver's - // `ProposedCandidate` wrapper. - expect(out.label).toBe('agentic-x-should-be-2') - expect(out.rationale).toBe('(high) x should be 2') - expect(dispositions).toEqual([{ kind: 'accepted', worktreePath: wt.path, verified: false }]) + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /requires the run-wide CostLedger/, + ) + expect(execute).not.toHaveBeenCalled() }) - it('retries up to maxShots when the harness produces no change, then gives up', async () => { - const dispositions: AgenticGeneratorShotDisposition[] = [] - const runHarness = vi.fn(async () => HARNESS_OK) // never edits the worktree - const gen = agenticGenerator({ - runHarness: runHarness as never, - onShotDisposition: (_receipt, disposition) => { - dispositions.push(disposition) + it('refuses a bridge placement that is not bound to the candidate worktree', async () => { + const generator = agenticGenerator({ + profile: { + ...PROFILE, + harness: 'pi', }, + executorForWorktree: () => ({ + backend: 'bridge', + bridgeUrl: 'https://bridge.invalid', + bridgeBearer: 'test', + cwd: '/tmp/not-the-candidate', + }), + buildPrompt, }) + const worktreePath = await candidateWorktree('wrong-cwd') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'noop' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 3, - signal: new AbortController().signal, - }) - - expect(runHarness).toHaveBeenCalledTimes(3) - expect(out.applied).toBe(false) - expect(dispositions).toEqual([ - { kind: 'clean', worktreePath: wt.path }, - { kind: 'clean', worktreePath: wt.path }, - { kind: 'clean', worktreePath: wt.path }, - ]) + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /cwd must equal the candidate worktree/, + ) }) - it('stops retrying as soon as a shot produces a change', async () => { - let calls = 0 - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - calls++ - if (calls === 2) writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK + it('rejects an empty caller-authored prompt before model dispatch', async () => { + const execute = vi.fn() + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(execute), + buildPrompt: () => ' ', }) - const gen = agenticGenerator({ runHarness: runHarness as never }) + const worktreePath = await candidateWorktree('empty-prompt') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'second' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 5, - signal: new AbortController().signal, - }) - - expect(calls).toBe(2) // stopped on the shot that changed the tree - expect(out.applied).toBe(true) + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /buildPrompt must return a non-empty string/, + ) + expect(execute).not.toHaveBeenCalled() }) - it('end-to-end through improvementDriver: harness edit → committed CodeSurface', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK - }) - const driver = improvementDriver({ - generator: agenticGenerator({ runHarness: runHarness as never }), - worktree: gitWorktreeAdapter({ repoRoot }), - baseRef: 'main', + it('retries a clean shot with explicit feedback and accepts the first edit', async () => { + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, body, call }) => { + const user = messages(body).findLast((message) => message.role === 'user') + prompts.push(String(user?.content ?? '')) + if (call === 2) writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }), + buildPrompt, }) + const worktreePath = await candidateWorktree('clean-retry') - const surfaces = await driver.propose(ctx(FINDINGS)) + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) - expect(surfaces).toHaveLength(1) - const proposed = surfaces[0]! - // The agentic generator attributes its change, so the driver returns a - // `ProposedCandidate` wrapper carrying {label, rationale} around the - // committed CodeSurface. - if (!isProposedCandidate(proposed)) throw new Error('expected ProposedCandidate') - expect(proposed.label).toBe('agentic-x-should-be-2') - expect(proposed.rationale).toBe('(high) x should be 2') - const surface = proposed.surface - if (typeof surface === 'string') throw new Error('expected CodeSurface') - expect(surface.kind).toBe('code') - // The harness's edit is committed on the candidate branch. - expect(git(['show', 'HEAD:app.ts'], surface.worktreeRef)).toBe('export const x = 2') - // main is untouched. - expect(git(['show', 'main:app.ts'], repoRoot)).toBe('export const x = 1') + expect(result.applied).toBe(true) + expect(prompts).toHaveLength(2) + expect(prompts[0]).not.toContain('left the working tree unchanged') + expect(prompts[1]).toContain('left the working tree unchanged') }) - it('discards the worktree when the agentic generator produces nothing', async () => { - const runHarness = vi.fn(async () => HARNESS_OK) - const driver = improvementDriver({ - generator: agenticGenerator({ runHarness: runHarness as never }), - worktree: gitWorktreeAdapter({ repoRoot }), - baseRef: 'main', - }) - - const surfaces = await driver.propose(ctx(FINDINGS, 2)) - expect(surfaces).toEqual([]) - expect(git(['worktree', 'list'], repoRoot).split('\n').length).toBe(1) - }) -}) - -describe('agenticGenerator — verify-in-session loop', () => { - const edits = (cwd: string, body: string) => writeFileSync(join(cwd, 'app.ts'), body) - - it('returns the candidate when a dirtying shot passes verification', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - edits(cwd, 'export const x = 2\n') - return HARNESS_OK + it('feeds verifier failure into the next shot and accepts only a passing tree', async () => { + const prompts: string[] = [] + let verifies = 0 + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, body, call }) => { + const user = messages(body).findLast((message) => message.role === 'user') + prompts.push(String(user?.content ?? '')) + writeFileSync( + join(worktreePath, 'app.ts'), + call === 1 ? 'export const x = broken\n' : 'export const x = 2\n', + ) + }), + buildPrompt, + verify() { + verifies += 1 + return verifies === 1 + ? { ok: false, feedback: 'TypeScript: cannot find name broken' } + : { ok: true } + }, }) - const verify = vi.fn(() => ({ ok: true })) - const gen = agenticGenerator({ runHarness: runHarness as never, verify }) + const worktreePath = await candidateWorktree('verify-retry') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'vok' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 3, - signal: new AbortController().signal, - }) + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 2)) - expect(runHarness).toHaveBeenCalledTimes(1) - expect(verify).toHaveBeenCalledTimes(1) - expect(out.applied).toBe(true) + expect(result.applied).toBe(true) + expect(verifies).toBe(2) + expect(prompts[1]).toContain('cannot find name broken') + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 2\n') }) - it('feeds the verifier failure into the next shot, then ships when it passes', async () => { - let shot = 0 - const prompts: string[] = [] + it('discards a dirty candidate that never verifies', async () => { const dispositions: AgenticGeneratorShotDisposition[] = [] - const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { - prompts.push(taskPrompt) - shot++ - edits(cwd, `export const x = ${100 + shot}\n`) // always differs from baseline (x=1) - return HARNESS_OK - }) - // Fail shot 1, pass shot 2. - const verify = vi.fn(() => - shot === 1 ? { ok: false, feedback: 'TS2322: x must be 2' } : { ok: true }, - ) - const gen = agenticGenerator({ - runHarness: runHarness as never, - verify, - onShotDisposition: (_receipt, disposition) => { - dispositions.push(disposition) - }, + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, call }) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = ${call + 1}\n`) + }), + buildPrompt, + verify: () => ({ ok: false, feedback: 'still failing' }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), }) + const worktreePath = await candidateWorktree('never-verifies') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'vresume' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 4, - signal: new AbortController().signal, - }) + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 2)) - expect(runHarness).toHaveBeenCalledTimes(2) - expect(out.applied).toBe(true) - // The second shot's prompt carries the verifier's failure (resume-with-error). - expect(prompts[1]).toContain('verification FAILED') - expect(prompts[1]).toContain('TS2322: x must be 2') - // The first shot's prompt is the clean base — no failure note yet. - expect(prompts[0]).not.toContain('verification FAILED') - expect(dispositions).toEqual([ - { - kind: 'rejected', - worktreePath: wt.path, - stage: 'verification', - feedback: 'TS2322: x must be 2', - }, - { kind: 'accepted', worktreePath: wt.path, verified: true }, - ]) + expect(result.applied).toBe(false) + expect(dispositions.map((disposition) => disposition.kind)).toEqual(['rejected', 'rejected']) }) - it('discards (applied:false) a candidate that never verifies within maxShots', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - edits(cwd, 'export const x = 9\n') // dirties every shot - return HARNESS_OK + it('emits the failed Runtime turn and rejects partial edits', async () => { + const receipts: AgenticGeneratorShotReceipt[] = [] + const executions: Array = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath }) => { + writeFileSync(join(worktreePath, 'app.ts'), 'export const partial = true\n') + throw new Error('provider unavailable') + }), + buildPrompt, + onShotCompleted(receipt, execution) { + receipts.push(receipt) + executions.push(execution) + }, }) - const verify = vi.fn(() => ({ ok: false, feedback: 'still broken' })) - const gen = agenticGenerator({ runHarness: runHarness as never, verify }) + const worktreePath = await candidateWorktree('failed-turn') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'vfail' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: FINDINGS, - maxShots: 3, - signal: new AbortController().signal, + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /author shot failed:.*provider unavailable/, + ) + expect(receipts[0]).toMatchObject({ + status: 'failed', + error: { message: expect.stringContaining('author shot failed') }, }) - - expect(runHarness).toHaveBeenCalledTimes(3) - expect(verify).toHaveBeenCalledTimes(3) - expect(out.applied).toBe(false) // an unverified tree is not a candidate - }) - - it('commandVerifier: exit 0 ⇒ ok, non-zero ⇒ feedback carries output', async () => { - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cmdv' }) - const pass = commandVerifier('true') - expect(await pass(wt.path)).toEqual({ ok: true }) - - const fail = commandVerifier('sh', ['-c', 'echo boom >&2; exit 1']) - const res = await fail(wt.path) - expect(res.ok).toBe(false) - expect(res.feedback).toContain('boom') - }) - - it('commandVerifier: a missing binary throws (setup bug, not a failed candidate)', async () => { - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cmdmiss' }) - const v = commandVerifier('definitely-not-a-real-binary-xyz') - await expect(v(wt.path)).rejects.toThrow(/not found in PATH/) + expect(executions[0]?.status).toBe('failed') + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const partial = true\n') }) - it('commandVerifier: rejects pre-abort before executing the command', async () => { - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cmdabort' }) - const marker = join(wt.path, 'must-not-exist') - const controller = new AbortController() - controller.abort(new Error('verifier pre-aborted')) - const v = commandVerifier(process.execPath, [ - '-e', - `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'ran')`, - ]) - - await expect(v(wt.path, controller.signal)).rejects.toThrow(/verifier pre-aborted/) - expect(existsSync(marker)).toBe(false) - }) - - it('commandVerifier: settles a TERM-ignoring descendant before rejecting abort', async () => { - if (process.platform === 'win32') return - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cmdtree' }) - const pidPath = join(wt.path, 'verifier-descendant.pid') - const lateWrite = join(wt.path, 'verifier-late-write') - const controller = new AbortController() - const childScript = [ - "const fs=require('node:fs')", - 'fs.writeFileSync(process.argv[1],String(process.pid))', - "process.on('SIGTERM',()=>{})", - "setTimeout(()=>fs.writeFileSync(process.argv[2],'late'),700)", - 'setInterval(()=>{},1000)', - ].join(';') - const v = commandVerifier('sh', [ - '-c', - `${JSON.stringify(process.execPath)} -e ${JSON.stringify(childScript)} ${JSON.stringify(pidPath)} ${JSON.stringify(lateWrite)} & wait`, - ]) - const running = v(wt.path, controller.signal) - await waitForPath(pidPath) - const descendantPid = Number(readFileSync(pidPath, 'utf8')) - controller.abort(new Error('cancel verifier tree')) - - await expect(running).rejects.toThrow(/cancel verifier tree/) - await delay(500) - expect(existsSync(lateWrite)).toBe(false) - expect(existsSync(`/proc/${descendantPid}`)).toBe(false) - }) -}) - -describe('agenticGenerator — raw-trace evidence discipline', () => { - const writeDiagnosis = (cwd: string, body: string) => { - mkdirSync(join(cwd, '.improve'), { recursive: true }) - writeFileSync(join(cwd, '.improve/raw-trace-diagnosis.md'), body) - } - - it('retries and discards a raw-trace candidate that edits code without citing inspected traces', async () => { - const prompts: string[] = [] - const dispositions: AgenticGeneratorShotDisposition[] = [] - const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { - prompts.push(taskPrompt) - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK - }) - const gen = agenticGenerator({ - runHarness: runHarness as never, - onShotDisposition: (_receipt, disposition) => { - dispositions.push(disposition) + it('fails closed when shot evidence persistence throws', async () => { + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath }) => { + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }), + buildPrompt, + onShotCompleted() { + throw new Error('receipt store unavailable') }, }) + const worktreePath = await candidateWorktree('receipt-failure') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'rt-miss' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: RAW_TRACE_FINDINGS, - maxShots: 2, - signal: new AbortController().signal, - }) - - expect(out.applied).toBe(false) - expect(runHarness).toHaveBeenCalledTimes(2) - expect(prompts[0]).toContain('Raw trace evidence requirement') - expect(prompts[0]).toContain('.improve/raw-trace-diagnosis.md') - expect(prompts[1]).toContain('raw-trace mode requires .improve/raw-trace-diagnosis.md') - expect(dispositions).toHaveLength(2) - expect(dispositions).toEqual([ - expect.objectContaining({ - kind: 'rejected', - worktreePath: wt.path, - stage: 'raw-trace-evidence', - feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), - }), - expect.objectContaining({ - kind: 'rejected', - worktreePath: wt.path, - stage: 'raw-trace-evidence', - feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), - }), - ]) + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /receipt store unavailable/, + ) }) - it('rejects a raw-trace candidate that only writes the diagnosis artifact', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeDiagnosis(cwd, `inspected: ${TRACE_PATH}\nmechanism: stale state\nchange: none yet\n`) - return HARNESS_OK - }) - const gen = agenticGenerator({ runHarness: runHarness as never }) - - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'rt-only' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: RAW_TRACE_FINDINGS, - maxShots: 1, - signal: new AbortController().signal, + it('fails closed when worktree disposition persistence throws', async () => { + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath }) => { + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }), + buildPrompt, + onShotDisposition() { + throw new Error('disposition store unavailable') + }, }) + const worktreePath = await candidateWorktree('disposition-failure') - expect(out.applied).toBe(false) - expect(runHarness).toHaveBeenCalledTimes(1) + await expect(generator.generate(generateArgs(worktreePath))).rejects.toThrow( + /disposition store unavailable/, + ) }) - it('accepts a raw-trace candidate with a substantive edit and diagnosis citing a real trace path', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - writeDiagnosis( - cwd, - [ - `inspected: ${TRACE_PATH}`, - 'mechanism: stale state was reused after a failed candidate', - 'change: reset the state before reuse', - '', - ].join('\n'), - ) - return HARNESS_OK + it('requires a substantive edit and exact trace citation in raw-trace mode', async () => { + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: routedExecutor(({ worktreePath, body, call }) => { + const user = messages(body).findLast((message) => message.role === 'user') + prompts.push(String(user?.content ?? '')) + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + if (call === 2) { + mkdirSync(join(worktreePath, '.improve'), { recursive: true }) + writeFileSync( + join(worktreePath, '.improve', 'raw-trace-diagnosis.md'), + `${TRACE_PATH}\nStale state caused the bad edit; app.ts now reads the current value.\n`, + ) + } + }), + buildPrompt, }) - const gen = agenticGenerator({ runHarness: runHarness as never }) + const worktreePath = await candidateWorktree('raw-trace') - const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'rt-ok' }) - const out = await gen.generate({ - worktreePath: wt.path, - findings: RAW_TRACE_FINDINGS, - maxShots: 1, - signal: new AbortController().signal, - }) + const result = await generator.generate(generateArgs(worktreePath, RAW_TRACE_FINDINGS, 2)) - expect(out.applied).toBe(true) - expect(runHarness).toHaveBeenCalledTimes(1) + expect(result.applied).toBe(true) + expect(prompts[1]).toContain('raw-trace-diagnosis.md') + expect( + readFileSync(join(worktreePath, '.improve', 'raw-trace-diagnosis.md'), 'utf8'), + ).toContain(TRACE_PATH) }) }) -describe('agenticGenerator — candidate attribution through the improvement loop', () => { - it('threads label + rationale into GenerationRecord and the loop winner', async () => { - const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { - writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') - return HARNESS_OK - }) - const driver = improvementDriver({ - generator: agenticGenerator({ runHarness: runHarness as never }), - worktree: gitWorktreeAdapter({ repoRoot }), - baseRef: 'main', - }) - const scenarios: Scenario[] = [{ id: 'task', kind: 'fixture' }] - // Rewards the agentic CodeSurface candidate over the string baseline so - // the candidate promotes and its attribution reaches the winner fields. - const judge: JudgeConfig<{ fromCode: boolean }, Scenario> = { - name: 'code-wins', - dimensions: [{ key: 'q', description: 'candidate quality' }], - score: ({ artifact }) => { - const composite = artifact.fromCode ? 1 : 0 - return { composite, dimensions: { q: composite }, notes: '' } - }, - } +describe('commandVerifier', () => { + it('reports command failure output and accepts exit zero', async () => { + const pass = commandVerifier(process.execPath, ['-e', 'process.exit(0)']) + const fail = commandVerifier(process.execPath, [ + '-e', + 'process.stderr.write("compile failed"); process.exit(1)', + ]) - const result = await runOptimization({ - baselineSurface: 'BASELINE', - scenarios, - dispatchWithSurface: async (surface) => ({ fromCode: typeof surface !== 'string' }), - dispatchRef: 'test:agentic-attribution', - judges: [judge], - proposer: driver, - findings: FINDINGS, - populationSize: 1, - maxGenerations: 1, - seed: 7, - reps: 1, - resumable: false, - runDir: '/agentic-attribution', - storage: inMemoryCampaignStorage(), - tracing: 'off', - expectUsage: 'off', + await expect(pass(repoRoot)).resolves.toEqual({ ok: true }) + await expect(fail(repoRoot)).resolves.toEqual({ + ok: false, + feedback: 'compile failed', }) + }) - const candidate = result.generations[0]!.record.candidates[0]! - expect(candidate.label).toBe('agentic-x-should-be-2') - expect(candidate.rationale).toBe('(high) x should be 2') - expect(result.winnerLabel).toBe('agentic-x-should-be-2') - expect(result.winnerRationale).toBe('(high) x should be 2') + it('throws when the verifier binary does not exist', async () => { + const verify = commandVerifier('definitely-not-a-real-verifier-binary') - await driver.cleanup() + await expect(verify(repoRoot)).rejects.toThrow(/not found in PATH/) }) }) diff --git a/tests/build-composition.test.ts b/tests/build-composition.test.ts index f613339c..2a879afc 100644 --- a/tests/build-composition.test.ts +++ b/tests/build-composition.test.ts @@ -10,11 +10,15 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { makeProposalFinding } from '@tangle-network/agent-eval' import { gitWorktreeAdapter } from '@tangle-network/agent-eval/campaign' +import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, beforeEach, expect, it, vi } from 'vitest' -import { agenticGenerator, commandVerifier } from '../src/improvement/agentic-generator' +import { + type AgenticGeneratorExecutorForWorktree, + agenticGenerator, + commandVerifier, +} from '../src/improvement/agentic-generator' import { mcpBuildPrompt, toolBuildPrompt } from '../src/improvement/build-prompts' import { mcpServeVerifier } from '../src/improvement/mcp-serve-verifier' -import type { LocalHarnessResult } from '../src/mcp/local-harness' function git(a: string[], cwd: string): string { return execFileSync('git', a, { cwd, encoding: 'utf8' }).trim() @@ -49,13 +53,33 @@ const FINDINGS = [ }), ] -const HARNESS_OK: LocalHarnessResult = { - exitCode: 0, - stdout: 'done', - stderr: '', - killedBySignal: null, - durationMs: 10, - timedOut: false, +const PROFILE: AgentProfile = { + name: 'artifact-author', + harness: 'cli-base', + model: { provider: 'offline', default: 'deterministic-author' }, + prompt: { systemPrompt: 'Build the requested artifact in this worktree.' }, +} + +function executorFor( + run: (input: { cwd: string; taskPrompt: string }) => void | Promise, +): AgenticGeneratorExecutorForWorktree { + return (cwd) => ({ + backend: 'router', + routerBaseUrl: 'https://offline.invalid/v1', + routerKey: 'test', + complete: async (body) => { + const messages = body.messages as Array<{ role?: string; content?: unknown }> + const taskPrompt = String( + messages.findLast((message) => message.role === 'user')?.content ?? '', + ) + await run({ cwd, taskPrompt }) + return { + model: body.model, + choices: [{ message: { role: 'assistant', content: 'done' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 2, cost_usd: 0.001 }, + } + }, + }) } const gen = (worktreePath: string) => ({ @@ -66,13 +90,13 @@ const gen = (worktreePath: string) => ({ }) it('build a tool: agenticGenerator + toolBuildPrompt + commandVerifier', async () => { - const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { + const run = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { expect(taskPrompt).toContain('building a new TOOL') writeFileSync(join(cwd, 'tool.ts'), 'export const ok = true\n') - return HARNESS_OK }) const g = agenticGenerator({ - runHarness: runHarness as never, + profile: PROFILE, + executorForWorktree: executorFor(run), buildPrompt: toolBuildPrompt, verify: commandVerifier('true'), }) @@ -89,13 +113,13 @@ it('build an MCP server: agenticGenerator + mcpBuildPrompt + mcpServeVerifier', ' if (m.method === "initialize") send({ jsonrpc:"2.0", id:m.id, result:{ protocolVersion:"2024-11-05", capabilities:{}, serverInfo:{name:"f",version:"0"} } })', ' else if (m.method === "tools/list") send({ jsonrpc:"2.0", id:m.id, result:{ tools:[{ name:"t", inputSchema:{type:"object"} }] } }) })', ].join('\n') - const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { + const run = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { expect(taskPrompt).toContain('MCP SERVER') writeFileSync(join(cwd, 'server.mjs'), server) - return HARNESS_OK }) const g = agenticGenerator({ - runHarness: runHarness as never, + profile: PROFILE, + executorForWorktree: executorFor(run), buildPrompt: mcpBuildPrompt, verify: mcpServeVerifier({ command: 'node', args: ['server.mjs'] }), }) From 449d76fb59b0cbeaa892279e16cb493baa25ba8d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:19:10 -0600 Subject: [PATCH 14/39] test(bench): use an exact solver profile --- bench/src/gate.test.mts | 8 +++++++- bench/src/gate.ts | 7 +++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bench/src/gate.test.mts b/bench/src/gate.test.mts index f90b55a3..a51417cb 100644 --- a/bench/src/gate.test.mts +++ b/bench/src/gate.test.mts @@ -11,6 +11,7 @@ */ import assert from 'node:assert/strict' +import type { AgentProfile } from '@tangle-network/agent-interface' import type { AgentSpec, DefaultVerdict, @@ -80,7 +81,12 @@ function stubAdapter(n: number): BenchmarkAdapter { } } -const profile = { name: 'stub-solver', model: { default: 'stub-model' } } as never +const profile = { + name: 'stub-solver', + harness: 'pi', + model: { provider: 'test', default: 'stub-model' }, + prompt: { systemPrompt: 'Solve the supplied benchmark task.' }, +} satisfies AgentProfile const report = await runGate({ adapter: stubAdapter(5), diff --git a/bench/src/gate.ts b/bench/src/gate.ts index b4e5a092..22e78b45 100644 --- a/bench/src/gate.ts +++ b/bench/src/gate.ts @@ -152,10 +152,9 @@ export function benchSolverRegistry(opts: BenchSolverOptions): ExecutorRegistry } } -/** Build the solver `Persona` from the developer's `AgentProfile` + a solve-and-grade registry. - * The deliverable type is the candidate text (`string`); `harness: null` is nominal — the - * supplied registry overrides resolution, so the root never falls through to the router/sandbox - * built-ins. */ +/** Build the solver `Persona` from the developer's exact `AgentProfile` + a solve-and-grade + * registry. The profile still declares its execution identity; `AgentSpec.harness: null` only + * says that the supplied registry owns placement instead of a built-in executor. */ export function defineSolverPersona( profile: AgentProfile, registry: ExecutorRegistry, From 4c66adf8f1204efed79e02af17298202a124917a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:37:39 -0600 Subject: [PATCH 15/39] refactor(runtime): enforce profiled model execution --- docs/api/mcp.md | 39 ---- docs/api/primitive-catalog.md | 5 +- docs/api/runtime.md | 29 +-- scripts/check-model-execution-boundary.mjs | 173 +++++++++++++++--- .../check-model-execution-boundary.test.mjs | 69 +++++++ src/mcp/index.ts | 1 - src/runtime/supervise/runtime.ts | 14 -- .../supervise/worktree-cli-executor.ts | 59 +++--- src/runtime/supervise/worktree-fanout.ts | 8 +- tests/kernel/worktree-loop.test.ts | 4 - tests/mcp/worktree-harness.test.ts | 4 +- tests/runtime/worktree-cli-executor.test.ts | 91 ++++----- 12 files changed, 318 insertions(+), 178 deletions(-) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 6cad9b6d..d5f90814 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -7045,45 +7045,6 @@ created, against the same table that emits the argv. *** -### runLocalHarness() - -> **runLocalHarness**(`options`): `Promise`\<[`LocalHarnessResult`](#localharnessresult)\> - -**`Experimental`** - -Spawn a local coding harness CLI as a subprocess + collect its output. - -NOT responsible for parsing the harness's output or extracting a diff — -the in-process executor's `streamPrompt` orchestrates `git diff` against -the worktree after this resolves. This function is intentionally narrow: -spawn, wait, capture, return. - -Fails loud — throws when: - - `cwd` doesn't exist (subprocess emits ENOENT; surfaced as Error) - - the harness binary is not on PATH (ENOENT) - - the caller signal was already aborted before process launch - -Does NOT throw when: - - the subprocess exits non-zero (`result.exitCode` carries the code) - - a non-reproducible subprocess is aborted / timed out (`result.aborted` / - `result.timedOut` carries the reason even when a TERM-aware child exits zero) - -Reproducible Codex additionally requires a terminal usage event. If cancellation -prevents that event, this rejects with `CodexExecutionDiagnosticError` instead of -returning an incomplete reproducibility receipt. - -#### Parameters - -##### options - -[`RunLocalHarnessOptions`](#runlocalharnessoptions) - -#### Returns - -`Promise`\<[`LocalHarnessResult`](#localharnessresult)\> - -*** - ### parseCodexTokenUsage() > **parseCodexTokenUsage**(`stdout`): [`CodexTokenUsage`](#codextokenusage) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 61e0a355..c5232f6b 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.128.1` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.129.0` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -1297,7 +1297,7 @@ Import from `@tangle-network/agent-runtime/testing` — 4 exports. ### MCP servers — delegate / coordination / detached-session -Import from `@tangle-network/agent-runtime/mcp` — 211 exports. +Import from `@tangle-network/agent-runtime/mcp` — 210 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1343,7 +1343,6 @@ Import from `@tangle-network/agent-runtime/mcp` — 211 exports. | `removeWorktree` | function | Remove a git worktree and delete its branch. Already-removed paths are harmless; every other | | `resolveMemoryFromEnv` | function | Resolve the bin's memory from `AGENT_MEMORY_FILE` (durable store) and/or | | `runDetachedTurn` | function | Dispatch one detached turn and advance it to a terminal state with | -| `runLocalHarness` | function | Spawn a local coding harness CLI as a subprocess + collect its output. | | `settleDetachedCoderTurn` | function | Settle a completed detached coder turn through the same gate the streaming | | `traceContextToEnv` | function | Build env vars to pass to a child subprocess so it inherits the current trace context. | | `validateDelegateArgs` | function | Parse and validate raw MCP tool input into typed `DelegateArgs`; throws `TypeError` on bad input. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 9f92a317..948e5d12 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -15922,14 +15922,6 @@ The supervisor-authored prompt/model plus materializable structural resources. provider extensions, and `resources.failOnError` fail before execution because this path cannot honor them. Harness-specific values the materializer cannot preserve also fail closed. -##### harness - -> **harness**: [`LocalHarness`](mcp.md#localharness) - -**`Experimental`** - -Local CLI for this leaf. This explicit choice overrides `profile.harness`. - ##### taskPrompt? > `optional` **taskPrompt?**: `string` @@ -15970,7 +15962,7 @@ Wall-clock cap per harness subprocess (ms). Default 5 min (the `runLocalHarness` **`Experimental`** Run Codex with an ephemeral session, isolated config/instructions, network disabled, and - JSONL usage capture. Requires `harness: 'codex'`; metered by default. + JSONL usage capture. Requires `profile.harness: 'codex'`; metered by default. ##### codexReadDeniedPaths? @@ -16088,8 +16080,8 @@ likewise return `LocalHarnessResult.usage`. **`Experimental`** -One authored harness profile in a worktree fanout: the §1.5 profile + which local - harness CLI drives it. The supervisor authors `profile` per sub-task; `harness` chooses the leaf. +One authored profile in a worktree fanout. Its exact `harness` field chooses the +local CLI; the supervisor authors the complete profile per sub-task. #### Properties @@ -16109,14 +16101,6 @@ A short label for the worktree branch + trace node. The supervisor-authored `AgentProfile` (systemPrompt + model reach the harness via §1.5). -##### harness - -> **harness**: [`LocalHarness`](mcp.md#localharness) - -**`Experimental`** - -Which local harness CLI drives this leaf. - ##### budgetExempt? > `optional` **budgetExempt?**: `boolean` @@ -25421,9 +25405,10 @@ caller's own seam env so a deliberately-set id wins (see the precedence note abo Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a fanout of N profiles = N parallel worktrees that never clobber each other. -Fail-loud: an empty `repoRoot`/`harness` or an explicitly empty `taskPrompt` throws at -construction. Calling `execute(undefined, signal)` without a configured prompt throws before a -worktree is created. `resultArtifact()` before `execute()` resolves throws. +Fail-loud: an empty `repoRoot`, an incomplete/unsupported profile, a separate harness override, +or an explicitly empty `taskPrompt` throws at construction. Calling `execute(undefined, signal)` +without a configured prompt throws before a worktree is created. `resultArtifact()` before +`execute()` resolves throws. #### Parameters diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index 5fd6134c..88d2f1a5 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -57,9 +57,41 @@ const lowLevelModelCalls = new Set([ 'routerChatWithTools', 'routerChatWithUsage', 'routerToolLoop', + 'runLocalHarness', 'streamRouterChatWithTools', ]) +// `runLocalHarness` is the physical process adapter under the profiled worktree executors. It is +// deliberately callable in exactly one implementation module and must never become a public or +// benchmark-facing shortcut around AgentProfile intake. +const lowLevelModelCallOwners = new Map([ + ['runLocalHarness', new Set(['src/mcp/worktree-harness.ts'])], +]) + +const forbiddenPublicModelCalls = new Set(['runLocalHarness']) + +// These two files are the implementation owners that translate a validated AgentProfile into a +// local process. Everywhere else, naming a coding-agent CLI in an executable command is a second +// model-execution path. +const directCliOwners = new Set([ + 'src/mcp/local-harness.ts', + 'scripts/check-model-execution-boundary.test.mjs', +]) + +const processLaunchCalls = new Set([ + 'dotenvxBash', + 'exec', + 'execFile', + 'execFileSync', + 'execSync', + 'run', + 'runOk', + 'runTb', + 'sh', + 'spawn', + 'spawnSync', +]) + const sourceExtensions = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.py', '.sh']) const ignoredDirectories = new Set([ '.git', @@ -116,7 +148,7 @@ export function checkJavaScript(path, text) { : ts.ScriptKind.JS const source = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, kind) const initializers = new Map() - const lowLevelBindings = new Set() + const lowLevelBindings = new Map() const lowLevelNamespaces = new Set() const failures = [] @@ -126,13 +158,38 @@ export function checkJavaScript(path, text) { ts.isIdentifier(node.name) && node.initializer !== undefined ) { - initializers.set(node.name.text, node.initializer) + const declarations = initializers.get(node.name.text) ?? [] + declarations.push(node) + initializers.set(node.name.text, declarations) } ts.forEachChild(node, collect) } collect(source) + function initializerFor(identifier) { + const declarations = initializers.get(identifier.text) ?? [] + let closest + for (const declaration of declarations) { + if (declaration.pos >= identifier.pos) continue + if (!closest || declaration.pos > closest.pos) closest = declaration + } + return closest?.initializer + } + for (const statement of source.statements) { + if (ts.isExportDeclaration(statement) && statement.exportClause) { + if (ts.isNamedExports(statement.exportClause)) { + for (const element of statement.exportClause.elements) { + const exported = element.propertyName?.text ?? element.name.text + if (forbiddenPublicModelCalls.has(exported) && !element.isTypeOnly) { + failures.push({ + node: element, + detail: `public low-level model executor ${JSON.stringify(exported)}`, + }) + } + } + } + } if (!ts.isImportDeclaration(statement)) continue const moduleName = ts.isStringLiteral(statement.moduleSpecifier) ? statement.moduleSpecifier.text @@ -147,7 +204,7 @@ export function checkJavaScript(path, text) { } else { for (const element of bindings.elements) { const imported = element.propertyName?.text ?? element.name.text - if (lowLevelModelCalls.has(imported)) lowLevelBindings.add(element.name.text) + if (lowLevelModelCalls.has(imported)) lowLevelBindings.set(element.name.text, imported) } } } @@ -175,7 +232,7 @@ export function checkJavaScript(path, text) { for (const element of node.name.elements) { if (!ts.isIdentifier(element.name)) continue const imported = element.propertyName?.getText(source) ?? element.name.text - if (lowLevelModelCalls.has(imported)) lowLevelBindings.add(element.name.text) + if (lowLevelModelCalls.has(imported)) lowLevelBindings.set(element.name.text, imported) } } } @@ -185,7 +242,7 @@ export function checkJavaScript(path, text) { isRequireCall(node.initializer.expression) && lowLevelModelCalls.has(node.initializer.name.text) ) { - lowLevelBindings.add(node.name.text) + lowLevelBindings.set(node.name.text, node.initializer.name.text) } } ts.forEachChild(node, collectCommonJs) @@ -196,7 +253,7 @@ export function checkJavaScript(path, text) { if (ts.isStringLiteralLike(node)) return node.text if (ts.isIdentifier(node)) { if (seen.has(node.text)) return undefined - const initializer = initializers.get(node.text) + const initializer = initializerFor(node) if (initializer === undefined) return undefined const nextSeen = new Set(seen) nextSeen.add(node.text) @@ -223,7 +280,7 @@ export function checkJavaScript(path, text) { function expressionText(node, seen = new Set()) { if (ts.isIdentifier(node)) { if (seen.has(node.text)) return node.getText(source) - const initializer = initializers.get(node.text) + const initializer = initializerFor(node) if (initializer !== undefined) { seen.add(node.text) return `${node.getText(source)}=${expressionText(initializer, seen)}` @@ -232,6 +289,48 @@ export function checkJavaScript(path, text) { return node.getText(source) } + function lowLevelModelCallName(node) { + if (ts.isIdentifier(node.expression)) return lowLevelBindings.get(node.expression.text) + if ( + ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + lowLevelNamespaces.has(node.expression.expression.text) && + lowLevelModelCalls.has(node.expression.name.text) + ) { + return node.expression.name.text + } + if ( + ts.isElementAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + lowLevelNamespaces.has(node.expression.expression.text) && + ts.isStringLiteral(node.expression.argumentExpression) && + lowLevelModelCalls.has(node.expression.argumentExpression.text) + ) { + return node.expression.argumentExpression.text + } + return undefined + } + + function isAllowedLowLevelModelCall(name) { + return lowLevelModelCallOwners.get(name)?.has(path) === true + } + + function isProcessLaunchCall(node) { + const callee = node.expression + if (ts.isIdentifier(callee)) return processLaunchCalls.has(callee.text) + if (ts.isPropertyAccessExpression(callee)) return processLaunchCalls.has(callee.name.text) + return false + } + + function isInsideProcessLaunchCall(node) { + let parent = node.parent + while (parent && !ts.isStatement(parent)) { + if (ts.isCallExpression(parent) && isProcessLaunchCall(parent)) return true + parent = parent.parent + } + return false + } + function inspect(node) { if (ts.isCallExpression(node)) { const callee = node.expression.getText(source) @@ -253,21 +352,35 @@ export function checkJavaScript(path, text) { /^(?:https?|request|axios)(?:\.|$)/.test(callee) && namesModelEndpoint(call) && !isLocalTestTarget(path, call) + const lowLevelCallName = lowLevelModelCallName(node) const lowLevelRuntimeCall = - (ts.isIdentifier(node.expression) && lowLevelBindings.has(callee)) || - (ts.isPropertyAccessExpression(node.expression) && - ts.isIdentifier(node.expression.expression) && - lowLevelNamespaces.has(node.expression.expression.text) && - lowLevelModelCalls.has(node.expression.name.text)) || - (ts.isElementAccessExpression(node.expression) && - ts.isIdentifier(node.expression.expression) && - lowLevelNamespaces.has(node.expression.expression.text) && - ts.isStringLiteral(node.expression.argumentExpression) && - lowLevelModelCalls.has(node.expression.argumentExpression.text)) - if (directFetch || providerSdk || rawHttp || lowLevelRuntimeCall) { + lowLevelCallName !== undefined && !isAllowedLowLevelModelCall(lowLevelCallName) + const cliLaunch = + !directCliOwners.has(path) && + isProcessLaunchCall(node) && + namesModelCliInvocation( + [callee, ...node.arguments.map((argument) => expressionText(argument))].join(' '), + ) + if (directFetch || providerSdk || rawHttp || lowLevelRuntimeCall || cliLaunch) { failures.push({ node, detail: call.slice(0, 180).replace(/\s+/g, ' ') }) } } + if ( + !directCliOwners.has(path) && + (ts.isStringLiteralLike(node) || ts.isNoSubstitutionTemplateLiteral(node)) && + !isInsideProcessLaunchCall(node) && + namesModelCliInvocation(node.text) + ) { + failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) + } + if ( + !directCliOwners.has(path) && + ts.isTemplateExpression(node) && + !isInsideProcessLaunchCall(node) && + namesModelCliInvocation(node.getText(source)) + ) { + failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) + } if ( ts.isNewExpression(node) && /^(?:OpenAI|Anthropic)$/.test(node.expression.getText(source)) @@ -280,6 +393,18 @@ export function checkJavaScript(path, text) { return failures.map(({ node, detail }) => ({ location: sourceLocation(source, node), detail })) } +function namesModelCliInvocation(text) { + const cli = + /(?:^|[^A-Za-z0-9_])(?:claude|codex|opencode|pi)(?:Bin|Path|Executable)?\b/i.exec(text) + if (!cli) return false + const afterCli = text.slice((cli.index ?? 0) + cli[0].length) + const namesExecutionMode = + /(?:^|[\s'"`,:[\]()])(?:-p|--print|exec|run)(?=$|[\s'"`,:[\]()])/i.test(afterCli) + const selectsCliAgent = + /--agent(?:-import-path)?[\s'"`,:[\]()]+(?:claude|codex|opencode|pi)\b/i.test(text) + return namesExecutionMode || selectsCliAgent +} + function executablePythonLines(text) { const lines = text.split(/\r?\n/) let quote = null @@ -314,11 +439,16 @@ function executablePythonLines(text) { export function checkPython(text) { const failures = [] - for (const [index, line] of executablePythonLines(text).entries()) { + const lines = executablePythonLines(text) + for (const [index, line] of lines.entries()) { + const commandWindow = lines.slice(index, index + 3).join(' ') if ( /chat\/completions|api\.anthropic\.com|\.chat\.completions\.create\s*\(|\.responses\.create\s*\(|\.messages\.create\s*\(/i.test( line, - ) + ) || + (/(?:^|[^A-Za-z0-9_])(?:claude|codex|opencode|pi)(?:Bin|Path|Executable)?\b/i.test( + line, + ) && namesModelCliInvocation(commandWindow)) ) { failures.push({ location: `${index + 1}:1`, detail: line.trim().slice(0, 180) }) } @@ -331,7 +461,8 @@ export function checkShell(text) { for (const [index, line] of text.split(/\r?\n/).entries()) { const code = line.replace(/^\s*#.*$/, '') if ( - /(?:curl|wget|http)\b.*(?:chat\/completions|\/responses\b|api\.anthropic\.com)/i.test(code) + /(?:curl|wget|http)\b.*(?:chat\/completions|\/responses\b|api\.anthropic\.com)/i.test(code) || + namesModelCliInvocation(code) ) { failures.push({ location: `${index + 1}:1`, detail: code.trim().slice(0, 180) }) } diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index 0355a219..90301404 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -102,6 +102,67 @@ describe('model execution boundary source check', () => { ).toHaveLength(1) }) + it('rejects direct coding-agent CLI launches in JavaScript and TypeScript', () => { + expect( + checkJavaScript( + 'bench/claude.ts', + `const args = ['-p', prompt, '--model', model]\nawait run('claude', args)`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/opencode.ts', + 'await dotenvxBash(secrets, `cd ${cwd} && opencode --model ${model} run ${prompt}`)', + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/sidecar.ts', + 'const command = `opencode --model ${model} --format json run ${prompt}`', + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/terminal.ts', + `const args = ['run', '--agent', 'opencode']\nawait runTb(args)`, + ), + ).toHaveLength(1) + }) + + it('rejects model execution hidden inside a custom CandidateGenerator callback', () => { + const violations = checkJavaScript( + 'bench/custom-generator.ts', + `import { runLocalHarness as launch } from '@tangle-network/agent-runtime/mcp' +const generator = { + kind: 'direct-model', + async generate({ worktreePath, findings }) { + await launch({ harness: 'opencode', cwd: worktreePath, taskPrompt: String(findings) }) + return { applied: true, summary: 'changed' } + }, +} satisfies CandidateGenerator`, + ) + expect(violations).toHaveLength(1) + expect(violations[0]?.detail).toContain('launch(') + }) + + it('rejects publishing the low-level local harness executor', () => { + const violations = checkJavaScript( + 'src/mcp/index.ts', + `export { runLocalHarness, type RunLocalHarnessOptions } from './local-harness'`, + ) + expect(violations).toHaveLength(1) + expect(violations[0]?.detail).toContain('public low-level model executor') + }) + + it('allows the one internal runLocalHarness owner', () => { + expect( + checkJavaScript( + 'src/mcp/worktree-harness.ts', + `import { runLocalHarness } from './local-harness'\nawait runLocalHarness(options)`, + ), + ).toEqual([]) + }) + it('ignores comments, inert strings, and ordinary HTTP', () => { const source = ` // fetch('https://api.openai.com/v1/chat/completions') @@ -125,6 +186,14 @@ describe('model execution boundary source check', () => { expect(checkPython(`# client.messages.create(model='x')`)).toEqual([]) expect(checkShell(`# curl https://api.openai.com/v1/chat/completions`)).toEqual([]) expect(checkShell(`curl https://api.openai.com/v1/chat/completions`)).toHaveLength(1) + expect(checkPython(`command = f"opencode --model {model} run {task}"`)).toHaveLength(1) + expect(checkShell(`opencode --model "$MODEL" run "$TASK"`)).toHaveLength(1) + expect(checkShell(`# opencode --model "$MODEL" run "$TASK"`)).toEqual([]) + }) + + it('allows non-inference CLI inspection', () => { + expect(checkJavaScript('bench/auth.ts', `await exec('codex', ['login', 'status'])`)).toEqual([]) + expect(checkShell(`claude --version`)).toEqual([]) }) it('does not scan generated Python virtual environments', () => { diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 60523287..fa160c18 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -106,7 +106,6 @@ export { LOCAL_HARNESSES, localHarnessExecutable, parseCodexTokenUsage, - runLocalHarness, } from './local-harness' export { type AgentMemorySpec, diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 49c0ed56..8dc07c00 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -45,7 +45,6 @@ import { defineProfileMaterializationContract, } from '../../agent/profile-materialization' import { ValidationError } from '../../errors' -import type { LocalHarness } from '../../mcp/local-harness' import { mergeTraceEnv } from '../../mcp/trace-propagation' import { captureWorktreeDiff, @@ -3445,16 +3444,9 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { if (seam.bridge) return bridgeWorktreeExecutor(spec, ctx, seam) const effectiveProfile = agentProfileSchema.parse(spec.profile) assertExecutableAgentProfile(effectiveProfile, 'cliWorktreeExecutor') - const harness = localWorktreeHarness(agentHarness(effectiveProfile.harness)) - if (!harness) { - throw new ValidationError( - 'cliWorktreeExecutor: AgentProfile.harness must select claude-code, codex, or opencode when bridge is not set', - ) - } return createWorktreeCliExecutor({ repoRoot: seam.repoRoot, profile: effectiveProfile, - harness, ...(seam.taskPrompt !== undefined ? { taskPrompt: seam.taskPrompt } : {}), ...(seam.runId ? { runId: seam.runId } : {}), ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), @@ -3472,12 +3464,6 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { }) as Executor } -function localWorktreeHarness(harness: string | undefined): LocalHarness | undefined { - return harness === 'claude-code' || harness === 'codex' || harness === 'opencode' - ? harness - : undefined -} - // ── createExecutor: the ONE built-in factory (backend as data) ────────────────── /** diff --git a/src/runtime/supervise/worktree-cli-executor.ts b/src/runtime/supervise/worktree-cli-executor.ts index fff4d050..fb299c5f 100644 --- a/src/runtime/supervise/worktree-cli-executor.ts +++ b/src/runtime/supervise/worktree-cli-executor.ts @@ -25,7 +25,7 @@ import { randomUUID } from 'node:crypto' import type { AgentProfile } from '@tangle-network/agent-interface' import { contentAddress } from '../../durable/spawn-journal' import { ValidationError } from '../../errors' -import type { LocalHarness, runLocalHarness } from '../../mcp/local-harness' +import { LOCAL_HARNESSES, type LocalHarness, type runLocalHarness } from '../../mcp/local-harness' import type { GitRunner } from '../../mcp/worktree' import { runWorktreeHarness, @@ -36,6 +36,7 @@ import { type WorktreeProfileMaterializationReceipt, worktreeProfileExecutionPlan, } from '../../mcp/worktree-harness' +import { executableAgentProfileSnapshot } from './executable-spec' import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import { concreteProfileModel } from './model-policy' import type { Executor, ExecutorResult, Spend } from './types' @@ -56,8 +57,6 @@ export interface WorktreeCliExecutorOptions { * cannot honor them. Harness-specific values the materializer cannot preserve also fail closed. */ profile: AgentProfile - /** Local CLI for this leaf. This explicit choice overrides `profile.harness`. */ - harness: LocalHarness /** Default instruction for direct `execute(undefined, signal)` calls. An execution-time task * is authoritative. Omit when the caller always supplies the task to `execute`. */ taskPrompt?: string @@ -68,7 +67,7 @@ export interface WorktreeCliExecutorOptions { /** Wall-clock cap per harness subprocess (ms). Default 5 min (the `runLocalHarness` default). */ harnessTimeoutMs?: number /** Run Codex with an ephemeral session, isolated config/instructions, network disabled, and - * JSONL usage capture. Requires `harness: 'codex'`; metered by default. */ + * JSONL usage capture. Requires `profile.harness: 'codex'`; metered by default. */ codexReproducible?: boolean /** Absolute host paths denied to reproducible Codex (for benchmark answer copies, credentials, * or other task-specific ambient state). */ @@ -105,9 +104,10 @@ export interface WorktreeCliExecutorOptions { * Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a * fanout of N profiles = N parallel worktrees that never clobber each other. * - * Fail-loud: an empty `repoRoot`/`harness` or an explicitly empty `taskPrompt` throws at - * construction. Calling `execute(undefined, signal)` without a configured prompt throws before a - * worktree is created. `resultArtifact()` before `execute()` resolves throws. + * Fail-loud: an empty `repoRoot`, an incomplete/unsupported profile, a separate harness override, + * or an explicitly empty `taskPrompt` throws at construction. Calling `execute(undefined, signal)` + * without a configured prompt throws before a worktree is created. `resultArtifact()` before + * `execute()` resolves throws. * * @experimental */ @@ -117,9 +117,13 @@ export function createWorktreeCliExecutor( if (!options.repoRoot) { throw new ValidationError('createWorktreeCliExecutor: repoRoot required') } - if (!options.harness) { - throw new ValidationError('createWorktreeCliExecutor: harness required') + if ('harness' in options) { + throw new ValidationError( + 'createWorktreeCliExecutor: separate harness is forbidden; set AgentProfile.harness', + ) } + const profile = executableAgentProfileSnapshot(options.profile, 'createWorktreeCliExecutor') + const harness = localHarnessFromProfile(profile) if ( options.taskPrompt !== undefined && (typeof options.taskPrompt !== 'string' || options.taskPrompt.length === 0) @@ -128,7 +132,7 @@ export function createWorktreeCliExecutor( } // KEPT harness-name test: `codexReproducible` is a codex-SPECIFIC public option, so this // asserts caller self-consistency and throws loudly instead of varying behavior by name. - if (options.codexReproducible && options.harness !== 'codex') { + if (options.codexReproducible && harness !== 'codex') { throw new ValidationError( 'createWorktreeCliExecutor: codexReproducible requires harness "codex"', ) @@ -150,8 +154,11 @@ export function createWorktreeCliExecutor( let run: WorktreeHarnessRun | undefined let artifact: ExecutorResult | undefined - const profilePlan = worktreeProfileExecutionPlan(options.profile, options.harness) - const profileModel = concreteProfileModel(options.profile) + const profilePlan = worktreeProfileExecutionPlan(profile, harness) + const profileModel = concreteProfileModel(profile) + if (!profileModel) { + throw new ValidationError('createWorktreeCliExecutor: exact profile model unexpectedly missing') + } return attestRuntimeOwnedExecutor( { runtime: 'cli', @@ -163,8 +170,8 @@ export function createWorktreeCliExecutor( run = await runWorktreeHarness({ repoRoot: options.repoRoot, - profile: options.profile, - harness: options.harness, + profile, + harness, taskPrompt, runId, ...(options.baseRef ? { baseRef: options.baseRef } : {}), @@ -238,17 +245,15 @@ export function createWorktreeCliExecutor( }, }, { - effectiveProfile: options.profile, - backend: `cli-worktree:${options.harness}`, - model: profileModel - ? { status: 'known', id: profileModel } - : { status: 'unknown', reason: `${options.harness} selected its configured default model` }, + effectiveProfile: profile, + backend: `cli-worktree:${harness}`, + model: { status: 'known', id: profileModel }, execution: { kind: 'worktree-run', id: runId }, materializer: 'agent-profile-worktree-plan', plan: { kind: 'worktree-cli', profilePlan, - harness: options.harness, + harness, baseRef: options.baseRef ?? 'HEAD', harnessTimeoutMs: options.harnessTimeoutMs ?? null, codexReproducible: options.codexReproducible === true, @@ -264,19 +269,27 @@ export function createWorktreeCliExecutor( binding: { repoRoot: options.repoRoot, runId, - harness: options.harness, - model: profileModel ?? null, + harness, + model: profileModel, baseRef: options.baseRef ?? 'HEAD', }, descriptor: { kind: 'worktree-cli-run', transport: 'process', - backend: options.harness, + backend: harness, }, }, ) } +function localHarnessFromProfile(profile: AgentProfile): LocalHarness { + const harness = profile.harness + if (LOCAL_HARNESSES.includes(harness as LocalHarness)) return harness as LocalHarness + throw new ValidationError( + `createWorktreeCliExecutor: AgentProfile.harness must select ${LOCAL_HARNESSES.join(', ')}`, + ) +} + /** A scoped execution task is authoritative. The configured prompt remains only as the * unambiguous direct-call default for existing `execute(undefined, signal)` consumers. */ function executionTaskPrompt(task: unknown, configuredPrompt: string | undefined): string { diff --git a/src/runtime/supervise/worktree-fanout.ts b/src/runtime/supervise/worktree-fanout.ts index dde2a467..ec7de1f9 100644 --- a/src/runtime/supervise/worktree-fanout.ts +++ b/src/runtime/supervise/worktree-fanout.ts @@ -15,7 +15,6 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' -import type { LocalHarness } from '../../mcp/local-harness' import { fanout, selectValidWinner } from '../personify/combinators' import type { CombinatorShape, WinnerStrategy } from '../personify/wave-types' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' @@ -27,15 +26,13 @@ import { type WorktreePatchArtifact, } from './worktree-cli-executor' -/** @experimental One authored harness profile in a worktree fanout: the §1.5 profile + which local - * harness CLI drives it. The supervisor authors `profile` per sub-task; `harness` chooses the leaf. */ +/** @experimental One authored profile in a worktree fanout. Its exact `harness` field chooses the + * local CLI; the supervisor authors the complete profile per sub-task. */ export interface AuthoredHarness { /** A short label for the worktree branch + trace node. */ name: string /** The supervisor-authored `AgentProfile` (systemPrompt + model reach the harness via §1.5). */ profile: AgentProfile - /** Which local harness CLI drives this leaf. */ - harness: LocalHarness /** Require measured usage from this leaf. Budgeted supervision refuses the default unmetered * local-CLI mode; set false only when the selected runner actually returns token usage. */ budgetExempt?: WorktreeCliExecutorOptions['budgetExempt'] @@ -104,7 +101,6 @@ export function worktreeFanout( createWorktreeCliExecutor({ repoRoot: options.repoRoot, profile: item.profile, - harness: item.harness, taskPrompt: options.taskPrompt, executionAttemptId: ctx.node.attemptId, ...(item.budgetExempt !== undefined ? { budgetExempt: item.budgetExempt } : {}), diff --git a/tests/kernel/worktree-loop.test.ts b/tests/kernel/worktree-loop.test.ts index ed7f10c6..9d5b408e 100644 --- a/tests/kernel/worktree-loop.test.ts +++ b/tests/kernel/worktree-loop.test.ts @@ -60,13 +60,11 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { { name: 'claude', profile: profile('claude', 'claude-code'), - harness: 'claude-code', budgetExempt: false, }, { name: 'opencode', profile: profile('opencode', 'opencode'), - harness: 'opencode', budgetExempt: false, }, ], @@ -97,7 +95,6 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { { name: 'claude', profile: profile('claude', 'claude-code'), - harness: 'claude-code', budgetExempt: false, }, ], @@ -124,7 +121,6 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { { name: 'claude', profile: profile('claude', 'claude-code'), - harness: 'claude-code', budgetExempt: false, }, ], diff --git a/tests/mcp/worktree-harness.test.ts b/tests/mcp/worktree-harness.test.ts index 5b9eed10..5d88cc73 100644 --- a/tests/mcp/worktree-harness.test.ts +++ b/tests/mcp/worktree-harness.test.ts @@ -184,7 +184,8 @@ describe('runWorktreeHarness profile materialization', () => { 'src/value.ts': 'export const value = 1\n', }) const profile: AgentProfile = { - model: { default: 'gpt-5.4', reasoningEffort: 'xhigh' }, + harness: 'codex', + model: { provider: 'openai', default: 'gpt-5.4', reasoningEffort: 'xhigh' }, prompt: { systemPrompt: systemMarker, instructions: [promptInstructionMarker], @@ -208,7 +209,6 @@ describe('runWorktreeHarness profile materialization', () => { const executor = createWorktreeCliExecutor({ repoRoot, profile, - harness: 'codex', taskPrompt: taskMarker, runId, codexReproducible: true, diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index 826adfd7..c494926a 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -64,11 +64,16 @@ vi.mock('node:http', async () => { } }) -import { type RunLocalHarnessOptions, runLocalHarness } from '../../src/mcp/local-harness' +import { + type LocalHarness, + type RunLocalHarnessOptions, + runLocalHarness, +} from '../../src/mcp/local-harness' import type { GitRunner } from '../../src/mcp/worktree' import { type AgentSpec, createExecutor } from '../../src/runtime' import { createWorktreeCliExecutor, + type WorktreeCliExecutorOptions, type WorktreePatchArtifact, } from '../../src/runtime/supervise/worktree-cli-executor' @@ -113,10 +118,15 @@ function freshGitState(overrides?: Partial): FakeGitState { } } -const authoredProfile: AgentProfile = { - name: 'careful-refactorer', - prompt: { systemPrompt: 'You are a careful refactorer. Keep diffs minimal.' }, - model: { default: 'deepseek/deepseek-v4-flash' }, +function authoredProfile(harness: LocalHarness = 'claude-code'): AgentProfile { + const provider = + harness === 'codex' ? 'openai' : harness === 'claude-code' ? 'anthropic' : 'deepseek' + return { + name: 'careful-refactorer', + harness, + prompt: { systemPrompt: 'You are a careful refactorer. Keep diffs minimal.' }, + model: { provider, default: 'deepseek/deepseek-v4-flash' }, + } } const reproducibleCodexProfile: AgentProfile = { @@ -217,8 +227,7 @@ describe('createWorktreeCliExecutor', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'fix the off-by-one', runGit: makeFakeGit(state), runHarness, @@ -248,8 +257,7 @@ describe('createWorktreeCliExecutor', () => { }) const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'codex', + profile: authoredProfile('codex'), taskPrompt: 'add a.ts', runGit: makeFakeGit(state), runHarness: vi.fn(async () => ({ @@ -277,8 +285,7 @@ describe('createWorktreeCliExecutor', () => { const state = freshGitState() const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'opencode', + profile: authoredProfile('opencode'), taskPrompt: 'noop', runGit: makeFakeGit(state), runHarness: vi.fn(async () => ({ @@ -303,8 +310,7 @@ describe('createWorktreeCliExecutor', () => { it('is budgetExempt by default (a harness CLI cannot account tokens)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', runGit: makeFakeGit(freshGitState()), runHarness: vi.fn(), @@ -319,8 +325,7 @@ describe('createWorktreeCliExecutor', () => { // a ceiling that can never fire while believing it is protected. const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', runGit: makeFakeGit(freshGitState()), runHarness: vi.fn(async () => ({ @@ -348,8 +353,7 @@ describe('createWorktreeCliExecutor', () => { expect(() => createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'codex', + profile: authoredProfile('codex'), taskPrompt: 'x', codexReadDeniedPaths: ['/usr/lib/example/gold.py'], runGit: makeFakeGit(freshGitState()), @@ -364,10 +368,9 @@ describe('createWorktreeCliExecutor', () => { createWorktreeCliExecutor({ repoRoot: '/workspace', profile: { - ...authoredProfile, + ...authoredProfile(), connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], }, - harness: 'claude-code', taskPrompt: 'x', runGit: makeFakeGit(state), runHarness: vi.fn(), @@ -386,7 +389,6 @@ describe('createWorktreeCliExecutor', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', profile: reproducibleCodexProfile, - harness: 'codex', taskPrompt: 'fix the bug', codexReproducible: true, codexReadDeniedPaths: ['/usr/lib/example/gold.py'], @@ -504,13 +506,20 @@ describe('createWorktreeCliExecutor', () => { }) }) - it('rejects contradictory reproducible Codex configuration', () => { + it('rejects a separate harness override and contradictory reproducible Codex configuration', () => { expect(() => createWorktreeCliExecutor({ repoRoot: '/workspace', profile: reproducibleCodexProfile, harness: 'claude-code', taskPrompt: 'x', + } as WorktreeCliExecutorOptions & { harness: LocalHarness }), + ).toThrow(/separate harness is forbidden/) + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: authoredProfile(), + taskPrompt: 'x', codexReproducible: true, }), ).toThrow(/requires harness "codex"/) @@ -518,7 +527,6 @@ describe('createWorktreeCliExecutor', () => { createWorktreeCliExecutor({ repoRoot: '/workspace', profile: reproducibleCodexProfile, - harness: 'codex', taskPrompt: 'x', codexReproducible: true, budgetExempt: true, @@ -531,7 +539,6 @@ describe('createWorktreeCliExecutor', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', profile: reproducibleCodexProfile, - harness: 'codex', taskPrompt: 'x', codexReproducible: true, runGit: makeFakeGit(state), @@ -553,8 +560,7 @@ describe('createWorktreeCliExecutor', () => { it('budgetExempt: false opts the leaf into metering (explicit, not a buried hardcode)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', budgetExempt: false, runGit: makeFakeGit(freshGitState()), @@ -583,8 +589,7 @@ describe('createWorktreeCliExecutor', () => { let seen: RunLocalHarnessOptions | undefined const executor = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), runGit: makeFakeGit(state), runHarness: vi.fn(async (options) => { seen = options @@ -609,8 +614,7 @@ describe('createWorktreeCliExecutor', () => { it('resultArtifact() before execute() resolves throws (fail loud, no fabricated artifact)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', runGit: makeFakeGit(freshGitState()), runHarness: vi.fn(), @@ -632,8 +636,7 @@ describe('createWorktreeCliExecutor', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'opencode', + profile: authoredProfile('opencode'), taskPrompt: 'do it', runGit: makeFakeGit(state), runHarness: realRunHarness, @@ -653,8 +656,7 @@ describe('createWorktreeCliExecutor', () => { const ranIn: { command: string; cwd: string }[] = [] const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'opencode', + profile: authoredProfile('opencode'), taskPrompt: 'do it', testCmd: 'pnpm test', typecheckCmd: 'pnpm typecheck', @@ -688,8 +690,7 @@ describe('createWorktreeCliExecutor', () => { it('omits `checks` entirely when no verification command is configured', async () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', runGit: makeFakeGit(freshGitState()), runHarness: vi.fn(async () => ({ @@ -740,7 +741,7 @@ describe('createWorktreeCliExecutor', () => { }) const spec: AgentSpec = { profile: { - ...authoredProfile, + ...authoredProfile('codex'), harness: 'codex', model: { provider: 'tangle-router', default: 'live' }, }, @@ -782,28 +783,32 @@ describe('createWorktreeCliExecutor', () => { expect(state.worktreesRemoved).toEqual(state.worktreesCreated) }) - it('fails loud on a missing repoRoot, harness, or both task sources', async () => { + it('fails loud on a missing repoRoot, incomplete profile, or both task sources', async () => { expect(() => createWorktreeCliExecutor({ repoRoot: '', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), taskPrompt: 'x', }), ).toThrow(/repoRoot required/) expect(() => createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: { ...authoredProfile(), harness: undefined }, + taskPrompt: 'x', + }), + ).toThrow(/AgentProfile\.harness must be explicit/) + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: authoredProfile(), taskPrompt: '', }), ).toThrow(/taskPrompt required/) const noTask = createWorktreeCliExecutor({ repoRoot: '/workspace', - profile: authoredProfile, - harness: 'claude-code', + profile: authoredProfile(), }) await expect(noTask.execute(undefined, new AbortController().signal)).rejects.toThrow( /execute task required/, From 3a2591273c86e0c0a017c14a179b24f95a06c479 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:46:40 -0600 Subject: [PATCH 16/39] fix(bench): route model execution through exact profiles --- bench/src/cloud-loop.mts | 9 +- bench/src/commit0-gate.mts | 77 ++++++--- bench/src/fleet.mts | 11 +- .../profiles/quant-researcher.profile.json | 7 +- bench/src/quant-arena/quant-loop.mts | 149 +++++++++++------- bench/src/sandbox-run.ts | 51 ++---- bench/src/search-bench/profiles.ts | 24 ++- bench/src/search-bench/run.mts | 7 +- bench/src/smoke-structural-rollout.mts | 26 ++- bench/src/swe-arena/arms.ts | 88 +++++++++-- bench/src/swe-emit-patch.mts | 31 +++- bench/src/swe-improve.mts | 35 +++- bench/src/swe-local-proof.mts | 32 +++- bench/src/swe-stream.mts | 46 +++++- bench/src/swe-structural.mts | 33 +++- bench/src/tb-container-executor.mts | 44 +++++- bench/src/tb-container-executor.test.mts | 14 +- bench/src/tb-supervisor-sidecar.mts | 38 ++--- bench/src/terminal-compare.ts | 48 ++++-- bench/tb_agents/opencode_router_agent.py | 126 ++++++++++----- 20 files changed, 617 insertions(+), 279 deletions(-) diff --git a/bench/src/cloud-loop.mts b/bench/src/cloud-loop.mts index b7dd73de..d44f0aac 100644 --- a/bench/src/cloud-loop.mts +++ b/bench/src/cloud-loop.mts @@ -72,6 +72,7 @@ function tools(events: ReadonlyArray): string[] { async function main(): Promise { const routerKey = env('TANGLE_API_KEY') const model = env('MODEL', 'deepseek-v4-flash') + const workerProvider = env('WORKER_PROVIDER', 'openai-compat') const routerBaseUrl = env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1') const rounds = Number(env('ROUNDS', '3')) const client = new Sandbox({ baseUrl: env('SANDBOX_BASE_URL', 'https://sandbox.tangle.tools'), apiKey: routerKey }) @@ -97,7 +98,13 @@ async function main(): Promise { let output = '' let events: unknown[] = [] try { - const agentRun = sandboxAgentRun({ model, routerBaseUrl, backendType: 'opencode', name: `worker-r${round}` }) + const agentRun = sandboxAgentRun({ + profile: { + name: `worker-r${round}`, + harness: 'opencode', + model: { provider: workerProvider, default: model }, + }, + }) const run = await openSandboxRun( client, { agentRun, signal: controller.signal }, diff --git a/bench/src/commit0-gate.mts b/bench/src/commit0-gate.mts index dd694e55..8be533d1 100644 --- a/bench/src/commit0-gate.mts +++ b/bench/src/commit0-gate.mts @@ -53,10 +53,13 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { + collectAgentTurn, + createExecutor, type AgentRunSpec, type Deliverable, openSandboxRun, type SandboxRun, + streamAgentTurn, } from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' import { createCommit0Adapter } from './benchmarks/commit0' @@ -153,8 +156,17 @@ interface ShotCfg { * registered model names (e.g. gpt-4.1). Override via WORKER_PROVIDER. */ provider: string timeoutMs: number - /** local-backend: the opencode CLI binary (cli-bridge fallback when the sandbox is down). */ - opencodeBin: string + /** Local Runtime bridge transport. The profile, not this transport, selects the harness/model. */ + bridgeUrl?: string + bridgeBearer?: string +} + +function workerProfile(cfg: ShotCfg, name: string) { + return { + name, + harness: 'opencode' as const, + model: { provider: cfg.provider, default: cfg.model }, + } } /** The diff the in-box agent produces, read back off the box FS (+ any stream error). */ @@ -189,21 +201,15 @@ async function runShot(task: BenchTask, attempt: number, cfg: ShotCfg, steer?: s // other rollout (the powered-run crash). const controller = new AbortController() const timer = cfg.timeoutMs > 0 ? setTimeout(() => controller.abort(), cfg.timeoutMs) : undefined - // backend.model pins provider/model/baseUrl only; the platform writes the in-box - // provider config keyed to the box's own OPENCODE_MODEL_API_KEY. The inline - // profile + backend override is the same generic AgentRunSpec the runAgentRounds kernel - // boots. Never inject an external key — the egress proxy 403s foreign credentials. + // The exact profile owns harness/provider/model. Sandbox overrides contain only box + // infrastructure; Runtime derives the backend from the profile and refuses conflicts. const agentRun: AgentRunSpec = { - profile: { name: 'commit0-worker', metadata: { backendType: 'opencode' } }, + profile: workerProfile(cfg, 'commit0-worker'), name: 'commit0-worker', taskToPrompt: () => '', // unused — the prompt is streamed directly by openSandboxRun sandboxOverrides: { name: `commit0-${task.id}-${attempt}-${randomSuffix()}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60), environment: 'universal', - backend: { - type: 'opencode', - model: { provider: cfg.provider, model: cfg.model, baseUrl: cfg.routerBaseUrl }, - }, }, } let run: SandboxRun | undefined @@ -310,15 +316,29 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee if (co.code !== 0) { return { task, attempt, diff: '', ok: false, events: 0, wallMs: Date.now() - startedAt, detail: `git checkout ${meta.baseCommit} failed: ${co.out.trim().slice(-180)}` } } - // openai/* → route through the router (OPENAI_* env); anything else → opencode's - // OWN configured auth (kimi-for-coding / zai coding-plan subscriptions). - const env = cfg.model.startsWith('openai/') - ? { ...process.env, OPENAI_API_KEY: cfg.routerKey, OPENAI_BASE_URL: cfg.routerBaseUrl } - : process.env const prompt = steer ? steeredPrompt(localRolloutPrompt(meta), steer) : localRolloutPrompt(meta) - const oc = await sh(cfg.opencodeBin, ['run', prompt, '-m', cfg.model, '--dir', dir], { timeoutMs: cfg.timeoutMs, env }) - const lines = oc.out.split('\n') - const events = lines.length + if (!cfg.bridgeUrl || !cfg.bridgeBearer) { + throw new Error('local rollout requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER') + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: cfg.bridgeUrl, + bridgeBearer: cfg.bridgeBearer, + cwd: dir, + ...(cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {}), + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile: workerProfile(cfg, `commit0-local-${attempt}`) }, + prompt, + cfg.timeoutMs > 0 ? { timeoutMs: cfg.timeoutMs } : {}, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `Runtime bridge ended with ${turn.status}`) + } + const traceEvents = turn.events.map((event) => JSON.stringify(event)) + const events = turn.events.length // Read the diff straight from git, scoped to src_dir (excludes the .venv the agent made). const diffRes = await sh('bash', ['-c', `cd ${JSON.stringify(dir)} && git add -- ${JSON.stringify(meta.srcDir)} && git diff --cached -- ${JSON.stringify(meta.srcDir)}`], { timeoutMs: 60_000 }) const diff = diffRes.out @@ -329,10 +349,10 @@ async function runShotLocal(task: BenchTask, attempt: number, cfg: ShotCfg, stee diff, ok, events, - traceEvents: lines.slice(-TRACE_EVENTS_TAIL), + traceEvents: traceEvents.slice(-TRACE_EVENTS_TAIL), ...(steer ? { steer } : {}), wallMs: Date.now() - startedAt, - ...(ok ? {} : { detail: `no diff (opencode rc=${oc.code}): ${oc.out.trim().slice(-160)}` }), + ...(ok ? {} : { detail: `no diff (Runtime bridge completed): ${turn.finalText.trim().slice(-160)}` }), } } catch (err) { return { task, attempt, diff: '', ok: false, events: 0, ...(steer ? { steer } : {}), wallMs: Date.now() - startedAt, detail: `local rollout error: ${(err instanceof Error ? err.message : String(err)).slice(0, 180)}` } @@ -349,7 +369,7 @@ async function main(): Promise { const backend = process.env.COMMIT0_BACKEND === 'local' ? 'local' : 'sandbox' const n = Number(process.env.N ?? 8) const k = Number(process.env.K ?? 4) - const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'gpt-4.1') + const model = process.env.WORKER_MODEL ?? (backend === 'local' ? 'kimi-for-coding/kimi-k2-thinking' : 'deepseek-v4-flash') const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' // The arms under test. `random` = K independent blind shots (the equal-compute // control); `refineAudit` = blind shot 0, then trace-only-analyst-steered shots. @@ -363,7 +383,6 @@ async function main(): Promise { const needsRouterKey = backend === 'sandbox' || model.startsWith('openai/') || armNames.includes('refineAudit') const routerKey = needsRouterKey ? must('TANGLE_API_KEY') : (process.env.TANGLE_API_KEY ?? '') const sandboxBaseUrl = process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools' - const opencodeBin = process.env.OPENCODE_BIN ?? join(process.env.HOME ?? '', '.local/bin/opencode') // openai-compat = generic passthrough so cheap router models resolve in-box; // `openai` rejects non-registered model names. Override via WORKER_PROVIDER. const provider = process.env.WORKER_PROVIDER ?? 'openai-compat' @@ -384,7 +403,17 @@ async function main(): Promise { // Phase 1 — rollouts, concurrent. sandbox = remote box; local = cli-bridge (opencode // in a tmpdir, diff read from git). Both fault-isolated → a failure is a NO-DIFF, never a throw. - const cfg: ShotCfg = { sandboxBaseUrl, sandboxKey: routerKey, routerBaseUrl, routerKey, model, provider, timeoutMs, opencodeBin } + const cfg: ShotCfg = { + sandboxBaseUrl, + sandboxKey: routerKey, + routerBaseUrl, + routerKey, + model, + provider, + timeoutMs, + bridgeUrl: process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL, + bridgeBearer: process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER, + } const runRollout = backend === 'local' ? runShotLocal : runShot const analyze: AnalystFn = llmAnalyst({ routerBaseUrl, routerKey, model: analystModel }) const logShot = (armName: string, s: Shot) => diff --git a/bench/src/fleet.mts b/bench/src/fleet.mts index 62831997..832df7f9 100644 --- a/bench/src/fleet.mts +++ b/bench/src/fleet.mts @@ -46,7 +46,7 @@ interface WorkerResult { async function runWorker( client: Sandbox, - cfg: { backendType: WorkerBackendType; model: string; routerBaseUrl: string }, + cfg: { backendType: WorkerBackendType; model: string; provider: string; routerBaseUrl: string }, id: string, task: string, priorLearnings: string, @@ -56,7 +56,13 @@ async function runWorker( const controller = new AbortController() const timer = setTimeout(() => controller.abort(), Number(process.env.TIMEOUT_MS ?? 240_000)) try { - const agentRun = sandboxAgentRun({ ...cfg, name: id }) + const agentRun = sandboxAgentRun({ + profile: { + name: id, + harness: cfg.backendType, + model: { provider: cfg.provider, default: cfg.model }, + }, + }) const run = await openSandboxRun( client, { agentRun, signal: controller.signal }, @@ -80,6 +86,7 @@ async function main(): Promise { const cfg = { backendType: env('BACKEND', 'opencode') as WorkerBackendType, model: env('MODEL', 'deepseek-v4-flash'), + provider: env('WORKER_PROVIDER', 'openai-compat'), routerBaseUrl: env('ROUTER_BASE_URL', 'https://router.tangle.tools/v1'), } const n = Math.min(Number(env('N', '2')), subtasks.length) diff --git a/bench/src/quant-arena/profiles/quant-researcher.profile.json b/bench/src/quant-arena/profiles/quant-researcher.profile.json index 900589ea..40421f93 100644 --- a/bench/src/quant-arena/profiles/quant-researcher.profile.json +++ b/bench/src/quant-arena/profiles/quant-researcher.profile.json @@ -1,6 +1,11 @@ { "name": "quant-researcher", - "description": "Strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. Same claude harness as default-author; only the system prompt differs, so any performance delta is attributable to the lens.", + "description": "Pi strategy-author lens that speaks quant language: regimes, turnover, capacity, transaction costs. The model and harness match the default author; only the system prompt differs.", + "harness": "pi", + "model": { + "provider": "tangle-router", + "default": "glm-5.2" + }, "prompt": { "systemPrompt": "You are a systematic-trading researcher at a mid-frequency equity desk. You think in regimes (trend vs chop, calm vs stress), you treat turnover as a cost center (every rebalance pays spread + impact), you size positions for capacity, and you are allergic to backtest overfitting — you prefer one robust effect over five fitted parameters. When you write a strategy you state, in comments, the economic rationale the rule is harvesting and why it should survive out-of-sample." } diff --git a/bench/src/quant-arena/quant-loop.mts b/bench/src/quant-arena/quant-loop.mts index e7f2518b..7758d467 100644 --- a/bench/src/quant-arena/quant-loop.mts +++ b/bench/src/quant-arena/quant-loop.mts @@ -1,6 +1,6 @@ /** * QUANT-ARENA campaign loop — the improvement loop embodied for trading - * strategies. One command runs: strategy authors (Claude, profile-pinned) + * strategies. One command runs: strategy authors (Runtime profile-pinned) * propose candidate strategies (v2 `onBar` contract, driven incrementally by * driver.ts) -> every candidate passes a two-stage leak audit -> survivors * are scored on K bootstrap in-sample windows against the pinned baselines @@ -15,7 +15,7 @@ * are never the acceptance currency. * * tsx src/quant-arena/quant-loop.mts --out [--candidates 2] [--seed 20260722] - * [--author-model sonnet] [--audit-model haiku] [--skip-llm-audit] + * [--author-model glm-5.2] [--audit-model glm-5.2] [--skip-llm-audit] * * Kernel reuse (import, not copy — see src/swe-arena/): * - cost accounting: the lib's durable CostLedger (createRunCostLedger) + @@ -38,12 +38,12 @@ import { join } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import process from 'node:process' import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { agentProfileSchema, type AgentProfile } from '@tangle-network/agent-interface' +import { collectAgentTurn, createExecutor, streamAgentTurn } from '@tangle-network/agent-runtime/kernel' import { loadCampaignCells } from '../swe-arena/cell-evidence.mts' import { reconcileCrashOrphansOnDisk } from '../swe-arena/ledger-orphans.mts' import { loadLedgerReceipts } from '../swe-arena/manifest.mts' -import { loadAuthorProfile, type ProposerSpec } from '../swe-arena/proposer-fanout.mts' -import { proposerShotEnv } from '../swe-arena/outer-loop.mts' -import { run } from '../swe-arena/proc.ts' +import { resolveAuthorProfile, type ProposerSpec } from '../swe-arena/proposer-fanout.mts' import { runBacktest, statsForRange, type BacktestConfig, type RangeStats } from './backtest.ts' import { loadInSample, type AlignedBars } from './data.ts' import { loadStrategyFile } from './driver.ts' @@ -57,6 +57,9 @@ import * as equalWeight from './strategies/equal-weight/strategy.ts' import * as smaCrossover from './strategies/sma-crossover/strategy.ts' export const QUANT_PROFILES_DIR = fileURLToPath(new URL('./profiles', import.meta.url)) +const DEFAULT_AUTHOR_PROFILE = fileURLToPath( + new URL('../swe-arena/profiles/default-author.profile.json', import.meta.url), +) // --------------------------------------------------------------------------- // Config. @@ -88,11 +91,17 @@ export const PINNED_BASELINES: Record = { /** The two demo author seats: the plain author and the quant lens. */ export function defaultQuantProposers(): ProposerSpec[] { return [ - { name: 'default-author', profile: 'default-author.profile.json', harness: 'claude-code' }, + { + name: 'default-author', + profile: DEFAULT_AUTHOR_PROFILE, + harness: 'pi', + model: 'glm-5.2', + }, { name: 'quant-researcher', profile: join(QUANT_PROFILES_DIR, 'quant-researcher.profile.json'), - harness: 'claude-code', + harness: 'pi', + model: 'glm-5.2', lens: 'Favor ONE economically-motivated effect (trend, mean reversion, vol targeting) with few parameters. ' + 'State the regime in which it should work and keep turnover low enough that 15bps a side cannot eat the edge.', @@ -110,8 +119,8 @@ export function defaultConfig(outDir: string): QuantLoopConfig { warmupDays: 120, costBps: 10, slippageBps: 5, - authorModel: 'sonnet', - auditModel: 'haiku', + authorModel: 'glm-5.2', + auditModel: 'glm-5.2', skipLlmAudit: false, authorTimeoutMs: 480_000, auditTimeoutMs: 240_000, @@ -183,10 +192,10 @@ export async function loadNotebookRows(notebookPath: string): Promise console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`) // --------------------------------------------------------------------------- -// Claude shots (author + auditor) — metered paid calls through the run ledger. +// Exact-profile shots (author + auditor) — metered paid calls through Runtime + the run ledger. // --------------------------------------------------------------------------- -interface ClaudeShotOutcome { +interface ProfileShotOutcome { text: string model: string inputTokens: number @@ -197,64 +206,81 @@ interface ClaudeShotOutcome { type Ledger = ReturnType -async function claudeShot(opts: { +function withModel(profile: AgentProfile, model: string, name = profile.name): AgentProfile { + return agentProfileSchema.parse({ + ...profile, + ...(name ? { name } : {}), + model: { ...profile.model, default: model }, + }) +} + +function quantAuditProfile(model: string): AgentProfile { + return agentProfileSchema.parse({ + name: 'quant-leak-auditor', + harness: 'pi', + model: { provider: 'tangle-router', default: model }, + prompt: { + systemPrompt: + 'Audit the supplied trading strategy for look-ahead bias and nondeterminism. Follow the requested JSON response contract exactly.', + }, + }) +} + +async function profileShot(opts: { prompt: string - model: string - systemPrompt?: string + profile: AgentProfile timeoutMs: number cwd: string -}): Promise { - const argv = [ - '-p', - '--output-format', - 'json', - '--model', - opts.model, - // The shot is pure text generation: no filesystem, no shell, no web. - '--disallowed-tools', - 'Bash,Edit,Write,Read,Glob,Grep,WebFetch,WebSearch,Task,NotebookEdit', - ...(opts.systemPrompt ? ['--append-system-prompt', opts.systemPrompt] : []), - ] - const res = await run('claude', argv, { - stdin: opts.prompt, +}): Promise { + const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL + const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER + if (!bridgeUrl || !bridgeBearer) { + throw new Error( + 'quant profile shots require CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER', + ) + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, cwd: opts.cwd, - env: proposerShotEnv('claude-code'), timeoutMs: opts.timeoutMs, }) - if (res.code !== 0) { - throw new Error(`claude shot exited ${res.code}${res.timedOut ? ' (timeout)' : ''}: ${(res.stderr || res.stdout).slice(0, 800)}`) - } - let parsed: Record - try { - parsed = JSON.parse(res.stdout) as Record - } catch { - throw new Error(`claude shot: unparseable --output-format json stdout: ${res.stdout.slice(0, 400)}`) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile: opts.profile }, + opts.prompt, + { timeoutMs: opts.timeoutMs }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `quant profile shot ended with ${turn.status}`) } - if (parsed.is_error === true) throw new Error(`claude shot errored: ${String(parsed.result).slice(0, 800)}`) - const usage = (parsed.usage ?? {}) as Record - const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0) + const cachedTokens = Number(turn.usage.promptCache?.readTokens ?? 0) return { - text: typeof parsed.result === 'string' ? parsed.result : '', - model: typeof parsed.model === 'string' ? parsed.model : opts.model, - inputTokens: num(usage.input_tokens), - outputTokens: num(usage.output_tokens), - cachedTokens: num(usage.cache_read_input_tokens), - costUsd: typeof parsed.total_cost_usd === 'number' ? parsed.total_cost_usd : null, + text: turn.finalText, + model: turn.usage.model ?? opts.profile.model?.default ?? 'unknown', + inputTokens: turn.usage.input, + outputTokens: turn.usage.output, + cachedTokens: Number.isFinite(cachedTokens) ? cachedTokens : 0, + costUsd: turn.usage.costUsd ?? null, } } -async function meteredClaudeShot( +async function meteredProfileShot( ledger: Ledger, meta: { phase: string; actor: string; tags: Record }, - opts: Parameters[0], -): Promise<{ outcome: ClaudeShotOutcome; costUsd: number | null }> { - const paid = await ledger.runPaidCall({ + opts: Parameters[0], +): Promise<{ outcome: ProfileShotOutcome; costUsd: number | null }> { + const model = opts.profile.model?.default + if (!model) throw new Error('meteredProfileShot: profile.model.default is required') + const paid = await ledger.runPaidCall({ channel: 'driver', phase: meta.phase, actor: meta.actor, - model: opts.model, + model, tags: meta.tags, - execute: () => claudeShot(opts), + execute: () => profileShot(opts), receipt: (v) => ({ model: v.model, inputTokens: v.inputTokens, @@ -379,12 +405,12 @@ async function llmLeakAudit( code: string, ): Promise<{ verdict: 'clean' | 'leak' | 'inconclusive'; evidence: string }> { for (let attempt = 0; attempt < 2; attempt++) { - const { outcome } = await meteredClaudeShot( + const { outcome } = await meteredProfileShot( ledger, - { phase: 'audit.leak', actor: 'leak-auditor:claude', tags: { candidateId, attempt: String(attempt) } }, + { phase: 'audit.leak', actor: 'leak-auditor:runtime', tags: { candidateId, attempt: String(attempt) } }, { prompt: AUDIT_PROMPT_HEADER + '```ts\n' + code + '\n```', - model: config.auditModel, + profile: quantAuditProfile(config.auditModel), timeoutMs: config.auditTimeoutMs, cwd: config.outDir, }, @@ -630,7 +656,9 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo const rows: CandidateRow[] = [] for (const proposer of config.proposers) { - const profile = loadAuthorProfile(proposer) + const sourceProfile = resolveAuthorProfile(proposer) + if (!sourceProfile) throw new Error(`quant proposer ${proposer.name}: exact profile is required`) + const profile = withModel(sourceProfile, config.authorModel, `quant-${proposer.name}`) for (let shot = 0; shot < config.candidatesPerProposer; shot++) { nTried += 1 const candidateId = `cand-${String(nTried).padStart(3, '0')}-${proposer.name}` @@ -642,7 +670,7 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo at: new Date().toISOString(), candidateId, proposer: proposer.name, - authorModel: config.authorModel, + authorModel: profile.model?.default ?? config.authorModel, strategyPath: null, sha256: null, authoringCostUsd: null, @@ -663,13 +691,12 @@ async function runQuantCampaignWithWorker(worker: VbtWorker, config: QuantLoopCo nTried, ...(proposer.lens ? { lens: proposer.lens } : {}), }) - const { outcome, costUsd } = await meteredClaudeShot( + const { outcome, costUsd } = await meteredProfileShot( ledger, { phase: 'search.proposal', actor: `proposer-shot:${proposer.name}`, tags: { candidateId } }, { prompt, - model: config.authorModel, - ...(profile?.prompt?.systemPrompt ? { systemPrompt: profile.prompt.systemPrompt } : {}), + profile, timeoutMs: config.authorTimeoutMs, cwd: config.outDir, }, @@ -798,7 +825,7 @@ if (isMain) { if (!outDir) { console.error( 'usage: tsx src/quant-arena/quant-loop.mts --out [--candidates 2] [--seed 20260722] ' + - '[--author-model sonnet] [--audit-model haiku] [--skip-llm-audit] # SPENDS: author + audit shots', + '[--author-model glm-5.2] [--audit-model glm-5.2] [--skip-llm-audit] # SPENDS: author + audit shots', ) process.exit(2) } diff --git a/bench/src/sandbox-run.ts b/bench/src/sandbox-run.ts index 7716a3b4..b3f9bd8e 100644 --- a/bench/src/sandbox-run.ts +++ b/bench/src/sandbox-run.ts @@ -14,10 +14,12 @@ import { type AgentRunSpec, type OutputAdapter, } from '@tangle-network/agent-runtime/kernel' -// `BackendType` is the sandbox SDK's harness union and its canonical home. agent-runtime consumes -// it from there too; it is not re-exported from the kernel barrel. +import { parseExactAgentProfile } from '@tangle-network/agent-runtime' +// `BackendType` is the sandbox SDK's harness union and its canonical home. Runtime consumes it +// from there too; benchmark profiles use the same values as their exact harness identity. import type { BackendType } from '@tangle-network/sandbox' -import { runBenchRouterTurn } from './router-turn' +import { assertExecutableAgentProfile } from '../../src/runtime/supervise/model-policy' +import { benchRouterProfile, runBenchRouterTurn } from './router-turn' /** Parse the agent's final answer from the event stream (harness-agnostic). * The default deliverable; a benchmark whose artifact is a file overrides via @@ -69,11 +71,7 @@ export const llmAnalyst = (cfg: { routerBaseUrl: string; routerKey: string; mode { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, - profile: { - name: 'sandbox-run-analyst', - model: { provider: 'tangle-router', default: cfg.model }, - prompt: { systemPrompt }, - }, + profile: benchRouterProfile('sandbox-run-analyst', cfg.model, { systemPrompt }), }, `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`, ) @@ -86,43 +84,24 @@ export const llmAnalyst = (cfg: { routerBaseUrl: string; routerKey: string; mode * agent runs — no per-backend worker. */ export type WorkerBackendType = BackendType -/** Build the standard sandbox `AgentRunSpec` for a benchmark — the worker the - * kernel injects. `backendType` is the cost dial. Model auth is the BOX'S OWN - * provisioned credential: `backend.model` pins provider/model/baseUrl only, and - * the platform generates the in-box provider config keyed to - * `{env:OPENCODE_MODEL_API_KEY}`. Never pass an external router key into the - * box — the egress proxy rejects foreign credentials (403, empty output). */ +/** Build the standard sandbox `AgentRunSpec` for a benchmark. The complete profile is the only + * behavioral input: its harness selects the box backend and its provider/model select inference. + * Extra sandbox env remains infrastructure, not a second model-selection path. */ export function sandboxAgentRun(opts: { - model: string - routerBaseUrl: string - backendType?: WorkerBackendType - /** In-box model provider. Default `openai` (registered models like gpt-4.1). - * Cheap router models (deepseek/kimi/glm) are not in opencode's `openai` - * registry and 404 in-box — pass `openai-compat` (generic passthrough). */ - provider?: string - name?: string + profile: AgentProfile taskToPrompt?: (task: string) => string /** Extra box-level env (e.g. `TANGLE_SEARCH_DEFAULT_PROVIDER` to pin the in-box * agent's web-search provider, provider keys like EXA_API_KEY). Allowlisted * keys only reach the spawned CLI. Must NOT carry router/model credentials. */ env?: Record - /** The developer's AgentProfile — the one knob for "which agent" (prompt / model / - * tools / mcp). Spread through verbatim; the backend cost-dial is tagged into - * metadata. Omitted ⇒ a minimal worker profile. */ - profile?: AgentProfile }): AgentRunSpec { - const backendType = opts.backendType ?? 'opencode' - const name = opts.profile?.name ?? opts.name ?? `${backendType}-worker` + const profile = parseExactAgentProfile(opts.profile, 'sandboxAgentRun profile') + assertExecutableAgentProfile(profile, 'sandboxAgentRun profile') + const name = profile.name ?? 'sandbox-worker' return { - profile: { ...opts.profile, name, metadata: { ...opts.profile?.metadata, backendType } }, + profile, name, taskToPrompt: opts.taskToPrompt ?? ((t) => t), - sandboxOverrides: { - ...(opts.env ? { env: opts.env } : {}), - backend: { - type: backendType, - model: { provider: opts.provider ?? 'openai', model: opts.model, baseUrl: opts.routerBaseUrl }, - }, - }, + ...(opts.env ? { sandboxOverrides: { env: opts.env } } : {}), } } diff --git a/bench/src/search-bench/profiles.ts b/bench/src/search-bench/profiles.ts index 5e5f81f4..434437ff 100644 --- a/bench/src/search-bench/profiles.ts +++ b/bench/src/search-bench/profiles.ts @@ -17,7 +17,7 @@ * shape — a `transport:'http'` server at the router's `/v1/search/mcp` endpoint, * provider pinned via the `?provider=` query param. */ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, type HarnessType, agentProfileSchema } from '@tangle-network/agent-interface' export type SearchArm = 'native' | 'off' | { provider: string } @@ -45,37 +45,45 @@ export interface BuildArmProfileArgs { * foreign router credentials (403), so the provider arm needs the box-side * credential flow before sandbox-backed provider runs are trustworthy. */ tangleApiKey: string + harness: HarnessType + model: string + provider: string /** Optional name/metadata to merge. */ name?: string metadata?: Record } /** - * Build the AgentProfile fragment (tools / permission / mcp) for one search arm. - * Returned as a partial profile to be spread into `sandboxAgentRun({ profile })`. + * Build the complete executable AgentProfile for one search arm. */ export function buildArmProfile(args: BuildArmProfileArgs): AgentProfile { const { arm, routerBaseUrl, tangleApiKey } = args const base: AgentProfile = { name: args.name ?? 'search-bench-worker', + harness: args.harness, + model: { provider: args.provider, default: args.model }, ...(args.metadata ? { metadata: args.metadata } : {}), - } as AgentProfile + } if (arm === 'native') { // Native web tools stay on (harness default). No search MCP. For codex, // whose web_search ships off, explicitly enable it so the native arm is real. - return { ...base, tools: { web_search: true } } as AgentProfile + return agentProfileSchema.parse({ ...base, tools: { web_search: true } }) } if (arm === 'off') { // No web access at all — the parametric floor (search contributes nothing). - return { ...base, tools: { ...nativeWebToolsDisabled }, permission: { webfetch: 'deny' } } as AgentProfile + return agentProfileSchema.parse({ + ...base, + tools: { ...nativeWebToolsDisabled }, + permission: { webfetch: 'deny' }, + }) } if (!tangleApiKey) { throw new Error(`buildArmProfile: provider arm "${arm.provider}" requires a tangleApiKey for the search MCP`) } - return { + return agentProfileSchema.parse({ ...base, tools: { ...nativeWebToolsDisabled }, permission: { webfetch: 'deny' }, @@ -87,7 +95,7 @@ export function buildArmProfile(args: BuildArmProfileArgs): AgentProfile { enabled: true, }, }, - } as AgentProfile + }) } /** Stable condition label for the corpus: `:`. */ diff --git a/bench/src/search-bench/run.mts b/bench/src/search-bench/run.mts index 10d97fbb..1b4895cc 100644 --- a/bench/src/search-bench/run.mts +++ b/bench/src/search-bench/run.mts @@ -135,14 +135,13 @@ async function runCell( const timer = setTimeout(() => controller.abort(), cfg.timeoutMs ?? 300_000) try { const agentRun = sandboxAgentRun({ - model: cfg.model, - routerBaseUrl: cfg.routerBaseUrl, - backendType: harness, - ...(cfg.provider ? { provider: cfg.provider } : {}), profile: buildArmProfile({ arm, routerBaseUrl: cfg.routerBaseUrl, tangleApiKey: cfg.tangleApiKey, + harness, + model: cfg.model, + provider: cfg.provider ?? 'openai', name: `search-bench-${harness}-${armId}`, metadata: { harness, arm: armId, taskId: task.id }, }), diff --git a/bench/src/smoke-structural-rollout.mts b/bench/src/smoke-structural-rollout.mts index c8c65525..dbdb8eb3 100644 --- a/bench/src/smoke-structural-rollout.mts +++ b/bench/src/smoke-structural-rollout.mts @@ -37,12 +37,14 @@ import { defaultStructuralRolloutPolicy, runAgentic, type StructuralRolloutResult, + defaultAnalystInstruction, sandboxCheckRunner, selectBestIndex, structuralRollout, visibleCheckScore, } from '../../src/runtime/index' import { basePrompt, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' +import { withBenchProfile } from './router-turn' function must(name: string): string { const v = process.env[name] @@ -173,20 +175,34 @@ async function runTask(t: HumanEvalTask): Promise { name: 'humaneval-inert', check: () => ({ passes: 0, total: 1, errored: 0 }), }) + const workerProfile = withBenchProfile( + { + name: 'humaneval-structural-worker', + harness: 'cli-base', + model: { provider: 'together', default: MODEL }, + tools: { submit_answer: true }, + }, + { systemPrompt, temperature: TEMP, maxTokens: MAX_TOKENS, maxTurns: 2 }, + ) + const analystProfile = withBenchProfile( + { + name: 'humaneval-structural-analyst', + harness: 'cli-base', + model: { provider: 'together', default: MODEL }, + }, + { systemPrompt: defaultAnalystInstruction, temperature: TEMP, maxTokens: MAX_TOKENS }, + ) const result = (await runAgentic({ surface, task: { id: t.taskId, - systemPrompt, userPrompt: basePrompt(t), meta: { entryPoint: t.entryPoint }, }, routerBaseUrl: BASE, routerKey: must('TOGETHER_API_KEY'), - model: MODEL, - temperature: TEMP, - maxTokens: MAX_TOKENS, - innerTurns: 2, + workerProfile, + analystProfile, strategy, // The strategy's documented sizing: k samples + repair rounds + the check-author consult. budget: policy.k + policy.repairRounds + 1, diff --git a/bench/src/swe-arena/arms.ts b/bench/src/swe-arena/arms.ts index 6e37e301..a81bc4a4 100644 --- a/bench/src/swe-arena/arms.ts +++ b/bench/src/swe-arena/arms.ts @@ -17,6 +17,13 @@ import { chmod, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises import { join, resolve } from 'node:path' import { homedir } from 'node:os' import { fileURLToPath } from 'node:url' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type CollectedAgentTurn, +} from '@tangle-network/agent-runtime/kernel' import { run, runOk, type RunResult, shq } from './proc' import { materializeWorkspace } from './materialize' import type { ArmSpec, SoloUsage } from './types' @@ -192,12 +199,15 @@ export interface SoloArmSpec { kind: 'solo' /** Ledger/run-dir label, e.g. 'SOLO'. */ name: string - /** opencode model id, e.g. 'zai-coding-plan/glm-5.2'. */ - model: string + /** Complete worker identity. Runtime reads harness/provider/model/prompt only from here. */ + profile: AgentProfile /** Appended to the problem statement. Default: the shared worker suffix. */ promptSuffix?: string /** Whole-run ceiling. solo.sh used `timeout 1000` (s). */ timeoutMs?: number + /** Runtime bridge transport. Defaults to the CLI_BRIDGE environment variables. */ + bridgeUrl?: string + bridgeBearer?: string } export interface SupervisorArmSpec { @@ -271,8 +281,8 @@ export async function toArmIdentity(spec: ExecutableArmSpec): Promise { return { name: spec.name, kind: 'solo', - env: { model: spec.model }, - provenance: { repo: 'opencode', commit: 'cli' }, + env: { model: spec.profile.model?.default ?? 'missing' }, + provenance: { repo: spec.profile.harness ?? 'missing', commit: 'runtime-bridge' }, } } const loopsRepo = spec.loopsRepo ?? DEFAULT_LOOPS_REPO @@ -450,7 +460,29 @@ async function verifyWorkspace( return res.code } -/** solo.sh port: materialize → prompt → opencode run → patch extract → verify. */ +function runtimeSoloUsage(turn: CollectedAgentTurn): SoloUsage { + const calls = turn.events.filter((event) => event.type === 'llm_call') + const maxCtx = calls.reduce( + (max, event) => Math.max(max, (event.tokensIn ?? 0) + (event.tokensOut ?? 0)), + 0, + ) + const cacheRead = Number(turn.usage.promptCache?.readTokens ?? 0) + const cacheWrite = Number(turn.usage.promptCache?.writeTokens ?? 0) + const reasoning = turn.usage.reasoningTokens ?? 0 + return { + steps: calls.length, + in: turn.usage.input, + out: turn.usage.output, + reasoning, + cache_w: Number.isFinite(cacheWrite) ? cacheWrite : 0, + cache_r: Number.isFinite(cacheRead) ? cacheRead : 0, + max_ctx: maxCtx, + oc_cost: turn.usage.costUsd ?? turn.usage.estimatedCostUsd ?? 0, + total_io: turn.usage.input + turn.usage.output + reasoning, + } +} + +/** Solo arm: materialize → exact Runtime profile turn → patch extract → verify. */ export async function runSoloArm(spec: SoloArmSpec, ctx: ArmRunContext): Promise { const runDir = join(ctx.outDir, 'runs', ctx.instanceId, spec.name) const ws = join(runDir, 'ws') @@ -469,18 +501,44 @@ export async function runSoloArm(spec: SoloArmSpec, ctx: ArmRunContext): Promise }) const promptFile = join(runDir, 'prompt.txt') - await writeFile(promptFile, ctx.problemStatement + (spec.promptSuffix ?? WORKER_PROMPT_SUFFIX)) + const prompt = ctx.problemStatement + (spec.promptSuffix ?? WORKER_PROMPT_SUFFIX) + await writeFile(promptFile, prompt) + + const bridgeUrl = spec.bridgeUrl ?? process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL + const bridgeBearer = + spec.bridgeBearer ?? process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER + if (!bridgeUrl || !bridgeBearer) { + throw new Error( + 'runSoloArm requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER', + ) + } const t0 = Date.now() - const oc = await dotenvxBash( - ctx.secrets, - `cd ${shq(ws)} && opencode run "$(cat ${shq(promptFile)})" -m ${shq(spec.model)} --format json`, - { timeoutMs: spec.timeoutMs ?? 1_000_000, env: cell.env, signal: ctx.signal }, + const timeoutMs = spec.timeoutMs ?? 1_000_000 + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, + cwd: ws, + timeoutMs, + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile: spec.profile }, + prompt, + { timeoutMs, ...(ctx.signal ? { signal: ctx.signal } : {}) }, + ), ) const wall_s = Math.round((Date.now() - t0) / 1000) - await writeFile(join(runDir, 'oc.jsonl'), oc.stdout) - await writeFile(join(runDir, 'oc.err'), oc.stderr) - assertSoloProcessCompleted(oc, `runSoloArm ${ctx.instanceId}/${spec.name}`) + await writeFile( + join(runDir, 'runtime-events.jsonl'), + turn.events.map((event) => JSON.stringify(event)).join('\n') + '\n', + ) + if (turn.status !== 'completed') { + throw new Error( + `runSoloArm ${ctx.instanceId}/${spec.name}: ${turn.error?.message ?? turn.status}`, + ) + } if (ctx.signal?.aborted) throw ctx.signal.reason const patch = await extractPatch(ws, ctx.baseCommit, ctx.excludes, ctx.signal) @@ -492,12 +550,12 @@ export async function runSoloArm(spec: SoloArmSpec, ctx: ArmRunContext): Promise return { arm: spec.name, iid: ctx.instanceId, - oc_rc: oc.code, + oc_rc: 0, wall_s, patch_lines: patchLineCount(patch), verify_rc, verify_pass: verify_rc === 0, - usage: parseOcUsage(oc.stdout), + usage: runtimeSoloUsage(turn), patchPath, ws, } diff --git a/bench/src/swe-emit-patch.mts b/bench/src/swe-emit-patch.mts index 51d7e124..b6b902a1 100644 --- a/bench/src/swe-emit-patch.mts +++ b/bench/src/swe-emit-patch.mts @@ -23,8 +23,9 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel' -import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel' +import { defaultAnalystInstruction, refine, runAgentic } from '@tangle-network/agent-runtime/kernel' import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' +import { benchRouterProfile, withBenchProfile } from './router-turn' const exec = promisify(execFile) @@ -56,10 +57,31 @@ async function main(): Promise { const task = { id: bt.id, - systemPrompt: enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT, userPrompt: bt.prompt, meta: { instanceId: bt.id }, } + const workerProfile = withBenchProfile( + { + name: 'swe-emit-patch-worker', + harness: 'cli-base', + model: { provider: 'tangle-router', default: model }, + tools: { + list_files: true, + read_file: true, + edit_file: true, + ...(enableRun ? { run: true } : {}), + }, + }, + { + systemPrompt: enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT, + maxTokens, + maxTurns: innerTurns, + }, + ) + const analystProfile = benchRouterProfile('swe-emit-patch-analyst', model, { + systemPrompt: defaultAnalystInstruction, + maxTokens, + }) // Capture the patch from inside score() (called during the refine loop, BEFORE the surface closes // and rms the checkout). Keep the LATEST non-empty diff so accumulated refinements win and a later @@ -85,9 +107,8 @@ async function main(): Promise { strategy: refine, routerBaseUrl, routerKey, - model, - maxTokens, - innerTurns, + workerProfile, + analystProfile, budget, }) const files = capturedPatch ? [...capturedPatch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : [] diff --git a/bench/src/swe-improve.mts b/bench/src/swe-improve.mts index 53b98f10..4fac9204 100644 --- a/bench/src/swe-improve.mts +++ b/bench/src/swe-improve.mts @@ -30,10 +30,11 @@ import { } from '@tangle-network/agent-runtime' import { canonicalCandidateDigest, + agentProfileSchema, type AgentProfile, } from '@tangle-network/agent-interface' import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel' -import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel' +import { defaultAnalystInstruction, refine, runAgentic } from '@tangle-network/agent-runtime/kernel' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { createSweBenchAdapter } from './benchmarks/swe-bench' import type { BenchTask } from './benchmarks/types' @@ -43,6 +44,7 @@ import { requiredTokenPricing, } from './official-optimizer-config.mjs' import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' +import { benchRouterProfile, withBenchProfile } from './router-turn' const exec = promisify(execFile) @@ -70,6 +72,24 @@ async function main(): Promise { const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase()) const SEED_PROMPT = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT const allIds = [...new Set([...trainIds, ...selectionIds, ...testIds])] + const workerProfile = withBenchProfile( + { + name: 'swe-agent-worker', + harness: 'cli-base', + model: { provider: 'tangle-router', default: workerModel }, + tools: { + list_files: true, + read_file: true, + edit_file: true, + ...(enableRun ? { run: true } : {}), + }, + }, + { systemPrompt: SEED_PROMPT, maxTokens: workerMaxTokens, maxTurns: innerTurns }, + ) + const analystProfile = benchRouterProfile('swe-agent-analyst', workerModel, { + systemPrompt: defaultAnalystInstruction, + maxTokens: workerMaxTokens, + }) console.log('=== SWE-bench prompt optimization with official GEPA ===') console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl}`) @@ -103,11 +123,12 @@ async function main(): Promise { // score() BEFORE runAgentic closes (rm) the workspace; its score is a cheap // patch-exists proxy so the ONLY Docker run per cell is the improve judge. const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise => { - const promptText = candidate.prompt?.systemPrompt + const exactCandidate = agentProfileSchema.parse(candidate) + const promptText = exactCandidate.prompt?.systemPrompt if (promptText === undefined) throw new Error('agent: candidate profile has no system prompt') const bt = byId.get(scenario.id) if (!bt) throw new Error(`agent: unknown scenario ${scenario.id}`) - const task = { id: bt.id, systemPrompt: promptText, userPrompt: bt.prompt, meta: { instanceId: bt.id } } + const task = { id: bt.id, userPrompt: bt.prompt, meta: { instanceId: bt.id } } let capturedPatch = '' const stats = { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 } const proxy: AgenticSurface = { @@ -143,9 +164,8 @@ async function main(): Promise { strategy: refine, routerBaseUrl, routerKey, - model: workerModel, - maxTokens: workerMaxTokens, - innerTurns, + workerProfile: exactCandidate, + analystProfile, budget: budgetShots, }), receipt: (result) => { @@ -201,10 +221,9 @@ async function main(): Promise { }, } - const profile: AgentProfile = { name: 'swe-agent-glm46', prompt: { systemPrompt: SEED_PROMPT } } const scenario = (id: string): Scenario => ({ id, kind: 'swe-bench-verified' }) - const out = await improve(profile, { + const out = await improve(workerProfile, { surface: 'prompt', executionRef: canonicalCandidateDigest({ callback: 'bench/swe-improve', diff --git a/bench/src/swe-local-proof.mts b/bench/src/swe-local-proof.mts index 6847a083..78f78956 100644 --- a/bench/src/swe-local-proof.mts +++ b/bench/src/swe-local-proof.mts @@ -21,9 +21,10 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel' -import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel' +import { defaultAnalystInstruction, refine, runAgentic } from '@tangle-network/agent-runtime/kernel' import type { BenchScore } from './benchmarks/types' -import { createSweBenchEnvironment } from './swe-bench-env' +import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' +import { benchRouterProfile, withBenchProfile } from './router-turn' const exec = promisify(execFile) @@ -45,6 +46,28 @@ async function main(): Promise { console.log(`router=${routerBaseUrl}`) const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun }) + const workerProfile = withBenchProfile( + { + name: 'swe-local-proof-worker', + harness: 'cli-base', + model: { provider: 'tangle-router', default: model }, + tools: { + list_files: true, + read_file: true, + edit_file: true, + ...(enableRun ? { run: true } : {}), + }, + }, + { + systemPrompt: enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT, + maxTokens, + maxTurns: innerTurns, + }, + ) + const analystProfile = benchRouterProfile('swe-local-proof-analyst', model, { + systemPrompt: defaultAnalystInstruction, + maxTokens, + }) const taskList = await tasks(0, ids.length) const benchTaskById = new Map( (await adapter.loadTasks({ ids, split: 'test' })).map((task) => [task.id, task]), @@ -145,9 +168,8 @@ async function main(): Promise { strategy: refine, routerBaseUrl, routerKey, - model, - maxTokens, - innerTurns, + workerProfile, + analystProfile, budget, }) const rec = captured.get(task.id) diff --git a/bench/src/swe-stream.mts b/bench/src/swe-stream.mts index d812f4e1..f0129744 100644 --- a/bench/src/swe-stream.mts +++ b/bench/src/swe-stream.mts @@ -66,7 +66,7 @@ import { join } from 'node:path' import { promisify } from 'node:util' import { applyKnowledgeWriteBlocks, buildKnowledgeIndex, initKnowledgeBase, searchKnowledge } from '@tangle-network/agent-knowledge' import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel' -import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel' +import { defaultAnalystInstruction, refine, runAgentic } from '@tangle-network/agent-runtime/kernel' import type { BenchTask } from './benchmarks/types' import { createSweBenchEnvironment, resolveImageForMetadata, SWE_SEED_PROMPT } from './swe-bench-env' import { @@ -82,6 +82,7 @@ import { tail, zaiChatRaw, } from './swe-jail' +import { withBenchProfile } from './router-turn' const exec = promisify(execFile) @@ -286,6 +287,7 @@ const makeTransport = { ...body, ...WORKER_REASONING }, { name: 'swe-stream-worker', + harness: 'cli-base', model: { provider: 'zai', default: model, @@ -364,10 +366,44 @@ async function emitAttempt( } const task: AgenticTask = { id: bt.id, - systemPrompt: SWE_SEED_PROMPT, userPrompt: cfg.promptAppendix ? `${bt.prompt}\n\n${cfg.promptAppendix}` : bt.prompt, meta: { instanceId: bt.id }, } + const workerProfile = withBenchProfile( + { + name: 'swe-stream-worker', + harness: 'cli-base', + model: { + provider: 'zai', + default: WORKER_MODEL, + reasoningEffort: REASONING_ON ? 'high' : 'none', + }, + tools: { list_files: true, read_file: true, edit_file: true }, + }, + { + systemPrompt: SWE_SEED_PROMPT, + maxTokens: MAX_TOKENS, + maxTurns: INNER_TURNS, + temperature: cfg.temperature, + extraBody: WORKER_REASONING, + }, + ) + const analystProfile = withBenchProfile( + { + name: 'swe-stream-analyst', + harness: 'cli-base', + model: { + provider: 'zai', + default: WORKER_MODEL, + reasoningEffort: REASONING_ON ? 'high' : 'none', + }, + }, + { + systemPrompt: defaultAnalystInstruction, + maxTokens: MAX_TOKENS, + extraBody: WORKER_REASONING, + }, + ) let error: string | undefined try { const r = await runAgentic({ @@ -376,10 +412,8 @@ async function emitAttempt( strategy: refine, routerBaseUrl: 'zai-direct', // unused: the `complete` transport short-circuits the router routerKey: 'zai-direct', - model: WORKER_MODEL, - maxTokens: MAX_TOKENS, - temperature: cfg.temperature, - innerTurns: INNER_TURNS, + workerProfile, + analystProfile, budget: 1, complete: makeTransport(cfg.marks, counter, guard), }) diff --git a/bench/src/swe-structural.mts b/bench/src/swe-structural.mts index 2b5cde4a..2b6f5cd5 100644 --- a/bench/src/swe-structural.mts +++ b/bench/src/swe-structural.mts @@ -50,7 +50,7 @@ import { appendFileSync, existsSync, readFileSync, rmSync, writeFileSync } from import { join } from 'node:path' import { promisify } from 'node:util' import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/kernel' -import { refine, runAgentic } from '@tangle-network/agent-runtime/kernel' +import { defaultAnalystInstruction, refine, runAgentic } from '@tangle-network/agent-runtime/kernel' import type { BenchTask } from './benchmarks/types' import { createSweBenchEnvironment, resolveImageForMetadata, SWE_SEED_PROMPT } from './swe-bench-env' import { @@ -63,6 +63,7 @@ import { tail, zaiChatRaw, } from './swe-jail' +import { withBenchProfile } from './router-turn' const exec = promisify(execFile) @@ -150,6 +151,7 @@ const makeTransport = body, { name: 'swe-structural-worker', + harness: 'cli-base', model: { provider: 'zai', default: model, reasoningEffort: 'high' }, ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), ...(toolNames.length > 0 @@ -222,10 +224,31 @@ async function emitAttempt( } const task: AgenticTask = { id: bt.id, - systemPrompt: SWE_SEED_PROMPT, userPrompt: cfg.promptAppendix ? `${bt.prompt}\n\n${cfg.promptAppendix}` : bt.prompt, meta: { instanceId: bt.id }, } + const workerProfile = withBenchProfile( + { + name: 'swe-structural-worker', + harness: 'cli-base', + model: { provider: 'zai', default: MODEL, reasoningEffort: 'high' }, + tools: { list_files: true, read_file: true, edit_file: true }, + }, + { + systemPrompt: SWE_SEED_PROMPT, + maxTokens: MAX_TOKENS, + maxTurns: INNER_TURNS, + temperature: cfg.temperature, + }, + ) + const analystProfile = withBenchProfile( + { + name: 'swe-structural-analyst', + harness: 'cli-base', + model: { provider: 'zai', default: MODEL, reasoningEffort: 'high' }, + }, + { systemPrompt: defaultAnalystInstruction, maxTokens: MAX_TOKENS }, + ) let error: string | undefined try { const r = await runAgentic({ @@ -234,10 +257,8 @@ async function emitAttempt( strategy: refine, routerBaseUrl: 'zai-direct', // unused: the `complete` transport short-circuits the router routerKey: 'zai-direct', - model: MODEL, - maxTokens: MAX_TOKENS, - temperature: cfg.temperature, - innerTurns: INNER_TURNS, + workerProfile, + analystProfile, budget: 1, complete: makeTransport(cfg.marks, counter), }) diff --git a/bench/src/tb-container-executor.mts b/bench/src/tb-container-executor.mts index 5d42043e..341521f0 100644 --- a/bench/src/tb-container-executor.mts +++ b/bench/src/tb-container-executor.mts @@ -5,6 +5,7 @@ * provided container id and reports the captured process artifact back through the runtime. */ import { spawn } from 'node:child_process' +import { agentProfileSchema } from '@tangle-network/agent-interface' import type { Agent, AgentProfile, @@ -17,6 +18,8 @@ import type { Runtime, Spend, } from '../../src/runtime/index' +import { harnessInvocation, type LocalHarness } from '../../src/mcp/local-harness' +import { assertExecutableAgentProfile } from '../../src/runtime/supervise/model-policy' export interface TbExecOutput { /** Primary artifact consumed by drivers that read `{ content }`. */ @@ -40,7 +43,8 @@ export interface TbContainerConfig { readonly workdir?: string readonly shell?: string readonly env?: Readonly> - readonly wrapCommand?: (task: unknown) => string + /** Shell setup required inside the existing task container before the canonical invocation. */ + readonly commandPrefix?: ReadonlyArray readonly parseUsage?: ParseUsage readonly budgetExempt?: boolean readonly dockerBin?: string @@ -62,7 +66,7 @@ function resolveContainerId(config: TbContainerConfig): string { return id.trim() } -function taskToCommand(task: unknown): string { +function taskToPrompt(task: unknown): string { if (typeof task === 'string') return task if (task && typeof task === 'object') { const obj = task as Record @@ -73,6 +77,31 @@ function taskToCommand(task: unknown): string { return JSON.stringify(task) } +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'` +} + +function profileCommand( + profile: AgentProfile, + task: unknown, + prefix: ReadonlyArray = [], +): string { + const harness = profile.harness + if (harness !== 'claude-code' && harness !== 'codex' && harness !== 'opencode') { + throw new Error( + `tbContainerExecutor: profile harness ${JSON.stringify(harness)} is not available in the task container`, + ) + } + const invocation = harnessInvocation( + harness as LocalHarness, + profile, + taskToPrompt(task), + { dangerouslySkipPermissions: true }, + ) + const run = [invocation.command, ...invocation.args].map(shellQuote).join(' ') + return [...prefix, `exec ${run}`].join('; ') +} + /** * Compose the `docker exec` argv that runs `command` inside `containerId` via ` -c`. * Exported so tests can verify the container path without a live Docker daemon. @@ -113,6 +142,8 @@ export function createTbContainerExecutor( config: TbContainerConfig = {}, ): ExecutorFactory { return (_spec: AgentSpec, ctx: ExecutorContext): Executor => { + const profile = agentProfileSchema.parse(_spec.profile) + assertExecutableAgentProfile(profile, 'tbContainerExecutor') const containerId = resolveContainerId(config) const dockerBin = config.dockerBin ?? 'docker' const metered = config.parseUsage !== undefined @@ -133,7 +164,7 @@ export function createTbContainerExecutor( runtime, budgetExempt, execute(task, signal): Promise> { - const command = (config.wrapCommand ?? taskToCommand)(task) + const command = profileCommand(profile, task, config.commandPrefix) const args = buildTbDockerExecArgs(containerId, command, { ...(config.shell ? { shell: config.shell } : {}), ...(config.workdir ? { workdir: config.workdir } : {}), @@ -221,9 +252,10 @@ export function createTbContainerExecutor( export function makeTbContainerWorkerAgent(config: TbContainerConfig = {}): MakeWorkerAgent { const factory = createTbContainerExecutor(config) return (rawProfile) => { - const p = (rawProfile ?? {}) as { name?: unknown } - const name = typeof p.name === 'string' && p.name.length > 0 ? p.name : 'tb-worker' - const spec: AgentSpec = { profile: rawProfile as AgentProfile, harness: null } + const profile = agentProfileSchema.parse(rawProfile) + assertExecutableAgentProfile(profile, 'tbContainerWorkerAgent') + const name = profile.name ?? 'tb-worker' + const spec: AgentSpec = { profile, harness: null } const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } const executor = factory(spec, ctx) return { name, act: async () => '', executorSpec: { ...spec, executor } } as Agent< diff --git a/bench/src/tb-container-executor.test.mts b/bench/src/tb-container-executor.test.mts index 3ffc125d..ac3de897 100644 --- a/bench/src/tb-container-executor.test.mts +++ b/bench/src/tb-container-executor.test.mts @@ -14,7 +14,14 @@ import { type TbExecOutput, } from './tb-container-executor.mts' -const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null } +const spec: AgentSpec = { + profile: { + name: 'tb-test-worker', + harness: 'opencode', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }, + harness: null, +} function context(): ExecutorContext { return { signal: new AbortController().signal, seams: {} } @@ -78,10 +85,11 @@ printf 'argv:%s\\n' "$*" assert.equal(metered.budgetExempt, false, 'usage parser makes the executor metered') const meteredResult = await executeOneShot(metered, { command: 'echo hello' }) assert.equal(meteredResult.out.containerId, 'cid') - assert.equal(meteredResult.out.command, 'echo hello') + assert.match(meteredResult.out.command, /'echo hello'/) + assert.match(meteredResult.out.command, /'tangle-router\/deepseek-v4-flash'/) assert.match( meteredResult.out.stdout, - /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c echo hello/, + /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c exec 'opencode' 'run' 'echo hello' '-m' 'tangle-router\/deepseek-v4-flash'/, ) assert.deepEqual(meteredResult.spent.tokens, { input: 7, output: 11 }) assert.equal(meteredResult.spent.usd, 0.004) diff --git a/bench/src/tb-supervisor-sidecar.mts b/bench/src/tb-supervisor-sidecar.mts index 52445328..823dd5b5 100644 --- a/bench/src/tb-supervisor-sidecar.mts +++ b/bench/src/tb-supervisor-sidecar.mts @@ -20,7 +20,6 @@ import { makeTbContainerWorkerAgent, type ParseUsage } from './tb-container-exec const CONTAINER_ID = (process.env.TB_TARGET_CONTAINER ?? '').trim() const ROUTER_KEY = process.env.OPENAI_API_KEY ?? '' const ROUTER_BASE = process.env.OPENAI_BASE_URL ?? 'https://router.tangle.tools/v1' -const WORKER_MODEL = process.env.WORKER_MODEL ?? 'zai-coding-plan/glm-5.2' const DOCKER_BRIDGE_GATEWAY = process.env.DOCKER_BRIDGE_GATEWAY ?? '172.17.0.1' const PORT_FILE = process.env.TB_SIDECAR_PORT_FILE ?? '.tb-sidecar-port' const LOG_FILE = process.env.TB_SIDECAR_LOG ?? '.tb-sidecar-events.jsonl' @@ -42,30 +41,6 @@ function logEvent(kind: string, payload: unknown): void { console.error(`[sidecar] ${kind} ${JSON.stringify(payload).slice(0, 300)}`) } -function workerWrapCommand(task: unknown): string { - const instr = - typeof task === 'string' - ? task - : task && typeof task === 'object' - ? ((): string => { - const o = task as Record - for (const k of ['task', 'prompt', 'content', 'command', 'message', 'instruction']) { - if (typeof o[k] === 'string') return o[k] as string - } - return JSON.stringify(task) - })() - : String(task) - const q = `'${instr.replace(/'/g, `'\\''`)}'` - return [ - 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', - '{ [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; } 2>/dev/null || true', - '{ [ -s /root/.nvm/nvm.sh ] && . /root/.nvm/nvm.sh; } 2>/dev/null || true', - `export OPENCODE_CONFIG=${WORKER_CONFIG_PATH}`, - `cd ${WORKER_WORKDIR} 2>/dev/null || true`, - `opencode --model ${WORKER_MODEL} --format json run ${q}`, - ].join('; ') -} - const parseWorkerUsage: ParseUsage = ({ stdout }) => { let input = 0 let output = 0 @@ -115,9 +90,18 @@ async function main(): Promise { const makeWorkerAgent = makeTbContainerWorkerAgent({ containerId: CONTAINER_ID, workdir: WORKER_WORKDIR, - wrapCommand: workerWrapCommand, + commandPrefix: [ + 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', + '{ [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; } 2>/dev/null || true', + '{ [ -s /root/.nvm/nvm.sh ] && . /root/.nvm/nvm.sh; } 2>/dev/null || true', + ], parseUsage: parseWorkerUsage, - env: { OPENAI_API_KEY: ROUTER_KEY, OPENAI_BASE_URL: ROUTER_BASE, HOME: '/root' }, + env: { + OPENAI_API_KEY: ROUTER_KEY, + OPENAI_BASE_URL: ROUTER_BASE, + OPENCODE_CONFIG: WORKER_CONFIG_PATH, + HOME: '/root', + }, runtime: 'tb-container', }) diff --git a/bench/src/terminal-compare.ts b/bench/src/terminal-compare.ts index 30303893..1d873571 100644 --- a/bench/src/terminal-compare.ts +++ b/bench/src/terminal-compare.ts @@ -7,7 +7,7 @@ * judge here. * * Per task: - * ROUND 1 (blind) — `tb run --agent opencode` → parse results.json → + * ROUND 1 (blind) — Terminal-Bench runs one exact AgentProfile → parse results.json → * resolved_1. This IS blind pass@1. * ROUND r (refine, only if blind failed AND rounds>1) — extract a compact * summary of round 1's commands + terminal state + the @@ -26,11 +26,12 @@ */ import { execFile } from 'node:child_process' -import { mkdir, readFile, readdir, stat } from 'node:fs/promises' +import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { homedir } from 'node:os' +import { agentProfileSchema } from '@tangle-network/agent-interface' import { appendRunRecord, type AttemptRecord, type RunRecord } from './corpus' import { runPool } from './run-pool' @@ -38,7 +39,7 @@ import { runPool } from './run-pool' const BENCH_ROOT = fileURLToPath(new URL('..', import.meta.url)) const TB = join(BENCH_ROOT, '.venv', 'bin', 'tb') const RUNS_DIR = join(BENCH_ROOT, 'runs') -const REFINE_IMPORT = 'tb_agents.opencode_refine_agent:OpenCodeRefineAgent' +const PROFILE_AGENT_IMPORT = 'tb_agents.opencode_router_agent:OpenCodeRouterAgent' // The durable learning-flywheel corpus (docs/learning-flywheel.md). terminal-bench // is bench-orchestrated (tb owns the containers) so it cannot use buildRunRecord, // which consumes runAgentRounds Iterations; instead each task's per-round tb artifacts are @@ -47,7 +48,27 @@ const REFINE_IMPORT = 'tb_agents.opencode_refine_agent:OpenCodeRefineAgent' const CORPUS = process.env.CORPUS ?? join(BENCH_ROOT, 'corpus', 'terminal.jsonl') const DATASET = process.env.TB_DATASET ?? 'terminal-bench-core==0.1.1' -const MODEL = process.env.TB_MODEL ?? 'deepseek/deepseek-v4-pro' +const MODEL = process.env.TB_MODEL ?? 'deepseek-v4-flash' +const PROVIDER = process.env.TB_PROVIDER ?? 'tangle-router' +const PROFILE = agentProfileSchema.parse({ + name: 'terminal-compare-worker', + harness: 'opencode', + model: { provider: PROVIDER, default: MODEL }, + prompt: { + systemPrompt: + 'Solve the Terminal-Bench task completely in the provided container and verify the result before finishing.', + }, + permission: { + edit: 'allow', + bash: 'allow', + webfetch: 'allow', + read: 'allow', + write: 'allow', + external_directory: 'allow', + }, +}) +const PROFILE_MODEL = MODEL.includes('/') ? MODEL : `${PROVIDER}/${MODEL}` +const PROFILE_PATH = join(RUNS_DIR, 'terminal-compare.profile.json') const ROUNDS = Math.max(1, Number(process.env.ROUNDS ?? 2)) const CONCURRENCY = Math.max(1, Number(process.env.CONCURRENCY ?? 2)) // Per-round wall-clock cap (ms). tb itself enforces task timeouts; this is the @@ -330,7 +351,7 @@ async function captureRunRecord( benchmark: 'terminal-bench', instanceId: taskId, condition: ROUNDS > 1 ? `refine@${ROUNDS}` : 'blind', - model: MODEL, + model: PROFILE.model?.default ?? MODEL, blindResolved, resolved: last ? last.resolved : blindResolved, attempts, @@ -392,7 +413,7 @@ async function runRound(taskId: string, round: number, prior: string): Promise 1 ? ['--agent-kwarg', `prior_attempt_hex=${priorHex}`] : []), + ] await runTb(args, runId) const resolved = await readResolved(outputDir, taskId) const trialDir = await findTrialDir(outputDir, taskId) @@ -457,6 +476,7 @@ async function solveTask(taskId: string): Promise<{ async function main(): Promise { await mkdir(RUNS_DIR, { recursive: true }) + await writeFile(PROFILE_PATH, JSON.stringify(PROFILE, null, 2) + '\n') // Validate auth up front — fail loud before spending a single container. must('OPENAI_API_KEY') diff --git a/bench/tb_agents/opencode_router_agent.py b/bench/tb_agents/opencode_router_agent.py index 6d504fbc..0206c6bb 100644 --- a/bench/tb_agents/opencode_router_agent.py +++ b/bench/tb_agents/opencode_router_agent.py @@ -66,6 +66,15 @@ _USAGE_ERR_PATH = f"{_CONTAINER_DIR}/opencode-run.err" +REFINE_DIRECTIVE = ( + "REFINE PASS — your previous attempt failed the task's automated tests. " + "Use the evidence below to correct the concrete failure without discarding work " + "that was already right. Verify the task's success criteria before finishing.\n\n" + "\n{prior}\n\n\n" + "--- ORIGINAL TASK ---\n{instruction}" +) + + class OpenCodeRouterAgent(OpenCodeAgent): """`OpenCodeAgent` + Tangle-router routing + real token metering. @@ -77,27 +86,71 @@ class OpenCodeRouterAgent(OpenCodeAgent): a real `AgentResult`. """ - def __init__(self, model_name: str, *args, **kwargs): - # The stock __init__ does `self._provider, _ = model_name.split("/")`, which - # raises on a 3-segment or bare id. Derive the provider robustly ourselves, - # then hand the base class a value it can split without raising. - self._model_name = model_name - provider, _, sub = model_name.partition("/") + def __init__( + self, + model_name: str, + profile_path: str, + prior_attempt_hex: str | None = None, + *args, + **kwargs, + ): + self._profile = json.loads(Path(profile_path).read_text()) + if self._profile.get("harness") != "opencode": + raise ValueError("AgentProfile.harness must be 'opencode'") + model = self._profile.get("model") + if not isinstance(model, dict): + raise ValueError("AgentProfile.model is required") + provider = model.get("provider") + declared = model.get("default") + if not isinstance(provider, str) or not provider.strip(): + raise ValueError("AgentProfile.model.provider must be explicit") + if not isinstance(declared, str) or not declared.strip() or declared == "runtime-selected": + raise ValueError("AgentProfile.model.default must be concrete") + wire_model = declared if "/" in declared else f"{provider}/{declared}" + if model_name != wire_model: + raise ValueError( + f"Terminal-Bench model {model_name!r} conflicts with AgentProfile {wire_model!r}" + ) + self._model_name = wire_model self._provider = provider - # Model id opencode sees, minus the provider segment (e.g. "glm-5.2"). - self._provider_model = sub or provider + self._provider_model = declared.split("/", 1)[-1] + self._prior_attempt = self._decode_prior(prior_attempt_hex) + prompt = self._profile.get("prompt") + self._profile_instructions: list[str] = [] + if isinstance(prompt, dict): + system = prompt.get("systemPrompt") + if isinstance(system, str) and system.strip(): + self._profile_instructions.append(system.strip()) + instructions = prompt.get("instructions") + if isinstance(instructions, list): + self._profile_instructions.extend( + value.strip() + for value in instructions + if isinstance(value, str) and value.strip() + ) self._router_base_url = ( os.environ.get("OPENAI_BASE_URL") or os.environ.get("ROUTER_BASE_URL") or DEFAULT_ROUTER_BASE_URL ) self._router_api_key = os.environ.get("OPENAI_API_KEY", "") - # Call the grandparent (AbstractInstalledAgent) init via OpenCodeAgent, but - # avoid the base split-crash by not delegating provider derivation to it. - super(OpenCodeAgent, self).__init__(*args, **kwargs) - self._version = kwargs.get("version", "latest") + super().__init__(wire_model, *args, **kwargs) self._logger = logger.getChild(__name__) + @staticmethod + def _decode_prior(value: str | None) -> str: + if not value: + return "" + raw = str(value).strip() + if raw.startswith("h"): + raw = raw[1:] + if not raw: + return "" + try: + return bytes.fromhex(raw).decode("utf-8").strip() + except ValueError as error: + raise ValueError("prior_attempt_hex must be 'h' + hex-encoded UTF-8") from error + @staticmethod def name() -> str: return "agent-runtime-opencode-router" @@ -136,16 +189,6 @@ def _env(self) -> dict[str, str]: "OPENAI_BASE_URL": self._router_base_url, "OPENCODE_CONFIG": _CONFIG_PATH, } - # Pass through any provider-native key the user already exported, so the same - # agent also works when pointed at a vendor endpoint rather than the router. - for passthrough in ( - "ANTHROPIC_API_KEY", - "DEEPSEEK_API_KEY", - "GROQ_API_KEY", - "ZAI_API_KEY", - ): - if passthrough in os.environ: - env.setdefault(passthrough, os.environ[passthrough]) return {k: v for k, v in env.items() if v} def _build_router_config(self) -> str: @@ -159,14 +202,7 @@ def _build_router_config(self) -> str: "$schema": "https://opencode.ai/config.json", # Headless benchmark runs cannot answer interactive permission prompts. # Keep this identical for raw and supervisor arms. - "permission": { - "edit": "allow", - "bash": "allow", - "webfetch": "allow", - "read": "allow", - "write": "allow", - "external_directory": "allow", - }, + "permission": self._profile.get("permission", {}), "provider": { self._provider: { "npm": "@ai-sdk/openai-compatible", @@ -184,25 +220,24 @@ def _build_router_config(self) -> str: return json.dumps(config) def _run_agent_commands(self, instruction: str): - """Same command as the stock agent, but `--format json` with the event - stream captured to a file so we can meter token usage after the run.""" + """Instrument Terminal-Bench's stock command without selecting a model here.""" from terminal_bench.terminal.models import TerminalCommand - escaped_instruction = shlex.quote(instruction) - command = ( - f"opencode --model {self._model_name} --format json " - f"run {escaped_instruction} " - f"> {_USAGE_OUT_PATH} 2> {_USAGE_ERR_PATH}" - ) - return [ - TerminalCommand( + instrumented = [] + for base in super()._run_agent_commands(instruction): + marker = " run " + if marker not in base.command: + raise ValueError("stock OpenCodeAgent command has no run subcommand") + command = base.command.replace(marker, " --format json run ", 1) + command += f" > {_USAGE_OUT_PATH} 2> {_USAGE_ERR_PATH}" + instrumented.append(TerminalCommand( command=command, min_timeout_sec=0.0, max_timeout_sec=float("inf"), block=True, append_enter=True, - ) - ] + )) + return instrumented def _write_config_to_container(self, session: TmuxSession) -> None: config_json = self._build_router_config() @@ -365,6 +400,13 @@ def perform_task( "OPENAI_API_KEY (router key) is not set; opencode has no credential " "to reach the router." ) + if self._profile_instructions: + instruction = "\n\n".join([*self._profile_instructions, instruction]) + if self._prior_attempt: + instruction = REFINE_DIRECTIVE.format( + prior=self._prior_attempt, + instruction=instruction, + ) # Publish container id + inject router config BEFORE the base flow runs the # agent. The base perform_task copies the install script into _CONTAINER_DIR, # sources setup-env.sh (which exports OPENCODE_CONFIG), installs opencode, and From 63c2d27d61302d27cd2536af4905c67e03b22128 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 22:58:48 -0600 Subject: [PATCH 17/39] fix(bench): align exact shared-package contracts --- bench/src/benchmarks/appworld.test.mts | 6 +- bench/src/clbench-context-gate.mts | 2 +- bench/src/david-attribution.mts | 23 +++--- bench/src/hev-improve.mts | 5 +- bench/src/humaneval-object-ablation.mts | 3 + .../backfill-swe-arena.test.mts | 9 ++- bench/src/search-bench/bridge.ts | 52 +++++++++++-- bench/src/search-bench/profiles.ts | 11 ++- bench/src/swe-arena/gepa-seat.mts | 15 +++- bench/src/swe-arena/gepa-seat.test.mts | 78 +++++++++++-------- bench/src/swe-arena/outer-loop.mts | 25 +++--- .../src/swe-arena/premeasured-from-cells.mts | 17 +++- .../swe-arena/premeasured-from-cells.test.mts | 22 +++++- bench/src/swe-arena/proposer-provenance.mts | 2 +- 14 files changed, 193 insertions(+), 77 deletions(-) diff --git a/bench/src/benchmarks/appworld.test.mts b/bench/src/benchmarks/appworld.test.mts index af3fd490..334424f8 100644 --- a/bench/src/benchmarks/appworld.test.mts +++ b/bench/src/benchmarks/appworld.test.mts @@ -109,9 +109,9 @@ test('react client refuses an already-aborted round before a Python session or m model: 'offline-model', routerBaseUrl: 'https://router.invalid', routerKey: 'offline', - runWorldSession: async () => { + runWorldSession: async () => { sessionCalls += 1 - return undefined as T + throw new Error('unexpected world session') }, complete: async () => { modelCalls += 1 @@ -137,7 +137,7 @@ test('react client threads late abort to both the world session and Router call' model: 'offline-model', routerBaseUrl: 'https://router.invalid', routerKey: 'offline', - runWorldSession: async (_taskId, _split, signal, fn): Promise => { + runWorldSession: async (_taskId, _split, signal, fn) => { sessionSignal = signal try { return await fn(async () => ({ success: true, num_tests: 1, passes: 1 }), 'offline task') diff --git a/bench/src/clbench-context-gate.mts b/bench/src/clbench-context-gate.mts index 44969336..de53edc9 100644 --- a/bench/src/clbench-context-gate.mts +++ b/bench/src/clbench-context-gate.mts @@ -53,7 +53,7 @@ function must(name: string): string { return v } -interface ChatMessage { +interface ChatMessage extends Readonly> { role: string content: string } diff --git a/bench/src/david-attribution.mts b/bench/src/david-attribution.mts index 081ffb52..52f0e1df 100644 --- a/bench/src/david-attribution.mts +++ b/bench/src/david-attribution.mts @@ -20,10 +20,15 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval' -import { runBenchRouterTurn } from './router-turn' +import { benchRouterProfile, runBenchRouterTurn } from './router-turn' -const KEY = process.env.TANGLE_API_KEY -if (!KEY) throw new Error('TANGLE_API_KEY required') +function requiredEnv(name: string): string { + const value = process.env[name] + if (!value) throw new Error(`${name} required`) + return value +} + +const KEY = requiredEnv('TANGLE_API_KEY') const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant' const N = Number(process.env.N ?? 8) @@ -40,13 +45,11 @@ async function chat(messages: { role: string; content: string }[], temp: number) { routerBaseUrl: ROUTER, routerKey: KEY, - profile: { - name: 'david-attribution-worker', - model: { provider: 'tangle-router', default: DAVID }, - ...(system ? { prompt: { systemPrompt: system } } : {}), - }, - temperature: temp, - maxTokens: MAX_TOKENS, + profile: benchRouterProfile('david-attribution-worker', DAVID, { + ...(system ? { systemPrompt: system } : {}), + temperature: temp, + maxTokens: MAX_TOKENS, + }), timeoutMs: LLM_TIMEOUT_MS, }, { messages: messages.filter((message) => message.role !== 'system') }, diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts index 3aab911a..2f4a09d1 100644 --- a/bench/src/hev-improve.mts +++ b/bench/src/hev-improve.mts @@ -19,6 +19,7 @@ import { import { canonicalCandidateDigest, type AgentProfile, + agentProfileSchema, } from '@tangle-network/agent-interface' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' @@ -122,11 +123,11 @@ async function main(): Promise { const t = byId.get(scenario.id) if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) const prompt = `\`\`\`python\n${t.prompt}\`\`\`` - const executionProfile: AgentProfile = { + const executionProfile: AgentProfile = agentProfileSchema.parse({ ...candidate, name: candidate.name ?? 'humaneval-improvement-worker', model: { ...candidate.model, provider: 'tangle-router', default: workerModel }, - } + }) const t0 = Date.now() const paid = await ctx.cost.runPaidCall({ channel: 'agent', diff --git a/bench/src/humaneval-object-ablation.mts b/bench/src/humaneval-object-ablation.mts index 45242094..3907df82 100644 --- a/bench/src/humaneval-object-ablation.mts +++ b/bench/src/humaneval-object-ablation.mts @@ -75,6 +75,9 @@ async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: s { messages: messages.filter((message) => message.role !== 'system') }, ) const toolCalls = result.toolCalls.map((call) => { + if (call.id === undefined) { + throw new Error(`router tool call '${call.name}' omitted its required id`) + } let args: Record = {} try { args = JSON.parse(call.arguments) as Record diff --git a/bench/src/rollout-ledger/backfill-swe-arena.test.mts b/bench/src/rollout-ledger/backfill-swe-arena.test.mts index e98d646c..c6b22293 100644 --- a/bench/src/rollout-ledger/backfill-swe-arena.test.mts +++ b/bench/src/rollout-ledger/backfill-swe-arena.test.mts @@ -2,7 +2,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { assertRolloutLine, claudeProjectSlug, toSftRows } from '@tangle-network/agent-eval/rollout' +import { + assertMintedLines, + assertRolloutLine, + claudeProjectSlug, + toSftRows, +} from '@tangle-network/agent-eval/rollout' import { backfillSweArena } from './backfill-swe-arena.mts' let dir: string @@ -262,7 +267,7 @@ describe('backfillSweArena', () => { // The dataset is immediately trainable: reward==1 trainable-split lines // with transcripts → SFT rows (candidate worker + proposer here). - expect(toSftRows(lines)).toHaveLength(2) + expect(toSftRows(assertMintedLines(lines))).toHaveLength(2) }) it('joins a respawned worker cwd once, not once per recovered entry', async () => { diff --git a/bench/src/search-bench/bridge.ts b/bench/src/search-bench/bridge.ts index 6223bb3b..bb0e8c96 100644 --- a/bench/src/search-bench/bridge.ts +++ b/bench/src/search-bench/bridge.ts @@ -13,7 +13,12 @@ * `opencode/zai-coding-plan/glm-5.1`), so `harness` here is just the label. */ import { createExecutor } from '@tangle-network/agent-runtime/kernel' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + defineAgentProfileSecretRef, + harnessProviders, + type AgentProfile, + type HarnessType, +} from '@tangle-network/agent-interface' import type { SearchArm } from './profiles' import { armLabel } from './profiles' import type { SearchCellResult } from './run.mts' @@ -28,22 +33,58 @@ function bridgeProfile( routerSearchMcp: string, tangleApiKey: string, label: string, - model: string, + harness: HarnessType, + wireModel: string, ): AgentProfile { - const identity = { name: `search-bench-${label}`, model: { default: model } } + const prefix = `${harness}/` + if (!wireModel.startsWith(prefix)) { + throw new Error( + `bridgeProfile: wire model '${wireModel}' must start with harness '${prefix}'`, + ) + } + const modelPath = wireModel.slice(prefix.length) + const segments = modelPath.split('/').filter(Boolean) + if (segments.length === 0) { + throw new Error(`bridgeProfile: wire model '${wireModel}' has no model id`) + } + let provider: string + let model: string + if (segments.length === 1) { + const providers = harnessProviders(harness) + if (providers?.length !== 1) { + throw new Error( + `bridgeProfile: wire model '${wireModel}' must include a provider for harness '${harness}'`, + ) + } + provider = providers[0]! + model = segments[0]! + } else { + provider = segments[0]! + model = segments.slice(1).join('/') + } + const identity: AgentProfile = { + name: `search-bench-${label}`, + harness, + model: { provider, default: model }, + } if (arm === 'native') return identity const base: AgentProfile = { ...identity, metadata: { disallowedTools: nativeWebDisallowed }, } if (arm === 'off') return base + if (!tangleApiKey) { + throw new Error(`bridgeProfile: provider arm requires TANGLE_API_KEY`) + } return { ...base, mcp: { tangle_search: { transport: 'http', url: `${routerSearchMcp}?provider=${encodeURIComponent(arm.provider)}`, - headers: { Authorization: `Bearer ${tangleApiKey}` }, + headers: { + Authorization: defineAgentProfileSecretRef('TANGLE_API_KEY', 'bearer'), + }, enabled: true, }, }, @@ -69,7 +110,7 @@ export interface BridgeCfg { export async function runBridgeCell( cfg: BridgeCfg, task: SearchTask, - harness: string, + harness: HarnessType, arm: SearchArm, ): Promise { const startedAt = Date.now() @@ -92,6 +133,7 @@ export async function runBridgeCell( cfg.routerSearchMcp, cfg.tangleApiKey, `${harness}-${armId}`, + harness, cfg.bridgeModels[harness] ?? harness, ) const exec = createExecutor({ diff --git a/bench/src/search-bench/profiles.ts b/bench/src/search-bench/profiles.ts index 434437ff..6c238058 100644 --- a/bench/src/search-bench/profiles.ts +++ b/bench/src/search-bench/profiles.ts @@ -17,7 +17,12 @@ * shape — a `transport:'http'` server at the router's `/v1/search/mcp` endpoint, * provider pinned via the `?provider=` query param. */ -import { type AgentProfile, type HarnessType, agentProfileSchema } from '@tangle-network/agent-interface' +import { + type AgentProfile, + type HarnessType, + agentProfileSchema, + defineAgentProfileSecretRef, +} from '@tangle-network/agent-interface' export type SearchArm = 'native' | 'off' | { provider: string } @@ -91,7 +96,9 @@ export function buildArmProfile(args: BuildArmProfileArgs): AgentProfile { tangle_search: { transport: 'http', url: routerSearchMcpUrl(arm.provider, routerBaseUrl), - headers: { Authorization: `Bearer ${tangleApiKey}` }, + headers: { + Authorization: defineAgentProfileSecretRef('TANGLE_API_KEY', 'bearer'), + }, enabled: true, }, }, diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index ed262ff4..25b6c6d0 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -41,6 +41,7 @@ import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promise import { dirname, join } from 'node:path' import { type DispatchContext, + type ComparisonCost, createRunCostLedger, fsCampaignStorage, type GepaOptimizationMethodConfig, @@ -326,6 +327,7 @@ export interface GepaSeatInnerRun { tokenUsage: OptimizationTokenUsage artifactDir: string totalCostUsd: number + costProvenance: ComparisonCost['costProvenance'] accountingComplete: boolean incompleteReasons: string[] durationMs: number @@ -532,10 +534,7 @@ function completeProvenance( } } -function assertCompleteCost( - cost: { totalCostUsd: number; accountingComplete: boolean; incompleteReasons: string[] }, - seatName: string, -): void { +function assertCompleteCost(cost: ComparisonCost, seatName: string): void { if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) { throw new Error(`gepa seat '${seatName}': optimizer returned invalid total cost`) } @@ -550,6 +549,13 @@ function assertCompleteCost( if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) { throw new Error(`gepa seat '${seatName}': optimizer returned inconsistent cost accounting`) } + if (cost.costProvenance.kind === 'uncaptured') { + if (cost.accountingComplete) { + throw new Error(`gepa seat '${seatName}': complete cost cannot have uncaptured provenance`) + } + } else if (cost.costProvenance.usd !== cost.totalCostUsd) { + throw new Error(`gepa seat '${seatName}': cost provenance does not match total cost`) + } if (!cost.accountingComplete) { throw new Error( `gepa seat '${seatName}': cost accounting is incomplete: ${cost.incompleteReasons.join('; ') || 'no reason provided'}`, @@ -823,6 +829,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut : null, ...completeProvenance(result.provenance, spec.name), totalCostUsd: result.cost.totalCostUsd, + costProvenance: result.cost.costProvenance, accountingComplete: result.cost.accountingComplete, incompleteReasons: [...result.cost.incompleteReasons], durationMs: Date.now() - started, diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index 7e8ea872..e4558033 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -34,6 +34,7 @@ import { import { defaultRound4Config, type OuterLoopConfig } from './outer-loop.mts' import { fanOutLoopsGenerator, type ProposerSpec, type SmokeRunner, type SmokeVerdict } from './proposer-fanout.mts' import { captureProposerProvenance } from './proposer-provenance.mts' +import { officialOptimizerModel } from '../official-optimizer-config.mts' import { runOk } from './proc.ts' const SURFACE = 'extensions/pi/prompts/worker-coding-system.md' @@ -370,6 +371,7 @@ describe('recordGepaSeatInnerRun', () => { tokenUsage: result.tokenUsage, artifactDir: result.artifactDir, totalCostUsd: 0.25, + costProvenance: { kind: 'observed', usd: 0.25 }, accountingComplete: true, incompleteReasons: [], durationMs: 5, @@ -426,24 +428,26 @@ describe('recordGepaSeatInnerRun', () => { // --------------------------------------------------------------------------- const fakeCtx = {} as unknown as DispatchContext -const testOptimizer = { +const testOptimizerEnv: NodeJS.ProcessEnv = { + TEST_OPTIMIZER_INPUT_USD_PER_MILLION: '1', + TEST_OPTIMIZER_CACHED_INPUT_USD_PER_MILLION: '0.1', + TEST_OPTIMIZER_CACHE_WRITE_USD_PER_MILLION: '1.25', + TEST_OPTIMIZER_OUTPUT_USD_PER_MILLION: '5', + TEST_OPTIMIZER_MAX_REQUESTS: '10', + TEST_OPTIMIZER_MAX_REQUEST_BYTES: '100000', + TEST_OPTIMIZER_MAX_RESPONSE_BYTES: '100000', +} + +const testOptimizer = officialOptimizerModel({ + env: testOptimizerEnv, + envPrefix: 'TEST_OPTIMIZER', model: 'optimizer-model', baseUrl: 'http://127.0.0.1:1/v1', apiKey: 'optimizer-key', - budget: { - maxCostUsd: 1, - maxRequests: 10, - maxRequestBytes: 100_000, - maxResponseBytes: 100_000, - maxOutputTokensPerRequest: 2_000, - pricing: { - inputUsdPerMillion: 1, - cachedInputUsdPerMillion: 0.1, - cacheWriteUsdPerMillion: 1.25, - outputUsdPerMillion: 5, - }, - }, -} + maxCostUsd: 1, + maxOutputTokensPerRequest: 2_000, + callRef: 'test:optimizer-model', +}) const fullProvenance = ( runId = 'gepa-run', @@ -504,7 +508,12 @@ const fakeGepaFactory = } return { winnerSurface: best.surface, - cost: { totalCostUsd: 0.125, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0.125, + costProvenance: { kind: 'observed', usd: 0.125 }, + accountingComplete: true, + incompleteReasons: [], + }, durationMs: 1, provenance: fullProvenance(), } @@ -737,7 +746,12 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { ]) return { winnerSurface: candidateA, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, durationMs: 1, provenance: fullProvenance('parallel-gepa', { evaluationCount: 2 }), } @@ -823,6 +837,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { winnerSurface: WINNER, cost: { totalCostUsd: 0.25, + costProvenance: { kind: 'uncaptured', usd: null }, accountingComplete: false, incompleteReasons: ['optimizer model receipt missing'], }, @@ -873,7 +888,12 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { if (rejected) throw rejected.reason return { winnerSurface: SEED, - cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'uncaptured', usd: null }, + accountingComplete: false, + incompleteReasons: [], + }, durationMs: 1, provenance: fullProvenance('runaway'), } @@ -1009,24 +1029,16 @@ describe('integration: real adapter roundtrip', () => { }, smokeInstanceId: 'astropy__astropy-13033', scoreSplit: null, - gepaOptimizer: { + gepaOptimizer: officialOptimizerModel({ + env: testOptimizerEnv, + envPrefix: 'TEST_OPTIMIZER', model: 'test-optimizer', baseUrl: `http://127.0.0.1:${address.port}/v1`, apiKey: 'local-test-key', - budget: { - maxCostUsd: 1, - maxRequests: 10, - maxRequestBytes: 100_000, - maxResponseBytes: 100_000, - maxOutputTokensPerRequest: 2_000, - pricing: { - inputUsdPerMillion: 1, - cachedInputUsdPerMillion: 0.1, - cacheWriteUsdPerMillion: 1.25, - outputUsdPerMillion: 5, - }, - }, - }, + maxCostUsd: 1, + maxOutputTokensPerRequest: 2_000, + callRef: 'test:integration-optimizer', + }), }, ) const result = await gen.generate({ diff --git a/bench/src/swe-arena/outer-loop.mts b/bench/src/swe-arena/outer-loop.mts index bb4e5cff..7f683d66 100644 --- a/bench/src/swe-arena/outer-loop.mts +++ b/bench/src/swe-arena/outer-loop.mts @@ -69,6 +69,7 @@ import { improve, rawTraceDistiller, type CandidateGenerator, + type ImproveCodeRunOptions, type Verifier, } from '@tangle-network/agent-runtime' import { canonicalCandidateDigest, type AgentProfile } from '@tangle-network/agent-interface' @@ -2032,12 +2033,9 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P evidence_refs: [], proposal_origin: 'search', }) - const analyzeGeneration = async (input: { - generation: number - runDir: string - candidates: Array<{ surfaceHash: string; composite: number; campaign: unknown }> - history: unknown[] - }): Promise => { + const analyzeGeneration: NonNullable< + ImproveCodeRunOptions['analyzeGeneration'] + > = async (input): Promise => { signal?.throwIfAborted() const runs: SupRunArtifacts[] = [] if (input.generation === -1) { @@ -2061,9 +2059,11 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P // Candidate failure artifacts come from the LIB's campaign cells (the // artifacts name their own runDir/patch) — resume-replayed cells included, // which the old recorder-based lookup silently dropped. - const worstFirst = [...input.candidates].sort((a, b) => a.composite - b.composite).slice(0, 4) + const worstFirst = [...input.candidates] + .sort((a, b) => (a.composite ?? -Infinity) - (b.composite ?? -Infinity)) + .slice(0, 4) for (const cand of worstFirst) { - const cells = cellsFromCampaign(cand.campaign as CampaignResult) + const cells = cellsFromCampaign(cand.campaign) for (const cell of cells) { const a = cell.artifact if (a === null || a.kind !== 'swe-arm' || !a.runDir) continue @@ -2114,16 +2114,17 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P : { ...input, candidates: input.candidates.map((cand) => { - const campaign = cand.campaign as { cells?: Array<{ scenarioId: string }> } | null - if (campaign === null || typeof campaign !== 'object' || !Array.isArray(campaign.cells)) return cand return { ...cand, - campaign: { ...campaign, cells: campaign.cells.filter((c) => !privateIids.has(c.scenarioId)) }, + campaign: { + ...cand.campaign, + cells: cand.campaign.cells.filter((cell) => !privateIids.has(cell.scenarioId)), + }, } }), } signal?.throwIfAborted() - const rawFindings = await rawTrace(censoredInput as Parameters[0]) + const rawFindings = await rawTrace(censoredInput) signal?.throwIfAborted() return [steeringFinding, ...ensembleFindings, ...rawFindings] } diff --git a/bench/src/swe-arena/premeasured-from-cells.mts b/bench/src/swe-arena/premeasured-from-cells.mts index 6f9468af..d164a34b 100644 --- a/bench/src/swe-arena/premeasured-from-cells.mts +++ b/bench/src/swe-arena/premeasured-from-cells.mts @@ -153,6 +153,16 @@ export function buildPremeasuredFromCells(input: { if (cell.artifact === null || cell.artifact.kind !== 'swe-arm') { throw new Error(`premeasured-from-cells: cell ${cell.cellId} has no swe-arm artifact`) } + if (cell.costProvenance.kind === 'uncaptured') { + throw new Error(`premeasured-from-cells: cell ${cell.cellId} has uncaptured cost`) + } + if ( + cell.costProvenance.usd !== cell.costUsd + ) { + throw new Error( + `premeasured-from-cells: cell ${cell.cellId} cost provenance does not match costUsd`, + ) + } } const scenarios: Scenario[] = instances.map((iid) => ({ id: iid, kind: 'swe-instance' })) @@ -180,6 +190,11 @@ export function buildPremeasuredFromCells(input: { }), ) const totalCostUsd = sorted.reduce((s, c) => s + c.costUsd, 0) + const costProvenance: CampaignCellResult['costProvenance'] = sorted.some( + (cell) => cell.costProvenance.kind === 'estimated', + ) + ? { kind: 'estimated', usd: totalCostUsd } + : { kind: 'observed', usd: totalCostUsd } const inputTokens = sorted.reduce((s, c) => s + c.tokenUsage.input, 0) const outputTokens = sorted.reduce((s, c) => s + c.tokenUsage.output, 0) const totalCalls = sorted.reduce((s, c) => s + (c.costCallIds?.length ?? 0), 0) @@ -213,6 +228,7 @@ export function buildPremeasuredFromCells(input: { outputTokens, cachedTokens: 0, totalCostUsd, + costProvenance, byChannel: [ { channel: 'agent', @@ -231,7 +247,6 @@ export function buildPremeasuredFromCells(input: { accountingComplete: true, incompleteReasons: [], }, - totalCostUsd, cellsExecuted: sorted.length, cellsSkipped: 0, cellsCached: sorted.length, diff --git a/bench/src/swe-arena/premeasured-from-cells.test.mts b/bench/src/swe-arena/premeasured-from-cells.test.mts index a788e54d..e5805baf 100644 --- a/bench/src/swe-arena/premeasured-from-cells.test.mts +++ b/bench/src/swe-arena/premeasured-from-cells.test.mts @@ -49,6 +49,7 @@ const cell = (iid: string, rep: number, resolved: boolean, over: Partial { expect(out.campaign.cells.every((c) => c.cached)).toBe(true) expect(out.campaign.cells.every((c) => !('mtimeMs' in c))).toBe(true) // Honest spend rollup from the real cells. - expect(out.campaign.aggregates.totalCostUsd).toBeCloseTo(0.2) + expect(out.campaign.aggregates.cost.totalCostUsd).toBeCloseTo(0.2) + expect(out.campaign.aggregates.cost.costProvenance).toEqual({ + kind: 'observed', + usd: 0.2, + }) expect(out.campaign.aggregates.cost.inputTokens).toBe(400) expect(out.campaign.runDir).toBe('/prior/baseline') // Window reconstructed from mtimes: ends at the last cell write. @@ -126,6 +131,21 @@ describe('buildPremeasuredFromCells', () => { expect(() => buildPremeasuredFromCells({ cells: errored, instances: INSTANCES, reps: 2, surfaceHash: 'x', sourceDir: '/p' }), ).toThrow(/carries an error/) + const uncapturedCost = [ + cell('inst-a', 0, false, { costProvenance: { kind: 'uncaptured', usd: null } }), + cell('inst-a', 1, true), + cell('inst-b', 0, true), + cell('inst-b', 1, true), + ] + expect(() => + buildPremeasuredFromCells({ + cells: uncapturedCost, + instances: INSTANCES, + reps: 2, + surfaceHash: 'x', + sourceDir: '/p', + }), + ).toThrow(/has uncaptured cost/) }) }) diff --git a/bench/src/swe-arena/proposer-provenance.mts b/bench/src/swe-arena/proposer-provenance.mts index 0a2bc94f..fb3d1c61 100644 --- a/bench/src/swe-arena/proposer-provenance.mts +++ b/bench/src/swe-arena/proposer-provenance.mts @@ -120,7 +120,7 @@ export async function captureProposerProvenance( for (const harness of harnesses) { // The harness id is not the binary name (`claude-code` runs `claude`); read the executable // from the runtime's harness table rather than spawning the id. - const executable = localHarnessExecutable(harness) + const executable = harness === 'pi' ? 'pi' : localHarnessExecutable(harness) const res = await exec(executable, ['--version']) if (res.code !== 0) { throw new Error( From 87348de3783aca547fe4e3d33970acd410bc23a1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:01:57 -0600 Subject: [PATCH 18/39] fix(runtime): enforce exact profile model calls --- src/runtime/local-sandbox-client.ts | 32 +++--- src/runtime/profile-chat-client.test.ts | 124 +++++++++++++++++++++- src/runtime/profile-chat-client.ts | 47 ++++---- src/runtime/router-client.ts | 12 ++- src/runtime/stdio-mcp-client.test.ts | 46 ++++++++ src/runtime/stream-agent-turn.test.ts | 40 +++++++ src/runtime/stream-agent-turn.ts | 13 +-- src/runtime/supervise/supervisor-agent.ts | 3 + tests/kernel/router-brain.test.ts | 23 ++++ tests/kernel/supervisor-agent.test.ts | 44 ++++++++ 10 files changed, 337 insertions(+), 47 deletions(-) diff --git a/src/runtime/local-sandbox-client.ts b/src/runtime/local-sandbox-client.ts index 88f7ed2f..29e6b0f6 100644 --- a/src/runtime/local-sandbox-client.ts +++ b/src/runtime/local-sandbox-client.ts @@ -20,7 +20,6 @@ import { type AgentProfile, type AgentProfileSecurityPolicy, - agentProfileSchema, canonicalAgentProfileDigest, } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' @@ -28,11 +27,8 @@ import { ValidationError } from '../errors' import type { KeyProvider } from './key-provider' import { routerBrain } from './router-client' import { materializeLocalMcp } from './stdio-mcp-client' -import { - assertExecutableAgentProfile, - profileModelExecutionSettings, - profileProviderModel, -} from './supervise/model-policy' +import { executableAgentProfileSnapshot } from './supervise/executable-spec' +import { profileModelExecutionSettings, profileProviderModel } from './supervise/model-policy' import { runBrainLoop, type ToolLoopChat } from './tool-loop' import type { SandboxClient } from './types' @@ -52,22 +48,27 @@ export interface LocalSandboxClientOptions { /** A same-host `SandboxClient` adapter with no process isolation. Local MCP is * refused unless the caller explicitly supplies a policy that allows it. */ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClient { - if (opts.profileSecurityPolicy?.allowLocalMcp && opts.profile === undefined) { + const defaultProfile = + opts.profile === undefined + ? undefined + : executableAgentProfileSnapshot(opts.profile, 'localSandboxClient default profile') + const router = Object.freeze({ ...opts.router }) + if (opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile === undefined) { throw new ValidationError( 'localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox', ) } const trustedProfileDigest = - opts.profileSecurityPolicy?.allowLocalMcp && opts.profile !== undefined - ? canonicalAgentProfileDigest(opts.profile) + opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile !== undefined + ? canonicalAgentProfileDigest(defaultProfile) : undefined let seq = 0 return { async create(options?: CreateSandboxOptions): Promise { - const profile = agentProfileSchema.parse( - (options?.backend as { profile?: AgentProfile } | undefined)?.profile ?? opts.profile, + const profile = executableAgentProfileSnapshot( + (options?.backend as { profile?: AgentProfile } | undefined)?.profile ?? defaultProfile, + 'localSandboxClient', ) - assertExecutableAgentProfile(profile, 'localSandboxClient') const profileModel = profileProviderModel(profile) const model = profileModel! const settings = profileModelExecutionSettings(profile, 'localSandboxClient') @@ -85,8 +86,8 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie }) const brain: ToolLoopChat = routerBrain( { - routerBaseUrl: opts.router.baseUrl, - routerKey: opts.router.key, + routerBaseUrl: router.baseUrl, + routerKey: router.key, model, ...(settings.maxAttempts !== undefined ? { maxAttempts: settings.maxAttempts } : {}), ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), @@ -94,6 +95,9 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie }, { ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), + ...(settings.seed !== undefined ? { seed: settings.seed } : {}), + ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}), + ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}), ...(profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}), diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts index e28aa255..db92a67c 100644 --- a/src/runtime/profile-chat-client.test.ts +++ b/src/runtime/profile-chat-client.test.ts @@ -1,4 +1,4 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalAgentProfileDigest } from '@tangle-network/agent-interface' import { describe, expect, it, vi } from 'vitest' import { profileChatClient, @@ -132,6 +132,41 @@ describe('profileChatClient exact Runtime adapter', () => { expect(complete).toHaveBeenCalledOnce() }) + it('materializes profile instructions once instead of duplicating the system prompt', async () => { + const complete = vi.fn(async (body: Record) => { + expect(body.messages).toEqual([ + { role: 'system', content: 'Base policy.\nStanding rule.' }, + { role: 'system', content: 'Task-specific context.' }, + { role: 'user', content: 'candidate text' }, + ]) + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'improved candidate' } }], + } + }) + const client = profileChatClient({ + profile: { + ...profile, + prompt: { systemPrompt: 'Base policy.', instructions: ['Standing rule.'] }, + }, + context: 'profile prompt materialization test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) + + await client.chat({ + messages: [ + { role: 'system', content: 'Task-specific context.' }, + { role: 'user', content: 'candidate text' }, + ], + }) + expect(complete).toHaveBeenCalledOnce() + }) + it('fails loud on unenforced maxCostUsd instead of silently treating it as a limit', async () => { const complete = vi.fn(async () => ({ model: 'deepseek-v4-flash', @@ -227,6 +262,93 @@ describe('profileChatClient exact Runtime adapter', () => { expect(complete).toHaveBeenCalledOnce() }) + it('captures the profile and reusable transport before caller mutation', async () => { + const providerOptions = { mode: 'before' } + const mutableProfile: AgentProfile = { + ...profile, + model: { + ...profile.model, + metadata: { extraBody: { provider_options: providerOptions } }, + }, + } + const expectedProfileDigest = canonicalAgentProfileDigest(mutableProfile) + let sent: Record | undefined + const originalComplete = vi.fn(async (body: Record) => { + sent = body + providerOptions.mode = 'during-call' + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'stable response' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + } + }) + const replacementComplete = vi.fn(async (_body: Record) => ({ + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'wrong transport' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + })) + const executor = { + backend: 'router' as const, + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete: originalComplete, + } + const call = profileOptimizerModelCall({ + profile: mutableProfile, + context: 'profile optimizer snapshot test', + executor, + }) + + providerOptions.mode = 'after-bind' + executor.complete = replacementComplete + const result = await call({ + callId: 'optimizer-snapshot', + request: { ...request, model: 'deepseek-v4-flash' }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(true) + expect(originalComplete).toHaveBeenCalledOnce() + expect(replacementComplete).not.toHaveBeenCalled() + expect(sent).toMatchObject({ provider_options: { mode: 'before' } }) + expect(result.execution).toMatchObject({ profileDigest: expectedProfileDigest }) + }) + + it('returns the same custom-priced cost in the optimizer response and receipt', async () => { + const pricing = { inputUsdPerMillion: 2, outputUsdPerMillion: 4 } + const call = profileOptimizerModelCall({ + profile: { + ...profile, + model: { provider: 'custom-provider', default: 'unpriced-custom-model' }, + }, + context: 'profile optimizer custom pricing test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete: async () => ({ + model: 'unpriced-custom-model', + choices: [{ message: { content: 'priced response' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }), + }, + pricing, + }) + + const result = await call({ + callId: 'optimizer-custom-pricing', + request: { ...request, model: 'unpriced-custom-model' }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(true) + if (!result.succeeded) throw new Error(result.error) + expect(result.receipt).toMatchObject({ customTokenPricing: pricing }) + expect(result.response.costUsd).toBeCloseTo(0.000014) + }) + it('keeps paid-call evidence when cache classification fails after transport', async () => { const complete = vi.fn(async () => ({ model: 'deepseek-v4-flash', diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts index f4333520..cfb765c2 100644 --- a/src/runtime/profile-chat-client.ts +++ b/src/runtime/profile-chat-client.ts @@ -5,6 +5,7 @@ import type { CostReceiptInput, CustomTokenPricing, } from '@tangle-network/agent-eval' +import { costForTokenPricing } from '@tangle-network/agent-eval' import type { ExternalOptimizerModelCall, ExternalOptimizerModelCallRequest, @@ -16,13 +17,17 @@ import { canonicalCandidateDigest, } from '@tangle-network/agent-interface' import { type CollectedAgentTurn, collectAgentTurn, streamAgentTurn } from './stream-agent-turn' +import { executableAgentProfileSnapshot } from './supervise/executable-spec' import { - assertExecutableAgentProfile, concreteModelId, type ProfileModelExecutionSettings, profileModelExecutionSettings, } from './supervise/model-policy' -import { createExecutor, type ExecutorConfig } from './supervise/runtime' +import { + captureReusableExecutorConfig, + createExecutor, + type ExecutorConfig, +} from './supervise/runtime' /** Profile-exact adapter for packages that consume agent-eval's ChatClient contract. * Every call still enters Runtime through createExecutor -> streamAgentTurn, and every @@ -93,7 +98,12 @@ export function profileOptimizerModelCall(args: { try { const receipt = optimizerReceipt(binding.model, run, args.pricing) return run.succeeded - ? { succeeded: true, response: run.response, receipt, execution } + ? { + succeeded: true, + response: { ...run.response, costUsd: optimizerResponseCostUsd(receipt) }, + receipt, + execution, + } : { succeeded: false, error: run.error, receipt, execution } } catch (error) { const message = `profile optimizer receipt normalization failed after execution: ${errorMessage(error)}` @@ -117,7 +127,6 @@ interface BoundProfileChat { readonly context: string readonly model: string readonly settings: ProfileModelExecutionSettings - readonly systemPrompt: string } export type ProfileChatRun = @@ -129,17 +138,15 @@ export function bindProfileChat(args: { executor: ExecutorConfig context: string }): BoundProfileChat { - const profile = agentProfileSchema.parse(args.profile) - assertExecutableAgentProfile(profile, args.context) + const profile = executableAgentProfileSnapshot(args.profile, args.context) const model = concreteModelId(profile.model?.default) if (!model) throw new Error(`${args.context}: AgentProfile.model.default must be concrete`) return { profile, - executor: args.executor, + executor: captureReusableExecutorConfig(args.executor, args.context), context: args.context, model, settings: profileModelExecutionSettings(profile, args.context), - systemPrompt: profile.prompt?.systemPrompt ?? '', } } @@ -156,19 +163,6 @@ export async function runBoundProfileChat( binding.context, ) assertSupportedChatCallOptions(callOpts, binding.context) - // The profile prompt is the immutable leading policy. A caller's system-role messages are - // per-call task context: preserve them after that policy rather than letting them replace it or - // forcing dynamic task text into AgentProfile identity. - const profilePromptAlreadyLeading = - binding.systemPrompt.length > 0 && - req.messages[0]?.role === 'system' && - req.messages[0].content === binding.systemPrompt - const messages = [ - ...(binding.systemPrompt && !profilePromptAlreadyLeading - ? [{ role: 'system' as const, content: binding.systemPrompt }] - : []), - ...req.messages, - ] const turnProfile = responseProfile(binding.profile, req, binding.context) const startedAt = performance.now() const turn = await collectAgentTurn( @@ -178,7 +172,7 @@ export async function runBoundProfileChat( profile: turnProfile, factory: createExecutor(binding.executor), }, - { messages: messages as Array<{ role: string; content: unknown }> }, + { messages: req.messages as Array<{ role: string; content: unknown }> }, { ...(req.timeoutMs !== undefined ? { timeoutMs: req.timeoutMs } : {}), ...(callOpts?.signal ? { signal: callOpts.signal } : {}), @@ -445,6 +439,15 @@ function unknownOptimizerReceipt(model: string): CostReceiptInput { } } +function optimizerResponseCostUsd(receipt: CostReceiptInput): number | null { + if (receipt.actualCostUsd !== undefined) return receipt.actualCostUsd + if (receipt.estimatedCostUsd !== undefined) return receipt.estimatedCostUsd + if (receipt.customTokenPricing !== undefined && receipt.usageUnknown !== true) { + return costForTokenPricing(receipt.customTokenPricing, receipt) + } + return null +} + function optimizerTokenCount(value: unknown, label: string): number | undefined { if (value === undefined) return undefined if (!Number.isSafeInteger(value) || (value as number) < 0) { diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index e778b8bf..1c3e4f25 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -1024,16 +1024,24 @@ export async function routerToolLoop( */ export function routerBrain( cfg: RouterConfig, - opts: { temperature?: number; reasoningEffort?: ReasoningEffort } = {}, + opts: { + temperature?: number + reasoningEffort?: ReasoningEffort + seed?: number + toolChoice?: 'auto' | 'required' | 'none' + extraBody?: Readonly> + } = {}, ): ToolLoopChat { const temperature = opts.temperature ?? 0.4 return (messages, tools) => chatWithTools(cfg, messages, tools, { temperature, - toolChoice: 'auto', + toolChoice: opts.toolChoice ?? 'auto', // The config's ceiling reaches the completion, so a caller driving a reasoning model can // raise it. Without this a router-brained supervisor is stuck on the 8192 default. ...(cfg.maxTokens !== undefined ? { maxTokens: cfg.maxTokens } : {}), + ...(opts.seed !== undefined ? { seed: opts.seed } : {}), + ...(opts.extraBody !== undefined ? { extraBody: opts.extraBody } : {}), ...(opts.reasoningEffort ? { reasoningEffort: opts.reasoningEffort } : {}), }) } diff --git a/src/runtime/stdio-mcp-client.test.ts b/src/runtime/stdio-mcp-client.test.ts index edc3bb61..063d5b7e 100644 --- a/src/runtime/stdio-mcp-client.test.ts +++ b/src/runtime/stdio-mcp-client.test.ts @@ -142,6 +142,52 @@ describe('connectStdioMcp', () => { }) describe('materializeLocalMcp', () => { + it('the same-host client snapshots and forwards profile-owned Router settings', async () => { + const providerOptions = { prompt_cache: true } + const profile: AgentProfile = { + name: 'profile-settings-worker', + harness: 'cli-base', + model: { + provider: 'offline', + default: 'offline-test-model', + metadata: { seed: 42, extraBody: { provider_options: providerOptions } }, + }, + } + let sent: Record | undefined + const fetchMock = vi.fn(async (_url: string, init: { body?: string }) => { + sent = JSON.parse(init.body ?? '{}') as Record + return { + ok: true, + status: 200, + json: async () => ({ + model: 'offline-test-model', + choices: [{ message: { content: 'done' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }), + text: async () => '', + } + }) + vi.stubGlobal('fetch', fetchMock) + const client = localSandboxClient({ + router: { baseUrl: 'https://router.invalid', key: 'unused' }, + profile, + }) + providerOptions.prompt_cache = false + const box = await client.create() + try { + for await (const _event of box.streamPrompt('work')) { + // Draining the stream executes the Router turn. + } + expect(sent).toMatchObject({ + seed: 42, + provider_options: { prompt_cache: true }, + }) + } finally { + await box.delete() + vi.unstubAllGlobals() + } + }) + it('spawns each enabled stdio server and namespaces its tools __', async () => { const mat = await materializeLocalMcp( { diff --git a/src/runtime/stream-agent-turn.test.ts b/src/runtime/stream-agent-turn.test.ts index 5b4fb5cd..e366761c 100644 --- a/src/runtime/stream-agent-turn.test.ts +++ b/src/runtime/stream-agent-turn.test.ts @@ -13,6 +13,7 @@ import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { inProcessSandboxClient } from './in-process-sandbox-client' import { collectAgentTurn, streamAgentTurn, streamObservedAgentTurn } from './stream-agent-turn' import { attestRuntimeOwnedExecutor } from './supervise/materialization' +import { createExecutor } from './supervise/runtime' import type { Executor, ExecutorFactory, ExecutorResult } from './supervise/types' const TEST_PROFILE = { @@ -529,6 +530,45 @@ describe('streamAgentTurn: executor backend', () => { expect(toreDown).toBe(1) }) + it('uses one detached profile snapshot even when the caller mutates nested input mid-turn', async () => { + const providerOptions = { mode: 'before' } + const profile = { + ...TEST_PROFILE, + model: { + ...TEST_PROFILE.model, + metadata: { extraBody: { provider_options: providerOptions } }, + }, + } + let sent: Record | undefined + const turn = await collectAgentTurn( + streamAgentTurn( + { + kind: 'executor', + profile, + factory: createExecutor({ + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete: async (body) => { + sent = body + providerOptions.mode = 'after-transport-started' + return { + model: 'offline-test-model', + choices: [{ message: { content: 'stable response' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + } + }, + }), + }, + 'ping', + ), + ) + + expect(turn.status).toBe('completed') + expect(turn.finalText).toBe('stable response') + expect(sent).toMatchObject({ provider_options: { mode: 'before' } }) + }) + it('abort reaches the executor signal and terminates with status aborted', async () => { let toreDown = 0 const controller = new AbortController() diff --git a/src/runtime/stream-agent-turn.ts b/src/runtime/stream-agent-turn.ts index b68f7a6b..1643b745 100644 --- a/src/runtime/stream-agent-turn.ts +++ b/src/runtime/stream-agent-turn.ts @@ -48,11 +48,7 @@ */ import { scoreKnowledgeReadiness } from '@tangle-network/agent-eval' -import { - type AgentProfile, - agentProfileSchema, - canonicalCandidateDigest, -} from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import type { PromptOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { normalizeBackendStreamEvent } from '../backends' import { BackendTransportError, ValidationError } from '../errors' @@ -67,6 +63,7 @@ import type { RuntimeStreamEvent, } from '../types' import { createSandboxToolPartState, mapSandboxEvent, mapSandboxToolEvent } from './sandbox-events' +import { executableAgentProfileSnapshot } from './supervise/executable-spec' import { authoredProfileDigest, knownExecutionBindingReceipt, @@ -78,7 +75,6 @@ import { unknownMaterializationReceipt, } from './supervise/materialization' import { - assertExecutableAgentProfile, concreteModelId, profileBridgeWireModel, profileProviderModel, @@ -476,8 +472,9 @@ async function* streamAgentTurnInternal( ): AsyncGenerator { const label = backend.kind === 'chat' ? backend.backend.kind : backend.kind const profile = - backend.kind === 'executor' ? agentProfileSchema.parse(backend.profile) : undefined - if (profile) assertExecutableAgentProfile(profile, 'streamAgentTurn') + backend.kind === 'executor' + ? executableAgentProfileSnapshot(backend.profile, 'streamAgentTurn') + : undefined const profileDigest = profile ? authoredProfileDigest(profile) : undefined assertTurnIdentity(opts.callId, 'callId') assertTurnIdentity(opts.correlationId, 'correlationId') diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index f7493fc6..25ac124b 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -668,6 +668,9 @@ function routerBrainFromProfile( }, { ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), + ...(settings.seed !== undefined ? { seed: settings.seed } : {}), + ...(settings.toolChoice !== undefined ? { toolChoice: settings.toolChoice } : {}), + ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}), ...(profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}), }, ) diff --git a/tests/kernel/router-brain.test.ts b/tests/kernel/router-brain.test.ts index 4f3f12e7..8f738f3c 100644 --- a/tests/kernel/router-brain.test.ts +++ b/tests/kernel/router-brain.test.ts @@ -159,6 +159,29 @@ describe('routerBrain — the production ToolLoopChat seam over the router tool- expect(sent.tool_choice).toBeUndefined() }) + it('forwards profile-owned seed, tool choice, and provider request fields', async () => { + stubRouter({ choices: [{ message: { content: 'x' } }] }) + const tools = [ + { + type: 'function' as const, + function: { name: 'inspect', parameters: { type: 'object' } }, + }, + ] + await routerBrain(cfg, { + seed: 42, + toolChoice: 'none', + extraBody: { provider_options: { prompt_cache: true } }, + })([], tools) + + const init = fetchMock.mock.calls[0]![1] as { body: string } + const sent = JSON.parse(init.body) + expect(sent).toMatchObject({ + seed: 42, + tool_choice: 'none', + provider_options: { prompt_cache: true }, + }) + }) + it('forwards the router usage + cost so the driver can meter its inference', async () => { stubRouter({ choices: [{ message: { content: 'x', tool_calls: [] } }], diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 1d800aff..9e681c3b 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -184,6 +184,50 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen ).toThrow(/router/) }) + it('lowers every profile-owned Router setting into the supervisor request', async () => { + const blobs = new InMemoryResultBlobStore() + let sent: Record | undefined + const root = supervisorAgent( + testAgentProfile('root', { + harness: 'cli-base', + model: { + provider: 'offline', + default: 'offline-test-model', + metadata: { + seed: 42, + toolChoice: 'none', + extraBody: { provider_options: { prompt_cache: true } }, + }, + }, + }), + { + blobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + maxTurns: 1, + router: { + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete: async (body) => { + sent = body + return { + model: 'offline-test-model', + choices: [{ message: { content: 'done' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + } + }, + }, + }, + ) + + await runSupervisor(root, blobs, new InMemorySpawnJournal()) + expect(sent).toMatchObject({ + seed: 42, + tool_choice: 'none', + provider_options: { prompt_cache: true }, + }) + }) + it('binds the same node-scoped product tool to router and external managers with trusted context', async () => { const identity = { profileDigest: `sha256:${'a'.repeat(64)}`, From 54462fde0d76a545ccdf3bdf96eca0a0d1312db9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:10:09 -0600 Subject: [PATCH 19/39] fix(runtime): harden model execution boundary scan --- scripts/check-model-execution-boundary.mjs | 368 +++++++++++++++--- .../check-model-execution-boundary.test.mjs | 110 +++++- 2 files changed, 413 insertions(+), 65 deletions(-) diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index 88d2f1a5..daa4ae89 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -8,16 +8,6 @@ import ts from 'typescript' const root = resolve(import.meta.dirname, '..') const sourceRoots = ['src', 'bench', 'examples', 'scripts'] -// Runtime owns provider transport. This list is deliberately limited to implementation adapters; -// benchmarks, examples, and probes must enter through an exact AgentProfile. -const directTransportOwners = new Set([ - 'src/runtime/local-sandbox-client.ts', - 'src/runtime/router-client.ts', - 'src/runtime/router-client.complete.test.ts', - 'src/runtime/supervise/runtime.ts', - 'src/runtime/supervise/supervisor-agent.ts', -]) - // These files use caller-supplied URLs for non-inference HTTP (OAuth, telemetry, MCP, search, or // public benchmark data). Everywhere else a global fetch target must be statically readable so a // computed provider endpoint cannot hide from this check. @@ -65,9 +55,48 @@ const lowLevelModelCalls = new Set([ // deliberately callable in exactly one implementation module and must never become a public or // benchmark-facing shortcut around AgentProfile intake. const lowLevelModelCallOwners = new Map([ + [ + 'routerBrain', + new Set([ + 'src/runtime/local-sandbox-client.ts', + 'src/runtime/supervise/supervisor-agent.ts', + ]), + ], + [ + 'routerChatWithTools', + new Set([ + 'src/runtime/router-client.complete.test.ts', + 'src/runtime/supervise/runtime.ts', + ]), + ], + [ + 'routerChatWithUsage', + new Set([ + 'src/runtime/router-client.complete.test.ts', + 'src/runtime/supervise/runtime.ts', + ]), + ], + [ + 'streamRouterChatWithTools', + new Set(['src/runtime/supervise/runtime.ts']), + ], ['runLocalHarness', new Set(['src/mcp/worktree-harness.ts'])], ]) +// The Router adapter owns exactly three fetch sites. The function name and count are both checked: +// adding another raw provider request anywhere in the file fails this check instead of inheriting a +// whole-file exemption. +const ownedModelFetchAllowances = new Map([ + [ + 'src/runtime/router-client.ts', + new Map([ + ['routerChatWithUsage', 1], + ['routerChatWithTools', 1], + ['streamRouterChatWithTools', 1], + ]), + ], +]) + const forbiddenPublicModelCalls = new Set(['runLocalHarness']) // These two files are the implementation owners that translate a validated AgentProfile into a @@ -92,7 +121,20 @@ const processLaunchCalls = new Set([ 'spawnSync', ]) -const sourceExtensions = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.py', '.sh']) +const childProcessModules = new Set(['child_process', 'node:child_process']) + +const sourceExtensions = new Set([ + '.js', + '.mjs', + '.cjs', + '.jsx', + '.ts', + '.mts', + '.cts', + '.tsx', + '.py', + '.sh', +]) const ignoredDirectories = new Set([ '.git', '.venv', @@ -122,7 +164,7 @@ export function findSourceFiles(path) { } function isTestFile(path) { - return /(?:^|\/)[^/]+\.(?:test|spec)\.[cm]?[jt]s$/.test(path) + return /(?:^|\/)[^/]+\.(?:test|spec)\.[cm]?[jt]sx?$/.test(path) } function sourceLocation(source, node) { @@ -138,18 +180,35 @@ function namesModelEndpoint(text) { ) } +function isLowLevelModelModule(moduleName) { + return ( + moduleName === '@tangle-network/agent-eval' || + moduleName.startsWith('@tangle-network/agent-eval/') || + moduleName === '@tangle-network/agent-runtime' || + moduleName.startsWith('@tangle-network/agent-runtime/') || + /(?:^|\/)(?:local-harness|router-client)(?:\.[cm]?[jt]sx?)?$/.test(moduleName) + ) +} + function isLocalTestTarget(path, target) { return isTestFile(path) && /(?:localhost|127\.0\.0\.1|\[::1\]|\.test)(?::\d+)?\//i.test(target) } export function checkJavaScript(path, text) { - const kind = path.endsWith('.ts') || path.endsWith('.mts') || path.endsWith('.cts') - ? ts.ScriptKind.TS - : ts.ScriptKind.JS + const kind = path.endsWith('.tsx') + ? ts.ScriptKind.TSX + : path.endsWith('.jsx') + ? ts.ScriptKind.JSX + : path.endsWith('.ts') || path.endsWith('.mts') || path.endsWith('.cts') + ? ts.ScriptKind.TS + : ts.ScriptKind.JS const source = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, kind) const initializers = new Map() const lowLevelBindings = new Map() const lowLevelNamespaces = new Set() + const processLaunchBindings = new Set() + const childProcessNamespaces = new Set() + const ownedModelFetchCounts = new Map() const failures = [] function collect(node) { @@ -200,11 +259,29 @@ export function checkJavaScript(path, text) { if (!statement.importClause?.namedBindings) continue const bindings = statement.importClause.namedBindings if (ts.isNamespaceImport(bindings)) { - lowLevelNamespaces.add(bindings.name.text) + if (moduleName !== undefined && isLowLevelModelModule(moduleName)) { + lowLevelNamespaces.add(bindings.name.text) + } + if (moduleName !== undefined && childProcessModules.has(moduleName)) { + childProcessNamespaces.add(bindings.name.text) + } } else { for (const element of bindings.elements) { const imported = element.propertyName?.text ?? element.name.text - if (lowLevelModelCalls.has(imported)) lowLevelBindings.set(element.name.text, imported) + if ( + moduleName !== undefined && + isLowLevelModelModule(moduleName) && + lowLevelModelCalls.has(imported) + ) { + lowLevelBindings.set(element.name.text, imported) + } + if ( + moduleName !== undefined && + childProcessModules.has(moduleName) && + processLaunchCalls.has(imported) + ) { + processLaunchBindings.add(element.name.text) + } } } } @@ -221,18 +298,45 @@ export function checkJavaScript(path, text) { if (ts.isVariableDeclaration(node) && node.initializer !== undefined) { if (isRequireCall(node.initializer)) { const moduleArg = node.initializer.arguments[0] - if (ts.isStringLiteral(moduleArg) && providerSdkModules.has(moduleArg.text)) { + const moduleName = ts.isStringLiteral(moduleArg) ? moduleArg.text : undefined + if (moduleName !== undefined && providerSdkModules.has(moduleName)) { failures.push({ node: node.initializer, - detail: `provider SDK require ${JSON.stringify(moduleArg.text)}`, + detail: `provider SDK require ${JSON.stringify(moduleName)}`, }) } - if (ts.isIdentifier(node.name)) lowLevelNamespaces.add(node.name.text) + if ( + moduleName !== undefined && + isLowLevelModelModule(moduleName) && + ts.isIdentifier(node.name) + ) { + lowLevelNamespaces.add(node.name.text) + } + if ( + moduleName !== undefined && + childProcessModules.has(moduleName) && + ts.isIdentifier(node.name) + ) { + childProcessNamespaces.add(node.name.text) + } if (ts.isObjectBindingPattern(node.name)) { for (const element of node.name.elements) { if (!ts.isIdentifier(element.name)) continue const imported = element.propertyName?.getText(source) ?? element.name.text - if (lowLevelModelCalls.has(imported)) lowLevelBindings.set(element.name.text, imported) + if ( + moduleName !== undefined && + isLowLevelModelModule(moduleName) && + lowLevelModelCalls.has(imported) + ) { + lowLevelBindings.set(element.name.text, imported) + } + if ( + moduleName !== undefined && + childProcessModules.has(moduleName) && + processLaunchCalls.has(imported) + ) { + processLaunchBindings.add(element.name.text) + } } } } @@ -240,10 +344,24 @@ export function checkJavaScript(path, text) { ts.isIdentifier(node.name) && ts.isPropertyAccessExpression(node.initializer) && isRequireCall(node.initializer.expression) && + node.initializer.expression.arguments[0] !== undefined && + ts.isStringLiteral(node.initializer.expression.arguments[0]) && + isLowLevelModelModule(node.initializer.expression.arguments[0].text) && lowLevelModelCalls.has(node.initializer.name.text) ) { lowLevelBindings.set(node.name.text, node.initializer.name.text) } + if ( + ts.isIdentifier(node.name) && + ts.isPropertyAccessExpression(node.initializer) && + isRequireCall(node.initializer.expression) && + node.initializer.expression.arguments[0] !== undefined && + ts.isStringLiteral(node.initializer.expression.arguments[0]) && + childProcessModules.has(node.initializer.expression.arguments[0].text) && + processLaunchCalls.has(node.initializer.name.text) + ) { + processLaunchBindings.add(node.name.text) + } } ts.forEachChild(node, collectCommonJs) } @@ -315,20 +433,67 @@ export function checkJavaScript(path, text) { return lowLevelModelCallOwners.get(name)?.has(path) === true } - function isProcessLaunchCall(node) { - const callee = node.expression - if (ts.isIdentifier(callee)) return processLaunchCalls.has(callee.text) - if (ts.isPropertyAccessExpression(callee)) return processLaunchCalls.has(callee.name.text) - return false + function isProcessLaunchExpression(node, seen = new Set()) { + if (ts.isIdentifier(node)) { + if (processLaunchCalls.has(node.text) || processLaunchBindings.has(node.text)) return true + if (seen.has(node.text)) return false + const initializer = initializerFor(node) + if (initializer === undefined) return false + const nextSeen = new Set(seen) + nextSeen.add(node.text) + return isProcessLaunchExpression(initializer, nextSeen) + } + return ( + ts.isPropertyAccessExpression(node) && + processLaunchCalls.has(node.name.text) && + (!ts.isIdentifier(node.expression) || childProcessNamespaces.has(node.expression.text)) + ) + } + + function isGlobalFetchExpression(node, seen = new Set()) { + if (ts.isIdentifier(node)) { + if (node.text === 'fetch') return true + if (seen.has(node.text)) return false + const initializer = initializerFor(node) + if (initializer === undefined) return false + const nextSeen = new Set(seen) + nextSeen.add(node.text) + return isGlobalFetchExpression(initializer, nextSeen) + } + return ( + ts.isPropertyAccessExpression(node) && + node.name.text === 'fetch' && + ts.isIdentifier(node.expression) && + ['global', 'globalThis', 'window'].includes(node.expression.text) + ) } - function isInsideProcessLaunchCall(node) { + function enclosingFunctionName(node) { let parent = node.parent - while (parent && !ts.isStatement(parent)) { - if (ts.isCallExpression(parent) && isProcessLaunchCall(parent)) return true + while (parent) { + if (ts.isFunctionDeclaration(parent) && parent.name) return parent.name.text + if ( + (ts.isFunctionExpression(parent) || ts.isArrowFunction(parent)) && + ts.isVariableDeclaration(parent.parent) && + ts.isIdentifier(parent.parent.name) + ) { + return parent.parent.name.text + } parent = parent.parent } - return false + return undefined + } + + function isAllowedOwnedModelFetch(node) { + if (!ts.isIdentifier(node.expression) || node.expression.text !== 'fetch') return false + const functionName = enclosingFunctionName(node) + if (functionName === undefined) return false + const maximum = ownedModelFetchAllowances.get(path)?.get(functionName) + if (maximum === undefined) return false + const key = `${path}:${functionName}` + const count = (ownedModelFetchCounts.get(key) ?? 0) + 1 + ownedModelFetchCounts.set(key, count) + return count <= maximum } function inspect(node) { @@ -338,12 +503,13 @@ export function checkJavaScript(path, text) { const target = first === undefined ? '' : expressionText(first) const resolvedTarget = first === undefined ? undefined : staticString(first) const call = node.getText(source) - const isGlobalFetch = callee === 'fetch' + const isGlobalFetch = isGlobalFetchExpression(node.expression) const directFetch = isGlobalFetch && (namesModelEndpoint(resolvedTarget ?? target) || (resolvedTarget === undefined && !dynamicNonModelFetchOwners.has(path))) && - !isLocalTestTarget(path, resolvedTarget ?? target) + !isLocalTestTarget(path, resolvedTarget ?? target) && + !isAllowedOwnedModelFetch(node) const providerSdk = /(?:^|\.)(?:chat\.completions\.create|responses\.(?:create|stream)|messages\.(?:create|stream)|generateContent)$/.test( callee, @@ -357,7 +523,7 @@ export function checkJavaScript(path, text) { lowLevelCallName !== undefined && !isAllowedLowLevelModelCall(lowLevelCallName) const cliLaunch = !directCliOwners.has(path) && - isProcessLaunchCall(node) && + isProcessLaunchExpression(node.expression) && namesModelCliInvocation( [callee, ...node.arguments.map((argument) => expressionText(argument))].join(' '), ) @@ -365,22 +531,6 @@ export function checkJavaScript(path, text) { failures.push({ node, detail: call.slice(0, 180).replace(/\s+/g, ' ') }) } } - if ( - !directCliOwners.has(path) && - (ts.isStringLiteralLike(node) || ts.isNoSubstitutionTemplateLiteral(node)) && - !isInsideProcessLaunchCall(node) && - namesModelCliInvocation(node.text) - ) { - failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) - } - if ( - !directCliOwners.has(path) && - ts.isTemplateExpression(node) && - !isInsideProcessLaunchCall(node) && - namesModelCliInvocation(node.getText(source)) - ) { - failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) - } if ( ts.isNewExpression(node) && /^(?:OpenAI|Anthropic)$/.test(node.expression.getText(source)) @@ -437,34 +587,127 @@ function executablePythonLines(text) { }) } +function namesComputedModelEndpoint(text) { + const compact = text.replace(/[\s'"`+(){}]/g, '') + return namesModelEndpoint(compact) +} + +function expandNamedExpressions(text, expressions, tokenPattern, seen = new Set()) { + return text.replace(tokenPattern, (token, braced, plain) => { + const name = braced ?? plain ?? token + if (seen.has(name) || !expressions.has(name)) return token + const nextSeen = new Set(seen) + nextSeen.add(name) + return `(${expandNamedExpressions(expressions.get(name), expressions, tokenPattern, nextSeen)})` + }) +} + +const pythonProviderImport = + /^(?:from\s+(?:openai|anthropic|google\.generativeai|cohere|groq|mistralai)(?:\.[A-Za-z0-9_.]+)?\s+import\b|import\s+(?:openai|anthropic|google\.generativeai|cohere|groq|mistralai)\b)/i +const pythonProviderCall = + /(?:^|\.)(?:chat\.completions\.(?:create|stream)|responses\.(?:create|stream)|messages\.(?:create|stream)|generate_content)\s*\(/i +const pythonHttpCall = + /\b(?:requests|httpx|urllib3|session|client)\.(?:post|request|open|urlopen)\s*\(/i + export function checkPython(text) { const failures = [] const lines = executablePythonLines(text) + const expressions = new Map() + const modelCallBindings = new Set() + const processLaunchBindings = new Set(['call', 'check_call', 'check_output', 'Popen', 'run']) + const processNamespaces = new Set(['os', 'subprocess']) for (const [index, line] of lines.entries()) { - const commandWindow = lines.slice(index, index + 3).join(' ') - if ( - /chat\/completions|api\.anthropic\.com|\.chat\.completions\.create\s*\(|\.responses\.create\s*\(|\.messages\.create\s*\(/i.test( - line, + const code = line.trim() + if (code.length === 0) continue + + let violation = pythonProviderImport.test(code) + const subprocessImport = /^from\s+subprocess\s+import\s+(.+)$/.exec(code) + if (subprocessImport) { + for (const binding of subprocessImport[1].split(',')) { + const match = /^\s*(?:run|Popen|call|check_call|check_output)(?:\s+as\s+([A-Za-z_]\w*))?\s*$/.exec( + binding, + ) + if (match) processLaunchBindings.add(match[1] ?? binding.trim()) + } + } + const namespaceImport = /^import\s+(os|subprocess)(?:\s+as\s+([A-Za-z_]\w*))?\s*$/.exec(code) + if (namespaceImport) processNamespaces.add(namespaceImport[2] ?? namespaceImport[1]) + + const assignment = /^([A-Za-z_]\w*)\s*=\s*(?!=)(.+)$/.exec(code) + if (assignment) { + const [, name, expression] = assignment + expressions.set(name, expression) + if ( + /(?:^|\.)(?:chat\.completions\.(?:create|stream)|responses\.(?:create|stream)|messages\.(?:create|stream)|generate_content)\s*$/.test( + expression.trim(), + ) + ) { + modelCallBindings.add(name) + } + if (violation) { + failures.push({ location: `${index + 1}:1`, detail: code.slice(0, 180) }) + } + continue + } + + const expanded = expandNamedExpressions(code, expressions, /\b([A-Za-z_]\w*)\b/g) + const modelAliasCall = [...modelCallBindings].some((name) => + new RegExp(`(?:^|[^A-Za-z0-9_])${name}\\s*\\(`).test(code), + ) + const processCall = + [...processNamespaces].some((name) => + new RegExp(`(?:^|[^A-Za-z0-9_])${name}\\.(?:run|Popen|call|check_call|check_output|system|popen)\\s*\\(`).test( + code, + ), ) || - (/(?:^|[^A-Za-z0-9_])(?:claude|codex|opencode|pi)(?:Bin|Path|Executable)?\b/i.test( - line, - ) && namesModelCliInvocation(commandWindow)) - ) { - failures.push({ location: `${index + 1}:1`, detail: line.trim().slice(0, 180) }) + [...processLaunchBindings].some((name) => + new RegExp(`(?:^|[^A-Za-z0-9_])${name}\\s*\\(`).test(code), + ) + violation ||= + pythonProviderCall.test(code) || + modelAliasCall || + (pythonHttpCall.test(code) && namesComputedModelEndpoint(expanded)) || + (processCall && namesModelCliInvocation(expanded)) + if (violation) { + failures.push({ location: `${index + 1}:1`, detail: code.slice(0, 180) }) } } return failures } +function shellExecutesModelCli(code) { + return code.split(/&&|\|\||;|\|/).some((segment) => { + const command = segment.trim().replace(/^(?:[A-Za-z_]\w*=\S+\s+)+/, '') + return ( + (/^(?:command\s+)?(?:claude|codex|opencode|pi)(?:Bin|Path|Executable)?\b/i.test(command) && + namesModelCliInvocation(command)) || + /^(?:eval|exec|bash\s+-c|sh\s+-c|xargs)\b/i.test(command) && + namesModelCliInvocation(command) + ) + }) +} + export function checkShell(text) { const failures = [] + const expressions = new Map() for (const [index, line] of text.split(/\r?\n/).entries()) { - const code = line.replace(/^\s*#.*$/, '') + const code = line.replace(/^\s*#.*$/, '').trim() + if (code.length === 0) continue + const assignment = /^(?:(?:export|local|readonly)\s+)?([A-Za-z_]\w*)=(.*)$/.exec(code) + if (assignment) { + expressions.set(assignment[1], assignment[2]) + continue + } + const expanded = expandNamedExpressions( + code, + expressions, + /\$\{([A-Za-z_]\w*)\}|\$([A-Za-z_]\w*)/g, + ) if ( - /(?:curl|wget|http)\b.*(?:chat\/completions|\/responses\b|api\.anthropic\.com)/i.test(code) || - namesModelCliInvocation(code) + (/\b(?:curl|wget|http)\b/i.test(code) && namesComputedModelEndpoint(expanded)) || + shellExecutesModelCli(expanded) ) { - failures.push({ location: `${index + 1}:1`, detail: code.trim().slice(0, 180) }) + failures.push({ location: `${index + 1}:1`, detail: code.slice(0, 180) }) } } return failures @@ -476,7 +719,6 @@ export function scanRepository() { const path = resolve(root, sourceRoot) for (const file of findSourceFiles(path)) { const repoPath = relative(root, file).replaceAll('\\', '/') - if (directTransportOwners.has(repoPath)) continue const text = readFileSync(file, 'utf8') const ext = extension(file) const failures = diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index 90301404..9ed44cca 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -48,6 +48,21 @@ describe('model execution boundary source check', () => { expect(violations[0]?.location).toBe('3:7') }) + it('rejects qualified and aliased global fetch calls', () => { + expect( + checkJavaScript( + 'examples/direct.ts', + `await globalThis.fetch('https://router.tangle.tools/v1/chat/completions')`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'examples/direct.ts', + `const request = globalThis.fetch\nawait request('https://router.tangle.tools/v1/chat/completions')`, + ), + ).toHaveLength(1) + }) + it('rejects low-level Runtime model clients outside Runtime-owned adapters', () => { expect( checkJavaScript( @@ -102,6 +117,21 @@ describe('model execution boundary source check', () => { ).toHaveLength(1) }) + it('does not treat an unrelated namespace as a Runtime model client', () => { + expect( + checkJavaScript( + 'bench/utility.ts', + `import * as utility from './utility'\nawait utility.routerBrain()`, + ), + ).toEqual([]) + expect( + checkJavaScript( + 'bench/utility.cjs', + `const utility = require('./utility')\nutility.routerBrain()`, + ), + ).toEqual([]) + }) + it('rejects direct coding-agent CLI launches in JavaScript and TypeScript', () => { expect( checkJavaScript( @@ -118,7 +148,7 @@ describe('model execution boundary source check', () => { expect( checkJavaScript( 'bench/sidecar.ts', - 'const command = `opencode --model ${model} --format json run ${prompt}`', + 'const command = `opencode --model ${model} --format json run ${prompt}`\nawait exec(command)', ), ).toHaveLength(1) expect( @@ -129,6 +159,21 @@ describe('model execution boundary source check', () => { ).toHaveLength(1) }) + it('rejects child-process aliases but allows inert CLI documentation', () => { + expect( + checkJavaScript( + 'bench/aliased-launch.ts', + `import { spawn as launch } from 'node:child_process'\nlaunch('opencode', ['run', prompt])`, + ), + ).toHaveLength(1) + expect( + checkJavaScript( + 'bench/documentation.ts', + `const help = 'Use opencode run to start a session'`, + ), + ).toEqual([]) + }) + it('rejects model execution hidden inside a custom CandidateGenerator callback', () => { const violations = checkJavaScript( 'bench/custom-generator.ts', @@ -163,6 +208,27 @@ const generator = { ).toEqual([]) }) + it('checks transport-owner files and admits only the existing owned call sites', () => { + expect( + checkJavaScript( + 'src/runtime/local-sandbox-client.ts', + `import { routerBrain } from './router-client'\nrouterBrain(config)\nglobalThis.fetch('https://router.test/v1/chat/completions')`, + ), + ).toEqual([ + expect.objectContaining({ + detail: `globalThis.fetch('https://router.test/v1/chat/completions')`, + }), + ]) + + const routerOwner = ` + export async function routerChatWithUsage() { + await fetch(url) + await fetch(url) + } + ` + expect(checkJavaScript('src/runtime/router-client.ts', routerOwner)).toHaveLength(1) + }) + it('ignores comments, inert strings, and ordinary HTTP', () => { const source = ` // fetch('https://api.openai.com/v1/chat/completions') @@ -186,11 +252,31 @@ const generator = { expect(checkPython(`# client.messages.create(model='x')`)).toEqual([]) expect(checkShell(`# curl https://api.openai.com/v1/chat/completions`)).toEqual([]) expect(checkShell(`curl https://api.openai.com/v1/chat/completions`)).toHaveLength(1) - expect(checkPython(`command = f"opencode --model {model} run {task}"`)).toHaveLength(1) + expect( + checkPython( + `import subprocess\ncommand = f"opencode --model {model} run {task}"\nsubprocess.run(command, shell=True)`, + ), + ).toHaveLength(1) expect(checkShell(`opencode --model "$MODEL" run "$TASK"`)).toHaveLength(1) expect(checkShell(`# opencode --model "$MODEL" run "$TASK"`)).toEqual([]) }) + it('rejects aliased Python SDK calls and computed Python and shell model URLs', () => { + expect( + checkPython( + `from openai import OpenAI\nclient = OpenAI()\ninvoke = client.responses.create\ninvoke(model='x', input='y')`, + ).length, + ).toBeGreaterThan(0) + expect( + checkPython( + `import requests\nurl = base + '/chat/' + 'completions'\nrequests.post(url, json={})`, + ), + ).toHaveLength(1) + expect( + checkShell(`route='/v1/chat/completions'\ncurl "$base$route"`), + ).toHaveLength(1) + }) + it('allows non-inference CLI inspection', () => { expect(checkJavaScript('bench/auth.ts', `await exec('codex', ['login', 'status'])`)).toEqual([]) expect(checkShell(`claude --version`)).toEqual([]) @@ -210,4 +296,24 @@ const generator = { rmSync(fixture, { force: true, recursive: true }) } }) + + it('discovers TSX and JSX source files', () => { + const fixture = mkdtempSync(join(tmpdir(), 'model-boundary-jsx-')) + try { + const tsx = join(fixture, 'agent.tsx') + const jsx = join(fixture, 'agent.jsx') + writeFileSync(tsx, `export const Agent = () =>
\n`) + writeFileSync(jsx, `export const Agent = () =>
\n`) + + expect(findSourceFiles(fixture).sort()).toEqual([jsx, tsx].sort()) + expect( + checkJavaScript( + 'examples/agent.tsx', + `export const Agent = () => `, + ), + ).toHaveLength(1) + } finally { + rmSync(fixture, { force: true, recursive: true }) + } + }) }) From ea3fc2a0e072df9ffb79f710235a822876c1f679 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:29:00 -0600 Subject: [PATCH 20/39] feat(runtime): bind exact profiles to task backends --- docs/api/primitive-catalog.md | 10 +- examples/driver-loop/driver-loop.ts | 8 +- examples/quickstart/quickstart.ts | 8 +- src/conversation/index.ts | 1 + src/conversation/run-persona.ts | 17 +-- src/index.ts | 1 + src/runtime/profile-execution-backend.test.ts | 70 +++++++++++ src/runtime/profile-execution-backend.ts | 118 ++++++++++++++++++ src/runtime/supervise/types.ts | 6 + 9 files changed, 222 insertions(+), 17 deletions(-) create mode 100644 src/runtime/profile-execution-backend.test.ts create mode 100644 src/runtime/profile-execution-backend.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index c5232f6b..9fd27356 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.129.0` and `@tangle-network/agent-eval@0.144.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.129.0` and `@tangle-network/agent-eval@0.144.3` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 406 exports. +Import from `@tangle-network/agent-runtime` — 407 exports. | Symbol | Kind | Summary | |---|---|---| @@ -47,6 +47,7 @@ Import from `@tangle-network/agent-runtime` — 406 exports. | `createKnowledgeImprovementActivationExecutor` | function | Apply or restore one local knowledge candidate through the shared activation contract. | | `createOpenInferenceFileExporter` | function | Create an exporter that APPENDS spans to a local OpenInference-JSONL file, one complete span per | | `createOtelExporter` | function | Create an OTEL exporter. Returns undefined when no endpoint is configured. | +| `createProfileExecutionBackend` | function | Bind one exact profile and Runtime executor to the stable `AgentExecutionBackend` contract used | | `createProtectedAgentCandidateModelPort` | function | Bind a protected model-grant service to the immutable candidate runtime. | | `createRuntimeEventCollector` | function | Build an in-memory collector that sanitizes and accumulates `AgentRuntimeEvent`s for inspection. | | `createRuntimeStreamEventCollector` | function | Streaming-event counterpart of `createRuntimeEventCollector`. Pass each | @@ -300,13 +301,14 @@ Import from `@tangle-network/agent-runtime/agent` — 48 exports. ### Multi-turn conversations -Import from `@tangle-network/agent-runtime/conversation` — 53 exports. +Import from `@tangle-network/agent-runtime/conversation` — 54 exports. | Symbol | Kind | Summary | |---|---|---| | `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | | `computeBackoff` | function | Compute the delay before the next attempt. Default: 250ms exponential with jitter. | | `createConversationBackend` | function | Adapt a multi-participant conversation into the standard execution backend contract. | +| `createProfileExecutionBackend` | function | Bind one exact profile and Runtime executor to the stable `AgentExecutionBackend` contract used | | `d1ToSqlAdapter` | function | Adapt a Cloudflare D1 binding to the SqlAdapter shape. Lives here so D1 | | `defineConversation` | function | Validate and define a conversation before execution. | | `isDepthExceeded` | function | Refuse further forwarding when the inbound depth has reached the limit. | @@ -1669,7 +1671,7 @@ Import from `@tangle-network/agent-eval/campaign` — 353 exports. | `FileSearchLedger` | class | Append-only file-backed search ledger with idempotent writes and replay. | | `FsLabeledScenarioStore` | class | Filesystem `LabeledScenarioStore`: appends one JSONL file per source with provenance and | | `LabeledScenarioStoreError` | class | Typed rejection from a labeled-scenario store (bad provenance, rate limit, invalid sample args) — carries a stable string `code`. | -| `ProfileMatrixError` | class | Thrown when the matrix is misconfigured (no profiles, a profile whose model | +| `ProfileMatrixError` | class | Thrown when the matrix is misconfigured (no profiles, missing resolved model evidence, | | `SearchLedgerConflictError` | class | Error raised when an event identifier is reused with different content. | | `SearchLedgerError` | class | Base error for invalid search-ledger input or operations. | | `SearchLedgerIntegrityError` | class | Error raised when durable search-ledger data fails an integrity check. | diff --git a/examples/driver-loop/driver-loop.ts b/examples/driver-loop/driver-loop.ts index ad3f555a..cff80bfb 100644 --- a/examples/driver-loop/driver-loop.ts +++ b/examples/driver-loop/driver-loop.ts @@ -59,6 +59,12 @@ import { // non-terminal 'refine' to keep going, and only the terminal 'pick-winner'/'fail' when truly done. type NoteDecision = 'refine' | 'pick-winner' | 'fail' +const noteWriterProfile = { + name: 'note-writer', + harness: 'cli-base', + model: { provider: 'scripted', default: 'scripted/note-writer' }, +} satisfies AgentProfile + function refineDriver(maxShots: number): Driver { return { name: 'refine', @@ -116,7 +122,7 @@ async function main(): Promise { const result = await runAgentRounds({ driver: refineDriver(3), agentRun: { - profile: { name: 'note-writer' } as AgentProfile, + profile: noteWriterProfile, // Each shot's task carries the prompt the driver authored; this is how the rewritten // instruction actually reaches the worker. taskToPrompt: (t) => t.prompt, diff --git a/examples/quickstart/quickstart.ts b/examples/quickstart/quickstart.ts index 5d37f96d..1fa6f0ad 100644 --- a/examples/quickstart/quickstart.ts +++ b/examples/quickstart/quickstart.ts @@ -17,6 +17,12 @@ import type { SandboxEvent } from '@tangle-network/sandbox' type Task = { prompt: string } type Note = { note: string } +const noteWriterProfile = { + name: 'note-writer', + harness: 'cli-base', + model: { provider: 'scripted', default: 'scripted/note-writer' }, +} satisfies AgentProfile + // A stand-in worker: it obeys the prompt it is given. Swap for a real backend later. const worker = inProcessSandboxClient({ onPrompt: (prompt): SandboxEvent[] => [ @@ -51,7 +57,7 @@ const result = await runAgentRounds t.prompt }, + agentRun: { profile: noteWriterProfile, taskToPrompt: (t) => t.prompt }, output: { parse: (events) => { for (const ev of events) { diff --git a/src/conversation/index.ts b/src/conversation/index.ts index 15bbb0d2..b9370675 100644 --- a/src/conversation/index.ts +++ b/src/conversation/index.ts @@ -14,6 +14,7 @@ * See `docs/agent-bus-protocol.md` for the cross-gateway header contract. */ +export { createProfileExecutionBackend } from '../runtime/profile-execution-backend' export { type BackendCallPolicy, type CircuitBreakerConfig, diff --git a/src/conversation/run-persona.ts b/src/conversation/run-persona.ts index ca13d48b..5c3577ae 100644 --- a/src/conversation/run-persona.ts +++ b/src/conversation/run-persona.ts @@ -22,7 +22,7 @@ import type { Scenario, } from '@tangle-network/agent-eval/campaign' import { createIterableBackend } from '../backends' -import { streamAgentTurn } from '../runtime/stream-agent-turn' +import { createProfileExecutionBackend } from '../runtime/profile-execution-backend' import type { ExecutorFactory } from '../runtime/supervise/types' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { defineConversation } from './define-conversation' @@ -85,18 +85,13 @@ function profileRuntimeBackend( factory: ExecutorFactory, counter?: UsageCounter, ): AgentExecutionBackend { + const backend = createProfileExecutionBackend({ profile, executor: factory }) + if (!counter) return backend return { - kind: 'runtime-profile', + ...backend, async *stream(input, context) { - const turnInput = input.messages - ? { messages: input.messages } - : (input.message ?? context.task.intent) - for await (const event of streamAgentTurn({ kind: 'executor', profile, factory }, turnInput, { - signal: context.signal, - ...(context.turnId ? { callId: context.turnId } : {}), - ...(context.runId ? { correlationId: context.runId } : {}), - })) { - if (counter && event.type === 'llm_call') { + for await (const event of backend.stream(input, context)) { + if (event.type === 'llm_call') { counter.sawLlmCall = true counter.tokensIn += event.tokensIn ?? 0 counter.tokensOut += event.tokensOut ?? 0 diff --git a/src/index.ts b/src/index.ts index 8bacf07a..98531f07 100644 --- a/src/index.ts +++ b/src/index.ts @@ -68,6 +68,7 @@ export { CircuitOpenError, computeBackoff, createConversationBackend, + createProfileExecutionBackend, DEFAULT_MAX_DEPTH, DeadlineExceededError, d1ToSqlAdapter, diff --git a/src/runtime/profile-execution-backend.test.ts b/src/runtime/profile-execution-backend.test.ts new file mode 100644 index 00000000..72851913 --- /dev/null +++ b/src/runtime/profile-execution-backend.test.ts @@ -0,0 +1,70 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it, vi } from 'vitest' +import { runAgentTaskStream } from '../run' +import type { RuntimeStreamEvent } from '../types' +import { createProfileExecutionBackend } from './profile-execution-backend' +import { createExecutor } from './supervise/runtime' + +const profile = { + name: 'profile-backend-test', + harness: 'cli-base', + model: { provider: 'offline', default: 'offline/profile-backend' }, +} satisfies AgentProfile + +describe('createProfileExecutionBackend', () => { + it('runs through runAgentTaskStream with one outer terminal event and the exact profile', async () => { + const complete = vi.fn(async (_body: Record) => ({ + model: 'offline/profile-backend', + choices: [{ message: { content: 'profile-owned answer' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 4, completion_tokens: 2 }, + })) + const backend = createProfileExecutionBackend({ + profile, + executor: createExecutor({ + backend: 'router', + routerBaseUrl: 'http://offline.invalid/v1', + routerKey: 'offline', + complete, + }), + }) + + const events: RuntimeStreamEvent[] = [] + for await (const event of runAgentTaskStream({ + task: { id: 'profile-task', intent: 'answer from the declared profile' }, + backend, + })) { + events.push(event) + } + + expect(complete).toHaveBeenCalledOnce() + expect(complete.mock.calls[0]?.[0]).toMatchObject({ + model: 'offline/profile-backend', + messages: [{ role: 'user', content: 'answer from the declared profile' }], + }) + expect(events.filter((event) => event.type === 'final')).toHaveLength(1) + expect(events.at(-1)).toMatchObject({ + type: 'final', + status: 'completed', + text: 'profile-owned answer', + }) + const modelCall = events.find((event) => event.type === 'llm_call') + expect(modelCall).toMatchObject({ + type: 'llm_call', + task: { id: 'profile-task' }, + model: 'offline/profile-backend', + tokensIn: 4, + tokensOut: 2, + }) + }) + + it('rejects an incomplete profile before constructing an executor', () => { + const executor = vi.fn() + expect(() => + createProfileExecutionBackend({ + profile: { name: 'incomplete' } as AgentProfile, + executor, + }), + ).toThrow(/AgentProfile\.harness must be explicit/u) + expect(executor).not.toHaveBeenCalled() + }) +}) diff --git a/src/runtime/profile-execution-backend.ts b/src/runtime/profile-execution-backend.ts new file mode 100644 index 00000000..6e206cef --- /dev/null +++ b/src/runtime/profile-execution-backend.ts @@ -0,0 +1,118 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { BackendTransportError, ValidationError } from '../errors' +import type { + AgentBackendContext, + AgentExecutionBackend, + BackendErrorDetail, + RuntimeStreamEvent, +} from '../types' +import { streamAgentTurn } from './stream-agent-turn' +import { executableAgentProfileSnapshot } from './supervise/executable-spec' +import type { ExecutorFactory } from './supervise/types' + +/** + * Bind one exact profile and Runtime executor to the stable `AgentExecutionBackend` contract used + * by `runAgentTaskStream` and conversations. + * + * Runtime still owns the model call through `streamAgentTurn`. + * The adapter only translates the two stream protocols and carries the caller's request headers + * into `ExecutorContext` so an HTTP executor can preserve authorization, recursion depth, and + * trace identity. + * + * @stable + */ +export function createProfileExecutionBackend(options: { + profile: AgentProfile + executor: ExecutorFactory +}): AgentExecutionBackend { + const profile = executableAgentProfileSnapshot(options.profile, 'createProfileExecutionBackend') + const executor = options.executor + + return { + kind: 'runtime-profile', + async *stream(input, context) { + const propagatedHeaders = snapshotPropagatedHeaders(context.propagatedHeaders) + const contextualExecutor: ExecutorFactory = (spec, executorContext) => + executor(spec, { + ...executorContext, + ...(propagatedHeaders === undefined ? {} : { propagatedHeaders }), + }) + const turnInput = input.messages + ? { messages: input.messages.map((message) => ({ ...message })) } + : (input.message ?? context.task.intent) + let terminal = false + let emittedText = false + + for await (const event of streamAgentTurn( + { kind: 'executor', profile, factory: contextualExecutor }, + turnInput, + { + signal: context.signal, + ...(context.turnId ? { callId: context.turnId } : {}), + ...(context.runId ? { correlationId: context.runId } : {}), + }, + )) { + if (event.type === 'backend_start') continue + if (event.type === 'backend_error') throw backendError(event.error, event.message) + if (event.type === 'final') { + terminal = true + if (event.status !== 'completed') { + throw backendError(event.error, event.reason) + } + if (!emittedText && event.text) { + yield { + type: 'text_delta', + task: context.task, + session: context.session, + text: event.text, + timestamp: event.timestamp, + } + } + continue + } + if (event.type === 'text_delta') emittedText = true + yield rebindEvent(event, context) + } + + if (!terminal) { + throw new ValidationError( + 'createProfileExecutionBackend: streamAgentTurn ended without a terminal event', + ) + } + }, + } +} + +function snapshotPropagatedHeaders( + headers: Readonly> | undefined, +): Readonly> | undefined { + if (headers === undefined) return undefined + const snapshot: Record = {} + for (const [name, value] of Object.entries(headers)) { + if (typeof value !== 'string') { + throw new ValidationError( + `createProfileExecutionBackend: propagated header ${JSON.stringify(name)} must be a string`, + ) + } + snapshot[name] = value + } + return Object.freeze(snapshot) +} + +function backendError(detail: BackendErrorDetail | undefined, message: string): Error { + if (detail?.kind === 'transport') { + return new BackendTransportError('runtime-profile', detail.message, { + status: detail.status, + body: detail.body, + }) + } + return new Error(detail?.message ?? message) +} + +function rebindEvent(event: RuntimeStreamEvent, context: AgentBackendContext): RuntimeStreamEvent { + return { + ...event, + task: context.task, + session: context.session, + } as RuntimeStreamEvent +} diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index a22517df..3d0e0bb9 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -408,6 +408,12 @@ export type ExecutorFactory = (spec: AgentSpec, ctx: ExecutorContext) => Ex * the factory reaching into module globals. */ export interface ExecutorContext { readonly signal: AbortSignal + /** + * Request headers inherited from an enclosing task or conversation. + * Network executors forward these after their own connection headers so caller authorization, + * recursion depth, and trace identity survive the profile-to-executor boundary. + */ + readonly propagatedHeaders?: Readonly> /** Present when Scope constructs the executor for a supervised node. */ readonly node?: ExecutorNodeContext /** Opaque seams the registry threads through; a built-in narrows what it needs. */ From 37c2b20ff42ff941c06729f7fda84f7af7f7db2d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:29:37 -0600 Subject: [PATCH 21/39] fix(runtime): preserve profile retry controls --- bench/src/aec-gate.mts | 2 +- bench/src/router-turn.ts | 11 +- src/runtime/local-sandbox-client.ts | 2 +- src/runtime/profile-chat-client.test.ts | 43 ++++ src/runtime/profile-chat-client.ts | 4 +- src/runtime/router-client.complete.test.ts | 166 +++++++++++++- src/runtime/router-client.ts | 222 ++++++++++++------- src/runtime/router-retry-policy.test.ts | 69 ++++++ src/runtime/router-retry-policy.ts | 125 +++++++++++ src/runtime/supervise/model-policy.ts | 14 +- src/runtime/supervise/runtime.ts | 19 +- src/runtime/supervise/supervisor-agent.ts | 2 +- tests/kernel/router-brain.test.ts | 50 ++++- tests/runtime/executor-profile-model.test.ts | 61 ++++- 14 files changed, 670 insertions(+), 120 deletions(-) create mode 100644 src/runtime/router-retry-policy.test.ts create mode 100644 src/runtime/router-retry-policy.ts diff --git a/bench/src/aec-gate.mts b/bench/src/aec-gate.mts index aae061ab..ccf264f0 100644 --- a/bench/src/aec-gate.mts +++ b/bench/src/aec-gate.mts @@ -194,7 +194,7 @@ async function main(): Promise { routerBaseUrl, routerKey, profile: benchRouterProfile('aec-worker', model, { - maxRetries: Number(process.env.MAX_RETRIES ?? 2), + retry: { maxAttempts: Number(process.env.MAX_ATTEMPTS ?? 3) }, }), } const bench = process.env.BENCH ?? 'aec-bench' diff --git a/bench/src/router-turn.ts b/bench/src/router-turn.ts index fcf2a909..6021ec60 100644 --- a/bench/src/router-turn.ts +++ b/bench/src/router-turn.ts @@ -28,7 +28,14 @@ export interface BenchProfileSettings { systemPrompt?: string temperature?: number maxTokens?: number - maxRetries?: number + retry?: { + maxAttempts?: number + initialBackoffMs?: number + maxBackoffMs?: number + jitter?: number + retryStatuses?: ReadonlyArray + requestTimeoutMs?: number + } maxTurns?: number seed?: number reasoningEffort?: ReasoningEffort @@ -62,7 +69,7 @@ export function withBenchProfile( ...(base.model?.metadata ?? {}), ...(settings.temperature !== undefined ? { temperature: settings.temperature } : {}), ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), - ...(settings.maxRetries !== undefined ? { maxRetries: settings.maxRetries } : {}), + ...(settings.retry !== undefined ? { retry: settings.retry } : {}), ...(settings.maxTurns !== undefined ? { maxTurns: settings.maxTurns } : {}), ...(settings.seed !== undefined ? { seed: settings.seed } : {}), ...(settings.extraBody !== undefined ? { extraBody: settings.extraBody } : {}), diff --git a/src/runtime/local-sandbox-client.ts b/src/runtime/local-sandbox-client.ts index 29e6b0f6..28606c52 100644 --- a/src/runtime/local-sandbox-client.ts +++ b/src/runtime/local-sandbox-client.ts @@ -89,7 +89,7 @@ export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClie routerBaseUrl: router.baseUrl, routerKey: router.key, model, - ...(settings.maxAttempts !== undefined ? { maxAttempts: settings.maxAttempts } : {}), + ...(settings.retry !== undefined ? { retry: settings.retry } : {}), ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), ...(settings.stream !== undefined ? { stream: settings.stream } : {}), }, diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts index db92a67c..5c73e836 100644 --- a/src/runtime/profile-chat-client.test.ts +++ b/src/runtime/profile-chat-client.test.ts @@ -71,6 +71,49 @@ describe('profileChatClient exact Runtime adapter', () => { expect(response.durationMs).toBeGreaterThanOrEqual(0) }) + it('carries the exact profile retry policy through the injected Router transport', async () => { + let attempts = 0 + const complete = vi.fn(async () => { + attempts += 1 + if (attempts === 1) throw new TypeError('fetch failed: injected reset') + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'retried response' } }], + } + }) + const client = profileChatClient({ + profile: { + ...profile, + model: { + ...profile.model, + metadata: { + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + requestTimeoutMs: 0, + }, + }, + }, + }, + context: 'profile retry test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete, + }, + }) + + const response = await client.chat(request) + + expect(client.maximumAttempts).toBe(2) + expect(response.content).toBe('retried response') + expect(response.raw).toMatchObject({ transportAttempts: 2 }) + expect(complete).toHaveBeenCalledTimes(2) + }) + it('refuses a provider-reported model different from the exact profile', async () => { await expect( clientWith(async () => ({ diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts index cfb765c2..b2917bde 100644 --- a/src/runtime/profile-chat-client.ts +++ b/src/runtime/profile-chat-client.ts @@ -42,8 +42,8 @@ export function profileChatClient(args: { return { transport: 'custom', defaultModel: binding.model, - ...(binding.settings.maxAttempts !== undefined - ? { maximumAttempts: binding.settings.maxAttempts } + ...(binding.settings.retry?.maxAttempts !== undefined + ? { maximumAttempts: binding.settings.retry.maxAttempts } : {}), async chat(req, callOpts) { const run = await runBoundProfileChat(binding, req, callOpts) diff --git a/src/runtime/router-client.complete.test.ts b/src/runtime/router-client.complete.test.ts index 458ed8be..ccf391e8 100644 --- a/src/runtime/router-client.complete.test.ts +++ b/src/runtime/router-client.complete.test.ts @@ -5,7 +5,10 @@ import { routerChatWithTools, routerChatWithUsage } from './router-client' // clients call it with the OpenAI request body and parse what it returns, INSTEAD of `fetch`-ing // the router. The offline-benchmark path — a deterministic in-process responder, no network. -afterEach(() => vi.unstubAllGlobals()) +afterEach(() => { + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) describe('RouterConfig.complete — the injected completion transport', () => { it('routerChatWithUsage uses `complete` and never touches fetch', async () => { @@ -91,17 +94,11 @@ describe('RouterConfig.complete — the injected completion transport', () => { expect(fetchSpy).toHaveBeenCalledOnce() }) - it('reports the exact transport count and honors a caller-owned retry total', async () => { + it('buffered chat retries a thrown network failure and reports the exact attempt count', async () => { let calls = 0 const fetchSpy = vi.fn(async () => { calls += 1 - if (calls === 1) { - return { - ok: false, - status: 503, - text: async (): Promise => 'capacity', - } - } + if (calls === 1) throw new TypeError('fetch failed: socket reset') return { ok: true, status: 200, @@ -119,7 +116,13 @@ describe('RouterConfig.complete — the injected completion transport', () => { routerBaseUrl: 'http://router.test/v1', routerKey: 'k', model: 'deepseek-v4-flash', - maxAttempts: 2, + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + requestTimeoutMs: 0, + }, }, [{ role: 'user', content: 'recover' }], ) @@ -129,6 +132,147 @@ describe('RouterConfig.complete — the injected completion transport', () => { expect(result.content).toBe('recovered') }) + it('fails loud with the final network cause after the configured attempts are exhausted', async () => { + const fetchSpy = vi.fn(async () => { + throw new TypeError('fetch failed: connection refused') + }) + vi.stubGlobal('fetch', fetchSpy) + + await expect( + routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + requestTimeoutMs: 0, + }, + }, + [{ role: 'user', content: 'cannot connect' }], + ), + ).rejects.toThrow(/router network failure: fetch failed: connection refused/u) + expect(fetchSpy).toHaveBeenCalledTimes(2) + }) + + it('enforces the per-attempt header timeout and retries the timed-out request', async () => { + let calls = 0 + const fetchSpy = vi.fn(async (_url: unknown, init: { signal?: AbortSignal }) => { + calls += 1 + if (calls === 1) { + return new Promise((_resolve, reject) => { + init.signal?.addEventListener( + 'abort', + () => reject(init.signal?.reason ?? new Error('aborted')), + { once: true }, + ) + }) + } + return { + ok: true, + status: 200, + json: async () => ({ choices: [{ message: { content: 'after timeout' } }] }), + } + }) + vi.stubGlobal('fetch', fetchSpy) + + const result = await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + requestTimeoutMs: 10, + }, + }, + [{ role: 'user', content: 'timeout once' }], + ) + + expect(result.content).toBe('after timeout') + expect(result.transportAttempts).toBe(2) + expect(fetchSpy).toHaveBeenCalledTimes(2) + }) + + it('passes initial delay, exponential cap, and jitter to the shared retry primitive', async () => { + vi.spyOn(Math, 'random').mockReturnValue(1) + const timerSpy = vi.spyOn(globalThis, 'setTimeout') + let calls = 0 + const fetchSpy = vi.fn(async () => { + calls += 1 + if (calls < 3) throw new TypeError('fetch failed') + return { + ok: true, + status: 200, + json: async () => ({ choices: [{ message: { content: 'third try' } }] }), + } + }) + vi.stubGlobal('fetch', fetchSpy) + + await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + retry: { + maxAttempts: 3, + initialBackoffMs: 10, + maxBackoffMs: 15, + jitter: 0.2, + requestTimeoutMs: 0, + }, + }, + [{ role: 'user', content: 'back off' }], + ) + + expect(timerSpy.mock.calls.map((call) => call[1])).toEqual([12, 18]) + }) + + it('cancels pending backoff and never starts another request after caller abort', async () => { + const controller = new AbortController() + const timerSpy = vi.spyOn(globalThis, 'setTimeout') + let firstAttemptStarted = () => {} + const started = new Promise((resolve) => { + firstAttemptStarted = resolve + }) + const fetchSpy = vi.fn(async () => { + firstAttemptStarted() + throw new TypeError('fetch failed: offline') + }) + vi.stubGlobal('fetch', fetchSpy) + + const pending = routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + retry: { + maxAttempts: 3, + initialBackoffMs: 60_000, + maxBackoffMs: 60_000, + jitter: 0, + requestTimeoutMs: 0, + }, + }, + [{ role: 'user', content: 'stop retrying' }], + { signal: controller.signal }, + ) + await started + await Promise.resolve() + await Promise.resolve() + expect(timerSpy).toHaveBeenCalledWith(expect.any(Function), 60_000) + controller.abort(new Error('caller stopped')) + + await expect(pending).rejects.toThrow(/caller stopped/u) + expect(fetchSpy).toHaveBeenCalledOnce() + }) + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( 'rejects invalid maxAttempts %s before dispatch', async (maxAttempts) => { @@ -140,7 +284,7 @@ describe('RouterConfig.complete — the injected completion transport', () => { routerBaseUrl: 'http://router.test/v1', routerKey: 'k', model: 'deepseek-v4-flash', - maxAttempts, + retry: { maxAttempts }, }, [{ role: 'user', content: 'do not dispatch' }], ), diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 1c3e4f25..f6f59150 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -10,9 +10,11 @@ * (a phantom 0 reads as a free call downstream, which the gate would act on). */ +import { type RetryConfig, SDKError, withRetry } from '@tangle-network/agent-core' import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import type { ReasoningEffort } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' +import { type RouterRetryPolicy, resolveRouterRetryPolicy } from './router-retry-policy' import { runBrainLoop, type ToolLoopChat } from './tool-loop' /** @@ -43,12 +45,8 @@ export interface RouterTransportConfig { */ export interface RouterConfig extends RouterTransportConfig { model: string - /** - * Total HTTP attempts for one completion, including the first request. - * Defaults to 5. Set 1 when an outer workflow owns retries so the two - * policies cannot multiply invisibly. Any positive safe integer is allowed. - */ - maxAttempts?: number + /** Exact retry controls lowered from `AgentProfile.model.metadata.retry`. */ + retry?: RouterRetryPolicy /** * Optional ceiling for one completion, forwarded as `max_tokens`. * @@ -168,44 +166,33 @@ export async function routerChatWithUsage( ...(opts?.seed !== undefined ? { seed: opts.seed } : {}), ...(opts?.reasoningEffort ? { reasoning_effort: opts.reasoningEffort } : {}), }) - const maxAttempts = routerMaxAttempts(cfg) + const retry = resolveRouterRetryPolicy(cfg.retry, 'RouterConfig.retry') // Injected transport short-circuits the network: the offline benchmark seam. It owns its own - // determinism, so the fetch-specific transient-retry/temperature-handling below does not apply. - if (cfg.complete) { - return parseChatResult( - await cfg.complete(body(), { + // determinism while sharing the HTTP path's retry, timeout, and cancellation behavior. + const complete = cfg.complete + if (complete) { + const { value, attempts } = await retryRouterOperation(retry, opts?.signal, (signal) => + complete(body(), { headers, - ...(opts?.signal ? { signal: opts.signal } : {}), + signal, }), - cfg.model, - 1, ) + return parseChatResult(value, cfg.model, attempts) } // Retry transient upstream failures (429/5xx) with backoff so a single capacity // hiccup does not kill a whole multi-model benchmark run. Provider requests to // change temperature fail: generation behavior belongs to the exact profile. - let lastErr = '' - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const res = await fetch(url, { + const { response, attempts } = await fetchRouterResponse( + url, + { method: 'POST', headers, body: JSON.stringify(body()), - ...(opts?.signal ? { signal: opts.signal } : {}), - }) - if (res.ok) return parseChatResult(await res.json(), cfg.model, attempt) - const status = res.status - const text = (await res.text()).slice(0, 200) - lastErr = `router ${status}: ${text}` - // Non-retryable (auth/quota/malformed) fails loud immediately; retryable - // statuses back off and continue until the loop's attempt bound, then the - // post-loop throw is the honest "exhausted retries" terminal. 408/425 + the - // Cloudflare-origin family (520/522/524) are transient under heavy parallel - // load — a fleet of concurrent gate runs hits 524 ("origin timeout") and must - // retry, not crash the whole run. - if (!isTransientRouterStatus(status)) throw new Error(lastErr) - if (attempt < maxAttempts) await new Promise((r) => setTimeout(r, 800 * 2 ** (attempt - 1))) - } - throw new Error(`${lastErr} (exhausted retries)`) + }, + opts?.signal, + retry, + ) + return parseChatResult(await response.json(), cfg.model, attempts) } function parseChatResult( @@ -345,34 +332,32 @@ export async function routerChatWithTools( }, ): Promise { const body = toolCompletionBody(cfg, messages, tools, opts) - const maxAttempts = routerMaxAttempts(cfg) + const retry = resolveRouterRetryPolicy(cfg.retry, 'RouterConfig.retry') // Injected transport short-circuits the network — the offline benchmark seam (see RouterConfig.complete). let transportAttempts = 1 const headers = routerRequestHeaders(cfg, opts) - const raw = cfg.complete - ? await cfg.complete(structuredClone(body), { - headers, - ...(opts?.signal ? { signal: opts.signal } : {}), - }) + const complete = cfg.complete + const raw = complete + ? await (async () => { + const result = await retryRouterOperation(retry, opts?.signal, (signal) => + complete(structuredClone(body), { headers, signal }), + ) + transportAttempts = result.attempts + return result.value + })() : await (async () => { - let lastErr = '' - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - transportAttempts = attempt - const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { + const result = await fetchRouterResponse( + `${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, + { method: 'POST', headers, body: JSON.stringify(body), - ...(opts?.signal ? { signal: opts.signal } : {}), - }) - if (res.ok) return res.json() - const status = res.status - lastErr = `router ${status}: ${(await res.text()).slice(0, 200)}` - if (!isTransientRouterStatus(status)) throw new Error(lastErr) - if (attempt < maxAttempts) { - await new Promise((resolve) => setTimeout(resolve, 800 * 2 ** (attempt - 1))) - } - } - throw new Error(`${lastErr} (exhausted retries)`) + }, + opts?.signal, + retry, + ) + transportAttempts = result.attempts + return result.response.json() })() const data = raw as { model?: unknown @@ -658,7 +643,7 @@ export async function streamRouterChatWithTools( correlationId?: string }, ): Promise { - const maxAttempts = routerMaxAttempts(cfg) + const retry = resolveRouterRetryPolicy(cfg.retry, 'RouterConfig.retry') if (cfg.complete) { throw new ValidationError( 'streamRouterChatWithTools: RouterConfig.complete is a BUFFERED transport (it returns one ' + @@ -674,31 +659,19 @@ export async function streamRouterChatWithTools( // fact by `usageUnknown`, not assumed away. stream_options: { include_usage: true }, } - let res: Response | undefined - let transportAttempts = 0 - let lastErr = '' - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - transportAttempts = attempt - const candidate = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { + const { response: res, attempts: transportAttempts } = await fetchRouterResponse( + `${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, + { method: 'POST', headers: { ...routerRequestHeaders(cfg, opts), accept: 'text/event-stream', }, body: JSON.stringify(body), - ...(opts?.signal ? { signal: opts.signal } : {}), - }) - if (candidate.ok) { - res = candidate - break - } - lastErr = `router ${candidate.status}: ${(await candidate.text()).slice(0, 200)}` - if (!isTransientRouterStatus(candidate.status)) throw new Error(lastErr) - if (attempt < maxAttempts) { - await new Promise((resolve) => setTimeout(resolve, 800 * 2 ** (attempt - 1))) - } - } - if (!res) throw new Error(`${lastErr} (exhausted retries)`) + }, + opts?.signal, + retry, + ) if (!res.body) { throw new ValidationError( `router ${res.status}: streamed completion returned no response body to read`, @@ -811,16 +784,103 @@ function reportedModel(value: unknown): string | undefined { return typeof value === 'string' && value.length > 0 ? value : undefined } -function routerMaxAttempts(cfg: RouterConfig): number { - const value = cfg.maxAttempts ?? 5 - if (!Number.isSafeInteger(value) || value < 1) { - throw new ValidationError('RouterConfig.maxAttempts must be a positive safe integer') +async function fetchRouterResponse( + url: string, + init: Omit, + callerSignal: AbortSignal | undefined, + retry: ReturnType, +): Promise<{ response: Response; attempts: number }> { + const result = await retryRouterOperation(retry, callerSignal, async (signal) => { + const candidate = await fetch(url, { ...init, signal }) + if (candidate.ok) return candidate + const text = await candidate.text().catch(() => '') + throw new SDKError(`router ${candidate.status}: ${text.slice(0, 200)}`, { + code: 'UNKNOWN', + status: candidate.status, + retryable: retry.retryStatuses.includes(candidate.status), + context: { body: text.slice(0, 200) }, + }) + }) + return { response: result.value, attempts: result.attempts } +} + +async function retryRouterOperation( + retry: ReturnType, + callerSignal: AbortSignal | undefined, + operation: (signal: AbortSignal) => Promise, +): Promise<{ value: T; attempts: number }> { + let attempts = 0 + const value = await withRetry( + async (attempt) => { + attempts = attempt + 1 + const attemptSignal = withRouterRequestTimeout(callerSignal, retry.requestTimeoutMs) + try { + return await operation(attemptSignal.signal) + } catch (error) { + if (callerSignal?.aborted) throw callerSignal.reason ?? error + if (attemptSignal.signal.aborted) { + throw new SDKError(`router request timeout after ${retry.requestTimeoutMs}ms`, { + code: 'TIMEOUT', + cause: error instanceof Error ? error : new Error(String(error)), + }) + } + if (error instanceof TypeError && error.message.includes('fetch')) { + throw new SDKError(`router network failure: ${error.message}`, { + code: 'NETWORK', + cause: error, + }) + } + throw error + } finally { + attemptSignal.dispose() + } + }, + agentCoreRetryConfig(retry, callerSignal), + ) + return { value, attempts } +} + +function agentCoreRetryConfig( + retry: ReturnType, + callerSignal: AbortSignal | undefined, +): RetryConfig { + return { + maxAttempts: retry.maxAttempts, + initialDelayMs: retry.initialBackoffMs, + maxDelayMs: retry.maxBackoffMs, + multiplier: 2, + jitter: retry.jitter, + ...(callerSignal ? { signal: callerSignal } : {}), + shouldRetry: (error) => { + if (callerSignal?.aborted) return false + if (error.status !== undefined) return retry.retryStatuses.includes(error.status) + return error.code === 'NETWORK' || error.code === 'TIMEOUT' + }, } - return value } -function isTransientRouterStatus(status: number): boolean { - return [408, 425, 429, 500, 502, 503, 504, 520, 522, 524].includes(status) +function withRouterRequestTimeout( + callerSignal: AbortSignal | undefined, + timeoutMs: number, +): { signal: AbortSignal; dispose: () => void } { + if (timeoutMs === 0) { + return { signal: callerSignal ?? new AbortController().signal, dispose: () => undefined } + } + const controller = new AbortController() + const timeout = setTimeout( + () => controller.abort(new Error(`router request timeout after ${timeoutMs}ms`)), + timeoutMs, + ) + const onCallerAbort = () => controller.abort(callerSignal?.reason ?? new Error('aborted')) + if (callerSignal?.aborted) onCallerAbort() + else callerSignal?.addEventListener('abort', onCallerAbort, { once: true }) + return { + signal: controller.signal, + dispose: () => { + clearTimeout(timeout) + callerSignal?.removeEventListener('abort', onCallerAbort) + }, + } } function idAlreadyOpen(calls: Map, id: string): boolean { diff --git a/src/runtime/router-retry-policy.test.ts b/src/runtime/router-retry-policy.test.ts new file mode 100644 index 00000000..e8605529 --- /dev/null +++ b/src/runtime/router-retry-policy.test.ts @@ -0,0 +1,69 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { profileModelExecutionSettings } from './supervise/model-policy' + +const profileWithRetry = (retry: unknown): Pick => ({ + model: { + provider: 'tangle-router', + default: 'test-model', + metadata: { retry }, + }, +}) + +describe('AgentProfile.model.metadata.retry', () => { + it('resolves every caller control into one immutable current policy', () => { + const settings = profileModelExecutionSettings( + profileWithRetry({ + maxAttempts: 3, + initialBackoffMs: 40, + maxBackoffMs: 90, + jitter: 0.2, + retryStatuses: [409, 429], + requestTimeoutMs: 250, + }), + 'retry profile test', + ) + + expect(settings.retry).toEqual({ + maxAttempts: 3, + initialBackoffMs: 40, + maxBackoffMs: 90, + jitter: 0.2, + retryStatuses: [409, 429], + requestTimeoutMs: 250, + }) + expect(Object.isFrozen(settings.retry)).toBe(true) + expect(Object.isFrozen(settings.retry?.retryStatuses)).toBe(true) + }) + + it.each([ + ['a non-object policy', 'bad', /retry.*must be an object/u], + ['an unknown field', { legacyRetries: 2 }, /unsupported fields: legacyRetries/u], + ['a zero attempt total', { maxAttempts: 0 }, /maxAttempts.*positive safe integer/u], + ['a negative initial delay', { initialBackoffMs: -1 }, /initialBackoffMs.*nonnegative/u], + ['an oversized backoff cap', { maxBackoffMs: 2_147_483_648 }, /maxBackoffMs.*2147483647/u], + ['jitter above one', { jitter: 1.1 }, /jitter.*0 through 1/u], + ['a non-status value', { retryStatuses: [99] }, /retryStatuses.*100 through 599/u], + ['duplicate statuses', { retryStatuses: [429, 429] }, /must not contain duplicates/u], + ['a negative request deadline', { requestTimeoutMs: -1 }, /requestTimeoutMs.*nonnegative/u], + ])('fails before execution for %s', (_label, retry, message) => { + expect(() => + profileModelExecutionSettings(profileWithRetry(retry), 'invalid retry profile'), + ).toThrow(message) + }) + + it('rejects the removed maxRetries alias instead of silently changing its meaning', () => { + expect(() => + profileModelExecutionSettings( + { + model: { + provider: 'tangle-router', + default: 'test-model', + metadata: { maxRetries: 2 }, + }, + }, + 'legacy retry profile', + ), + ).toThrow(/unsupported AgentProfile\.model\.metadata fields: maxRetries/u) + }) +}) diff --git a/src/runtime/router-retry-policy.ts b/src/runtime/router-retry-policy.ts new file mode 100644 index 00000000..e6d9fb68 --- /dev/null +++ b/src/runtime/router-retry-policy.ts @@ -0,0 +1,125 @@ +import { ValidationError } from '../errors' + +/** Exact retry controls accepted at `AgentProfile.model.metadata.retry`. */ +export interface RouterRetryPolicy { + /** Total attempts, including the first request. */ + readonly maxAttempts?: number + /** Delay before the second attempt. Later delays grow exponentially. */ + readonly initialBackoffMs?: number + /** Maximum delay between attempts. */ + readonly maxBackoffMs?: number + /** Symmetric random variation around each delay, from 0 through 1. */ + readonly jitter?: number + /** HTTP statuses that may be retried. */ + readonly retryStatuses?: ReadonlyArray + /** Deadline for receiving one attempt's response headers. Zero disables it. */ + readonly requestTimeoutMs?: number +} + +export interface ResolvedRouterRetryPolicy { + readonly maxAttempts: number + readonly initialBackoffMs: number + readonly maxBackoffMs: number + readonly jitter: number + readonly retryStatuses: ReadonlyArray + readonly requestTimeoutMs: number +} + +const maximumTimerMs = 2_147_483_647 +const defaultRetryStatuses = Object.freeze([408, 425, 429, 500, 502, 503, 504, 520, 522, 524]) +const retryPolicyKeys = new Set([ + 'initialBackoffMs', + 'jitter', + 'maxAttempts', + 'maxBackoffMs', + 'requestTimeoutMs', + 'retryStatuses', +]) + +/** Parse untrusted profile/config data once and return an immutable complete policy. */ +export function resolveRouterRetryPolicy( + input: unknown, + context: string, +): ResolvedRouterRetryPolicy { + if ( + input !== undefined && + (typeof input !== 'object' || input === null || Array.isArray(input)) + ) { + throw new ValidationError(`${context} must be an object`) + } + const policy = input === undefined ? {} : Object.fromEntries(Object.entries(input)) + const unknown = Object.keys(policy).filter((key) => !retryPolicyKeys.has(key)) + if (unknown.length > 0) { + throw new ValidationError(`${context} has unsupported fields: ${unknown.join(', ')}`) + } + const maxAttempts = positiveInteger(policy.maxAttempts, 5, `${context}.maxAttempts`) + const initialBackoffMs = timerInteger( + policy.initialBackoffMs, + 1_000, + `${context}.initialBackoffMs`, + ) + const maxBackoffMs = timerInteger(policy.maxBackoffMs, 30_000, `${context}.maxBackoffMs`) + const requestTimeoutMs = timerInteger( + policy.requestTimeoutMs, + 120_000, + `${context}.requestTimeoutMs`, + ) + const jitter = policy.jitter ?? 0.25 + if (typeof jitter !== 'number' || !Number.isFinite(jitter) || jitter < 0 || jitter > 1) { + throw new ValidationError(`${context}.jitter must be a finite number from 0 through 1`) + } + const retryStatuses = parseRetryStatuses(policy.retryStatuses, context) + return Object.freeze({ + maxAttempts, + initialBackoffMs, + maxBackoffMs, + jitter, + retryStatuses, + requestTimeoutMs, + }) +} + +function parseRetryStatuses(value: unknown, context: string): ReadonlyArray { + if (value === undefined) return defaultRetryStatuses + if (!Array.isArray(value)) { + throw new ValidationError(`${context}.retryStatuses must be an array of HTTP statuses`) + } + const statuses: number[] = [] + const seen = new Set() + for (const status of value) { + if (!Number.isSafeInteger(status) || status < 100 || status > 599) { + throw new ValidationError( + `${context}.retryStatuses must contain integer HTTP statuses from 100 through 599`, + ) + } + if (seen.has(status)) { + throw new ValidationError(`${context}.retryStatuses must not contain duplicates`) + } + seen.add(status) + statuses.push(status) + } + return Object.freeze(statuses) +} + +function positiveInteger(value: unknown, fallback: number, context: string): number { + const parsed = value ?? fallback + if (typeof parsed !== 'number' || !Number.isSafeInteger(parsed) || parsed < 1) { + throw new ValidationError(`${context} must be a positive safe integer`) + } + return parsed +} + +function timerInteger(value: unknown, fallback: number, context: string): number { + const parsed = value ?? fallback + if ( + typeof parsed !== 'number' || + !Number.isSafeInteger(parsed) || + parsed < 0 || + parsed > maximumTimerMs + ) { + throw new ValidationError( + `${context} must be a nonnegative safe integer no greater than ${maximumTimerMs}`, + ) + } + return parsed +} diff --git a/src/runtime/supervise/model-policy.ts b/src/runtime/supervise/model-policy.ts index 58d42d2b..b5f1cfef 100644 --- a/src/runtime/supervise/model-policy.ts +++ b/src/runtime/supervise/model-policy.ts @@ -8,6 +8,7 @@ import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { ConfigError } from '../../errors' import { agentHarness } from '../harness-role' +import { type ResolvedRouterRetryPolicy, resolveRouterRetryPolicy } from '../router-retry-policy' /** * Return the model id an executor may send to a provider. @@ -84,8 +85,7 @@ export function assertExecutableAgentProfile(profile: AgentProfile, context: str export interface ProfileModelExecutionSettings { readonly temperature?: number readonly maxTokens?: number - /** Total transport attempts including the first request. */ - readonly maxAttempts?: number + readonly retry?: ResolvedRouterRetryPolicy readonly seed?: number readonly toolChoice?: 'auto' | 'required' | 'none' readonly extraBody?: Readonly> @@ -96,9 +96,9 @@ export interface ProfileModelExecutionSettings { const PROFILE_MODEL_METADATA_KEYS = new Set([ 'extraBody', - 'maxRetries', 'maxTokens', 'maxTurns', + 'retry', 'seed', 'stream', 'temperature', @@ -122,7 +122,11 @@ export function profileModelExecutionSettings( } const temperature = finiteNumber(metadata.temperature, `${context}: temperature`) const maxTokens = positiveInteger(metadata.maxTokens, `${context}: maxTokens`) - const maxRetries = nonnegativeInteger(metadata.maxRetries, `${context}: maxRetries`) + const retryInput = metadata.retry + const retry = + retryInput === undefined + ? undefined + : resolveRouterRetryPolicy(retryInput, `${context}: AgentProfile.model.metadata.retry`) const seed = safeInteger(metadata.seed, `${context}: seed`) const maxTurns = nonnegativeInteger(metadata.maxTurns, `${context}: maxTurns`) const stream = optionalBoolean(metadata.stream, `${context}: stream`) @@ -145,7 +149,7 @@ export function profileModelExecutionSettings( return { ...(temperature !== undefined ? { temperature } : {}), ...(maxTokens !== undefined ? { maxTokens } : {}), - ...(maxRetries !== undefined ? { maxAttempts: maxRetries + 1 } : {}), + ...(retry !== undefined ? { retry } : {}), ...(seed !== undefined ? { seed } : {}), ...(toolChoice !== undefined ? { toolChoice } : {}), ...(extraBody !== undefined diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 8dc07c00..fd9d9c26 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -101,6 +101,7 @@ import { import { assertExecutableAgentProfile, concreteProfileModel, + type ProfileModelExecutionSettings, profileBridgeWireModel, profileModelExecutionSettings, } from './model-policy' @@ -469,8 +470,8 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model, - ...(profileExecution.maxAttempts !== undefined - ? { maxAttempts: profileExecution.maxAttempts } + ...(profileExecution.retry !== undefined + ? { retry: profileExecution.retry } : {}), ...(seam.complete ? { complete: seam.complete } : {}), }, @@ -497,8 +498,8 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model, - ...(profileExecution.maxAttempts !== undefined - ? { maxAttempts: profileExecution.maxAttempts } + ...(profileExecution.retry !== undefined + ? { retry: profileExecution.retry } : {}), ...(seam.complete ? { complete: seam.complete } : {}), }, @@ -575,7 +576,7 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { provider: spec.profile.model?.provider ?? null, temperature: profileExecution.temperature ?? null, maxTokens: profileExecution.maxTokens ?? null, - maxAttempts: profileExecution.maxAttempts ?? null, + retry: profileExecution.retry ?? null, seed: profileExecution.seed ?? null, reasoningEffort: profileExecution.reasoningEffort ?? null, extraBody: profileExecution.extraBody ?? null, @@ -766,8 +767,8 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model, - ...(profileExecution.maxAttempts !== undefined - ? { maxAttempts: profileExecution.maxAttempts } + ...(profileExecution.retry !== undefined + ? { retry: profileExecution.retry } : {}), ...(seam.complete ? { complete: seam.complete } : {}), }, @@ -967,7 +968,7 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = tools: seam.tools, temperature: profileExecution.temperature ?? null, maxTokens: profileExecution.maxTokens ?? null, - maxAttempts: profileExecution.maxAttempts ?? null, + retry: profileExecution.retry ?? null, toolChoice: profileExecution.toolChoice ?? null, extraBody: profileExecution.extraBody ?? null, reasoningEffort: profileExecution.reasoningEffort ?? null, @@ -3864,7 +3865,7 @@ interface RouterProfileExecution { reasoningEffort?: ReasoningEffort temperature?: number maxTokens?: number - maxAttempts?: number + retry?: ProfileModelExecutionSettings['retry'] seed?: number toolChoice?: 'auto' | 'required' | 'none' extraBody?: Readonly> diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 25ac124b..b4ca0367 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -662,7 +662,7 @@ function routerBrainFromProfile( { ...deps.router, model: modelId, - ...(settings.maxAttempts !== undefined ? { maxAttempts: settings.maxAttempts } : {}), + ...(settings.retry !== undefined ? { retry: settings.retry } : {}), ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), ...(settings.stream !== undefined ? { stream: settings.stream } : {}), }, diff --git a/tests/kernel/router-brain.test.ts b/tests/kernel/router-brain.test.ts index 8f738f3c..28b7f4b9 100644 --- a/tests/kernel/router-brain.test.ts +++ b/tests/kernel/router-brain.test.ts @@ -416,7 +416,55 @@ describe('streamRouterChatWithTools — the SSE tool-calling transport', () => { it('fails loud on a non-2xx with the same message shape as the buffered path', async () => { stubStream('gateway blew up', { status: 524 }) - await expect(streamRouterChatWithTools(cfg, [], [])).rejects.toThrow(/router 524/) + await expect( + streamRouterChatWithTools({ ...cfg, retry: { maxAttempts: 1 } }, [], []), + ).rejects.toThrow(/router 524/) + }) + + it('streamed chat honors a caller-owned status set and retries through the shared request path', async () => { + const encoder = new TextEncoder() + let attempts = 0 + fetchMock = vi.fn(async () => { + attempts += 1 + if (attempts === 1) { + return { ok: false, status: 409, text: async () => 'retry this conflict' } + } + return { + ok: true, + status: 200, + body: new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `${frame({ choices: [{ delta: { content: 'recovered' } }] })}data: [DONE]\n\n`, + ), + ) + controller.close() + }, + }), + } + }) + vi.stubGlobal('fetch', fetchMock) + + const result = await streamRouterChatWithTools( + { + ...cfg, + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + retryStatuses: [409], + requestTimeoutMs: 0, + }, + }, + [], + [], + ) + + expect(result.content).toBe('recovered') + expect(result.transportAttempts).toBe(2) + expect(fetchMock).toHaveBeenCalledTimes(2) }) it('refuses to pair the streamed transport with the buffered injected transport', async () => { diff --git a/tests/runtime/executor-profile-model.test.ts b/tests/runtime/executor-profile-model.test.ts index bbac2f02..d1df9fbd 100644 --- a/tests/runtime/executor-profile-model.test.ts +++ b/tests/runtime/executor-profile-model.test.ts @@ -15,17 +15,23 @@ const spec: AgentSpec = { profile, harness: null } let server: Server | undefined -async function startRouter(onRequest: (body: Record) => void): Promise { +async function startRouter( + onRequest: (body: Record) => void, + responseStatus: () => number = () => 200, +): Promise { server = createServer(async (request, response) => { const chunks: Buffer[] = [] for await (const chunk of request) chunks.push(Buffer.from(chunk)) onRequest(JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record) - response.writeHead(200, { 'content-type': 'application/json' }) + const status = responseStatus() + response.writeHead(status, { 'content-type': 'application/json' }) response.end( - JSON.stringify({ - choices: [{ message: { content: 'done', tool_calls: [] } }], - usage: { prompt_tokens: 3, completion_tokens: 2 }, - }), + status === 200 + ? JSON.stringify({ + choices: [{ message: { content: 'done', tool_calls: [] } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }) + : JSON.stringify({ error: 'caller-selected retry status' }), ) }) await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) @@ -73,6 +79,49 @@ describe('router executor exact-profile identity', () => { }, ) + it.each(['router', 'router-tools'] as const)( + 'carries AgentProfile.model.metadata.retry through the %s executor', + async (backend) => { + let requests = 0 + const routerBaseUrl = await startRouter( + () => { + requests += 1 + }, + () => (requests === 1 ? 409 : 200), + ) + const factory = createExecutor({ + backend, + routerBaseUrl, + routerKey: 'key', + ...(backend === 'router-tools' ? { tools: [], executeToolCall: async () => '' } : {}), + }) + const retryProfile: AgentProfile = { + ...profile, + model: { + ...profile.model, + metadata: { + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + retryStatuses: [409], + requestTimeoutMs: 0, + }, + }, + }, + } + const executor = factory( + { profile: retryProfile, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) + + await drainExecution(executor.execute('retry the task', new AbortController().signal)) + + expect(requests).toBe(2) + }, + ) + it.each(['router', 'router-tools'] as const)( 'refuses runtime-selected model markers on the %s backend before dispatch', async (backend) => { From dad23bf2133c33a40c500f0175b65f57dc50a350 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:30:19 -0600 Subject: [PATCH 22/39] test(runtime): pin the single router transport call --- scripts/check-model-execution-boundary.mjs | 8 ++------ scripts/check-model-execution-boundary.test.mjs | 9 ++++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index daa4ae89..21f2e895 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -83,17 +83,13 @@ const lowLevelModelCallOwners = new Map([ ['runLocalHarness', new Set(['src/mcp/worktree-harness.ts'])], ]) -// The Router adapter owns exactly three fetch sites. The function name and count are both checked: +// The Router adapter owns exactly one fetch site. The function name and count are both checked: // adding another raw provider request anywhere in the file fails this check instead of inheriting a // whole-file exemption. const ownedModelFetchAllowances = new Map([ [ 'src/runtime/router-client.ts', - new Map([ - ['routerChatWithUsage', 1], - ['routerChatWithTools', 1], - ['streamRouterChatWithTools', 1], - ]), + new Map([['fetchRouterResponse', 1]]), ], ]) diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index 9ed44cca..b2eae442 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -221,12 +221,19 @@ const generator = { ]) const routerOwner = ` - export async function routerChatWithUsage() { + async function fetchRouterResponse() { await fetch(url) await fetch(url) } ` expect(checkJavaScript('src/runtime/router-client.ts', routerOwner)).toHaveLength(1) + + expect( + checkJavaScript( + 'src/runtime/router-client.ts', + `export async function routerChatWithUsage() { await fetch(url) }`, + ), + ).toHaveLength(1) }) it('ignores comments, inert strings, and ordinary HTTP', () => { From caf1a4155ef6faa9cb8f3c50cda0b37ee2f844df Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:30:24 -0600 Subject: [PATCH 23/39] chore(release): align exact shared package cohort --- package.json | 5 +++-- pnpm-lock.yaml | 44 +++++++++++++++++++++++++++++--------------- pnpm-workspace.yaml | 5 +++-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 34facd7b..4d10a660 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.128.1", + "version": "0.129.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -169,7 +169,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.144.1 <0.145.0", + "@tangle-network/agent-eval": ">=0.144.3 <0.145.0", "@tangle-network/agent-interface": ">=0.43.0 <0.44.0", "@tangle-network/sandbox": ">=0.17.2 <0.18.0" }, @@ -179,6 +179,7 @@ } }, "dependencies": { + "@tangle-network/agent-core": "catalog:", "@tangle-network/agent-knowledge": "catalog:", "@tangle-network/agent-profile-materialize": "catalog:", "@tangle-network/agent-trace-contract": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a638e541..f7712278 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,15 +9,18 @@ catalogs: '@arethetypeswrong/cli': specifier: 0.18.5 version: 0.18.5 + '@tangle-network/agent-core': + specifier: 0.4.34 + version: 0.4.34 '@tangle-network/agent-eval': - specifier: 0.144.1 - version: 0.144.1 + specifier: 0.144.3 + version: 0.144.3 '@tangle-network/agent-interface': specifier: 0.43.0 version: 0.43.0 '@tangle-network/agent-knowledge': - specifier: 7.0.9 - version: 7.0.9 + specifier: 7.0.10 + version: 7.0.10 '@tangle-network/agent-profile-materialize': specifier: 0.10.2 version: 0.10.2 @@ -47,9 +50,12 @@ importers: .: dependencies: + '@tangle-network/agent-core': + specifier: 'catalog:' + version: 0.4.34 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.9 + version: 7.0.10 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.10.2(@tangle-network/agent-interface@0.43.0) @@ -68,7 +74,7 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.1 + version: 0.144.3 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.0 @@ -116,13 +122,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.1 + version: 0.144.3 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.9 + version: 7.0.10 '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1105,16 +1111,19 @@ packages: '@tangle-network/agent-core@0.4.33': resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-eval@0.144.1': - resolution: {integrity: sha512-OGwF96faPiiGzcb5QQnCvdzYdTCybJ4JBPLIjgTs21pBkJZkjb0Jqylcto4P31b3e3YWjryts0CALBSDr+0oLA==} + '@tangle-network/agent-core@0.4.34': + resolution: {integrity: sha512-OH+d9kx9yN7lOVPCKRt0VXYVO0RhQgRsqdAAG++iIAM9Ti6g7qnG1flEw1zah1LgntYR+uiGOae62MxwWLGR3w==} + + '@tangle-network/agent-eval@0.144.3': + resolution: {integrity: sha512-gkyeie5CCzUuEzTfuQKe6iikSsu0S9h6w8yRtfnvjDZopkJJ7Nn+VDzkWZrsLDQ31LtLaHqJ7I2FgYbPNvjW0Q==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.43.0': resolution: {integrity: sha512-t35nGOJ3sWouHoUM/5A8eUmsR+8JcuRF6VFQbT9cPzxIMUV3NsFMJgYXPGmyIj65vtvs73O6Mvijp47FAAGbuQ==} - '@tangle-network/agent-knowledge@7.0.9': - resolution: {integrity: sha512-Otlf8JqyKj+71m817cSDgjDtBZZalGw2F5fnK+YO5yyhc/EtP3s+aLf/9fkIxuq8dZwIlKFRDCJQJmBoq8o4cQ==} + '@tangle-network/agent-knowledge@7.0.10': + resolution: {integrity: sha512-igVEo0a6nnCZWmemvu9aLdIRMznAPKZpHcNWMdRJ5EyYXc5TnX0kwKTgTCzBHvg9Fqp8gfMttNHJzA4T+V3Yog==} engines: {node: '>=20.19.0'} hasBin: true @@ -2780,7 +2789,12 @@ snapshots: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.144.1': + '@tangle-network/agent-core@0.4.34': + dependencies: + '@tangle-network/agent-interface': 0.43.0 + zod: 4.4.3 + + '@tangle-network/agent-eval@0.144.3': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) @@ -2798,9 +2812,9 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@7.0.9': + '@tangle-network/agent-knowledge@7.0.10': dependencies: - '@tangle-network/agent-eval': 0.144.1 + '@tangle-network/agent-eval': 0.144.3 '@tangle-network/agent-interface': 0.43.0 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9e498731..8a6a38ad 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,10 +17,11 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 + '@tangle-network/agent-core': 0.4.34 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.144.1 + '@tangle-network/agent-eval': 0.144.3 '@tangle-network/agent-interface': 0.43.0 - '@tangle-network/agent-knowledge': 7.0.9 + '@tangle-network/agent-knowledge': 7.0.10 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': 0.17.3 From 20402202cf644b9a3a366977dd817dcaab306ef3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:31:47 -0600 Subject: [PATCH 24/39] fix(runtime): leave model request deadlines unlimited --- src/runtime/router-retry-policy.test.ts | 6 ++++++ src/runtime/router-retry-policy.ts | 6 +----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/runtime/router-retry-policy.test.ts b/src/runtime/router-retry-policy.test.ts index e8605529..3b480a2b 100644 --- a/src/runtime/router-retry-policy.test.ts +++ b/src/runtime/router-retry-policy.test.ts @@ -11,6 +11,12 @@ const profileWithRetry = (retry: unknown): Pick => ({ }) describe('AgentProfile.model.metadata.retry', () => { + it('leaves request duration unlimited unless the caller sets a deadline', () => { + const settings = profileModelExecutionSettings(profileWithRetry({}), 'default retry profile') + + expect(settings.retry?.requestTimeoutMs).toBe(0) + }) + it('resolves every caller control into one immutable current policy', () => { const settings = profileModelExecutionSettings( profileWithRetry({ diff --git a/src/runtime/router-retry-policy.ts b/src/runtime/router-retry-policy.ts index e6d9fb68..9123eebd 100644 --- a/src/runtime/router-retry-policy.ts +++ b/src/runtime/router-retry-policy.ts @@ -59,11 +59,7 @@ export function resolveRouterRetryPolicy( `${context}.initialBackoffMs`, ) const maxBackoffMs = timerInteger(policy.maxBackoffMs, 30_000, `${context}.maxBackoffMs`) - const requestTimeoutMs = timerInteger( - policy.requestTimeoutMs, - 120_000, - `${context}.requestTimeoutMs`, - ) + const requestTimeoutMs = timerInteger(policy.requestTimeoutMs, 0, `${context}.requestTimeoutMs`) const jitter = policy.jitter ?? 0.25 if (typeof jitter !== 'number' || !Number.isFinite(jitter) || jitter < 0 || jitter > 1) { throw new ValidationError(`${context}.jitter must be a finite number from 0 through 1`) From b65ee4dc9a8f6e6666a1146e5c26b558e3e300b6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:31:49 -0600 Subject: [PATCH 25/39] fix(runtime): forward conversation headers through profiles --- src/runtime/profile-execution-backend.test.ts | 119 +++++++++++++++++- src/runtime/router-client.ts | 15 ++- src/runtime/supervise/runtime.ts | 15 ++- 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/src/runtime/profile-execution-backend.test.ts b/src/runtime/profile-execution-backend.test.ts index 72851913..155162f6 100644 --- a/src/runtime/profile-execution-backend.test.ts +++ b/src/runtime/profile-execution-backend.test.ts @@ -1,7 +1,11 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it, vi } from 'vitest' +import { createIterableBackend } from '../backends' +import { defineConversation } from '../conversation/define-conversation' +import { FORWARD_HEADERS } from '../conversation/headers' +import { runConversation } from '../conversation/run-conversation' import { runAgentTaskStream } from '../run' -import type { RuntimeStreamEvent } from '../types' +import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { createProfileExecutionBackend } from './profile-execution-backend' import { createExecutor } from './supervise/runtime' @@ -11,6 +15,21 @@ const profile = { model: { provider: 'offline', default: 'offline/profile-backend' }, } satisfies AgentProfile +function fixedTextBackend(text: string): AgentExecutionBackend { + return createIterableBackend({ + kind: 'fixed-text', + async *stream(_input, context) { + yield { + type: 'text_delta', + task: context.task, + session: context.session, + text, + timestamp: new Date().toISOString(), + } + }, + }) +} + describe('createProfileExecutionBackend', () => { it('runs through runAgentTaskStream with one outer terminal event and the exact profile', async () => { const complete = vi.fn(async (_body: Record) => ({ @@ -67,4 +86,102 @@ describe('createProfileExecutionBackend', () => { ).toThrow(/AgentProfile\.harness must be explicit/u) expect(executor).not.toHaveBeenCalled() }) + + it('forwards conversation authorization, depth, and trace headers into the Router request', async () => { + let requestHeaders: Readonly> | undefined + const complete = vi.fn( + async ( + _body: Record, + request?: { + readonly headers: Readonly> + readonly signal?: AbortSignal + }, + ) => { + requestHeaders = request?.headers + return { + model: 'offline/profile-backend', + choices: [{ message: { content: 'ack' } }], + usage: { prompt_tokens: 2, completion_tokens: 1 }, + } + }, + ) + const backend = createProfileExecutionBackend({ + profile, + executor: createExecutor({ + backend: 'router', + routerBaseUrl: 'http://offline.invalid/v1', + routerKey: 'agent-key', + complete, + }), + }) + const conversation = defineConversation({ + participants: [ + { name: 'profile-worker', backend }, + { name: 'other', backend: fixedTextBackend('unused') }, + ], + policy: { maxTurns: 1 }, + }) + + await runConversation(conversation, { + seed: 'go', + runId: 'run-headers', + parentTurnId: 'parent-turn', + inboundDepth: 2, + propagatedHeaders: { + [FORWARD_HEADERS.authorization]: 'Bearer user-key', + }, + }) + + expect(complete).toHaveBeenCalledOnce() + expect(requestHeaders).toMatchObject({ + authorization: 'Bearer agent-key', + [FORWARD_HEADERS.authorization]: 'Bearer user-key', + [FORWARD_HEADERS.depth]: '3', + [FORWARD_HEADERS.runId]: 'run-headers', + [FORWARD_HEADERS.parentTurnId]: 'parent-turn', + [FORWARD_HEADERS.speaker]: 'profile-worker', + 'x-correlation-id': 'run-headers', + }) + expect(requestHeaders?.[FORWARD_HEADERS.turnId]).toBeDefined() + expect(requestHeaders?.['idempotency-key']).toBe(requestHeaders?.[FORWARD_HEADERS.turnId]) + }) + + it('keeps agent-owned Router credentials without forwarding the caller authorization', async () => { + let requestHeaders: Readonly> | undefined + const backend = createProfileExecutionBackend({ + profile, + executor: createExecutor({ + backend: 'router', + routerBaseUrl: 'http://offline.invalid/v1', + routerKey: 'agent-key', + complete: async (_body, request) => { + requestHeaders = request?.headers + return { + model: 'offline/profile-backend', + choices: [{ message: { content: 'ack' } }], + usage: { prompt_tokens: 2, completion_tokens: 1 }, + } + }, + }), + }) + const conversation = defineConversation({ + participants: [ + { name: 'profile-worker', backend, authSource: 'agent-owned' }, + { name: 'other', backend: fixedTextBackend('unused') }, + ], + policy: { maxTurns: 1 }, + }) + + await runConversation(conversation, { + seed: 'go', + runId: 'run-agent-owned', + propagatedHeaders: { + [FORWARD_HEADERS.authorization]: 'Bearer user-key', + }, + }) + + expect(requestHeaders?.authorization).toBe('Bearer agent-key') + expect(requestHeaders?.[FORWARD_HEADERS.authorization]).toBeUndefined() + expect(requestHeaders?.[FORWARD_HEADERS.depth]).toBe('1') + }) }) diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index f6f59150..e044d505 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -142,6 +142,8 @@ export async function routerChatWithUsage( callId?: string /** Caller trace correlation forwarded independently of idempotency. */ correlationId?: string + /** Headers inherited from an enclosing conversation or task. */ + propagatedHeaders?: Readonly> }, ): Promise { const url = `${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions` @@ -329,6 +331,8 @@ export async function routerChatWithTools( reasoningEffort?: ReasoningEffort callId?: string correlationId?: string + /** Headers inherited from an enclosing conversation or task. */ + propagatedHeaders?: Readonly> }, ): Promise { const body = toolCompletionBody(cfg, messages, tools, opts) @@ -641,6 +645,8 @@ export async function streamRouterChatWithTools( reasoningEffort?: ReasoningEffort callId?: string correlationId?: string + /** Headers inherited from an enclosing conversation or task. */ + propagatedHeaders?: Readonly> }, ): Promise { const retry = resolveRouterRetryPolicy(cfg.retry, 'RouterConfig.retry') @@ -770,9 +776,16 @@ export async function streamRouterChatWithTools( function routerRequestHeaders( cfg: Pick, - opts: { callId?: string; correlationId?: string } | undefined, + opts: + | { + callId?: string + correlationId?: string + propagatedHeaders?: Readonly> + } + | undefined, ): Record { return { + ...(opts?.propagatedHeaders ?? {}), 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}`, ...(opts?.callId ? { 'idempotency-key': opts.callId } : {}), diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index fd9d9c26..8e84ce75 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -1002,21 +1002,32 @@ function assertObservedRouterModel( function routerRequestIdentity(ctx: ExecutorContext): { readonly callId?: string readonly correlationId?: string + readonly propagatedHeaders?: Readonly> } { const correlation = ctx.node?.identity?.correlation return { ...(correlation?.callId ? { callId: correlation.callId } : {}), ...(correlation?.correlationId ? { correlationId: correlation.correlationId } : {}), + ...(ctx.propagatedHeaders ? { propagatedHeaders: ctx.propagatedHeaders } : {}), } } function routerTurnRequestIdentity( - identity: { readonly callId?: string; readonly correlationId?: string }, + identity: { + readonly callId?: string + readonly correlationId?: string + readonly propagatedHeaders?: Readonly> + }, turnIndex: number, -): { readonly callId?: string; readonly correlationId?: string } { +): { + readonly callId?: string + readonly correlationId?: string + readonly propagatedHeaders?: Readonly> +} { return { ...(identity.callId ? { callId: `${identity.callId}:turn:${turnIndex + 1}` } : {}), ...(identity.correlationId ? { correlationId: identity.correlationId } : {}), + ...(identity.propagatedHeaders ? { propagatedHeaders: identity.propagatedHeaders } : {}), } } From 76030cf65b290976f6159ee1d7715d847db56c79 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:36:29 -0600 Subject: [PATCH 26/39] test(candidate-execution): remove wall-clock flake --- tests/candidate-execution-cleanup.test.ts | 20 ++++++++++++++++++++ tests/candidate-execution-prepare.test.ts | 2 -- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/candidate-execution-cleanup.test.ts b/tests/candidate-execution-cleanup.test.ts index dba378e0..4412eb86 100644 --- a/tests/candidate-execution-cleanup.test.ts +++ b/tests/candidate-execution-cleanup.test.ts @@ -69,4 +69,24 @@ describe('candidate cleanup timer bounds', () => { ).rejects.toBeInstanceOf(CandidateResultTimeoutError) expect(observedSignal?.aborted).toBe(true) }) + + it('interrupts hanging cleanup at the frozen deadline', async () => { + vi.useFakeTimers({ now: 100 }) + let observedSignal: AbortSignal | undefined + const result = withinCandidateCleanupDeadline( + async (signal) => { + observedSignal = signal + return await new Promise(() => undefined) + }, + 120, + 'hanging cleanup', + ) + const rejected = expect(result).rejects.toBeInstanceOf(CandidateCleanupTimeoutError) + + await vi.advanceTimersByTimeAsync(20) + await rejected + expect(Date.now()).toBe(120) + expect(observedSignal?.aborted).toBe(true) + expect(observedSignal?.reason).toBeInstanceOf(CandidateCleanupTimeoutError) + }) }) diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index 7217b37b..7ca1ff46 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -474,7 +474,6 @@ describe('candidate execution preparation', () => { network: { mode: 'gateway-only', domains: ['router.tangle.tools'] }, }) hanging.ports.models.settleGrant = async () => await new Promise(() => undefined) - const startedAt = Date.now() await expect( prepareAgentCandidateExecution( await verifyAgentCandidateBundle(hanging.bundle, hanging.ports), @@ -483,7 +482,6 @@ describe('candidate execution preparation', () => { { cleanupTimeoutMs: 20 }, ), ).rejects.toThrow(/cleanup failed/) - expect(Date.now() - startedAt).toBeLessThan(250) const mismatched = fixture() mismatched.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ From aff0f84c89bcb73a9fc50ca9214cde59444ee793 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:37:06 -0600 Subject: [PATCH 27/39] docs(runtime): map exact profile execution replacements --- CHANGELOG.md | 20 +++++++++- README.md | 22 +++++++---- docs/agent-bus-protocol.md | 3 +- docs/api/conversation.md | 6 +++ docs/api/index.md | 37 ++++++++++++++++-- docs/api/runtime.md | 8 ++++ docs/archive/conversation-economics.md | 46 +++++++++++++++-------- docs/canonical-api.md | 5 ++- docs/learning-flywheel.md | 2 +- docs/roadmap-rsi.md | 2 +- examples/p1-parity/run-parity.ts | 2 +- examples/strategy-suite/strategy-suite.ts | 2 +- examples/supervisor-loop/shared.ts | 8 ++-- src/conversation/types.ts | 5 +-- 14 files changed, 127 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb5ac3d1..a5690493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,32 @@ # Changelog -## 0.128.1 +## 0.129.0 -- Require Agent Eval 0.144.1 and Agent Knowledge 7.0.9, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. +- Require Agent Eval 0.144.3 and Agent Knowledge 7.0.10, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. - Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. - Require `defineLeaderboard` callers to supply an exact `baseProfile`; remove its `modelBackend` override so each matrix cell's profile remains the only model authority. - Require generic coder, researcher, and supervised-knowledge paths to receive complete profiles; remove harness/model overlays and MCP environment alias ladders. - Resolve Sandbox execution only from `AgentProfile.harness`; a backend type may confirm that choice but cannot replace it. - Parse, detach, and deeply freeze a complete `AgentProfile` before Scope, registry, nested-driver, or personified execution can honor any built-in or caller-supplied executor. - Require `driverChild(profile, ...)` and `worktreeLoopRunner({ rootProfile, ... })`; remove name-only driver and personified-root shortcuts. +- Remove the public `runLocalHarness` process shortcut; use `createWorktreeCliExecutor({ profile, ... })` directly or `createExecutor({ backend: 'cli-worktree', ... })` inside a supervised run so the CLI and model derive from one exact `AgentProfile`. - Keep missing token usage and billed cost unknown, and report reasoning-token usage when the provider supplies it. +### Removed public execution shortcuts + +- Remove `driverLoopGenerator`, `DriverLoopGeneratorOptions`, `buildDriverSystem`, and `researchDriverNote`; use `improve({ surface: 'code', code: { profile, executorForWorktree, buildPrompt } })` or `agenticGenerator` so every authoring call uses the exact declared profile through Runtime. +- Remove `AGENTIC_PROFILE_RESOURCE_ROOT`; profile resources now travel inside the exact `AgentProfile` and Runtime materializes them at execution. +- Remove `AgentBackendKind`, `ResolveAgentBackendOptions`, `resolveAgentBackend`, `createOpenAICompatibleBackend`, and `BackendRetryPolicy`; select a built-in transport with `ExecutorConfig` and `createExecutor(...)`, bind it to an exact profile with `createProfileExecutionBackend(...)` for `runAgentTaskStream` or conversation APIs, or pass both directly to `streamAgentTurn`. +- Move retry configuration into `AgentProfile.model.metadata.retry`, including attempt count, per-attempt timeout, retryable status codes, exponential-backoff bounds, and jitter, so deleting the parallel backend API does not delete caller control. +- Remove the public Router chat and tool-loop API (`RouterConfig`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolCall`, `RouterToolLoopResult`, `routerChatWithUsage`, `routerChatWithTools`, `streamRouterChatWithTools`, `routerToolLoop`, and `routerBrain`); use `createExecutor(...)` plus `streamAgentTurn` and `collectAgentTurn`, use `ToolLoopToolCall` for the provider-neutral tool-call record, use `profileChatClient` for Eval integrations, or use `supervise(...)` for a profiled supervisor. +- Replace `ShotPersona` with `ShotSpec.profile`, which carries a complete exact profile instead of a prompt/model-only override. +- Remove `canonicalizeAuthoredProfile`; inputs must already satisfy `agentProfileSchema`, with no flat legacy spelling that Runtime silently repairs. +- Remove `authoredWorker`; use `workerFromBackend(...)`, which admits each complete profile through the same executor path as every other supervised worker. +- Remove the public `chatCompletionsTransport` constructor; inject an offline completion function through `ChatTransportExecutorOptions.complete`, or use an exact profile with `createExecutor(...)` for real execution. +- Remove `createPrimeIntellectBackend`; use `primeIntellectExecutorConfig(context)` with `createExecutor(...)`, then execute an exact profile through `streamAgentTurn` or bind it with `createProfileExecutionBackend(...)` when an `AgentExecutionBackend` is required. +- Remove `uiAuditorProfile`, `createInProcessUiAuditClient`, and their browser/judge option types; author a normal `AgentProfile`, execute it through Runtime, and keep using `UiAuditTask`, `encodeAuditTaskEnvelope`, `formatAuditorPrompt`, `parseAuditorEvents`, and `createUiAuditorValidator` from `/profiles`. +- Remove the bundled UI-audit example and the Playwright peer dependency because browser placement belongs to the caller-owned execution environment, not a profile-specific Runtime client. + ## 0.128.0 ### chat-transport executor: workers on a bare chat-completions transport diff --git a/README.md b/README.md index f5f6be88..9e59a4f6 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,15 @@ That file defines the scripted `worker`, `output`, and `validator` used below so Replace the scripted worker with a sandbox, CLI bridge, or router backend without changing the driver. ```ts +import type { AgentProfile } from '@tangle-network/agent-interface' import { inProcessSandboxClient, runAgentRounds } from '@tangle-network/agent-runtime/kernel' +const noteWriterProfile = { + name: 'note-writer', + harness: 'cli-base', + model: { provider: 'scripted', default: 'scripted/note-writer' }, +} satisfies AgentProfile + const result = await runAgentRounds({ task: { prompt: 'Write a one-line release note for one-click restore.' }, driver: { @@ -47,7 +54,7 @@ const result = await runAgentRounds history.some((shot) => shot.verdict?.valid) ? 'pick-winner' : history.length < 3 ? 'refine' : 'fail', }, - agentRun: { profile: { name: 'note-writer' } as AgentProfile, taskToPrompt: (t) => t.prompt }, + agentRun: { profile: noteWriterProfile, taskToPrompt: (t) => t.prompt }, output, // parses the worker's event stream into { note } validator, // pass/fail check: does the note mention "rollback"? ctx: { sandboxClient: worker }, @@ -60,9 +67,9 @@ Run it from a clone of this repo and you get exactly this: ```bash $ pnpm i && pnpm build $ pnpm tsx examples/quickstart/quickstart.ts -shot 0: reject: "Shipped one-click restore." -shot 1: PASS: "Shipped one-click restore with an instant rollback path." -decision: pick-winner: winner: shot 1 +shot 0: reject — "Shipped one-click restore." +shot 1: PASS — "Shipped one-click restore with an instant rollback path." +decision: pick-winner — winner: shot 1 ``` The annotated version is [`examples/driver-loop`](./examples/driver-loop). @@ -119,6 +126,7 @@ const result = await supervise( { name: 'supervisor', harness: 'cli-base', + model: { provider: 'tangle-router', default: process.env.TANGLE_MODEL! }, prompt: { systemPrompt: 'Delegate to workers; do not solve the task yourself.', }, @@ -234,7 +242,7 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.144.1" +python -m pip install "agent-eval-rpc==0.144.3" python -m pip install "gepa[full]==0.1.4" ``` @@ -248,7 +256,7 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.144.1" +python -m pip install "agent-eval-rpc==0.144.3" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` @@ -296,7 +304,7 @@ It uses Runtime's isolated git worktrees and coding-agent candidate execution: ```ts const result = await improve({ surface: 'code', - code: { repoRoot, baseRef, generator }, + code: { repoRoot, baseRef, profile, generator }, scenarios, judge, agent, diff --git a/docs/agent-bus-protocol.md b/docs/agent-bus-protocol.md index bdecb21a..3f1f33f7 100644 --- a/docs/agent-bus-protocol.md +++ b/docs/agent-bus-protocol.md @@ -61,7 +61,8 @@ A `@tangle-network/agent-gateway`-fronted endpoint: A `@tangle-network/agent-runtime` consumer (driver code): - Passes `propagatedHeaders` + `inboundDepth` + `parentTurnId` to `runConversation` / `runConversationStream` from its inbound request context. - The runtime derives `buildForwardHeaders(...)` and exposes the result as `AgentBackendContext.propagatedHeaders` on every participant backend call. -- A caller-owned HTTP backend MUST merge `context.propagatedHeaders` into its outbound request; Runtime does not issue provider HTTP on its behalf. +- `createProfileExecutionBackend(...)` passes those headers into its Runtime executor, and Runtime's Router executor merges them into the provider request automatically. +- A caller-owned HTTP backend MUST merge `context.propagatedHeaders` into its own outbound request. ## Reference diff --git a/docs/api/conversation.md b/docs/api/conversation.md index e2b79361..cff4e0a5 100644 --- a/docs/api/conversation.md +++ b/docs/api/conversation.md @@ -8,6 +8,12 @@ ## References +### createProfileExecutionBackend + +Re-exports [createProfileExecutionBackend](index.md#createprofileexecutionbackend) + +*** + ### BackendCallPolicy Re-exports [BackendCallPolicy](index.md#backendcallpolicy) diff --git a/docs/api/index.md b/docs/api/index.md index 6ca6a3b3..6f6b0065 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -3954,9 +3954,8 @@ within a `Conversation`. > **backend**: [`AgentExecutionBackend`](#agentexecutionbackend) Backend that runs this participant's turn. Reuses the existing -`AgentExecutionBackend` contract from `runAgentTaskStream`, so any -registered backend (iterable, sandbox, OpenAI-compatible) works without -adaptation. +`AgentExecutionBackend` contract from `runAgentTaskStream`, so an iterable, +sandbox, or profile-backed Runtime executor works through the same runner. ##### label? @@ -14273,6 +14272,38 @@ lifetime; consumers should not share it across requests. *** +### createProfileExecutionBackend() + +> **createProfileExecutionBackend**(`options`): [`AgentExecutionBackend`](#agentexecutionbackend) + +**`Stable`** + +Bind one exact profile and Runtime executor to the stable `AgentExecutionBackend` contract used +by `runAgentTaskStream` and conversations. + +Runtime still owns the model call through `streamAgentTurn`. +The adapter only translates the two stream protocols and carries the caller's request headers +into `ExecutorContext` so an HTTP executor can preserve authorization, recursion depth, and +trace identity. + +#### Parameters + +##### options + +###### profile + +`AgentProfile` + +###### executor + +[`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> + +#### Returns + +[`AgentExecutionBackend`](#agentexecutionbackend) + +*** + ### sanitizeKnowledgeReadinessReport() > **sanitizeKnowledgeReadinessReport**(`report`, `options?`): [`SanitizedKnowledgeReadinessReport`](#sanitizedknowledgereadinessreport) diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 948e5d12..bb803f61 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -14837,6 +14837,14 @@ Construction context handed to a `ExecutorFactory` — the seams a built-in need > `readonly` **signal**: `AbortSignal` +##### propagatedHeaders? + +> `readonly` `optional` **propagatedHeaders?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Request headers inherited from an enclosing task or conversation. +Network executors forward these after their own connection headers so caller authorization, +recursion depth, and trace identity survive the profile-to-executor boundary. + ##### node? > `readonly` `optional` **node?**: [`ExecutorNodeContext`](#executornodecontext) diff --git a/docs/archive/conversation-economics.md b/docs/archive/conversation-economics.md index 3fdf8c52..481ecb3b 100644 --- a/docs/archive/conversation-economics.md +++ b/docs/archive/conversation-economics.md @@ -12,11 +12,13 @@ But three real commercial shapes show up in multi-agent systems, and the runtime | Reseller | `'agent-owned'` | The agent | Bundled-price agents that absorb sub-agent costs | | Mixed | `(state) => 'forward-user' \| 'agent-owned'` | Decided per turn | Tiered services — base is agent-owned, premium add-ons forward the user | -The agent's *own* credentials (the sk-tan-AGENT or x402 wallet that pays when `authSource` is `agent-owned`) are configured **on the backend at construction**, not on this knob. This field is purely about whether to *additionally* forward the user's identity downstream. +The agent's *own* credentials (the sk-tan-AGENT or x402 wallet that pays when `authSource` is `agent-owned`) are configured **on the executor at construction**, not on this knob. +This field is purely about whether to *additionally* forward the user's identity downstream. ## How forwarding actually works -The conversation runner reads `propagatedHeaders` on every `runConversation` call (typically threaded in by the gateway middleware that received the inbound request) and emits the [agent-bus headers](./agent-bus-protocol.md) on every outbound participant call. The forwarded-authorization header — `x-tangle-forwarded-authorization` — is the one that determines downstream billing identity. +The conversation runner reads `propagatedHeaders` on every `runConversation` call (typically threaded in by the gateway middleware that received the inbound request) and emits the [agent-bus headers](../agent-bus-protocol.md) on every outbound participant call. +The forwarded-authorization header — `x-tangle-forwarded-authorization` — is the one that determines downstream billing identity. ``` user ──Bearer sk-tan-user-123──▶ gateway ──▶ runConversation({ @@ -36,7 +38,8 @@ user ──Bearer sk-tan-user-123──▶ gateway ──▶ runConversation({ }) ``` -When the participant's backend issues HTTP (e.g. `createOpenAICompatibleBackend`), `context.propagatedHeaders` is merged into the outbound request automatically — the receiving gateway sees the user's token and bills accordingly. +When a participant uses `createProfileExecutionBackend(...)` with Runtime's Router executor, `context.propagatedHeaders` is merged into the outbound request automatically — the receiving gateway sees the user's token and bills accordingly. +A caller-owned HTTP backend must merge those headers itself. ## Mode 1 — Pass-through (`forward-user`, default) @@ -65,21 +68,33 @@ Every outbound call from `researcher` and `critic` carries the user's `sk-tan-us ## Mode 2 — Reseller (`agent-owned`) -The participant pays its own bill. The user's auth is stripped from outbound calls; the participant's backend uses its own credentials (set when you constructed the backend). +The participant pays its own bill. +The user's auth is stripped from outbound calls; the participant's profile-backed executor uses its own credentials. Use when the participant is a paid service that bundles upstream costs into a fixed price. Inbound revenue (whatever the user paid for the outer conversation) minus outbound costs (what this participant spends on its sub-LLM calls) is the participant's margin. ```ts +import type { AgentProfile } from '@tangle-network/agent-interface' import { - createOpenAICompatibleBackend, + createProfileExecutionBackend, defineConversation, runConversation, } from '@tangle-network/agent-runtime' - -const researcher = createOpenAICompatibleBackend({ - baseURL: 'https://router.tangle.tools/v1', - apiKey: process.env.RESEARCHER_AGENT_SK_TAN!, // ← the AGENT's key, not the user's - model: 'openai/gpt-4o-mini', +import { createExecutor } from '@tangle-network/agent-runtime/kernel' + +const researcherProfile = { + name: 'researcher', + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'openai/gpt-4o-mini' }, +} satisfies AgentProfile + +const researcher = createProfileExecutionBackend({ + profile: researcherProfile, + executor: createExecutor({ + backend: 'router', + routerBaseUrl: 'https://router.tangle.tools/v1', + routerKey: process.env.RESEARCHER_AGENT_SK_TAN!, // ← the AGENT's key, not the user's + }), }) const conv = defineConversation({ @@ -87,7 +102,7 @@ const conv = defineConversation({ { name: 'researcher', backend: researcher, - authSource: 'agent-owned', // ← strip user auth on outbound; backend's apiKey takes over + authSource: 'agent-owned', // ← strip user auth; the executor's routerKey takes over }, { name: 'critic', backend: criticBackend /* forwards user */ }, ], @@ -102,7 +117,7 @@ await runConversation(conv, { What changes on the wire: -- `researcher`'s outbound call has **no** `x-tangle-forwarded-authorization`; the receiving gateway authenticates via the backend's `apiKey` (the agent's own sk-tan-AGENT) and bills the agent. +- `researcher`'s outbound call has **no** `x-tangle-forwarded-authorization`; the receiving gateway authenticates via the executor's `routerKey` (the agent's own sk-tan-AGENT) and bills the agent. - `critic`'s outbound call **still has** the user's forwarded-authorization; that hop bills the user. You can mix the two freely within one conversation — the runtime resolves `authSource` per participant per turn. @@ -139,7 +154,8 @@ The predicate receives `{ transcript, turnIndex, spentCreditsCents }` — enough ## What this is NOT -- **Not authentication.** The agent's credentials live on the backend (constructed once, baked in). `authSource` does not set or change those — it only decides whether to also forward the user's identity downstream. +- **Not authentication.** The agent's credentials live on the executor, which is constructed once and bound to the profile. + `authSource` does not set or change those credentials — it only decides whether to also forward the user's identity downstream. - **Not a policy engine.** The decision returns one of two strings; if you want richer routing (e.g. *which* downstream model to call), do that in your backend, not here. - **Not a substitute for `policy.maxCreditsCents`.** The hard credit ceiling still applies. A reseller participant that bills against its own creds is still counted against the conversation's `spentCreditsCents` budget so a runaway reseller can't drain the agent's wallet under cover of "the user isn't paying anyway." @@ -152,6 +168,6 @@ The simplest way to verify a configuration: inspect the propagated headers your ## Related -- [agent-bus-protocol.md](./agent-bus-protocol.md) — the wire-level header contract this builds on. -- [durability-adapters.md](./durability-adapters.md) — how to persist the resulting conversations across crashes. +- [agent-bus-protocol.md](../agent-bus-protocol.md) — the wire-level header contract this builds on. +- [durability-adapters.md](../durability-adapters.md) — how to persist the resulting conversations across crashes. - `src/conversation/types.ts` — full `AuthSource` and `ConversationDriveState` definitions. diff --git a/docs/canonical-api.md b/docs/canonical-api.md index d08519e3..b37866bf 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.128.1.** +> **Version 0.129.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.144.1 <0.145.0`. +> `agent-eval` must satisfy `>=0.144.3 <0.145.0`. > `sandbox` must satisfy `>=0.17.2 <0.18.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.0 <0.44.0`. > @@ -132,6 +132,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Run **agent-eval fixture folders** through Runtime `runAgentRounds` | agent-eval fixture loading/planning, then `loopCampaignDispatch(...)`: `/kernel`; it starts the Runtime cell inside Eval's paid-call lifecycle | a one-off `runCampaign` dispatch, or attaching a completed `LoopResult` after paid work already ran | | Run + **resume** ONE persistent box across turns | `openSandboxRun(client, opts, deliverable)`: `/kernel` | a per-domain `new Sandbox`+`box.fs.read`+delete copy | | Run **ONE agent turn** on any substrate: box (`streamPrompt`), cli-bridge/router `Executor`, or in-process chat backend: as ONE normalized `RuntimeStreamEvent` stream with a guaranteed terminal result+usage event; opt into in-stream `tool_call`/`tool_result` with `preserveToolParts`, or tap the raw sandbox events with `onRawEvent` | `streamAgentTurn(backend, prompt, { signal, timeoutMs, preserveToolParts?, onRawEvent? })` + `collectAgentTurn(stream)`: `/kernel` | a per-provider stream→event mapper zoo, a hand-faked box around a non-box executor, or raw fetch leaking through the turn abstraction | +| Use an exact profile and Runtime executor where `runAgentTaskStream` or a conversation expects an `AgentExecutionBackend` | `createProfileExecutionBackend({ profile, executor: createExecutor(config) })`: root `.`; the adapter preserves conversation authorization, recursion-depth, and trace headers | a provider-specific backend constructor or an adapter that reads a second model/prompt configuration | | Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })`: `/kernel` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | | Adapt an exact `AgentProfile` to agent-eval's `ChatClient` without moving credentials or execution policy into Eval | `profileChatClient({ profile, executor, context })`: `/kernel` | a provider fetch configured separately from the profile, or request fields that override the profile's model policy | | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor`: `/kernel` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | diff --git a/docs/learning-flywheel.md b/docs/learning-flywheel.md index 98427fce..f256b4e1 100644 --- a/docs/learning-flywheel.md +++ b/docs/learning-flywheel.md @@ -316,7 +316,7 @@ A separate final-test partition is required because source labels alone cannot p substrate (`src/runtime/supervise/`), with `runAgentic`/`defineStrategy`/`runPersonified`. - **The published optimization suite**: `@tangle-network/agent-runtime/kernel` (source: `src/runtime/`): - `Environment`/`Strategy`/`defineStrategy`/`ShotPersona` (`strategy.ts`), `runBenchmark` + `Environment`/`Strategy`/`defineStrategy`/`ShotSpec.profile` (`strategy.ts`), `runBenchmark` (`run-benchmark.ts`), `createVerifierEnvironment`/`createMcpEnvironment`, `harvestCorpus`, `authorStrategy` (`strategy-author.ts`), `auditIntent`, and `promotionGate` (`promotion-gate.ts` — the seeded paired-bootstrap holdout gate over diff --git a/docs/roadmap-rsi.md b/docs/roadmap-rsi.md index ff75dace..8d25c1ef 100644 --- a/docs/roadmap-rsi.md +++ b/docs/roadmap-rsi.md @@ -139,7 +139,7 @@ No benchmark adapter is removed — planned stubs (e.g. AppWorld) are kept. ## Evidence anchors - Agent-driver: `src/mcp/tools/coordination.ts` (`createCoordinationTools` — spawn · observe · steer · stop) over `src/runtime/supervise/` (`Scope`/`Supervisor`). -- Strategy program space: `src/runtime/strategy.ts` (`defineStrategy`/`ShotPersona`), `src/runtime/strategy-author.ts` (`authorStrategy`), `src/runtime/run-benchmark.ts` (`runBenchmark`/`Environment`). +- Strategy program space: `src/runtime/strategy.ts` (`defineStrategy`/`ShotSpec.profile`), `src/runtime/strategy-author.ts` (`authorStrategy`), `src/runtime/run-benchmark.ts` (`runBenchmark`/`Environment`). - Selection: `src/runtime/run-loop.ts:983` (`defaultSelectWinner`), `:797` (`branchPoint`), `:104` (`selectWinner` inject); deployable selector = `bench/src/selector.ts` replayed via `corpus-replay.mts --selector`. - Analyst proposal source: `src/analyst-loop/types.ts` (`KnowledgeProposalSource`); the trace observer feeding the canonical loop is `observe()` (`src/runtime/observe.ts`). - Shared loop: `bench/src/refine-loop.ts` (`RefineLoopSpec`). diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts index a9d86897..a9bf79fd 100644 --- a/examples/p1-parity/run-parity.ts +++ b/examples/p1-parity/run-parity.ts @@ -76,7 +76,7 @@ function parityCell(index: number, shots: number): CellSpec { // The coder model is PINNED on its profile (the arms refuse a model-less coder — a silent // fallback could let the two arms drift apart); offline it names the scripted transport. // The reviewer profile stays model-less: as the graph ROOT it is materialized by the driver - // brain, and the driver model is substrate config (multishot backend / graph RouterConfig). + // brain, and the driver model comes from the exact root profile in both arms. coderProfile: { name: 'coder', harness: 'cli-base', diff --git a/examples/strategy-suite/strategy-suite.ts b/examples/strategy-suite/strategy-suite.ts index e92733d8..8873f931 100644 --- a/examples/strategy-suite/strategy-suite.ts +++ b/examples/strategy-suite/strategy-suite.ts @@ -83,7 +83,7 @@ const doubleCheck = defineStrategy( ) // ── The offline worker: a deterministic `complete` transport (no server) ───── -// `worker.complete` is the injection seam (RouterConfig.complete): given the OpenAI request body +// `worker.complete` is the injected completion transport: given the OpenAI request body // it returns the parsed `/chat/completions` JSON the worker + analyst would have fetched. The same // fn serves BOTH legs — the worker's tool-calling turns and the refine analyst's chat-only steer — // exactly as a localhost mock endpoint would, but in-process. The live router is the drop-in upgrade. diff --git a/examples/supervisor-loop/shared.ts b/examples/supervisor-loop/shared.ts index 0cf060d2..bed7bce0 100644 --- a/examples/supervisor-loop/shared.ts +++ b/examples/supervisor-loop/shared.ts @@ -34,10 +34,10 @@ export const demoGoal = `Produce the exact line "${expectedAnswer}".` /** * A SCRIPTED `ToolLoopChat`: spawn `workerCount` workers (the "drive N workers" * shape), await each settlement, then stop. This is the exact contract - * `routerBrain` fills in production — here it returns a fixed turn sequence so the - * brain runs with no inference (the same offline seam the driver's own unit tests - * use). The brain still REASONS the loop (spawn → await → stop) against a live - * `Scope`; only the driver-LLM call is mocked. + * Runtime derives from the supervisor's exact profile in production; here the chat + * function returns a fixed turn sequence so the brain runs with no inference (the + * same offline seam the driver's own unit tests use). The brain still REASONS the + * loop (spawn → await → stop) against a live `Scope`; only the driver-LLM call is mocked. * * The canonical loop parses `toolCalls[].arguments` itself, so each scripted call * serializes its arguments to a JSON string; the loop JSON.parses them before diff --git a/src/conversation/types.ts b/src/conversation/types.ts index 04583505..e3cc61a7 100644 --- a/src/conversation/types.ts +++ b/src/conversation/types.ts @@ -26,9 +26,8 @@ export interface ConversationParticipant { name: string /** * Backend that runs this participant's turn. Reuses the existing - * `AgentExecutionBackend` contract from `runAgentTaskStream`, so any - * registered backend (iterable, sandbox, OpenAI-compatible) works without - * adaptation. + * `AgentExecutionBackend` contract from `runAgentTaskStream`, so an iterable, + * sandbox, or profile-backed Runtime executor works through the same runner. */ backend: AgentExecutionBackend /** From 3d019d99a7aa4fb2b129cad6d7f46a07e09d8426 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:41:23 -0600 Subject: [PATCH 28/39] chore(release): prepare exact profile cohort --- .github/workflows/ci.yml | 4 +-- .github/workflows/publish.yml | 4 +-- bench/CHANGELOG.md | 2 +- bench/src/swe-arena/gepa-seat.mts | 2 +- .../verify-official-optimizers-consumer.mjs | 30 +++++++++++++++++-- src/improvement/official-optimizers.ts | 2 +- src/runtime/index.ts | 12 ++++---- .../fixtures/agent-improvement-proposal.json | 10 +++---- .../agent-profile-improvement-proposal.json | 6 ++-- 9 files changed, 48 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9d66a96..e17d91c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 42a044f29725f0587d6ab0d9783f4e889d5fb8f7 # v0.143.0 + ref: 3f78bc880d7b2d3d477069ade50bb78f5c34bf72 # v0.144.3 path: .cohort/agent-eval persist-credentials: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 3a41d557a5549c324fad2b101d02f0bf30657cac # v7.0.8 + ref: 6dc011ba7452638fbdbf89f454fb21f30e397d54 # v7.0.10 path: .cohort/agent-knowledge persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e0fe8cfb..d4de9792 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -143,7 +143,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 42a044f29725f0587d6ab0d9783f4e889d5fb8f7 # v0.143.0 + ref: 3f78bc880d7b2d3d477069ade50bb78f5c34bf72 # v0.144.3 path: .cohort/agent-eval persist-credentials: false @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 3a41d557a5549c324fad2b101d02f0bf30657cac # v7.0.8 + ref: 6dc011ba7452638fbdbf89f454fb21f30e397d54 # v7.0.10 path: .cohort/agent-knowledge persist-credentials: false diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 4c2507e4..d80d05a9 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.7.2 -- Consume Runtime 0.128.1, Eval 0.144.1, Knowledge 7.0.9, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. +- Consume Runtime 0.129.0, Eval 0.144.3, Knowledge 7.0.10, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. ## 0.7.1 diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 25b6c6d0..a4e002d9 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -234,7 +234,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.144.1`, then install ' + + 'install `agent-eval-rpc==0.144.3`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/scripts/verify-official-optimizers-consumer.mjs b/scripts/verify-official-optimizers-consumer.mjs index 014eacde..530aa1bd 100644 --- a/scripts/verify-official-optimizers-consumer.mjs +++ b/scripts/verify-official-optimizers-consumer.mjs @@ -8,10 +8,12 @@ import { officialGepa, officialSkillOpt, } from '@tangle-network/agent-runtime' +import { profileOptimizerModelCall } from '@tangle-network/agent-runtime/kernel' import { createOptimizationActivationReceipt, optimizationActivationReceiptFromMetadata, } from '@tangle-network/agent-runtime/intelligence' +import { canonicalAgentProfileDigest } from '@tangle-network/agent-interface' const python = process.env.AGENT_EVAL_TEST_PYTHON?.trim() if (!python) throw new Error('AGENT_EVAL_TEST_PYTHON is required') @@ -490,13 +492,35 @@ const requiredRuleJudge = { } function optimizerModel(baseUrl, maxOutputTokensPerRequest) { + const profile = { + name: 'packed-official-optimizer', + harness: 'cli-base', + model: { + provider: 'local-openai-compatible', + default: 'local-model', + metadata: { maxTokens: maxOutputTokensPerRequest }, + }, + } + const profileDigest = canonicalAgentProfileDigest(profile) return { model: 'local-model', - baseUrl, - apiKey: 'provider-secret', + callRef: `packed-official-optimizers:${profileDigest}:${new URL(baseUrl).origin}`, + call: profileOptimizerModelCall({ + profile, + context: 'packed official optimizer verification', + executor: { + backend: 'router', + routerBaseUrl: baseUrl, + routerKey: 'provider-secret', + }, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + }), budget: { maxCostUsd: 1, - maxRequests: 10, + maxRequests: 100, maxRequestBytes: 100_000, maxResponseBytes: 100_000, maxOutputTokensPerRequest, diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index b9968efa..025e95e7 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -25,7 +25,7 @@ import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.1"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.3"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' diff --git a/src/runtime/index.ts b/src/runtime/index.ts index d6835134..6c9d07d5 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -356,9 +356,9 @@ export { printBenchmarkReport, runBenchmark, } from './run-benchmark' -// `runAgentRounds` is the multi-agent fanout/vote/refine kernel (many sandbox sessions per -// call). It is NOT `runToolLoop`/`streamToolLoop` (`/tool-loop`: one chat turn, tool calls -// folded back in) and NOT `routerToolLoop` (also on this subpath — router chat + tools). +// `runAgentRounds` is the multi-agent fanout/vote/refine kernel over many sandbox sessions. +// It is distinct from `runToolLoop`/`streamToolLoop`, which execute one chat turn and fold +// tool results back into that same conversation. export { defaultSelectWinner, type RunAgentRoundsOptions, runAgentRounds } from './run-loop' export { type AcquireOptions, acquireSandbox } from './sandbox-acquire' export { @@ -953,9 +953,9 @@ export { superviseSurface, } from './supervise-surface' export type { SandboxControlClient } from './tangle-sandbox-exact-process-provider' -// The driver-brain seam type a consumer scripts (a mock) or passes (`routerBrain`) into -// `DriverAgentOptions.brain` — the canonical one-inference-turn tool-loop chat. `ToolLoopCompaction` -// is the self-compaction config that bounds the brain's own context window (the supervisor chapter-close). +// The driver-brain type a test or advanced caller may implement and pass into +// `DriverAgentOptions.brain`. Production supervisors derive this chat function from their exact +// profile. `ToolLoopCompaction` bounds the driver's own context window. export type { ToolLoopChat, ToolLoopCompaction, diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 5d5ed3af..6733d57b 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:ae45e6d96a4931bf38572396e968de5c4b753b05c9e3a5608c0d3b0f27e1d4c7", + "digest": "sha256:51691ad716939f633997479cb6a54458806eeb9d693a520e64b4400c035cc187", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.128.1" + "runtimeVersion": "0.129.0" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:f6030055252c51ab3c035cc2dded16b24d98cc6ecbb0630e0cd5b8f5be4db34e", - "runId": "agent-runtime-0.128.1-proposal-fixture", + "recordDigest": "sha256:aa49f5a490a6cab44580a20bf8f454862eff990823e576d7d063ad3c59c951f2", + "runId": "agent-runtime-0.129.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.128.1-proposal-fixture" + "runId": "agent-runtime-0.129.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 42d4c917..988f9d8f 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:f5f780797333cfb8abfe9bfefe46d70faa412d804b997626d8a1d1316d9c2b68", + "digest": "sha256:ac0b55438c591e740c19eb7b39e119f99f1d0e220c3cb3f785a7d1a3f49bfb48", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.128.1" + "runtimeVersion": "0.129.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:6c9c0e3a1cf1e01b2ee759dc67e0ac6d61fde0da14fe22d15d11532bdc63870e", + "recordDigest": "sha256:c9bb01de88046bd5c02d32f323f02651614a2870a8af3ffe9a9f0bceb4a0aa63", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From bcaa068fccd0eff4e2da8427e125b2b9c13c63f4 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:46:13 -0600 Subject: [PATCH 29/39] fix(runtime): preserve active model cancellation --- pnpm-lock.yaml | 12 ++++++------ pnpm-workspace.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f7712278..aad70079 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,8 +10,8 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-core': - specifier: 0.4.34 - version: 0.4.34 + specifier: 0.4.35 + version: 0.4.35 '@tangle-network/agent-eval': specifier: 0.144.3 version: 0.144.3 @@ -52,7 +52,7 @@ importers: dependencies: '@tangle-network/agent-core': specifier: 'catalog:' - version: 0.4.34 + version: 0.4.35 '@tangle-network/agent-knowledge': specifier: 'catalog:' version: 7.0.10 @@ -1111,8 +1111,8 @@ packages: '@tangle-network/agent-core@0.4.33': resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-core@0.4.34': - resolution: {integrity: sha512-OH+d9kx9yN7lOVPCKRt0VXYVO0RhQgRsqdAAG++iIAM9Ti6g7qnG1flEw1zah1LgntYR+uiGOae62MxwWLGR3w==} + '@tangle-network/agent-core@0.4.35': + resolution: {integrity: sha512-qLPdWUO7wCPQ86m+DE3jjk1hV0WGbmO/DkSfNcxZ0Shxx7vsNIgp8vasOsDOwfZzHlugZb9PemL0JZ5ulc8R4g==} '@tangle-network/agent-eval@0.144.3': resolution: {integrity: sha512-gkyeie5CCzUuEzTfuQKe6iikSsu0S9h6w8yRtfnvjDZopkJJ7Nn+VDzkWZrsLDQ31LtLaHqJ7I2FgYbPNvjW0Q==} @@ -2789,7 +2789,7 @@ snapshots: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 - '@tangle-network/agent-core@0.4.34': + '@tangle-network/agent-core@0.4.35': dependencies: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8a6a38ad..41fea6e0 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,7 +17,7 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 - '@tangle-network/agent-core': 0.4.34 + '@tangle-network/agent-core': 0.4.35 '@types/node': 26.1.1 '@tangle-network/agent-eval': 0.144.3 '@tangle-network/agent-interface': 0.43.0 From 996e15f81a488e4e0a286cac8fe49284b1c228b1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:51:59 -0600 Subject: [PATCH 30/39] fix(release): report packed cohort cost provenance --- scripts/fixtures/packed-cohort-consumer.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/fixtures/packed-cohort-consumer.ts b/scripts/fixtures/packed-cohort-consumer.ts index ad392033..3ee4c57f 100644 --- a/scripts/fixtures/packed-cohort-consumer.ts +++ b/scripts/fixtures/packed-cohort-consumer.ts @@ -97,7 +97,12 @@ const method: ImproveMethodFactory = (context) => ({ } return { winnerSurface: 'CANDIDATE', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, }) From e8dfede2bd42bac305fe3609f23ccd055bdf77be Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 23:56:59 -0600 Subject: [PATCH 31/39] fix(runtime): seal router retry authority --- src/runtime/router-client.complete.test.ts | 42 ++++++++++++++++++- src/runtime/router-client.ts | 13 +++++- src/runtime/supervise/supervisor-agent.ts | 47 ++++++++++++++++++++-- tests/kernel/supervisor-agent.test.ts | 26 ++++++++++++ 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/src/runtime/router-client.complete.test.ts b/src/runtime/router-client.complete.test.ts index ccf391e8..ceaf87df 100644 --- a/src/runtime/router-client.complete.test.ts +++ b/src/runtime/router-client.complete.test.ts @@ -96,7 +96,9 @@ describe('RouterConfig.complete — the injected completion transport', () => { it('buffered chat retries a thrown network failure and reports the exact attempt count', async () => { let calls = 0 - const fetchSpy = vi.fn(async () => { + const idempotencyKeys: string[] = [] + const fetchSpy = vi.fn(async (_url: unknown, init: RequestInit) => { + idempotencyKeys.push((init.headers as Record)['idempotency-key'] ?? '') calls += 1 if (calls === 1) throw new TypeError('fetch failed: socket reset') return { @@ -128,10 +130,48 @@ describe('RouterConfig.complete — the injected completion transport', () => { ) expect(fetchSpy).toHaveBeenCalledTimes(2) + expect(idempotencyKeys[0]).toMatch(/^idem_/u) + expect(idempotencyKeys).toEqual([idempotencyKeys[0], idempotencyKeys[0]]) expect(result.transportAttempts).toBe(2) expect(result.content).toBe('recovered') }) + it('keeps a caller-supplied logical-call id authoritative across retries', async () => { + const seen: string[] = [] + let calls = 0 + const complete = vi.fn( + async ( + _body: Record, + request?: { headers: Readonly> }, + ) => { + seen.push(request?.headers['idempotency-key'] ?? '') + calls += 1 + if (calls === 1) throw new TypeError('fetch failed: accepted response lost') + return { choices: [{ message: { content: 'recovered' } }] } + }, + ) + + await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + retry: { + maxAttempts: 2, + initialBackoffMs: 0, + maxBackoffMs: 0, + jitter: 0, + requestTimeoutMs: 0, + }, + }, + [{ role: 'user', content: 'recover' }], + { callId: 'trusted-call-1' }, + ) + + expect(seen).toEqual(['trusted-call-1', 'trusted-call-1']) + }) + it('fails loud with the final network cause after the configured attempts are exhausted', async () => { const fetchSpy = vi.fn(async () => { throw new TypeError('fetch failed: connection refused') diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index e044d505..03c3e553 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -10,7 +10,12 @@ * (a phantom 0 reads as a free call downstream, which the gate would act on). */ -import { type RetryConfig, SDKError, withRetry } from '@tangle-network/agent-core' +import { + generateIdempotencyKey, + type RetryConfig, + SDKError, + withRetry, +} from '@tangle-network/agent-core' import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import type { ReasoningEffort } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' @@ -784,11 +789,15 @@ function routerRequestHeaders( } | undefined, ): Record { + // Every POST is retryable. Mint the logical-call identity once while the request headers are + // assembled, outside `withRetry`, so an accepted response whose connection dies cannot make a + // retry look like a new billable completion. A trusted caller identity remains authoritative. + const callId = opts?.callId ?? generateIdempotencyKey() return { ...(opts?.propagatedHeaders ?? {}), 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}`, - ...(opts?.callId ? { 'idempotency-key': opts.callId } : {}), + 'idempotency-key': callId, ...(opts?.correlationId ? { 'x-correlation-id': opts.correlationId } : {}), } } diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index b4ca0367..b2a4f28b 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -343,6 +343,41 @@ export interface SupervisorAgentDeps { readonly coordination?: CoordinationBinding } +const ROUTER_TRANSPORT_FIELDS = new Set(['routerBaseUrl', 'routerKey', 'complete']) + +/** Capture the transport-only Router seam before any profile lowering. TypeScript excess-property + * checks are not a runtime boundary: JavaScript and widened objects can otherwise smuggle private + * generation/retry fields through a spread and override an AgentProfile that omitted them. */ +function snapshotRouterTransportConfig(input: RouterTransportConfig): RouterTransportConfig { + if (typeof input !== 'object' || input === null || Array.isArray(input)) { + throw new ValidationError('supervisorAgent: deps.router must be a RouterTransportConfig object') + } + const unsupported = Object.keys(input).filter((field) => !ROUTER_TRANSPORT_FIELDS.has(field)) + if (unsupported.length > 0) { + throw new ValidationError( + `supervisorAgent: deps.router contains unsupported behavioral fields: ${unsupported.sort().join(', ')}`, + ) + } + if (typeof input.routerBaseUrl !== 'string' || input.routerBaseUrl.length === 0) { + throw new ValidationError( + 'supervisorAgent: deps.router.routerBaseUrl must be a non-empty string', + ) + } + if (typeof input.routerKey !== 'string' || input.routerKey.length === 0) { + throw new ValidationError('supervisorAgent: deps.router.routerKey must be a non-empty string') + } + if (input.complete !== undefined && typeof input.complete !== 'function') { + throw new ValidationError( + 'supervisorAgent: deps.router.complete must be a function when provided', + ) + } + return Object.freeze({ + routerBaseUrl: input.routerBaseUrl, + routerKey: input.routerKey, + ...(input.complete !== undefined ? { complete: input.complete } : {}), + }) +} + /** Build a supervisor `Agent` from its profile: the brain resolves from `profile.harness` (backend-as-data), the same resolution rule as every worker. */ export function supervisorAgent( profile: SupervisorProfile, @@ -351,6 +386,8 @@ export function supervisorAgent( const exactProfile = agentProfileSchema.parse(profile) assertExecutableAgentProfile(exactProfile, 'supervisorAgent') const stableProfile = detachedSnapshot(exactProfile, 'supervisorAgent profile') + const stableRouter = + deps.router === undefined ? undefined : snapshotRouterTransportConfig(deps.router) const resolveTools = deps.resolveSupervisorTools const observeNodeEvent = deps.observeNodeEvent const nodeContextSeed = @@ -396,7 +433,7 @@ export function supervisorAgent( if (harness === null) { // ROUTER arm: the in-process tool-loop. `routerBrain` is now an internal detail — the caller // passes a profile, not a hand-built brain (a test may still inject `deps.brain`). - const brain = deps.brain ?? routerBrainFromProfile(stableProfile, deps) + const brain = deps.brain ?? routerBrainFromProfile(stableProfile, stableRouter) const inbox = createInbox() const build = ( priorCoordination?: PriorCoordination, @@ -649,9 +686,9 @@ function bindSupervisorNodeObserver( function routerBrainFromProfile( profile: SupervisorProfile, - deps: SupervisorAgentDeps, + router: RouterTransportConfig | undefined, ): ToolLoopChat { - if (!deps.router) { + if (!router) { throw new ValidationError( 'supervisorAgent: a router-brained supervisor (harness omitted or cli-base) needs deps.router (or deps.brain)', ) @@ -660,7 +697,9 @@ function routerBrainFromProfile( const settings = profileModelExecutionSettings(profile, 'supervisorAgent') return routerBrain( { - ...deps.router, + routerBaseUrl: router.routerBaseUrl, + routerKey: router.routerKey, + ...(router.complete !== undefined ? { complete: router.complete } : {}), model: modelId, ...(settings.retry !== undefined ? { retry: settings.retry } : {}), ...(settings.maxTokens !== undefined ? { maxTokens: settings.maxTokens } : {}), diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 9e681c3b..66191991 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -2,6 +2,7 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { ConfigError } from '../../src/errors' +import type { RouterTransportConfig } from '../../src/runtime/router-client' import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' import { createRootHandle, createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -78,6 +79,31 @@ function runSupervisor( } describe('supervisorAgent — the brain is resolved from profile.harness (backend-as-data)', () => { + it('rejects behavioral fields smuggled through a widened Router transport object', () => { + const router = { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + maxTokens: 7, + retry: { maxAttempts: 1 }, + stream: true, + } as unknown as RouterTransportConfig + + expect(() => + supervisorAgent( + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }), + { + router, + blobs: new InMemoryResultBlobStore(), + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + }, + ), + ).toThrow(/unsupported behavioral fields: maxTokens, retry, stream/u) + }) + it('ROUTER arm (harness null): the in-process tool-loop drives a worker to delivery', async () => { const blobs = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() From 29646bf0e39fd25c8ecfbd4d3df70314e7139aaf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 00:16:53 -0600 Subject: [PATCH 32/39] fix(runtime): enforce profiled supervisor calls --- CHANGELOG.md | 2 + bench/src/agent-graphs-improve.mts | 7 +- docs/api/primitive-catalog.md | 23 +- docs/api/runtime.md | 383 +--- docs/api/testing.md | 1666 +++++++++++++++++ examples/graphs/analyst-agent-review.ts | 14 +- examples/graphs/best-of-n.ts | 14 +- examples/graphs/collaborates-review-loop.ts | 9 +- examples/graphs/shared.ts | 2 +- examples/graphs/shot-loop-resumed.ts | 9 +- examples/graphs/shot-loop.ts | 9 +- examples/graphs/user-sim-conversation.ts | 9 +- examples/graphs/watchdog-steer.ts | 15 +- examples/p1-parity/arms.ts | 11 +- examples/p1-parity/offline.ts | 3 +- examples/supervisor-loop/run.ts | 39 +- examples/supervisor-loop/shared.ts | 2 +- scripts/verify-package-exports.mjs | 43 +- src/runtime/index.ts | 16 +- src/runtime/router-client.complete.test.ts | 21 + src/runtime/router-client.ts | 32 +- src/runtime/supervise/coordination-driver.ts | 61 +- src/runtime/supervise/graph.ts | 48 +- src/runtime/supervise/supervise.ts | 48 +- src/runtime/supervise/supervisor-agent.ts | 45 +- src/runtime/tool-loop.ts | 17 +- src/testing/index.ts | 18 + tests/examples/graph-topologies.test.ts | 14 +- tests/examples/user-sim-conversation.test.ts | 4 +- tests/helpers/resume-driver-child.ts | 2 +- tests/helpers/runtime-with-test-brain.ts | 51 + tests/kernel/graph.test.ts | 3 +- .../nested-coordination-durability.test.ts | 2 +- tests/kernel/supervise-convenience.test.ts | 133 +- .../supervise-full-profile-bridge.test.ts | 2 +- .../supervise-global-concurrency.test.ts | 3 +- tests/kernel/supervise-otel-spans.test.ts | 3 +- tests/kernel/supervise-worker-trace.test.ts | 2 +- tests/kernel/supervisor-agent.test.ts | 2 +- tests/runtime/mid-flight-steering.test.ts | 2 +- tests/runtime/supervisor-finalizer.test.ts | 2 +- tests/supervisor-loop-example.test.ts | 2 +- 42 files changed, 2302 insertions(+), 491 deletions(-) create mode 100644 tests/helpers/runtime-with-test-brain.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a5690493..e69eae1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ - Remove `createPrimeIntellectBackend`; use `primeIntellectExecutorConfig(context)` with `createExecutor(...)`, then execute an exact profile through `streamAgentTurn` or bind it with `createProfileExecutionBackend(...)` when an `AgentExecutionBackend` is required. - Remove `uiAuditorProfile`, `createInProcessUiAuditClient`, and their browser/judge option types; author a normal `AgentProfile`, execute it through Runtime, and keep using `UiAuditTask`, `encodeAuditTaskEnvelope`, `formatAuditorPrompt`, `parseAuditorEvents`, and `createUiAuditorValidator` from `/profiles`. - Remove the bundled UI-audit example and the Playwright peer dependency because browser placement belongs to the caller-owned execution environment, not a profile-specific Runtime client. +- Remove direct model injection from `SuperviseOptions`, `RunGraphOptions`, and `SupervisorAgentDeps`, and remove `driverAgent`, `DriverAgentOptions`, and `ToolLoopChat` from `/kernel`; production supervisor calls now derive model execution only from the exact root `AgentProfile`, while deterministic scripted calls live under the explicit `/testing` entrypoint. +- Bind every root Router turn to one Runtime-authored cancellation signal, call id, and correlation id; preserve provider-observed model, prompt-cache, retry-attempt, reasoning-token, and billed-cost evidence; meter a mismatched model response before refusing its output; and reuse one nonempty idempotency key across all physical retries of the logical call. ## 0.128.0 diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index bb48892d..37e2bf24 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -38,10 +38,9 @@ import { GraphEdgeCapError, type GraphResult, promptHandle, - type RunGraphOptions, - runGraph, streamAgentTurn, } from '../../src/runtime/index.ts' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing/index.ts' import { leafSeam, scriptedBrain, type ScriptedTurn } from './agent-graphs-improve/offline-seams.mts' const HERE = dirname(fileURLToPath(import.meta.url)) @@ -380,7 +379,7 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi maxTokens: spec.perWorker.maxTokens ?? Math.max(1, Math.floor(spec.budget.maxTokens / 4)), } - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId, maxLiveWorkers: Math.max(workerIds.length, 1), ...(perWorker !== undefined ? { perWorker } : {}), @@ -404,7 +403,7 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi ) t.unref?.() }) - const res: GraphResult = await Promise.race([runGraph(graph, opts), timeout]) + const res: GraphResult = await Promise.race([runGraphWithTestBrain(graph, opts), timeout]) return { resultKind: res.result.kind, ledger: res.ledger, exhaustedEdges: res.exhaustedEdges } } diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 9fd27356..3c88af52 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -504,7 +504,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 712 exports. +Import from `@tangle-network/agent-runtime/kernel` — 709 exports. | Symbol | Kind | Summary | |---|---|---| @@ -582,7 +582,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `depthStrategy` | function | DEPTH: one persistent artifact, carried across analyst-steered shots. | | `deterministicCompletion` | function | Completion for a DETERMINISTIC check (build/test/lint/citation/proof): done iff the check | | `discriminatingMeans` | function | Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field | -| `driverAgent` | function | Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a | | `effectiveConcurrency` | function | The ONE honest effective limit on simultaneous workers: the minimum of the caps that actually | | `envKeyProvider` | function | The env-backed provider: reads the (dotenvx-loaded) process env. Empty / | | `equalKOnCost` | function | Assert the arms are comparable at EQUAL conserved COST (tokens + usd), NOT raw iteration | @@ -692,9 +691,9 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `streamAgentTurn` | function | Run ONE agent turn on any backend kind and stream its events. Yields the | | `structuralRollout` | function | Build the structuralRollout `Strategy`: k shots → score each by the frozen visible | | `sumSandboxUsage` | function | Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an | -| `supervise` | function | One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. | +| `supervise` | function | One-call supervisor: build + run a supervisor from its exact profile. | | `superviseSurface` | function | Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and | -| `supervisorAgent` | function | Build a supervisor `Agent` from its profile: the brain resolves from `profile.harness` (backend-as-data), the same resolution rule as every worker. | +| `supervisorAgent` | function | Build a supervisor `Agent` from its profile: the brain resolves from `profile.harness` | | `supervisorInstructions` | function | The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable | | `supervisorRunDir` | function | The run directory every artifact of one supervisor run lives under. | | `supervisorRunsRoot` | function | The root every supervisor run of one workspace lives under. | @@ -1051,7 +1050,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `SupervisorNodeContextSeed` | type | Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. | | `SupervisorProfile` | type | A supervisor is an exact canonical AgentProfile; no looser model/prompt shape exists. | | `SupervisorSpanAttributes` | type | OTLP span attribute values. Exported because `SupervisorSpanOptions.attributes` is public and | -| `ToolLoopChat` | type | One inference turn over the running conversation + the tool specs → the model's text, any | | `ToolLoopCompactionOptions` | type | Public supervisor-facing compaction config: same knobs as the primitive, but `distill` is optional | | `ToolLoopMessageRecord` | type | Provider-neutral conversation record accepted by a tool-loop brain. | | `TrajectoryReportFn` | type | `trajectoryReport(...)` — the tree+cost reconstructor. Async (reads journal + optionally blobs). | @@ -1071,7 +1069,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 712 exports. | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge @@ -1288,14 +1286,25 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 108 exports. ### Testing fixtures — validated Runtime wire records -Import from `@tangle-network/agent-runtime/testing` — 4 exports. +Import from `@tangle-network/agent-runtime/testing` — 14 exports. | Symbol | Kind | Summary | |---|---|---| +| `driverAgent` | function | Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a | | `loadAgentImprovementProposalFixture` | function | Load an isolated, production-validated Runtime proposal for consumer tests. | | `loadAgentProfileImprovementFixture` | function | Load an isolated profile proposal and its private activation state for consumer tests. | +| `runGraphWithTestBrain` | function | Deterministic scripted-brain path for graph tests. Not exported from Runtime's main entry. | +| `superviseWithTestBrain` | function | Deterministic scripted-brain path for tests. Not exported from Runtime's main entry. | +| `supervisorAgentWithTestBrain` | function | Scripted-brain construction for deterministic tests. Not exported from Runtime's main entry. | | `AgentProfileImprovementFixture` | interface | Complete private state for exercising profile activation and restore in consumer tests. | +| `RunGraphTestOptions` | interface | Test-only graph options, exported only through the package's explicit `/testing` entry. | +| `SuperviseTestOptions` | interface | Test-only one-call shape, exported only through the package's explicit `/testing` entry. | +| `SupervisorAgentTestDeps` | interface | Test-only dependency shape. It is exported only through the package's explicit `/testing` | +| `ToolLoopCallContext` | interface | Runtime-owned identity and cancellation for one logical inference call. The wrapper is frozen | | `AgentProfileImprovementProposalFixture` | type | A proposal produced by Runtime's opaque profile-improvement path. | +| `ToolLoopChat` | type | One inference turn over the running conversation + the tool specs → the model's text, any | + +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `DriverAgentOptions`. ### MCP servers — delegate / coordination / detached-session diff --git a/docs/api/runtime.md b/docs/api/runtime.md index bb803f61..a0782b33 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -645,7 +645,7 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. ##### load() -> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination)\> ###### Parameters @@ -659,7 +659,7 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. ###### Returns -`Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +`Promise`\<[`PriorCoordination`](#priorcoordination)\> ###### Implementation of @@ -9833,262 +9833,6 @@ What the spawn was supposed to produce — surfaced in traces/reports. *** -### DriverAgentOptions - -#### Properties - -##### name - -> `readonly` **name**: `string` - -##### brain - -> `readonly` **brain**: [`ToolLoopChat`](#toolloopchat) - -The driver-LLM seam — ONE inference turn over the conversation + the coordination tool specs - (the canonical `ToolLoopChat`): a scripted mock offline, the router's tool-calling in - production, or a sandboxed harness. The same seam every tool-loop uses; no bespoke shape. - -##### blobs - -> `readonly` **blobs**: [`ResultBlobStore`](#resultblobstore) - -Shared blob store — `observe_agent` reads settled outputs through it. - -##### makeWorkerAgent - -> `readonly` **makeWorkerAgent**: [`MakeWorkerAgent`](#makeworkeragent) - -Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). - -##### authorizeDownMessage? - -> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](#authorizedownmessage) - -##### perWorker - -> `readonly` **perWorker**: [`Budget`](index.md#budget-4) - -Per-child budget reserved from the conserved pool on each spawn. - -##### deliverable? - -> `readonly` `optional` **deliverable?**: [`DeliverableSpec`](#deliverablespec)\<`unknown`\> - -Independent completion check for work the driver performs itself. When present, the driver - receives `submit_result`; the first passing submission ends the loop and becomes the output. - -##### maxLiveWorkers? - -> `readonly` `optional` **maxLiveWorkers?**: `number` - -Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in - flight (a concurrency fence on top of the conserved-pool fence). Omit/`<= 0` = no cap. - -##### analysts? - -> `readonly` `optional` **analysts?**: [`AnalystRegistry`](index.md#analystregistry) - -The analyst lenses available to the driver. Required for `analyzeOnSettle` (and `run_analyst`). - Unset → no analyst feed (status quo: the driver gets settled outputs, no findings). - -##### analyzeOnSettle? - -> `readonly` `optional` **analyzeOnSettle?**: readonly (`string` \| [`AnalyzeOnSettleRoute`](#analyzeonsettleroute))[] - -Analyst kind ids run AUTOMATICALLY when a worker settles `done` — each result re-enters as a - `finding` the driver pulls and composes its next steer from. The UP-leg of the self-improving - loop. Omit/empty = no auto-analysis (status quo). Requires `analysts`. - -##### watchWorkers? - -> `readonly` `optional` **watchWorkers?**: [`WorkerWatchOptions`](#workerwatchoptions) - -Run the ONLINE detector panel over each worker's LIVE tool trace and raise a `finding` the - moment it loops/error-storms — mid-run evidence to steer on, not a settle-time post-mortem. - Omit = no online watching. - -##### stallAfterMs? - -> `readonly` `optional` **stallAfterMs?**: `number` - -Idle time after which `observe_agent` reports a worker as stalled (a derived read; nothing is - killed). Omit = the runtime default. - -##### continuityByProfile? - -> `readonly` `optional` **continuityByProfile?**: `Readonly`\<`Record`\<`string`, [`ContinuityMode`](#continuitymode)\>\> - -Default continuity per worker PROFILE NAME — `'resume'` makes spawns of that name re-attach - to the node's latest settled worker (see - `CoordinationToolsOptions.continuityByProfile`); `spawn_agent`'s per-call `continuity` - argument overrides. Omit = every spawn fresh (status quo). - -##### systemPrompt - -> `readonly` **systemPrompt**: `string` \| ((`task`) => `string`) - -The driver's stance — a string, or built from the task (the worker-driver prompt / - the generator). INJECTED so the prompt is a pluggable, optimizable role. - -##### nodeTools? - -> `readonly` `optional` **nodeTools?**: readonly [`McpToolDescriptor`](mcp.md#mcptooldescriptor)[] - -Product-selected tools already bound to this exact supervisor node. The same descriptors are - served over MCP for external supervisors; this arm projects them into router ToolSpecs. - -##### extraTools? - -> `readonly` `optional` **extraTools?**: readonly `object`[] - -WORK tools the driver may call DIRECTLY (alongside the coordination verbs) — so the driver is - not a pure manager but a full agent that can ACT (do simple work itself) OR SPAWN (delegate). - Each is a router tool spec; their names must not collide with the coordination verbs. Pair with - `executeExtraTool`. Unset → coordination-only (the prior behavior). - -##### executeExtraTool? - -> `readonly` `optional` **executeExtraTool?**: (`name`, `args`) => `Promise`\<`string` \| `null` \| `undefined`\> - -Runs an `extraTools` call. Returns a string result, or null/undefined to signal "not handled" - so the call falls through to the coordination dispatch. Required iff `extraTools` is set. - -###### Parameters - -###### name - -`string` - -###### args - -`Record`\<`string`, `unknown`\> - -###### Returns - -`Promise`\<`string` \| `null` \| `undefined`\> - -##### maxTurns? - -> `readonly` `optional` **maxTurns?**: `number` - -Max driver turns before the loop force-finalizes on the best settled child. Default 16. - `0` lifts the turn-COUNT cap: the loop is bounded instead by the conserved budget pool, - an absolute deadline, the driver's own stop, and abort (checked in-loop). A finite - anti-runaway tripwire still guards a degenerate driver that loops on a no-spawn tool. - -##### now? - -> `readonly` `optional` **now?**: () => `number` - -Injected clock for the in-loop absolute-deadline guard — keeps the deadline check - deterministic in tests. Defaults to `Date.now`. - -###### Returns - -`number` - -##### stopRule? - -> `readonly` `optional` **stopRule?**: [`StopRule`](#stoprule-1) - -PROGRESS-derived stop (mechanic D). Today a run ends on a ceiling — iterations, tokens, -dollars, deadline, turn cap — which answers "may it continue?" and never "is it still getting -anywhere?". A stop rule reads the run's own progress (best-so-far over settled work, time -since the last settle, the live worker feed) and ends a run that has stopped learning BEFORE -it exhausts a budget. - -Composes with, and can never override, the hard guards: `poolStarved` / `deadlinePassed` / -abort / the driver's own stop are evaluated first, so a rule can only ADD a stop. - -THRESHOLDS are the caller's judgment, not this module's — build the rule with -`plateau({window, minDelta})` / `noProgressFor({...})` / `allWorkersStalled({...})` from -`supervise/stop-rules`. Omit ⇒ ceilings only (unchanged behavior). - -##### onProgressStop? - -> `readonly` `optional` **onProgressStop?**: (`reason`) => `void` - -Called once with the rule's reason when a `stopRule` ends the run — so a caller can record - WHY a run stopped early instead of inferring it from an unexhausted budget. - -###### Parameters - -###### reason - -`string` - -###### Returns - -`void` - -##### compaction? - -> `readonly` `optional` **compaction?**: [`ToolLoopCompactionOptions`](#toolloopcompactionoptions) - -Give the driver brain a chapter-lifecycle on its OWN context window. The LLM-brain front doors - lose to a dumb-Ralph respawn because the brain re-bills its whole coordination transcript every - turn — the same context overflow a single steered agent suffers, one level up. With this set, - once the brain's running conversation exceeds `thresholdTokens` it distills the accumulated - history to a compact progress note and continues fresh: the supervisor analog of respawning - against external tracking state, except the live `Scope` roster IS the durable state. Default - off (no behavior change). `distill` defaults to a self-summary authored by the brain combined - with the factual settled-worker roster; override to supply your own. - -##### onEvent? - -> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> - -Pass-through subscriber for every coordination bus event: settled/question/finding, - pre-delivery instruction receipts, and steer/answer delivery outcomes. A durable caller uses - this to append the coordination log. Omit = no observer. - -###### Parameters - -###### event - -[`CoordinationEvent`](index.md#coordinationevent) - -###### record - -[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> - -###### Returns - -`void` \| `Promise`\<`void`\> - -##### replaySettlements? - -> `readonly` `optional` **replaySettlements?**: `boolean` - -Re-publish resume-time settlements through the awaited observer before the first brain turn. - -##### priorCoordination? - -> `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](#priorcoordination-1) - -Questions, findings, and authorized continuation receipts loaded from a prior process. - Questions seed the ledger (`list_questions`, blocking-stop policy); all three feed the resume - brief. Continuation receipts are evidence only and are never auto-delivered. Omit = fresh. - -##### finalizer? - -> `readonly` `optional` **finalizer?**: [`SupervisorFinalizer`](index.md#supervisorfinalizer) - -How the settled-worker ledger becomes the run's output. Default `bestDelivered` — the single - highest-scoring DELIVERED child (the exact keep-best every existing caller had). Runs under - the delivered-only invariant (`runFinalizer`): whatever the finalizer, an undelivered or - invalid child's output stays unreachable. - -##### inbox? - -> `readonly` `optional` **inbox?**: [`Inbox`](#inbox-1) - -Optional shared manager inbox used by a wrapper that must accept messages before async node -setup finishes. Ordinary callers omit it and the driver owns a fresh inbox. - -*** - ### PriorCoordination Coordination evidence loaded from prior processes of one durable supervised run. @@ -10166,7 +9910,7 @@ The durable coordination side-log seam. `append` records one bus event (kinds it ##### load() -> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination)\> ###### Parameters @@ -10180,7 +9924,7 @@ The durable coordination side-log seam. `append` records one bus event (kinds it ###### Returns -`Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +`Promise`\<[`PriorCoordination`](#priorcoordination)\> *** @@ -10904,6 +10648,10 @@ The concrete worker node id, once known. ### RunGraphOptions +#### Extended by + +- [`RunGraphTestOptions`](testing.md#rungraphtestoptions) + #### Properties ##### backend? @@ -10933,12 +10681,6 @@ Caller-side runtime hooks (telemetry, policy, product extensions). Composed AFTE graph's own spawn-binding hook on the SAME event stream — the graph never swallows the seam supervise() exposes. -##### brain? - -> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) - -Inject the driver brain directly (offline tests / advanced). - ##### analysts? > `readonly` `optional` **analysts?**: [`AnalystRegistry`](index.md#analystregistry) @@ -12556,7 +12298,7 @@ Drive the worker to settlement. `signal` is the spawn-scoped abort handed to `ex ##### inbox -> `readonly` **inbox**: [`Inbox`](#inbox-1) +> `readonly` **inbox**: [`Inbox`](#inbox) ##### taskToPrompt @@ -12995,7 +12737,7 @@ Evaluate a rule against the current view. ###### rule -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) ###### scope? @@ -13189,6 +12931,10 @@ caller that owns the code registers it here once and names it from data thereaft ### SuperviseOptions +#### Extended by + +- [`SuperviseTestOptions`](testing.md#supervisetestoptions) + #### Properties ##### budget @@ -13398,12 +13144,6 @@ Decide whether an authorized child becomes another supervisor. By default only The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's model wins. -##### brain? - -> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) - -Inject the supervisor brain directly (tests / advanced). - ##### driveHarness? > `readonly` `optional` **driveHarness?**: [`DriveHarness`](#driveharness-1) @@ -13601,7 +13341,7 @@ Predicate registry for `poll` wait-states (`Scope.wait`). A `poll` names its pre ##### stopRule? -> `readonly` `optional` **stopRule?**: [`StopRule`](#stoprule-1) +> `readonly` `optional` **stopRule?**: [`StopRule`](#stoprule) PROGRESS-derived stop rule (router-brained supervisor). Ends a run that has stopped LEARNING before it exhausts a ceiling — the answer to "a run should end because it is done or stuck, @@ -14111,6 +13851,10 @@ when no executor inbox is active instead of claiming a message was delivered. ### SupervisorAgentDeps +#### Extended by + +- [`SupervisorAgentTestDeps`](testing.md#supervisoragenttestdeps) + #### Properties ##### blobs @@ -14156,12 +13900,6 @@ Hard cap on simultaneously-LIVE workers across both arms — `spawn_agent` fails Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The profile's model wins. -##### brain? - -> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) - -Inject the brain directly (tests / advanced) instead of resolving `routerBrain` from the profile. - ##### driveHarness? > `readonly` `optional` **driveHarness?**: [`DriveHarness`](#driveharness-1) @@ -14256,7 +13994,7 @@ Default continuity per worker PROFILE NAME (both arms) — `'resume'` re-attache ##### stopRule? -> `readonly` `optional` **stopRule?**: [`StopRule`](#stoprule-1) +> `readonly` `optional` **stopRule?**: [`StopRule`](#stoprule) PROGRESS-derived stop rule (router arm). Ends a run that has stopped learning BEFORE it exhausts a ceiling; it can never keep a run alive past one. Build it with `plateau` / @@ -14314,7 +14052,7 @@ Pass-through subscriber for every coordination bus event (both arms) — the sea ##### priorCoordination? -> `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](#priorcoordination-1) +> `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](#priorcoordination) Questions, findings, and authorized continuation receipts loaded from a prior process. Router arm: questions seed the ledger and all evidence enters the resume brief. External arm: @@ -14322,7 +14060,7 @@ Questions, findings, and authorized continuation receipts loaded from a prior pr ##### loadPriorCoordination? -> `readonly` `optional` **loadPriorCoordination?**: () => `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +> `readonly` `optional` **loadPriorCoordination?**: () => `Promise`\<[`PriorCoordination`](#priorcoordination)\> Deferred owner-scoped replay for a recursive supervisor. Its stable owner is known while the parent authorizes the child, but loading remains asynchronous; Runtime calls this before the @@ -14330,7 +14068,7 @@ nested brain can publish or act on coordination state. ###### Returns -`Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +`Promise`\<[`PriorCoordination`](#priorcoordination)\> ##### finalizer? @@ -20004,29 +19742,6 @@ Provider-neutral conversation record accepted by a tool-loop brain. *** -### ToolLoopChat - -> **ToolLoopChat** = (`messages`, `tools`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; \}\> - -One inference turn over the running conversation + the tool specs → the model's text, any - tool calls, and token usage. The seam every brain satisfies. - -#### Parameters - -##### messages - -`ReadonlyArray`\<[`ToolLoopMessageRecord`](#toolloopmessagerecord)\> - -##### tools - -`ReadonlyArray`\<[`ToolSpec`](#toolspec)\> - -#### Returns - -`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; \}\> - -*** - ### ToolLoopCompactionOptions > **ToolLoopCompactionOptions** = `Omit`\<[`ToolLoopCompaction`](#toolloopcompaction), `"distill"`\> & `object` @@ -23412,25 +23127,6 @@ must not rebuild an Executor around a model transport merely to change `out`. *** -### driverAgent() - -> **driverAgent**(`opts`): [`Agent`](#agent-2)\<`unknown`, `unknown`\> - -Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a -`driverChild` (`driver-executor.ts`) to run it inside a nested scope, recursively. - -#### Parameters - -##### opts - -[`DriverAgentOptions`](#driveragentoptions) - -#### Returns - -[`Agent`](#agent-2)\<`unknown`, `unknown`\> - -*** - ### finalizeBestDelivered() > **finalizeBestDelivered**(`settled`, `blobs`): `Promise`\<`unknown`\> @@ -23926,13 +23622,13 @@ traversal is ledgered and journaled. ### createInbox() -> **createInbox**(): [`Inbox`](#inbox-1) +> **createInbox**(): [`Inbox`](#inbox) Create the worker-side inbox for the down-leg: the driver's `steer_agent` / `answer_question` messages queue here and the worker's loop drains them at step boundaries and before settle. #### Returns -[`Inbox`](#inbox-1) +[`Inbox`](#inbox) *** @@ -24619,7 +24315,7 @@ Build a `ProgressSample` from a scope settlement. The objective is the verdict s ### noProgressFor() -> **noProgressFor**(`opts`): [`StopRule`](#stoprule-1) +> **noProgressFor**(`opts`): [`StopRule`](#stoprule) "Nothing new has happened." Fires when the run has produced no new settled work for `ms`, or no IMPROVEMENT over the last `settles` settlements. @@ -24635,13 +24331,13 @@ on CI is not a run that stopped making progress, and killing it there would defe #### Returns -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) *** ### plateau() -> **plateau**(`opts`): [`StopRule`](#stoprule-1) +> **plateau**(`opts`): [`StopRule`](#stoprule) "The objective has stopped climbing." Fires when the best-so-far curve has risen by no more than `minDelta` across the last `window` settlements. @@ -24658,13 +24354,13 @@ run was flat. #### Returns -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) *** ### allWorkersStalled() -> **allWorkersStalled**(`opts?`): [`StopRule`](#stoprule-1) +> **allWorkersStalled**(`opts?`): [`StopRule`](#stoprule) "Everyone is stuck." Fires when every live worker reads `stalled` — no metered activity for longer than the stall threshold — and none of the tree is merely waiting. @@ -24680,13 +24376,13 @@ reads it. A tree with armed waits never fires: waiting is not stalling. #### Returns -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) *** ### anyOf() -> **anyOf**(...`rules`): [`StopRule`](#stoprule-1) +> **anyOf**(...`rules`): [`StopRule`](#stoprule) Stop when ANY rule stops — the ordinary composition (each rule is a separate reason to end). @@ -24694,17 +24390,17 @@ Stop when ANY rule stops — the ordinary composition (each rule is a separate r ##### rules -...readonly [`StopRule`](#stoprule-1)[] +...readonly [`StopRule`](#stoprule)[] #### Returns -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) *** ### allOf() -> **allOf**(...`rules`): [`StopRule`](#stoprule-1) +> **allOf**(...`rules`): [`StopRule`](#stoprule) Stop only when EVERY rule stops — for a conservative gate that needs corroboration. @@ -24712,11 +24408,11 @@ Stop only when EVERY rule stops — for a conservative gate that needs corrobora ##### rules -...readonly [`StopRule`](#stoprule-1)[] +...readonly [`StopRule`](#stoprule)[] #### Returns -[`StopRule`](#stoprule-1) +[`StopRule`](#stoprule) *** @@ -24760,7 +24456,7 @@ ahead of the worker seam. > **supervise**(`profile`, `task`, `opts`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<`unknown`\>\> -One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. +One-call supervisor: build + run a supervisor from its exact profile. #### Parameters @@ -24826,7 +24522,8 @@ explicit, recorded acknowledgment — never a silent bind. > **supervisorAgent**(`profile`, `deps`): [`Agent`](#agent-2)\<`unknown`, `unknown`\> -Build a supervisor `Agent` from its profile: the brain resolves from `profile.harness` (backend-as-data), the same resolution rule as every worker. +Build a supervisor `Agent` from its profile: the brain resolves from `profile.harness` +(backend-as-data), the same resolution rule as every worker. #### Parameters diff --git a/docs/api/testing.md b/docs/api/testing.md index 35f848f9..a6097ccb 100644 --- a/docs/api/testing.md +++ b/docs/api/testing.md @@ -8,6 +8,1556 @@ ## Interfaces +### DriverAgentOptions + +#### Properties + +##### name + +> `readonly` **name**: `string` + +##### brain + +> `readonly` **brain**: [`ToolLoopChat`](#toolloopchat) + +The driver-LLM seam — ONE inference turn over the conversation + the coordination tool specs + (the canonical `ToolLoopChat`): a scripted mock offline, the router's tool-calling in + production, or a sandboxed harness. The same seam every tool-loop uses; no bespoke shape. + +##### expectedModel? + +> `readonly` `optional` **expectedModel?**: `string` + +Profile-declared model for a production Router brain. When set, every turn must report this +exact provider-observed model before its output is accepted. Omitted by scripted test brains. + +##### blobs + +> `readonly` **blobs**: [`ResultBlobStore`](runtime.md#resultblobstore) + +Shared blob store — `observe_agent` reads settled outputs through it. + +##### makeWorkerAgent + +> `readonly` **makeWorkerAgent**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). + +##### authorizeDownMessage? + +> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](runtime.md#authorizedownmessage) + +##### perWorker + +> `readonly` **perWorker**: [`Budget`](index.md#budget-4) + +Per-child budget reserved from the conserved pool on each spawn. + +##### deliverable? + +> `readonly` `optional` **deliverable?**: [`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> + +Independent completion check for work the driver performs itself. When present, the driver + receives `submit_result`; the first passing submission ends the loop and becomes the output. + +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in + flight (a concurrency fence on top of the conserved-pool fence). Omit/`<= 0` = no cap. + +##### analysts? + +> `readonly` `optional` **analysts?**: [`AnalystRegistry`](index.md#analystregistry) + +The analyst lenses available to the driver. Required for `analyzeOnSettle` (and `run_analyst`). + Unset → no analyst feed (status quo: the driver gets settled outputs, no findings). + +##### analyzeOnSettle? + +> `readonly` `optional` **analyzeOnSettle?**: readonly (`string` \| [`AnalyzeOnSettleRoute`](runtime.md#analyzeonsettleroute))[] + +Analyst kind ids run AUTOMATICALLY when a worker settles `done` — each result re-enters as a + `finding` the driver pulls and composes its next steer from. The UP-leg of the self-improving + loop. Omit/empty = no auto-analysis (status quo). Requires `analysts`. + +##### watchWorkers? + +> `readonly` `optional` **watchWorkers?**: [`WorkerWatchOptions`](runtime.md#workerwatchoptions) + +Run the ONLINE detector panel over each worker's LIVE tool trace and raise a `finding` the + moment it loops/error-storms — mid-run evidence to steer on, not a settle-time post-mortem. + Omit = no online watching. + +##### stallAfterMs? + +> `readonly` `optional` **stallAfterMs?**: `number` + +Idle time after which `observe_agent` reports a worker as stalled (a derived read; nothing is + killed). Omit = the runtime default. + +##### continuityByProfile? + +> `readonly` `optional` **continuityByProfile?**: `Readonly`\<`Record`\<`string`, [`ContinuityMode`](runtime.md#continuitymode)\>\> + +Default continuity per worker PROFILE NAME — `'resume'` makes spawns of that name re-attach + to the node's latest settled worker (see + `CoordinationToolsOptions.continuityByProfile`); `spawn_agent`'s per-call `continuity` + argument overrides. Omit = every spawn fresh (status quo). + +##### systemPrompt + +> `readonly` **systemPrompt**: `string` \| ((`task`) => `string`) + +The driver's stance — a string, or built from the task (the worker-driver prompt / + the generator). INJECTED so the prompt is a pluggable, optimizable role. + +##### nodeTools? + +> `readonly` `optional` **nodeTools?**: readonly [`McpToolDescriptor`](mcp.md#mcptooldescriptor)[] + +Product-selected tools already bound to this exact supervisor node. The same descriptors are + served over MCP for external supervisors; this arm projects them into router ToolSpecs. + +##### extraTools? + +> `readonly` `optional` **extraTools?**: readonly `object`[] + +WORK tools the driver may call DIRECTLY (alongside the coordination verbs) — so the driver is + not a pure manager but a full agent that can ACT (do simple work itself) OR SPAWN (delegate). + Each is a router tool spec; their names must not collide with the coordination verbs. Pair with + `executeExtraTool`. Unset → coordination-only (the prior behavior). + +##### executeExtraTool? + +> `readonly` `optional` **executeExtraTool?**: (`name`, `args`) => `Promise`\<`string` \| `null` \| `undefined`\> + +Runs an `extraTools` call. Returns a string result, or null/undefined to signal "not handled" + so the call falls through to the coordination dispatch. Required iff `extraTools` is set. + +###### Parameters + +###### name + +`string` + +###### args + +`Record`\<`string`, `unknown`\> + +###### Returns + +`Promise`\<`string` \| `null` \| `undefined`\> + +##### maxTurns? + +> `readonly` `optional` **maxTurns?**: `number` + +Max driver turns before the loop force-finalizes on the best settled child. Default 16. + `0` lifts the turn-COUNT cap: the loop is bounded instead by the conserved budget pool, + an absolute deadline, the driver's own stop, and abort (checked in-loop). A finite + anti-runaway tripwire still guards a degenerate driver that loops on a no-spawn tool. + +##### now? + +> `readonly` `optional` **now?**: () => `number` + +Injected clock for the in-loop absolute-deadline guard — keeps the deadline check + deterministic in tests. Defaults to `Date.now`. + +###### Returns + +`number` + +##### stopRule? + +> `readonly` `optional` **stopRule?**: [`StopRule`](runtime.md#stoprule) + +PROGRESS-derived stop (mechanic D). Today a run ends on a ceiling — iterations, tokens, +dollars, deadline, turn cap — which answers "may it continue?" and never "is it still getting +anywhere?". A stop rule reads the run's own progress (best-so-far over settled work, time +since the last settle, the live worker feed) and ends a run that has stopped learning BEFORE +it exhausts a budget. + +Composes with, and can never override, the hard guards: `poolStarved` / `deadlinePassed` / +abort / the driver's own stop are evaluated first, so a rule can only ADD a stop. + +THRESHOLDS are the caller's judgment, not this module's — build the rule with +`plateau({window, minDelta})` / `noProgressFor({...})` / `allWorkersStalled({...})` from +`supervise/stop-rules`. Omit ⇒ ceilings only (unchanged behavior). + +##### onProgressStop? + +> `readonly` `optional` **onProgressStop?**: (`reason`) => `void` + +Called once with the rule's reason when a `stopRule` ends the run — so a caller can record + WHY a run stopped early instead of inferring it from an unexhausted budget. + +###### Parameters + +###### reason + +`string` + +###### Returns + +`void` + +##### compaction? + +> `readonly` `optional` **compaction?**: [`ToolLoopCompactionOptions`](runtime.md#toolloopcompactionoptions) + +Give the driver brain a chapter-lifecycle on its OWN context window. The LLM-brain front doors + lose to a dumb-Ralph respawn because the brain re-bills its whole coordination transcript every + turn — the same context overflow a single steered agent suffers, one level up. With this set, + once the brain's running conversation exceeds `thresholdTokens` it distills the accumulated + history to a compact progress note and continues fresh: the supervisor analog of respawning + against external tracking state, except the live `Scope` roster IS the durable state. Default + off (no behavior change). `distill` defaults to a self-summary authored by the brain combined + with the factual settled-worker roster; override to supply your own. + +##### onEvent? + +> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> + +Pass-through subscriber for every coordination bus event: settled/question/finding, + pre-delivery instruction receipts, and steer/answer delivery outcomes. A durable caller uses + this to append the coordination log. Omit = no observer. + +###### Parameters + +###### event + +[`CoordinationEvent`](index.md#coordinationevent) + +###### record + +[`BusRecord`](runtime.md#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### replaySettlements? + +> `readonly` `optional` **replaySettlements?**: `boolean` + +Re-publish resume-time settlements through the awaited observer before the first brain turn. + +##### priorCoordination? + +> `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](runtime.md#priorcoordination) + +Questions, findings, and authorized continuation receipts loaded from a prior process. + Questions seed the ledger (`list_questions`, blocking-stop policy); all three feed the resume + brief. Continuation receipts are evidence only and are never auto-delivered. Omit = fresh. + +##### finalizer? + +> `readonly` `optional` **finalizer?**: [`SupervisorFinalizer`](index.md#supervisorfinalizer) + +How the settled-worker ledger becomes the run's output. Default `bestDelivered` — the single + highest-scoring DELIVERED child (the exact keep-best every existing caller had). Runs under + the delivered-only invariant (`runFinalizer`): whatever the finalizer, an undelivered or + invalid child's output stays unreachable. + +##### inbox? + +> `readonly` `optional` **inbox?**: [`Inbox`](runtime.md#inbox) + +Optional shared manager inbox used by a wrapper that must accept messages before async node +setup finishes. Ordinary callers omit it and the driver owns a fresh inbox. + +*** + +### RunGraphTestOptions + +Test-only graph options, exported only through the package's explicit `/testing` entry. + +#### Extends + +- [`RunGraphOptions`](runtime.md#rungraphoptions) + +#### Properties + +##### backend? + +> `readonly` `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) + +WHERE worker nodes run — the executor backend. Provide this OR `makeWorkerAgent`. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`backend`](runtime.md#backend-3) + +##### makeWorkerAgent? + +> `readonly` `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Leaf-execution override (offline tests / advanced). `runGraph` still owns node pinning, + directive delivery, and the edge ledger AROUND this seam — only the leaf `act` is yours. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`makeWorkerAgent`](runtime.md#makeworkeragent-1) + +##### router? + +> `readonly` `optional` **router?**: [`RouterTransportConfig`](runtime.md#routertransportconfig) + +The driver brain's router substrate (`profile.harness` omitted or `cli-base`). + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`router`](runtime.md#router-4) + +##### hooks? + +> `readonly` `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) + +Caller-side runtime hooks (telemetry, policy, product extensions). Composed AFTER the + graph's own spawn-binding hook on the SAME event stream — the graph never swallows the + seam supervise() exposes. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`hooks`](runtime.md#hooks-5) + +##### analysts? + +> `readonly` `optional` **analysts?**: [`AnalystRegistry`](index.md#analystregistry) + +The analyst lens registry `analyzes` edges resolve against. ENVIRONMENT — needed only for + lens analysts; an analyzes edge naming a graph NODE as its analyst needs no registry. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`analysts`](runtime.md#analysts-1) + +##### watchWorkers? + +> `readonly` `optional` **watchWorkers?**: [`WorkerWatchOptions`](runtime.md#workerwatchoptions) + +Watch every worker's LIVE tool trace with the online detector panel and raise a `finding` + on the bus the moment one loops or error-storms — forwarded to `supervise()` verbatim (see + `SuperviseOptions.watchWorkers`). Online findings (`analyst: 'online:'`) are bus + events for the driver, not graph edges, so they are never ledgered as traversals. Omit = + off (no online watching, no extra events). + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`watchWorkers`](runtime.md#watchworkers) + +##### registry? + +> `readonly` `optional` **registry?**: [`PromptRegistry`](runtime.md#promptregistry) + +Directive registry. Default: the seeded kernel registry (`kernelPromptRegistry()`). + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`registry`](runtime.md#registry-1) + +##### journal? + +> `readonly` `optional` **journal?**: [`SpawnJournal`](runtime.md#spawnjournal) + +The run journal the edge ledger and every spawn/settle ride. Default: in-memory. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`journal`](runtime.md#journal-1) + +##### blobs? + +> `readonly` `optional` **blobs?**: [`ResultBlobStore`](runtime.md#resultblobstore) + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`blobs`](runtime.md#blobs-1) + +##### runId? + +> `readonly` `optional` **runId?**: `string` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`runId`](runtime.md#runid-10) + +##### perWorker? + +> `readonly` `optional` **perWorker?**: [`Budget`](index.md#budget-4) + +Per-child budget reserved from the conserved pool on each spawn. + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`perWorker`](runtime.md#perworker) + +##### maxTurns? + +> `readonly` `optional` **maxTurns?**: `number` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`maxTurns`](runtime.md#maxturns) + +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`maxLiveWorkers`](runtime.md#maxliveworkers-2) + +##### authorizeMessage? + +> `readonly` `optional` **authorizeMessage?**: (`input`) => [`AuthorizedDownMessage`](runtime.md#authorizeddownmessage) + +Product authority over every steer/answer instruction (the filter seam). `runGraph` observes + what it CHANGES: a narrowed instruction ledgers its steer traversal as `stripped`. + +###### Parameters + +###### input + +[`DownMessageAuthorizationInput`](runtime.md#downmessageauthorizationinput) & `object` + +###### Returns + +[`AuthorizedDownMessage`](runtime.md#authorizeddownmessage) + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`authorizeMessage`](runtime.md#authorizemessage) + +##### signal? + +> `readonly` `optional` **signal?**: `AbortSignal` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`signal`](runtime.md#signal-11) + +##### now? + +> `readonly` `optional` **now?**: () => `number` + +###### Returns + +`number` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`now`](runtime.md#now-4) + +##### otel? + +> `readonly` `optional` **otel?**: `Omit`\<[`SupervisorSpanOptions`](runtime.md#supervisorspanoptions), `"runId"` \| `"now"`\> + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`otel`](runtime.md#otel) + +##### stallAfterMs? + +> `readonly` `optional` **stallAfterMs?**: `number` + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`stallAfterMs`](runtime.md#stallafterms) + +##### allowedModels? + +> `readonly` `optional` **allowedModels?**: readonly `string`[] + +###### Inherited from + +[`RunGraphOptions`](runtime.md#rungraphoptions).[`allowedModels`](runtime.md#allowedmodels-1) + +##### brain + +> `readonly` **brain**: [`ToolLoopChat`](#toolloopchat) + +*** + +### SuperviseTestOptions + +Test-only one-call shape, exported only through the package's explicit `/testing` entry. + +#### Extends + +- [`SuperviseOptions`](runtime.md#superviseoptions) + +#### Properties + +##### budget + +> `readonly` **budget**: [`Budget`](index.md#budget-4) + +The conserved compute pool for the whole run. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`budget`](runtime.md#budget-15) + +##### rootHandle? + +> `readonly` `optional` **rootHandle?**: [`RootHandle`](runtime.md#roothandle-1)\<`unknown`\> + +Caller-created live handle for observing, steering, or cancelling this root manager. Runtime +attaches it before execution and detaches it after the join barrier. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`rootHandle`](runtime.md#roothandle) + +##### signal? + +> `readonly` `optional` **signal?**: `AbortSignal` + +Caller-owned cancellation for the complete recursive run. Aborting it cascades through the +root scope and every live child, including acquisition and backend execution. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`signal`](runtime.md#signal-13) + +##### execution? + +> `readonly` `optional` **execution?**: [`AgentExecutionRef`](runtime.md#agentexecutionref) + +Trusted candidate and pursuit attribution for the root. The runtime derives profile/task +digests itself from the exact detached values it executes. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`execution`](runtime.md#execution-1) + +##### backend? + +> `readonly` `optional` **backend?**: [`ExecutorConfig`](runtime.md#executorconfig) + +WHERE workers run — derives the worker seam. Provide this OR an explicit `makeWorkerAgent`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`backend`](runtime.md#backend-4) + +##### deliverable? + +> `readonly` `optional` **deliverable?**: `string` \| [`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> + +The independent completion check for backend-derived workers and direct supervisor + submissions. Strongly recommended: without it the supervisor cannot submit its own work and + backend-derived workers fall back to their own validity signal. A `string` names an entry in + `registry.deliverables`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`deliverable`](runtime.md#deliverable-4) + +##### resolveDeliverable? + +> `readonly` `optional` **resolveDeliverable?**: (`input`) => [`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> \| `undefined` + +Resolve the completion check for one exact authorized backend-derived leaf. The callback runs +after spawn authorization and driver classification, receives a detached immutable context, +and may return `undefined` to use the run-wide `deliverable`. Driver profiles never call it. + +###### Parameters + +###### input + +[`AuthorizedSpawnContext`](runtime.md#authorizedspawncontext) + +###### Returns + +[`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> \| `undefined` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`resolveDeliverable`](runtime.md#resolvedeliverable) + +##### registry? + +> `readonly` `optional` **registry?**: [`SuperviseRegistry`](runtime.md#superviseregistry) + +Name→value tables for the four code-valued options, so a recorded run configuration can name + them instead of carrying closures. See [SuperviseRegistry](runtime.md#superviseregistry). + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`registry`](runtime.md#registry-3) + +##### coordination? + +> `readonly` `optional` **coordination?**: [`CoordinationBinding`](runtime.md#coordinationbinding) + +Where the coordination MCP binds when the supervisor is harness-driven. Omit = an ephemeral + port on `127.0.0.1`, which an off-host root cannot reach. A non-loopback host is refused + unless `allowUnauthenticatedRemote` acknowledges that the verbs are unauthenticated. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`coordination`](runtime.md#coordination) + +##### makeWorkerAgent? + +> `readonly` `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. + This is caller-owned execution: profile security, spawn authorization, and recursive-driver + selection below apply only to the backend-derived worker path. `authorizeMessage` still + governs continuations sent through Runtime's coordination tools. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`makeWorkerAgent`](runtime.md#makeworkeragent-2) + +##### driverBackend? + +> `readonly` `optional` **driverBackend?**: [`ExecutorConfig`](runtime.md#executorconfig) + +Run harness-brained supervisors here. Automatic execution supports a local `bridge`; a remote + sandbox requires an explicit `driveHarness` with a reachable coordination relay or tunnel. + Defaults to `backend`; separate it when managers and workers use different services. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`driverBackend`](runtime.md#driverbackend) + +##### profileSecurity? + +> `readonly` `optional` **profileSecurity?**: `AgentProfileSecurityPolicy` + +Security policy applied to every manager-authored child profile before budget reservation. + The default blocks local and remote MCP, hooks, and connection grants. Pass an explicit + allowlist to grant remote MCP hosts or other author-controlled capabilities. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`profileSecurity`](runtime.md#profilesecurity) + +##### authorizeSpawn? + +> `readonly` `optional` **authorizeSpawn?**: (`input`) => [`AuthorizedSpawn`](runtime.md#authorizedspawn) + +Product authority over one complete manager-authored spawn. The callback sees the detached, + immutable profile, task, budget, label, and key together, so approving a profile cannot + authorize a different task. Return the exact allowed profile (which may be narrowed) plus + trusted candidate/pursuit attribution, or throw to refuse the whole spawn before reservation. + +###### Parameters + +###### input + +###### profile + +`AgentProfile` + +###### parent + +`AgentProfile` + +###### parentIdentity + +[`NodeExecutionIdentity`](runtime.md#nodeexecutionidentity) + +Trusted identity of the manager authorizing this exact child. + +###### parentNodeId + +`string` + +Concrete manager node; never accepted from model-authored tool arguments. + +###### assignmentId + +`string` + +Stable manager-scoped assignment, including deterministic unkeyed siblings. + +###### task + +`unknown` + +###### budget + +[`Budget`](index.md#budget-4) + +###### label + +`string` + +###### key? + +`string` + +###### depth + +`number` + +###### Returns + +[`AuthorizedSpawn`](runtime.md#authorizedspawn) + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`authorizeSpawn`](runtime.md#authorizespawn) + +##### authorizeMessage? + +> `readonly` `optional` **authorizeMessage?**: (`input`) => [`AuthorizedDownMessage`](runtime.md#authorizeddownmessage) + +Product authority over every continuation sent to a live child. When spawn authorization is +enabled, omitting this refuses steer/answer instructions instead of silently extending the +authorized task. The exact worker identity and detached bytes are recorded before delivery. + +###### Parameters + +###### input + +[`DownMessageAuthorizationInput`](runtime.md#downmessageauthorizationinput) & `object` + +###### Returns + +[`AuthorizedDownMessage`](runtime.md#authorizeddownmessage) + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`authorizeMessage`](runtime.md#authorizemessage-1) + +##### isDriverProfile? + +> `readonly` `optional` **isDriverProfile?**: (`input`) => `boolean` + +Decide whether an authorized child becomes another supervisor. By default only + `metadata.role === 'driver'` does. Products receive the same frozen post-authorization + context as `resolveDeliverable`, so trusted execution/assignment authority can override + model-authored metadata without a side channel. + +###### Parameters + +###### input + +[`AuthorizedSpawnContext`](runtime.md#authorizedspawncontext) + +###### Returns + +`boolean` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`isDriverProfile`](runtime.md#isdriverprofile) + +##### router? + +> `readonly` `optional` **router?**: [`RouterTransportConfig`](runtime.md#routertransportconfig) + +The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's + model wins. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`router`](runtime.md#router-5) + +##### driveHarness? + +> `readonly` `optional` **driveHarness?**: [`DriveHarness`](runtime.md#driveharness-1) + +Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a + caller-owned override for a local bridge. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`driveHarness`](runtime.md#driveharness) + +##### resolveDriveHarness? + +> `readonly` `optional` **resolveDriveHarness?**: [`ResolveDriveHarness`](runtime.md#resolvedriveharness-1) + +Resolve one custom external-harness session per trusted manager identity. Use this instead of +`driveHarness` when recursive managers must be independently steerable. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`resolveDriveHarness`](runtime.md#resolvedriveharness) + +##### driveHarnessMaterialization? + +> `readonly` `optional` **driveHarnessMaterialization?**: [`ProfileMaterializationContract`](agent.md#profilematerializationcontract) + +Required with a custom `driveHarness` or `resolveDriveHarness`: declares which complete +AgentProfile axes that path really applies. Built-in bridge driving supplies its own +full-profile contract. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`driveHarnessMaterialization`](runtime.md#driveharnessmaterialization) + +##### resolveSupervisorTools? + +> `readonly` `optional` **resolveSupervisorTools?**: [`ResolveSupervisorTools`](runtime.md#resolvesupervisortools-1) + +Resolve product-owned tools from the exact trusted manager context. The same descriptors and +handlers are bound to router and external-harness managers; resolution happens once per node. +Each handler receives that manager scope's live cancellation signal in its trusted invocation +context, including recursive parent and root cascades. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`resolveSupervisorTools`](runtime.md#resolvesupervisortools) + +##### onCoordinationEvent? + +> `readonly` `optional` **onCoordinationEvent?**: (`context`, `eventId`, `record`) => `void` \| `Promise`\<`void`\> + +Awaited product transaction hook for every coordination record. `eventId` is stable across a +lost acknowledgement and durable restart; the record is not pull-visible until this commits. + +###### Parameters + +###### context + +[`SupervisorNodeContext`](runtime.md#supervisornodecontext) + +###### eventId + +`` `sha256:${string}` `` + +###### record + +[`BusRecord`](runtime.md#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + +###### Returns + +`void` \| `Promise`\<`void`\> + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`onCoordinationEvent`](runtime.md#oncoordinationevent) + +##### extraTools? + +> `readonly` `optional` **extraTools?**: readonly `object`[] + +WORK tools the supervisor may call DIRECTLY — so a recursive atom can ACT (do simple work + itself) OR SPAWN (delegate when it needs parallelism), not be a pure manager. Pair with + `executeExtraTool`. Router arm only (`profile.harness` omitted or `cli-base`). + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`extraTools`](runtime.md#extratools) + +##### executeExtraTool? + +> `readonly` `optional` **executeExtraTool?**: (`name`, `args`) => `Promise`\<`string` \| `null` \| `undefined`\> + +Runs an `extraTools` call; null/undefined falls through to the coordination dispatch. + +###### Parameters + +###### name + +`string` + +###### args + +`Record`\<`string`, `unknown`\> + +###### Returns + +`Promise`\<`string` \| `null` \| `undefined`\> + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`executeExtraTool`](runtime.md#executeextratool) + +##### perWorker? + +> `readonly` `optional` **perWorker?**: [`Budget`](index.md#budget-4) + +Per-child budget reserved on each spawn. Defaults to a quarter of the pool's tokens. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`perWorker`](runtime.md#perworker-1) + +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +Hard cap on simultaneously executing spawned workers across the WHOLE recursive tree. The + root is excluded; nested drivers and leaves share one allocation, so recursion cannot multiply + the cap. Omit/`<= 0` = no cap (the conserved pool stays the only bound). + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`maxLiveWorkers`](runtime.md#maxliveworkers-4) + +##### analysts? + +> `readonly` `optional` **analysts?**: `string` \| [`AnalystRegistry`](index.md#analystregistry) + +Analyst lenses available to the driver. Required for `analyzeOnSettle`. Unset → status quo + (the driver receives settled worker outputs, no analyst findings). A `string` names an entry in + `registry.analysts`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`analysts`](runtime.md#analysts-3) + +##### analyzeOnSettle? + +> `readonly` `optional` **analyzeOnSettle?**: readonly (`string` \| [`AnalyzeOnSettleRoute`](runtime.md#analyzeonsettleroute))[] + +Analyst kind ids run AUTOMATICALLY when a worker settles `done` — each re-enters as a `finding` + the driver pulls (`await_event`) and composes its next steer from. The self-improving UP-leg, + threaded to the driver at this level (propagate to sub-drivers via a recursive `makeWorkerAgent`). + Omit/empty = status quo (no analyst feed). Requires `analysts`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`analyzeOnSettle`](runtime.md#analyzeonsettle) + +##### watchWorkers? + +> `readonly` `optional` **watchWorkers?**: [`WorkerWatchOptions`](runtime.md#workerwatchoptions) + +Watch every worker's LIVE tool trace with the online detector panel and raise a `finding` the +moment one loops or error-storms — so the supervisor learns it mid-run (via `await_event`) +instead of at settle. Pairs with a steerable worker: the finding is the evidence, `steer_agent` +is the correction. Requires a backend whose executor exposes a trace source (the steerable +sandbox worker and the pi wrapper do); other runtimes are simply not watched. + +Omit = off (status quo — no online watching, no extra events). + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`watchWorkers`](runtime.md#watchworkers-1) + +##### stallAfterMs? + +> `readonly` `optional` **stallAfterMs?**: `number` + +Idle time after which `observe_agent` reports a running worker as `stalled`. A derived read + at observation time — nothing is killed or retried. Omit = the runtime default. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`stallAfterMs`](runtime.md#stallafterms-3) + +##### continuityByProfile? + +> `readonly` `optional` **continuityByProfile?**: `Readonly`\<`Record`\<`string`, [`ContinuityMode`](runtime.md#continuitymode)\>\> + +Default continuity per worker PROFILE NAME: `'resume'` makes each spawn of that name after + the first re-attach to the node's most recent SETTLED worker — a NEW live worker whose spawn + context carries the prior worker's identity (`WorkerSpawnContext.resume`), which the executor + seam re-attaches with. `spawn_agent`'s per-call `continuity` argument overrides in either + direction; `runGraph` derives this from delegates-edge `continuity`. Omit = every spawn is + `'fresh'` (status quo). See `CoordinationToolsOptions.continuityByProfile` for the + refusal semantics (no-prior / while-live / with-key) and the process-local resume boundary. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`continuityByProfile`](runtime.md#continuitybyprofile) + +##### blobs? + +> `readonly` `optional` **blobs?**: [`ResultBlobStore`](runtime.md#resultblobstore) + +Worker output store. Defaults to in-memory. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`blobs`](runtime.md#blobs-4) + +##### runDir? + +> `readonly` `optional` **runDir?**: `string` + +Make the run DURABLE: journal + result blobs + the coordination side-log are file-backed under +this directory (`createFileRunContext`), fsynced per write, and the supervisor reads the prior +tree first. Re-running with the same `runDir` AND the same `runId` resumes only when the exact +root profile/task identity and declared budget match. The original absolute deadline and prior +measured spend are restored before new admission. The built-in driver is resume-aware: children +that already settled, including their exact execution identities, are replayed onto +`Scope.resume` (and into the driver's settled ledger + its first context), keyed assignments +(`spawn_agent`'s `key`) resolve to their committed results instead of re-running, pending +waits re-arm on their original deadlines, and the coordination log loads prior questions, +findings, and instruction receipts. The router arm receives all three in its resume brief; the +external arm seeds prior questions while findings and receipts remain in the durable log. +Instruction receipts are evidence and are never delivered automatically to a replacement +worker. The final result spans both processes' work. Unset = in-memory, fresh every call. + +The boundary that remains: work that was IN FLIGHT when the process died is not recovered — +the built-in executors cannot re-attach to a dead process's executions. Each such assignment +resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, +and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining +capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. + +`runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single +resumable run per directory but collides across concurrent runs sharing one `runDir`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`runDir`](runtime.md#rundir-1) + +##### journal? + +> `readonly` `optional` **journal?**: [`SpawnJournal`](runtime.md#spawnjournal) + +Override the spawn journal directly (advanced; `runDir` is the ordinary durable path). Pair + with `blobs` — a journal whose result payloads live in a different store cannot replay. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`journal`](runtime.md#journal-4) + +##### probes? + +> `readonly` `optional` **probes?**: `string` \| [`WaitProbeRegistry`](runtime.md#waitproberegistry) + +Predicate registry for `poll` wait-states (`Scope.wait`). A `poll` names its predicate so the + wait survives a restart; this is what the name resolves against. Unset ⇒ `poll` waits are + refused `unknown-probe` and `timer` waits still work. A `string` names an entry in + `registry.probes`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`probes`](runtime.md#probes-2) + +##### stopRule? + +> `readonly` `optional` **stopRule?**: [`StopRule`](runtime.md#stoprule) + +PROGRESS-derived stop rule (router-brained supervisor). Ends a run that has stopped LEARNING +before it exhausts a ceiling — the answer to "a run should end because it is done or stuck, +not because it ran out". It composes with the budget guards and can never override one. + +Build it from `supervise/stop-rules`: `plateau({window, minDelta})`, +`noProgressFor({ms, settles})`, `allWorkersStalled({...})`, combined with `anyOf`/`allOf`. The +thresholds are policy and stay with you; the enforcement lives in the runtime. Omit = ceilings +only (unchanged behavior). + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`stopRule`](runtime.md#stoprule-1) + +##### onProgressStop? + +> `readonly` `optional` **onProgressStop?**: (`reason`) => `void` + +One-shot notification of WHY a `stopRule` ended the run — so a caller records the reason + instead of inferring an early stop from an unexhausted budget. + +###### Parameters + +###### reason + +`string` + +###### Returns + +`void` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`onProgressStop`](runtime.md#onprogressstop) + +##### maxDepth? + +> `readonly` `optional` **maxDepth?**: `number` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`maxDepth`](runtime.md#maxdepth-2) + +##### maxTurns? + +> `readonly` `optional` **maxTurns?**: `number` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`maxTurns`](runtime.md#maxturns-2) + +##### compaction? + +> `readonly` `optional` **compaction?**: [`ToolLoopCompactionOptions`](runtime.md#toolloopcompactionoptions) + +Give the supervisor brain a chapter-lifecycle on its OWN context window (router arm only): once + its coordination transcript exceeds `thresholdTokens` it distills to a compact progress note and + continues, instead of re-billing the whole transcript every turn (the cost that makes the LLM-brain + front door lose to a dumb-Ralph respawn). The live `Scope` roster is the durable state across + chapters. Default off. `distill` defaults to a brain self-summary + the settled-worker roster. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`compaction`](runtime.md#compaction) + +##### runId? + +> `readonly` `optional` **runId?**: `string` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`runId`](runtime.md#runid-14) + +##### now? + +> `readonly` `optional` **now?**: () => `number` + +###### Returns + +`number` + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`now`](runtime.md#now-10) + +##### allowedModels? + +> `readonly` `optional` **allowedModels?**: readonly `string`[] + +Restrict the run to this subset of models. When set, every configured model — the + supervisor router model, the profile's model, and the backend's model — must be a member, + or `supervise()` throws a `ConfigError` before any compute is spent. Unset = unrestricted. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`allowedModels`](runtime.md#allowedmodels-2) + +##### finalizer? + +> `readonly` `optional` **finalizer?**: `string` \| [`SupervisorFinalizer`](index.md#supervisorfinalizer) + +How the settled-worker ledger becomes the run's output. Default `bestDelivered` — the single + highest-scoring DELIVERED child (the exact behavior every existing caller had). Alternatives: + `collectDelivered` (every verified distinct output with provenance — a Pareto set / recorded + disagreement) or a custom `SupervisorFinalizer`. Whatever the finalizer, it operates on + structurally DELIVERED outputs only — an undelivered or invalid child stays ineligible. A + `string` names an entry in `registry.finalizers`. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`finalizer`](runtime.md#finalizer) + +##### hooks? + +> `readonly` `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) + +Lifecycle observers for the whole recursive tree (`Scope` re-seeds them into every nested + scope). Composed with the `otel` recorder below when both are set. Omit = no observers, which + is the behavior every existing caller has. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`hooks`](runtime.md#hooks-8) + +##### otel? + +> `readonly` `optional` **otel?**: `Omit`\<[`SupervisorSpanOptions`](runtime.md#supervisorspanoptions), `"runId"` \| `"now"`\> + +OPT-IN OTLP tracing: emit one span per supervised node (opened at spawn, closed at settle, +parented to its parent node's span) plus an `LLM` child span per metered driver turn, so the +tree is readable by any trace viewer instead of only by a journal parser. See `otel-spans.ts`. + +Omit and the run emits nothing, allocates no recorder, and installs no hook — telemetry is +never a default. Present with no reachable endpoint (no `exportConfig.endpoint` and no +`OTEL_EXPORTER_OTLP_ENDPOINT`) is also a no-op. The spawn journal is untouched either way: +spans are telemetry, never the replay/resume record. + +###### Inherited from + +[`SuperviseOptions`](runtime.md#superviseoptions).[`otel`](runtime.md#otel-1) + +##### brain + +> `readonly` **brain**: [`ToolLoopChat`](#toolloopchat) + +*** + +### SupervisorAgentTestDeps + +Test-only dependency shape. It is exported only through the package's explicit `/testing` +entry; production supervisor surfaces cannot replace profile-derived model execution. + +#### Extends + +- [`SupervisorAgentDeps`](runtime.md#supervisoragentdeps) + +#### Properties + +##### blobs + +> `readonly` **blobs**: [`ResultBlobStore`](runtime.md#resultblobstore) + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`blobs`](runtime.md#blobs-5) + +##### makeWorkerAgent + +> `readonly` **makeWorkerAgent**: [`MakeWorkerAgent`](runtime.md#makeworkeragent) + +Resolve a spawned worker `profile` to a leaf agent — the recursion seam (same for both arms). + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`makeWorkerAgent`](runtime.md#makeworkeragent-3) + +##### authorizeDownMessage? + +> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](runtime.md#authorizedownmessage) + +Product authorization for every down-leg continuation to a child. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`authorizeDownMessage`](runtime.md#authorizedownmessage-1) + +##### perWorker + +> `readonly` **perWorker**: [`Budget`](index.md#budget-4) + +Per-child budget reserved from the conserved pool on each spawn. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`perWorker`](runtime.md#perworker-2) + +##### deliverable? + +> `readonly` `optional` **deliverable?**: [`DeliverableSpec`](runtime.md#deliverablespec)\<`unknown`\> + +Independent completion check for direct driver work (`submit_result`). + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`deliverable`](runtime.md#deliverable-5) + +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +Hard cap on simultaneously-LIVE workers across both arms — `spawn_agent` fails closed once + this many are in flight (a concurrency fence on top of the conserved-pool fence; bounds live + boxes/sandboxes, not total work). Omit/`<= 0` = no cap. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`maxLiveWorkers`](runtime.md#maxliveworkers-5) + +##### router? + +> `readonly` `optional` **router?**: [`RouterTransportConfig`](runtime.md#routertransportconfig) + +Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The + profile's model wins. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`router`](runtime.md#router-6) + +##### driveHarness? + +> `readonly` `optional` **driveHarness?**: [`DriveHarness`](runtime.md#driveharness-1) + +Required to run an external-harness supervisor: runs the harness as the driver. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`driveHarness`](runtime.md#driveharness-3) + +##### nodeContext? + +> `readonly` `optional` **nodeContext?**: [`SupervisorNodeContextSeed`](runtime.md#supervisornodecontextseed) + +Trusted identity for this manager. Required with node-scoped tools or observation. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`nodeContext`](runtime.md#nodecontext) + +##### resolveSupervisorTools? + +> `readonly` `optional` **resolveSupervisorTools?**: [`ResolveSupervisorTools`](runtime.md#resolvesupervisortools-1) + +Resolve product-owned tools for this exact manager. Static `extraTools` remain a router-only + compatibility seam and deliberately receive no new recursive authority. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`resolveSupervisorTools`](runtime.md#resolvesupervisortools-2) + +##### observeNodeEvent? + +> `readonly` `optional` **observeNodeEvent?**: [`ObserveSupervisorNodeEvent`](runtime.md#observesupervisornodeevent) + +Awaited product observation, enriched with this manager's actual live node context. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`observeNodeEvent`](runtime.md#observenodeevent) + +##### replaySettlements? + +> `readonly` `optional` **replaySettlements?**: `boolean` + +Replay resume-time settlements through `observeNodeEvent` before the manager starts. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`replaySettlements`](runtime.md#replaysettlements) + +##### extraTools? + +> `readonly` `optional` **extraTools?**: readonly `object`[] + +WORK tools the supervisor may call DIRECTLY (router arm) — so it can do simple work ITSELF and + only delegate when it needs parallelism. Pair with `executeExtraTool`. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`extraTools`](runtime.md#extratools-1) + +##### executeExtraTool? + +> `readonly` `optional` **executeExtraTool?**: (`name`, `args`) => `Promise`\<`string` \| `null` \| `undefined`\> + +Runs an `extraTools` call; null/undefined falls through to the coordination dispatch. + +###### Parameters + +###### name + +`string` + +###### args + +`Record`\<`string`, `unknown`\> + +###### Returns + +`Promise`\<`string` \| `null` \| `undefined`\> + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`executeExtraTool`](runtime.md#executeextratool-1) + +##### analysts? + +> `readonly` `optional` **analysts?**: [`AnalystRegistry`](index.md#analystregistry) + +Analyst lenses available to the driver (both arms). Required for `analyzeOnSettle`. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`analysts`](runtime.md#analysts-4) + +##### analyzeOnSettle? + +> `readonly` `optional` **analyzeOnSettle?**: readonly (`string` \| [`AnalyzeOnSettleRoute`](runtime.md#analyzeonsettleroute))[] + +Analyst kinds run on each worker-settle → a `finding` the driver composes its next steer from + (the self-improving UP-leg). Unset/empty = status quo (no analyst feed). Requires `analysts`. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`analyzeOnSettle`](runtime.md#analyzeonsettle-1) + +##### watchWorkers? + +> `readonly` `optional` **watchWorkers?**: [`WorkerWatchOptions`](runtime.md#workerwatchoptions) + +Run the ONLINE detector panel over each worker's LIVE tool trace (both arms) so the driver + learns a worker is looping mid-run instead of at settle. Omit = no online watching. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`watchWorkers`](runtime.md#watchworkers-2) + +##### stallAfterMs? + +> `readonly` `optional` **stallAfterMs?**: `number` + +Idle time after which `observe_agent` reports a worker as stalled. Omit = runtime default. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`stallAfterMs`](runtime.md#stallafterms-4) + +##### continuityByProfile? + +> `readonly` `optional` **continuityByProfile?**: `Readonly`\<`Record`\<`string`, [`ContinuityMode`](runtime.md#continuitymode)\>\> + +Default continuity per worker PROFILE NAME (both arms) — `'resume'` re-attaches spawns of + that name to the node's latest settled worker; `spawn_agent`'s per-call `continuity` + overrides. Omit = every spawn fresh (status quo). + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`continuityByProfile`](runtime.md#continuitybyprofile-1) + +##### stopRule? + +> `readonly` `optional` **stopRule?**: [`StopRule`](runtime.md#stoprule) + +PROGRESS-derived stop rule (router arm). Ends a run that has stopped learning BEFORE it + exhausts a ceiling; it can never keep a run alive past one. Build it with `plateau` / + `noProgressFor` / `allWorkersStalled` from `supervise/stop-rules` — the thresholds are the + caller's judgment. Omit = ceilings only. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`stopRule`](runtime.md#stoprule-2) + +##### onProgressStop? + +> `readonly` `optional` **onProgressStop?**: (`reason`) => `void` + +One-shot notification of WHY a `stopRule` ended the run. + +###### Parameters + +###### reason + +`string` + +###### Returns + +`void` + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`onProgressStop`](runtime.md#onprogressstop-1) + +##### maxTurns? + +> `readonly` `optional` **maxTurns?**: `number` + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`maxTurns`](runtime.md#maxturns-3) + +##### compaction? + +> `readonly` `optional` **compaction?**: [`ToolLoopCompactionOptions`](runtime.md#toolloopcompactionoptions) + +Give the supervisor brain a chapter-lifecycle on its OWN context window (router arm only) — it + distills its coordination transcript to a compact progress note once it exceeds the threshold, + instead of re-billing the whole thing every turn. See `DriverAgentOptions.compaction`. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`compaction`](runtime.md#compaction-1) + +##### onEvent? + +> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> + +Pass-through subscriber for every coordination bus event (both arms) — the seam a durable + caller hooks its coordination log onto. + +###### Parameters + +###### event + +[`CoordinationEvent`](index.md#coordinationevent) + +###### record + +[`BusRecord`](runtime.md#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + +###### Returns + +`void` \| `Promise`\<`void`\> + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`onEvent`](runtime.md#onevent) + +##### priorCoordination? + +> `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](runtime.md#priorcoordination) + +Questions, findings, and authorized continuation receipts loaded from a prior process. + Router arm: questions seed the ledger and all evidence enters the resume brief. External arm: + questions seed the ledger; receipts remain durable evidence and are never auto-delivered. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`priorCoordination`](runtime.md#priorcoordination-1) + +##### loadPriorCoordination? + +> `readonly` `optional` **loadPriorCoordination?**: () => `Promise`\<[`PriorCoordination`](runtime.md#priorcoordination)\> + +Deferred owner-scoped replay for a recursive supervisor. Its stable owner is known while the +parent authorizes the child, but loading remains asynchronous; Runtime calls this before the +nested brain can publish or act on coordination state. + +###### Returns + +`Promise`\<[`PriorCoordination`](runtime.md#priorcoordination)\> + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`loadPriorCoordination`](runtime.md#loadpriorcoordination) + +##### finalizer? + +> `readonly` `optional` **finalizer?**: [`SupervisorFinalizer`](index.md#supervisorfinalizer) + +How the settled ledger becomes the run's output (both arms). Default `bestDelivered` — the + exact keep-best every existing caller had. Always runs under the delivered-only invariant. + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`finalizer`](runtime.md#finalizer-1) + +##### coordination? + +> `readonly` `optional` **coordination?**: [`CoordinationBinding`](runtime.md#coordinationbinding) + +Where the coordination MCP binds (external arm). Omit = an ephemeral loopback port, which is + unreachable from an off-host harness. A non-loopback host fails closed — see + [assertCoordinationBinding](runtime.md#assertcoordinationbinding). + +###### Inherited from + +[`SupervisorAgentDeps`](runtime.md#supervisoragentdeps).[`coordination`](runtime.md#coordination-1) + +##### brain + +> `readonly` **brain**: [`ToolLoopChat`](#toolloopchat) + +*** + +### ToolLoopCallContext + +Runtime-owned identity and cancellation for one logical inference call. The wrapper is frozen +before dispatch; a transport may observe the signal but cannot replace the authority it names. + +#### Properties + +##### signal + +> `readonly` **signal**: `AbortSignal` + +##### callId + +> `readonly` **callId**: `string` + +##### correlationId + +> `readonly` **correlationId**: `string` + +*** + ### AgentProfileImprovementFixture Complete private state for exercising profile activation and restore in consumer tests. @@ -32,6 +1582,33 @@ Complete private state for exercising profile activation and restore in consumer ## Type Aliases +### ToolLoopChat + +> **ToolLoopChat** = (`messages`, `tools`, `context?`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](runtime.md#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> + +One inference turn over the running conversation + the tool specs → the model's text, any + tool calls, and token usage. The seam every brain satisfies. + +#### Parameters + +##### messages + +`ReadonlyArray`\<[`ToolLoopMessageRecord`](runtime.md#toolloopmessagerecord)\> + +##### tools + +`ReadonlyArray`\<[`ToolSpec`](runtime.md#toolspec)\> + +##### context? + +[`ToolLoopCallContext`](#toolloopcallcontext) + +#### Returns + +`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](runtime.md#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> + +*** + ### AgentProfileImprovementProposalFixture > **AgentProfileImprovementProposalFixture** = `Omit`\<`AgentImprovementProposal`, `"evaluation"`\> & `object` @@ -46,6 +1623,95 @@ A proposal produced by Runtime's opaque profile-improvement path. ## Functions +### driverAgent() + +> **driverAgent**(`opts`): [`Agent`](runtime.md#agent-2)\<`unknown`, `unknown`\> + +Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a +`driverChild` (`driver-executor.ts`) to run it inside a nested scope, recursively. + +#### Parameters + +##### opts + +[`DriverAgentOptions`](#driveragentoptions) + +#### Returns + +[`Agent`](runtime.md#agent-2)\<`unknown`, `unknown`\> + +*** + +### runGraphWithTestBrain() + +> **runGraphWithTestBrain**(`graph`, `opts`): `Promise`\<[`GraphResult`](runtime.md#graphresult)\<`unknown`\>\> + +Deterministic scripted-brain path for graph tests. Not exported from Runtime's main entry. + +#### Parameters + +##### graph + +[`AgentGraph`](runtime.md#agentgraph) + +##### opts + +[`RunGraphTestOptions`](#rungraphtestoptions) + +#### Returns + +`Promise`\<[`GraphResult`](runtime.md#graphresult)\<`unknown`\>\> + +*** + +### superviseWithTestBrain() + +> **superviseWithTestBrain**(`profile`, `task`, `opts`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<`unknown`\>\> + +Deterministic scripted-brain path for tests. Not exported from Runtime's main entry. + +#### Parameters + +##### profile + +`AgentProfile` + +##### task + +`unknown` + +##### opts + +[`SuperviseTestOptions`](#supervisetestoptions) + +#### Returns + +`Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<`unknown`\>\> + +*** + +### supervisorAgentWithTestBrain() + +> **supervisorAgentWithTestBrain**(`profile`, `deps`): [`Agent`](runtime.md#agent-2)\<`unknown`, `unknown`\> + +Scripted-brain construction for deterministic tests. Not exported from Runtime's main entry. + +#### Parameters + +##### profile + +`AgentProfile` + +##### deps + +[`SupervisorAgentTestDeps`](#supervisoragenttestdeps) + +#### Returns + +[`Agent`](runtime.md#agent-2)\<`unknown`, `unknown`\> + +*** + ### loadAgentImprovementProposalFixture() > **loadAgentImprovementProposalFixture**(): `AgentImprovementProposal` diff --git a/examples/graphs/analyst-agent-review.ts b/examples/graphs/analyst-agent-review.ts index 91bec09e..6ea6b02f 100644 --- a/examples/graphs/analyst-agent-review.ts +++ b/examples/graphs/analyst-agent-review.ts @@ -16,18 +16,14 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' -import { - type AgentGraph, - promptHandle, - type RunGraphOptions, - runGraph, -} from '@tangle-network/agent-runtime/kernel' +import { type AgentGraph, promptHandle } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') const report = promptHandle('analyzes/findings-report/v1') -export function analystAgentReview(): { graph: AgentGraph; opts: RunGraphOptions } { +export function analystAgentReview(): { graph: AgentGraph; opts: RunGraphTestOptions } { // ── The topology: plain data (the analyst is the 'reviewer' NODE, not a registry lens) ── const graph: AgentGraph = { nodes: [ @@ -53,7 +49,7 @@ export function analystAgentReview(): { graph: AgentGraph; opts: RunGraphOptions } const received: AgentProfile[] = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'rev', makeWorkerAgent: leafSeam(received, { implementer: { withTrace: true }, @@ -83,7 +79,7 @@ export function analystAgentReview(): { graph: AgentGraph; opts: RunGraphOptions export async function main(): Promise { const { graph, opts } = analystAgentReview() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('analyst-agent-review', res) } diff --git a/examples/graphs/best-of-n.ts b/examples/graphs/best-of-n.ts index c8b66db8..f4546341 100644 --- a/examples/graphs/best-of-n.ts +++ b/examples/graphs/best-of-n.ts @@ -11,17 +11,13 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' -import { - type AgentGraph, - promptHandle, - type RunGraphOptions, - runGraph, -} from '@tangle-network/agent-runtime/kernel' +import { type AgentGraph, promptHandle } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') -export function bestOfN(): { graph: AgentGraph; opts: RunGraphOptions } { +export function bestOfN(): { graph: AgentGraph; opts: RunGraphTestOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ @@ -41,7 +37,7 @@ export function bestOfN(): { graph: AgentGraph; opts: RunGraphOptions } { } const received: AgentProfile[] = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'bon', maxLiveWorkers: 2, makeWorkerAgent: leafSeam(received, { @@ -73,7 +69,7 @@ export function bestOfN(): { graph: AgentGraph; opts: RunGraphOptions } { export async function main(): Promise { const { graph, opts } = bestOfN() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('best-of-n', res) } diff --git a/examples/graphs/collaborates-review-loop.ts b/examples/graphs/collaborates-review-loop.ts index af033615..5dd38151 100644 --- a/examples/graphs/collaborates-review-loop.ts +++ b/examples/graphs/collaborates-review-loop.ts @@ -23,9 +23,8 @@ import { type AgentGraph, type AnalystRegistry, promptHandle, - type RunGraphOptions, - runGraph, } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -43,7 +42,7 @@ const analysts: AnalystRegistry = { : { verdict: 'needs-changes', brief: 'add the missing tests' }, } -export function collaboratesReviewLoop(): { graph: AgentGraph; opts: RunGraphOptions } { +export function collaboratesReviewLoop(): { graph: AgentGraph; opts: RunGraphTestOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ @@ -68,7 +67,7 @@ export function collaboratesReviewLoop(): { graph: AgentGraph; opts: RunGraphOpt } const received: AgentProfile[] = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'collab', analysts, makeWorkerAgent: leafSeam(received, { @@ -133,7 +132,7 @@ export function collaboratesReviewLoop(): { graph: AgentGraph; opts: RunGraphOpt export async function main(): Promise { const { graph, opts } = collaboratesReviewLoop() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('collaborates-review-loop', res) } diff --git a/examples/graphs/shared.ts b/examples/graphs/shared.ts index 6a0d56de..2f5389db 100644 --- a/examples/graphs/shared.ts +++ b/examples/graphs/shared.ts @@ -23,10 +23,10 @@ import { type ExecutorResult, type GraphResult, type MakeWorkerAgent, - type ToolLoopChat, type TraceSource, type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' +import type { ToolLoopChat } from '../../src/testing' /** Exact profile identity for the examples' injected, deterministic executors. */ export function offlineProfile(name: string, systemPrompt: string): AgentProfile { diff --git a/examples/graphs/shot-loop-resumed.ts b/examples/graphs/shot-loop-resumed.ts index ddad4015..15059729 100644 --- a/examples/graphs/shot-loop-resumed.ts +++ b/examples/graphs/shot-loop-resumed.ts @@ -18,17 +18,16 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { type AgentGraph, promptHandle, - type RunGraphOptions, - runGraph, type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') export function shotLoopResumed(): { graph: AgentGraph - opts: RunGraphOptions + opts: RunGraphTestOptions contexts: Array } { // ── The topology: plain data — continuity is one field on the edge ── @@ -56,7 +55,7 @@ export function shotLoopResumed(): { const received: AgentProfile[] = [] const contexts: Array = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'rshots', makeWorkerAgent: leafSeam( received, @@ -108,7 +107,7 @@ export function shotLoopResumed(): { export async function main(): Promise { const { graph, opts, contexts } = shotLoopResumed() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('shot-loop-resumed', res) console.log('SPAWN CONTINUITY (what the executor seam received):') for (const context of contexts) { diff --git a/examples/graphs/shot-loop.ts b/examples/graphs/shot-loop.ts index 6373ff6c..8e581d74 100644 --- a/examples/graphs/shot-loop.ts +++ b/examples/graphs/shot-loop.ts @@ -19,9 +19,8 @@ import { type AgentGraph, type AnalystRegistry, promptHandle, - type RunGraphOptions, - runGraph, } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { leafSeam, offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -33,7 +32,7 @@ const analysts: AnalystRegistry = { run: async () => [{ check: 'test-suite', observed: 'see the settled output' }], } -export function shotLoop(): { graph: AgentGraph; opts: RunGraphOptions } { +export function shotLoop(): { graph: AgentGraph; opts: RunGraphTestOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ @@ -52,7 +51,7 @@ export function shotLoop(): { graph: AgentGraph; opts: RunGraphOptions } { } const received: AgentProfile[] = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'shots', analysts, makeWorkerAgent: leafSeam(received, { @@ -98,7 +97,7 @@ export function shotLoop(): { graph: AgentGraph; opts: RunGraphOptions } { export async function main(): Promise { const { graph, opts } = shotLoop() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('shot-loop', res) } diff --git a/examples/graphs/user-sim-conversation.ts b/examples/graphs/user-sim-conversation.ts index c186aae7..02b24bde 100644 --- a/examples/graphs/user-sim-conversation.ts +++ b/examples/graphs/user-sim-conversation.ts @@ -20,10 +20,9 @@ import { type AgentGraph, chatWorkerSeam, promptHandle, - type RunGraphOptions, - runGraph, type WorkerSpawnContext, } from '@tangle-network/agent-runtime/kernel' +import { type RunGraphTestOptions, runGraphWithTestBrain } from '../../src/testing' import { offlineProfile, printLedger, scriptedBrain } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -44,7 +43,7 @@ export const USER_TURNS = [ export function userSimConversation(): { graph: AgentGraph - opts: RunGraphOptions + opts: RunGraphTestOptions /** Every OpenAI-shape request body the product agent's transport received, in order — the * resumed message-history chain, captured at the wire. */ requests: Array> @@ -109,7 +108,7 @@ export function userSimConversation(): { }, }) const contexts: Array = [] - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'usim', makeWorkerAgent: (profile, context) => { contexts.push(context) @@ -132,7 +131,7 @@ export function userSimConversation(): { export async function main(): Promise { const { graph, opts, requests, contexts } = userSimConversation() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('user-sim-conversation', res) console.log('SPAWN CONTINUITY (what the executor seam received):') for (const context of contexts) { diff --git a/examples/graphs/watchdog-steer.ts b/examples/graphs/watchdog-steer.ts index b377537c..7ffcce70 100644 --- a/examples/graphs/watchdog-steer.ts +++ b/examples/graphs/watchdog-steer.ts @@ -14,13 +14,12 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentGraph, promptHandle } from '@tangle-network/agent-runtime/kernel' import { - type AgentGraph, - promptHandle, - type RunGraphOptions, - runGraph, + type RunGraphTestOptions, + runGraphWithTestBrain, type ToolLoopChat, -} from '@tangle-network/agent-runtime/kernel' +} from '../../src/testing' import { leafSeam, offlineProfile, printLedger } from './shared' const brief = promptHandle('delegates/worker-brief/v1') @@ -47,7 +46,7 @@ function onlineFinding( return undefined } -export function watchdogSteer(): { graph: AgentGraph; opts: RunGraphOptions } { +export function watchdogSteer(): { graph: AgentGraph; opts: RunGraphTestOptions } { // ── The topology: plain data ── const graph: AgentGraph = { nodes: [ @@ -119,7 +118,7 @@ export function watchdogSteer(): { graph: AgentGraph; opts: RunGraphOptions } { return { content: 'done', toolCalls: [] } } - const opts: RunGraphOptions = { + const opts: RunGraphTestOptions = { runId: 'wd', makeWorkerAgent: seam, brain, @@ -130,7 +129,7 @@ export function watchdogSteer(): { graph: AgentGraph; opts: RunGraphOptions } { export async function main(): Promise { const { graph, opts } = watchdogSteer() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) printLedger('watchdog-steer', res) } diff --git a/examples/p1-parity/arms.ts b/examples/p1-parity/arms.ts index 5916d5cd..e99a8087 100644 --- a/examples/p1-parity/arms.ts +++ b/examples/p1-parity/arms.ts @@ -49,8 +49,12 @@ import { type RunGraphOptions, runGraph, type Spend, - type ToolLoopChat, } from '@tangle-network/agent-runtime/kernel' +import { + type RunGraphTestOptions, + runGraphWithTestBrain, + type ToolLoopChat, +} from '../../src/testing' // ── The shared coder sampling (the F1 parity pin) ────────────────────────────── @@ -425,7 +429,7 @@ export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Pro } }, } - const opts: RunGraphOptions = + const opts: RunGraphOptions | RunGraphTestOptions = backend.kind === 'seam' ? { makeWorkerAgent: backend.makeWorkerAgent, @@ -448,7 +452,8 @@ export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Pro } const startedAt = Date.now() try { - const res = await runGraph(graph, opts) + const res = + 'brain' in opts ? await runGraphWithTestBrain(graph, opts) : await runGraph(graph, opts) return graphRecord( res.result.kind === 'winner', res.result.spentTotal, diff --git a/examples/p1-parity/offline.ts b/examples/p1-parity/offline.ts index 26850e0e..621eef27 100644 --- a/examples/p1-parity/offline.ts +++ b/examples/p1-parity/offline.ts @@ -21,7 +21,8 @@ import type { MultishotTransportRequest, } from '@tangle-network/agent-eval/multishot' import type { AgentProfile } from '@tangle-network/agent-interface' -import type { MakeWorkerAgent, ToolLoopChat } from '@tangle-network/agent-runtime/kernel' +import type { MakeWorkerAgent } from '@tangle-network/agent-runtime/kernel' +import type { ToolLoopChat } from '../../src/testing' import { type LeafShot, leafSeam, type ScriptedTurn, scriptedBrain } from '../graphs/shared' import type { CellSpec, GraphArmBackend, MultishotArmBackend } from './arms' diff --git a/examples/supervisor-loop/run.ts b/examples/supervisor-loop/run.ts index 7b745bb9..608f0a02 100644 --- a/examples/supervisor-loop/run.ts +++ b/examples/supervisor-loop/run.ts @@ -22,6 +22,7 @@ * pnpm test tests/kernel/coordination-driver.test.ts tests/supervisor-loop-example.test.ts */ import { supervise } from '@tangle-network/agent-runtime/kernel' +import { superviseWithTestBrain } from '../../src/testing' import { buildWorkerBackend, demoCheck, demoGoal, resolveSupervisorBrain } from './shared' async function main(): Promise { @@ -36,27 +37,25 @@ async function main(): Promise { console.log(`supervisor-loop · ${backend.backend.toUpperCase()} · driver=${label}`) - const result = await supervise( - profile, + const task = `${demoGoal}\nUse this exact worker execution identity in spawn_agent.profile: ` + - JSON.stringify({ harness: worker.profile.harness, model: worker.profile.model }), - { - backend, - deliverable: { check: demoCheck, describe: 'worker delivers the goal' }, - ...(brain ? { brain } : {}), - ...(!brain - ? { - router: { - routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', - routerKey: process.env.TANGLE_API_KEY!, - }, - } - : {}), - budget: { maxIterations: 100, maxTokens: 2_000_000, maxUsd: 2 }, - perWorker: { maxIterations: 1, maxTokens: 200_000 }, - runId: `supervisor-loop-${backend.backend}`, - }, - ) + JSON.stringify({ harness: worker.profile.harness, model: worker.profile.model }) + const common = { + backend, + deliverable: { check: demoCheck, describe: 'worker delivers the goal' }, + budget: { maxIterations: 100, maxTokens: 2_000_000, maxUsd: 2 }, + perWorker: { maxIterations: 1, maxTokens: 200_000 }, + runId: `supervisor-loop-${backend.backend}`, + } as const + const result = brain + ? await superviseWithTestBrain(profile, task, { ...common, brain }) + : await supervise(profile, task, { + ...common, + router: { + routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', + routerKey: process.env.TANGLE_API_KEY!, + }, + }) console.log( result.kind === 'winner' diff --git a/examples/supervisor-loop/shared.ts b/examples/supervisor-loop/shared.ts index bed7bce0..99bb8e30 100644 --- a/examples/supervisor-loop/shared.ts +++ b/examples/supervisor-loop/shared.ts @@ -11,9 +11,9 @@ import { type AgentProfile, harnessTypeSchema } from '@tangle-network/agent-inte import type { ExecutorConfig, SandboxClient as RuntimeSandboxClient, - ToolLoopChat, } from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' +import type { ToolLoopChat } from '../../src/testing' /** The marker every runner asks its workers to emit; the check confirms it landed. */ export const expectedAnswer = 'ANSWER=42' diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 63775b7e..6a1638ba 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -178,10 +178,17 @@ try { Sha256Digest, } from '@tangle-network/agent-interface' import { + driverAgent, type AgentProfileImprovementFixture, + type DriverAgentOptions, loadAgentImprovementProposalFixture, loadAgentProfileImprovementFixture, + type ToolLoopChat, } from '@tangle-network/agent-runtime/testing' + // @ts-expect-error Arbitrary driver construction is confined to the testing entrypoint. + import type { DriverAgentOptions as ForbiddenDriverAgentOptions } from '@tangle-network/agent-runtime/kernel' + // @ts-expect-error Arbitrary model callbacks are confined to the testing entrypoint. + import type { ToolLoopChat as ForbiddenToolLoopChat } from '@tangle-network/agent-runtime/kernel' import { deriveExecutionId, handleChatTurn, @@ -326,6 +333,11 @@ try { void durableExecutionId void durableTurnHandler void durableTurnResult + void driverAgent + void (undefined as unknown as DriverAgentOptions) + void (undefined as unknown as ToolLoopChat) + void (undefined as unknown as ForbiddenDriverAgentOptions) + void (undefined as unknown as ForbiddenToolLoopChat) `, ) // This fixture type-checks with its declared dev toolchain; ambient production @@ -534,16 +546,43 @@ try { ` const testing = await import('@tangle-network/agent-runtime/testing') const expected = [ + 'driverAgent', 'loadAgentImprovementProposalFixture', 'loadAgentProfileImprovementFixture', + 'runGraphWithTestBrain', + 'superviseWithTestBrain', + 'supervisorAgentWithTestBrain', ] const names = Object.keys(testing).sort() if (JSON.stringify(names) !== JSON.stringify(expected)) { - throw new Error('testing entrypoint must export only the proposal fixture loaders') + throw new Error('testing entrypoint must export exactly the declared test helpers') } for (const name of expected) { if (typeof testing[name] !== 'function') { - throw new Error('testing fixture export must be a function: ' + name) + throw new Error('testing export must be a function: ' + name) + } + } + const kernel = await import('@tangle-network/agent-runtime/kernel') + for (const name of [ + 'driverAgent', + 'runGraphWithTestBrain', + 'superviseWithTestBrain', + 'supervisorAgentWithTestBrain', + ]) { + if (name in kernel) { + throw new Error('test-only model execution export leaked into kernel: ' + name) + } + } + for (const [name, invoke] of [ + ['runGraph', () => kernel.runGraph({}, { brain: async () => ({ toolCalls: [] }) })], + ['supervise', () => kernel.supervise({}, null, { brain: async () => ({ toolCalls: [] }) })], + ['supervisorAgent', () => kernel.supervisorAgent({}, { brain: async () => ({ toolCalls: [] }) })], + ]) { + try { + invoke() + throw new Error(name + ' accepted direct model injection') + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('test-only')) throw error } } const first = testing.loadAgentImprovementProposalFixture() diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 6c9d07d5..018f4f94 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -540,15 +540,7 @@ export { gateOnDeliverable, mapExecutorResult, } from './supervise/completion-gate' -// The CHEAP / offline driver: an in-process router-tools loop that drives the coordination -// verbs over the Scope (no box, no creds). The CAPABLE driver is an external harness with the -// coordination verbs mounted as an MCP: `supervise()` wires a local bridge automatically, while a -// remote sandbox requires an explicit reachable `driveHarness`. -export { - type DriverAgentOptions, - driverAgent, - finalizeBestDelivered, -} from './supervise/coordination-driver' +export { finalizeBestDelivered } from './supervise/coordination-driver' // The durable coordination side-log a file-backed `RunContext` carries: questions, findings, answer // decisions, and authorized continuation receipts the spawn journal does not own. Receipts persist // as evidence and are never auto-delivered to a replacement worker. @@ -953,11 +945,9 @@ export { superviseSurface, } from './supervise-surface' export type { SandboxControlClient } from './tangle-sandbox-exact-process-provider' -// The driver-brain type a test or advanced caller may implement and pass into -// `DriverAgentOptions.brain`. Production supervisors derive this chat function from their exact -// profile. `ToolLoopCompaction` bounds the driver's own context window. +// Profile-owned supervisor configuration. The arbitrary model callback and raw driver constructor +// live only under `/testing`; production model execution enters through `supervise(AgentProfile)`. export type { - ToolLoopChat, ToolLoopCompaction, ToolLoopCompactionOptions, ToolLoopMessageRecord, diff --git a/src/runtime/router-client.complete.test.ts b/src/runtime/router-client.complete.test.ts index ceaf87df..3a164346 100644 --- a/src/runtime/router-client.complete.test.ts +++ b/src/runtime/router-client.complete.test.ts @@ -172,6 +172,27 @@ describe('RouterConfig.complete — the injected completion transport', () => { expect(seen).toEqual(['trusted-call-1', 'trusted-call-1']) }) + it.each(['', ' ', null])( + 'rejects an invalid supplied logical-call id %j before transport', + async (callId) => { + const complete = vi.fn(async () => ({ choices: [{ message: { content: 'unused' } }] })) + + await expect( + routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + }, + [{ role: 'user', content: 'do not dispatch' }], + { callId: callId as string }, + ), + ).rejects.toThrow(/callId must be a non-empty, non-whitespace string/u) + expect(complete).not.toHaveBeenCalled() + }, + ) + it('fails loud with the final network cause after the configured attempts are exhausted', async () => { const fetchSpy = vi.fn(async () => { throw new TypeError('fetch failed: connection refused') diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 03c3e553..747f2657 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -792,7 +792,14 @@ function routerRequestHeaders( // Every POST is retryable. Mint the logical-call identity once while the request headers are // assembled, outside `withRetry`, so an accepted response whose connection dies cannot make a // retry look like a new billable completion. A trusted caller identity remains authoritative. - const callId = opts?.callId ?? generateIdempotencyKey() + const suppliedCallId: unknown = opts?.callId + if ( + suppliedCallId !== undefined && + (typeof suppliedCallId !== 'string' || suppliedCallId.trim().length === 0) + ) { + throw new ValidationError('router request: callId must be a non-empty, non-whitespace string') + } + const callId = typeof suppliedCallId === 'string' ? suppliedCallId : generateIdempotencyKey() return { ...(opts?.propagatedHeaders ?? {}), 'content-type': 'application/json', @@ -1115,8 +1122,8 @@ export function routerBrain( } = {}, ): ToolLoopChat { const temperature = opts.temperature ?? 0.4 - return (messages, tools) => - chatWithTools(cfg, messages, tools, { + return async (messages, tools, context) => { + const result = await chatWithTools(cfg, messages, tools, { temperature, toolChoice: opts.toolChoice ?? 'auto', // The config's ceiling reaches the completion, so a caller driving a reasoning model can @@ -1125,5 +1132,24 @@ export function routerBrain( ...(opts.seed !== undefined ? { seed: opts.seed } : {}), ...(opts.extraBody !== undefined ? { extraBody: opts.extraBody } : {}), ...(opts.reasoningEffort ? { reasoningEffort: opts.reasoningEffort } : {}), + ...(context?.signal ? { signal: context.signal } : {}), + ...(context?.callId ? { callId: context.callId } : {}), + ...(context?.correlationId ? { correlationId: context.correlationId } : {}), }) + const hasBilledCost = result.billedCostUsd !== undefined + return { + content: result.content, + toolCalls: result.toolCalls, + ...(result.usage !== undefined ? { usage: result.usage } : {}), + ...(result.usageUnknown === true ? { usageUnknown: true as const } : {}), + ...(result.model !== undefined ? { model: result.model } : {}), + ...(result.cache !== undefined ? { promptCache: Object.freeze({ ...result.cache }) } : {}), + transportAttempts: result.transportAttempts, + ...(hasBilledCost + ? { costUsd: result.billedCostUsd, costProvenance: 'billing-receipt' as const } + : result.costUsd !== undefined + ? { costUsd: result.costUsd, costProvenance: result.costProvenance } + : {}), + } + } } diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index 0e973c77..5a33a581 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -44,6 +44,7 @@ import { import type { ToolSpec } from '../router-client' import { runBrainLoop, + type ToolLoopCallContext, type ToolLoopChat, type ToolLoopCompaction, type ToolLoopCompactionOptions, @@ -82,6 +83,9 @@ export interface DriverAgentOptions { * (the canonical `ToolLoopChat`): a scripted mock offline, the router's tool-calling in * production, or a sandboxed harness. The same seam every tool-loop uses; no bespoke shape. */ readonly brain: ToolLoopChat + /** Profile-declared model for a production Router brain. When set, every turn must report this + * exact provider-observed model before its output is accepted. Omitted by scripted test brains. */ + readonly expectedModel?: string /** Shared blob store — `observe_agent` reads settled outputs through it. */ readonly blobs: ResultBlobStore /** Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). */ @@ -413,14 +417,22 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // reader of `tokensLeft` can see the balance is a ceiling rather than a measurement. A // scripted/mock turn takes this path too, so offline equal-k still debits exactly zero tokens. let driverTurn = 0 + let driverCall = 0 const meteredBrain = async ( messages: ReadonlyArray>, tools: ReadonlyArray, detail: Record, ) => { let res: Awaited> + const call = driverCall + driverCall += 1 + const callContext: ToolLoopCallContext = Object.freeze({ + signal: scope.signal, + callId: `${scope.view.root}:brain:${crypto.randomUUID()}`, + correlationId: scope.view.root, + }) try { - res = await opts.brain(messages, tools) + res = await opts.brain(messages, tools, callContext) } catch (error) { await scope.meter( { @@ -431,10 +443,45 @@ export function driverAgent(opts: DriverAgentOptions): Agent { usdKnown: false, ms: 0, }, - { driver: opts.name, inferenceFailed: true, ...detail }, + { + driver: opts.name, + inferenceFailed: true, + call, + callId: callContext.callId, + correlationId: callContext.correlationId, + ...detail, + }, ) throw error } + let evidenceError: ValidationError | undefined + if (opts.expectedModel !== undefined) { + if (res.model === undefined) { + evidenceError = new ValidationError( + `driverAgent: Router response omitted model identity; expected ${JSON.stringify(opts.expectedModel)}`, + ) + } else if (res.model !== opts.expectedModel) { + evidenceError = new ValidationError( + `driverAgent: Router response reported model ${JSON.stringify(res.model)}; expected ${JSON.stringify(opts.expectedModel)}`, + ) + } + } + if ( + res.transportAttempts !== undefined && + (!Number.isSafeInteger(res.transportAttempts) || res.transportAttempts < 1) + ) { + evidenceError = new ValidationError( + 'driverAgent: transportAttempts must be a positive safe integer when reported', + ) + } + for (const [field, value] of Object.entries(res.promptCache ?? {})) { + if (typeof value === 'number' && (!Number.isFinite(value) || value < 0)) { + evidenceError = new ValidationError( + `driverAgent: prompt-cache field ${JSON.stringify(field)} must be finite and nonnegative`, + ) + break + } + } const trustedCost = res.costProvenance === 'provider-receipt' || res.costProvenance === 'billing-receipt' const turnSpend: Spend = { @@ -447,7 +494,16 @@ export function driverAgent(opts: DriverAgentOptions): Agent { } await scope.meter(turnSpend, { driver: opts.name, + call, + callId: callContext.callId, + correlationId: callContext.correlationId, toolCalls: (res.toolCalls ?? []).map((c) => c.name), + ...(res.model !== undefined ? { model: res.model } : {}), + ...(res.transportAttempts !== undefined + ? { transportAttempts: res.transportAttempts } + : {}), + ...(res.usage?.reasoning !== undefined ? { reasoningTokens: res.usage.reasoning } : {}), + ...(res.promptCache !== undefined ? { promptCache: res.promptCache } : {}), // The streamed transport says explicitly when it finished with no usage chunk (a broken // `include_usage` contract), which is a different fact from a brain that never reports // usage at all. Recorded on the turn so the journal distinguishes them. @@ -455,6 +511,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { ...(res.costProvenance === 'catalog-estimate' ? { estimatedCostUsd: res.costUsd } : {}), ...detail, }) + if (evidenceError !== undefined) throw evidenceError return res } const chat: ToolLoopChat = async (messages, tools) => { diff --git a/src/runtime/supervise/graph.ts b/src/runtime/supervise/graph.ts index 42db3feb..f22df299 100644 --- a/src/runtime/supervise/graph.ts +++ b/src/runtime/supervise/graph.ts @@ -71,7 +71,12 @@ import { type PromptRegistry, } from './prompt-registry' import type { ExecutorConfig } from './runtime' -import { type SuperviseOptions, supervise, workerFromBackend } from './supervise' +import { + type SuperviseOptions, + supervise, + superviseWithTestBrain, + workerFromBackend, +} from './supervise' import type { Budget, NodeId, ResultBlobStore, SpawnJournal, SupervisedResult } from './types' // ── The algebra ──────────────────────────────────────────────────────────────── @@ -207,8 +212,6 @@ export interface RunGraphOptions { * graph's own spawn-binding hook on the SAME event stream — the graph never swallows the * seam supervise() exposes. */ readonly hooks?: RuntimeHooks - /** Inject the driver brain directly (offline tests / advanced). */ - readonly brain?: ToolLoopChat /** The analyst lens registry `analyzes` edges resolve against. ENVIRONMENT — needed only for * lens analysts; an analyzes edge naming a graph NODE as its analyst needs no registry. */ readonly analysts?: AnalystRegistry @@ -252,6 +255,11 @@ export interface GraphResult { readonly runId: string } +/** Test-only graph options, exported only through the package's explicit `/testing` entry. */ +export interface RunGraphTestOptions extends RunGraphOptions { + readonly brain: ToolLoopChat +} + // ── Validation ───────────────────────────────────────────────────────────────── interface ValidatedGraph { @@ -517,6 +525,28 @@ function stringifyPayload(payload: unknown): string { * traversal is ledgered and journaled. */ export function runGraph(graph: AgentGraph, opts: RunGraphOptions): Promise { + if ('brain' in opts) { + throw new ValidationError( + 'runGraph: direct brain injection is test-only; production execution derives the model call from the root AgentProfile', + ) + } + return runGraphInternal(graph, opts) +} + +/** Deterministic scripted-brain path for graph tests. Not exported from Runtime's main entry. */ +export function runGraphWithTestBrain( + graph: AgentGraph, + opts: RunGraphTestOptions, +): Promise { + const { brain, ...runtimeOptions } = opts + return runGraphInternal(graph, runtimeOptions, brain) +} + +function runGraphInternal( + graph: AgentGraph, + opts: RunGraphOptions, + testBrain?: ToolLoopChat, +): Promise { const registry = opts.registry ?? kernelPromptRegistry() const { root, workers, delegatesByWorker, analyzes, analystNodes } = validateGraph( graph, @@ -916,7 +946,7 @@ export function runGraph(graph: AgentGraph, opts: RunGraphOptions): Promise => { - const result = await supervise(rootProfile, graphTask(graph, root), { + const superviseOptions = { budget: graph.budget, deliverable: graph.deliverable, makeWorkerAgent: graphWorker, @@ -933,7 +963,6 @@ export function runGraph(graph: AgentGraph, opts: RunGraphOptions): Promise 0 ? { continuityByProfile } : {}), ...(opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {}), ...(opts.router ? { router: opts.router } : {}), - ...(opts.brain ? { brain: opts.brain } : {}), ...(authorizeMessage ? { authorizeMessage } : {}), ...(opts.perWorker ? { perWorker: opts.perWorker } : {}), ...(opts.maxTurns !== undefined ? { maxTurns: opts.maxTurns } : {}), @@ -943,7 +972,14 @@ export function runGraph(graph: AgentGraph, opts: RunGraphOptions): Promise { + if ('brain' in deps) { + throw new ValidationError( + 'supervisorAgent: direct brain injection is test-only; production execution derives the model call from AgentProfile', + ) + } + return buildSupervisorAgent(profile, deps) +} + +/** Scripted-brain construction for deterministic tests. Not exported from Runtime's main entry. */ +export function supervisorAgentWithTestBrain( + profile: SupervisorProfile, + deps: SupervisorAgentTestDeps, +): Agent { + const { brain, ...runtimeDeps } = deps + return buildSupervisorAgent(profile, runtimeDeps, brain) +} + +function buildSupervisorAgent( + profile: SupervisorProfile, + deps: SupervisorAgentDeps, + testBrain?: ToolLoopChat, ): Agent { const exactProfile = agentProfileSchema.parse(profile) assertExecutableAgentProfile(exactProfile, 'supervisorAgent') @@ -431,9 +458,10 @@ export function supervisorAgent( } if (harness === null) { - // ROUTER arm: the in-process tool-loop. `routerBrain` is now an internal detail — the caller - // passes a profile, not a hand-built brain (a test may still inject `deps.brain`). - const brain = deps.brain ?? routerBrainFromProfile(stableProfile, stableRouter) + // ROUTER arm: the in-process tool-loop. `routerBrain` is an internal detail — a production + // caller passes a profile, never a hand-built brain. Deterministic source tests use the + // separately exported `/testing` constructor. + const brain = testBrain ?? routerBrainFromProfile(stableProfile, stableRouter) const inbox = createInbox() const build = ( priorCoordination?: PriorCoordination, @@ -443,6 +471,9 @@ export function supervisorAgent( driverAgent({ name, brain, + ...(testBrain === undefined + ? { expectedModel: resolveSupervisorModelId(stableProfile) } + : {}), blobs: deps.blobs, makeWorkerAgent: deps.makeWorkerAgent, ...(deps.authorizeDownMessage ? { authorizeDownMessage: deps.authorizeDownMessage } : {}), @@ -690,7 +721,7 @@ function routerBrainFromProfile( ): ToolLoopChat { if (!router) { throw new ValidationError( - 'supervisorAgent: a router-brained supervisor (harness omitted or cli-base) needs deps.router (or deps.brain)', + 'supervisorAgent: a router-brained supervisor (harness omitted or cli-base) needs deps.router', ) } const modelId = resolveSupervisorModelId(profile) diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index e7b995d5..0b95645a 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -23,15 +23,24 @@ export interface ToolLoopToolCall { arguments: string } +/** Runtime-owned identity and cancellation for one logical inference call. The wrapper is frozen + * before dispatch; a transport may observe the signal but cannot replace the authority it names. */ +export interface ToolLoopCallContext { + readonly signal: AbortSignal + readonly callId: string + readonly correlationId: string +} + /** One inference turn over the running conversation + the tool specs → the model's text, any * tool calls, and token usage. The seam every brain satisfies. */ export type ToolLoopChat = ( messages: ReadonlyArray, tools: ReadonlyArray, + context?: ToolLoopCallContext, ) => Promise<{ content?: string | null toolCalls: ToolLoopToolCall[] - usage?: { input: number; output: number } + usage?: { input: number; output: number; reasoning?: number } /** Dollar value reported for the turn. It is not billed spend unless provenance says so. */ costUsd?: number costProvenance?: 'provider-receipt' | 'billing-receipt' | 'catalog-estimate' @@ -39,6 +48,12 @@ export type ToolLoopChat = ( * router transport asks for usage and this says it never arrived). A metering caller records an * unknown turn on it; `runBrainLoop` itself ignores it. */ usageUnknown?: true + /** Provider-observed model identity. Profile-bound callers validate it before accepting output. */ + model?: string + /** Provider-reported prompt-cache evidence; missing fields remain missing. */ + promptCache?: Readonly> + /** Physical HTTP/injected-transport attempts spent by this one logical call. */ + transportAttempts?: number }> /** Optional per-loop concerns the metered/steerable call sites attach. The loop is one copy; diff --git a/src/testing/index.ts b/src/testing/index.ts index 78fea8b1..c1f5f718 100644 --- a/src/testing/index.ts +++ b/src/testing/index.ts @@ -13,6 +13,24 @@ import canonicalAgentImprovementProposalFixture from './fixtures/agent-improveme import canonicalAgentProfileImprovementProposalFixture from './fixtures/agent-profile-improvement-proposal.json' import canonicalAgentProfileImprovementStateFixture from './fixtures/agent-profile-improvement-state.json' +export { + type DriverAgentOptions, + driverAgent, +} from '../runtime/supervise/coordination-driver' +export { + type RunGraphTestOptions, + runGraphWithTestBrain, +} from '../runtime/supervise/graph' +export { + type SuperviseTestOptions, + superviseWithTestBrain, +} from '../runtime/supervise/supervise' +export { + type SupervisorAgentTestDeps, + supervisorAgentWithTestBrain, +} from '../runtime/supervise/supervisor-agent' +export type { ToolLoopCallContext, ToolLoopChat } from '../runtime/tool-loop' + const serializedAgentImprovementProposalFixture = JSON.stringify( canonicalAgentImprovementProposalFixture, ) diff --git a/tests/examples/graph-topologies.test.ts b/tests/examples/graph-topologies.test.ts index a0add5b6..4d950c0d 100644 --- a/tests/examples/graph-topologies.test.ts +++ b/tests/examples/graph-topologies.test.ts @@ -25,7 +25,7 @@ * the ledger and all three shots' spend in the one conserved pool. */ -import { runGraph } from '@tangle-network/agent-runtime/kernel' +import { runGraphWithTestBrain } from '@tangle-network/agent-runtime/testing' import { describe, expect, it } from 'vitest' import { analystAgentReview } from '../../examples/graphs/analyst-agent-review' import { bestOfN } from '../../examples/graphs/best-of-n' @@ -37,7 +37,7 @@ import { watchdogSteer } from '../../examples/graphs/watchdog-steer' describe('examples/graphs — the six topologies run offline with truthful ledgers', () => { it('collaborates-review-loop: every peer hop is mediated, ledgered, and addressed', async () => { const { graph, opts } = collaboratesReviewLoop() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') expect(res.result.out).toEqual({ revision: 2 }) @@ -66,7 +66,7 @@ describe('examples/graphs — the six topologies run offline with truthful ledge it('best-of-n: two delivered spawn traversals, winner decided by the deliverable', async () => { const { graph, opts } = bestOfN() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') expect(res.result.out).toEqual({ candidate: 'b', pass: true }) @@ -81,7 +81,7 @@ describe('examples/graphs — the six topologies run offline with truthful ledge it('watchdog-steer: the corrective steer is the mid-run leg of the delegates edge', async () => { const { graph, opts } = watchdogSteer() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') { @@ -102,7 +102,7 @@ describe('examples/graphs — the six topologies run offline with truthful ledge it('shot-loop: two shots under the 3-traversal cap, each with a delivered verify report', async () => { const { graph, opts } = shotLoop() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') expect(res.result.out).toEqual({ tests: 'pass' }) @@ -119,7 +119,7 @@ describe('examples/graphs — the six topologies run offline with truthful ledge it('shot-loop-resumed: shot 1 fresh, shots 2-3 resume the prior settled session, one conserved pool', async () => { const { graph, opts, contexts } = shotLoopResumed() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') expect(res.result.out).toEqual({ tests: 'pass' }) @@ -152,7 +152,7 @@ describe('examples/graphs — the six topologies run offline with truthful ledge it('analyst-agent-review: the reviewer NODE runs as the analyst and its output is the finding', async () => { const { graph, opts } = analystAgentReview() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) expect(res.result.kind).toBe('winner') if (res.result.kind === 'winner') { diff --git a/tests/examples/user-sim-conversation.test.ts b/tests/examples/user-sim-conversation.test.ts index 27bf197b..6ce1604e 100644 --- a/tests/examples/user-sim-conversation.test.ts +++ b/tests/examples/user-sim-conversation.test.ts @@ -20,12 +20,12 @@ import { USER_TURNS, userSimConversation, } from '../../examples/graphs/user-sim-conversation' -import { runGraph } from '../../src/runtime/supervise/graph' +import { runGraphWithTestBrain } from '../../src/testing' describe('examples/graphs/user-sim-conversation — turns are traversals, the session is one message list', () => { it('resumes one growing conversation across three workers, ledgered and metered', async () => { const { graph, opts, requests, contexts } = userSimConversation() - const res = await runGraph(graph, opts) + const res = await runGraphWithTestBrain(graph, opts) // ── 5. The confirming turn wins through the deliverable ── expect(res.result.kind).toBe('winner') diff --git a/tests/helpers/resume-driver-child.ts b/tests/helpers/resume-driver-child.ts index a625aaf7..aa463438 100644 --- a/tests/helpers/resume-driver-child.ts +++ b/tests/helpers/resume-driver-child.ts @@ -18,7 +18,6 @@ import { appendFileSync } from 'node:fs' import type { AgentProfile } from '@tangle-network/agent-interface' import { spendFromUsageEvents } from '../../src/runtime/supervise/budget' -import { supervise } from '../../src/runtime/supervise/supervise' import type { Agent, AgentSpec, @@ -27,6 +26,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' +import { supervise } from './runtime-with-test-brain' const [dir, runId, phase] = process.argv.slice(2) if ( diff --git a/tests/helpers/runtime-with-test-brain.ts b/tests/helpers/runtime-with-test-brain.ts new file mode 100644 index 00000000..e4b73e39 --- /dev/null +++ b/tests/helpers/runtime-with-test-brain.ts @@ -0,0 +1,51 @@ +import type { + AgentGraph, + RunGraphOptions, + RunGraphTestOptions, +} from '../../src/runtime/supervise/graph' +import { + runGraphWithTestBrain, + runGraph as runProfileGraph, +} from '../../src/runtime/supervise/graph' +import type { SuperviseOptions, SuperviseTestOptions } from '../../src/runtime/supervise/supervise' +import { + supervise as superviseProfile, + superviseWithTestBrain, +} from '../../src/runtime/supervise/supervise' +import type { + SupervisorAgentDeps, + SupervisorAgentTestDeps, + SupervisorProfile, +} from '../../src/runtime/supervise/supervisor-agent' +import { + supervisorAgent as profileSupervisorAgent, + supervisorAgentWithTestBrain, +} from '../../src/runtime/supervise/supervisor-agent' + +/** Route deterministic source tests through the explicit test-only brain entry. */ +export function supervise( + profile: SupervisorProfile, + task: unknown, + options: SuperviseOptions | SuperviseTestOptions, +) { + return 'brain' in options + ? superviseWithTestBrain(profile, task, options) + : superviseProfile(profile, task, options) +} + +/** Route deterministic graph tests through the explicit test-only brain entry. */ +export function runGraph(graph: AgentGraph, options: RunGraphOptions | RunGraphTestOptions) { + return 'brain' in options + ? runGraphWithTestBrain(graph, options) + : runProfileGraph(graph, options) +} + +/** Route deterministic supervisor construction through the explicit test-only brain entry. */ +export function supervisorAgent( + profile: SupervisorProfile, + deps: SupervisorAgentDeps | SupervisorAgentTestDeps, +) { + return 'brain' in deps + ? supervisorAgentWithTestBrain(profile, deps) + : profileSupervisorAgent(profile, deps) +} diff --git a/tests/kernel/graph.test.ts b/tests/kernel/graph.test.ts index 754d8f34..beb4cef2 100644 --- a/tests/kernel/graph.test.ts +++ b/tests/kernel/graph.test.ts @@ -43,7 +43,7 @@ import { describe, expect, it } from 'vitest' import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { ValidationError } from '../../src/errors' import type { MakeWorkerAgent, WorkerSpawnContext } from '../../src/mcp/tools/coordination' -import { type AgentGraph, GraphEdgeCapError, runGraph } from '../../src/runtime/supervise/graph' +import { type AgentGraph, GraphEdgeCapError } from '../../src/runtime/supervise/graph' import { analyzesFindingsReportPrompt, createPromptRegistry, @@ -61,6 +61,7 @@ import type { SpawnEvent, } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' +import { runGraph } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/kernel/nested-coordination-durability.test.ts b/tests/kernel/nested-coordination-durability.test.ts index 42d62b82..725daafe 100644 --- a/tests/kernel/nested-coordination-durability.test.ts +++ b/tests/kernel/nested-coordination-durability.test.ts @@ -4,12 +4,12 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { fullProfileMaterialization } from '../../src/agent/profile-materialization' import type { CoordinationEvent, QuestionRecord } from '../../src/mcp/tools/coordination' -import { supervise } from '../../src/runtime/supervise/supervise' import type { DriveHarness, DriveHarnessOwnerContext, } from '../../src/runtime/supervise/supervisor-agent' import type { ToolLoopChat } from '../../src/runtime/tool-loop' +import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index f2c581b6..4c9d162d 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -12,7 +12,6 @@ import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' import { type SuperviseOptions, type SuperviseRegistryTable, - supervise, workerFromBackend, } from '../../src/runtime/supervise/supervise' import type { @@ -23,6 +22,7 @@ import type { ExecutorResult, UsageEvent, } from '../../src/runtime/supervise/types' +import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' @@ -832,6 +832,137 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(result.kind).toBeDefined() }) + it('binds the root Router call to Runtime identity and preserves model, cache, attempts, reasoning, and billed cost', async () => { + const turns: Array> = [] + let requestHeaders: Readonly> | undefined + const result = await supervise( + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }), + 'finish without delegation', + { + budget, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + router: { + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'test', + complete: async (_body, request) => { + requestHeaders = request?.headers + return { + model: 'deepseek-v4-flash', + choices: [{ message: { content: 'done' } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + completion_tokens_details: { reasoning_tokens: 3 }, + prompt_tokens_details: { cached_tokens: 4 }, + cost_usd: 0.02, + }, + } + }, + }, + hooks: { + onEvent(event) { + if (event.target === 'agent.turn') { + turns.push(event.payload as Record) + } + }, + }, + }, + ) + + expect(result.spentTotal.tokens).toEqual({ input: 10, output: 5 }) + expect(result.spentTotal.usd).toBe(0.02) + expect(turns).toHaveLength(1) + expect(turns[0]).toMatchObject({ + model: 'deepseek-v4-flash', + promptCache: { readTokens: 4 }, + reasoningTokens: 3, + transportAttempts: 1, + }) + expect(Object.isFrozen(turns[0]?.promptCache)).toBe(true) + expect(requestHeaders?.['idempotency-key']).toBe(turns[0]?.callId) + expect(requestHeaders?.['x-correlation-id']).toBe(turns[0]?.correlationId) + }) + + it('accounts for a mismatched observed model and then refuses its output', async () => { + const result = await supervise( + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'declared-model' }, + }), + 'reject model drift', + { + budget, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + router: { + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'test', + complete: async () => ({ + model: 'ambient-model', + choices: [{ message: { content: 'must not be accepted' } }], + usage: { prompt_tokens: 7, completion_tokens: 2, cost_usd: 0.01 }, + }), + }, + }, + ) + + expect(result.kind).toBe('no-winner') + expect(result.kind === 'no-winner' && result.reason).toBe('driver-failed') + expect(result.kind === 'no-winner' && result.error?.message).toMatch( + /reported model "ambient-model"; expected "declared-model"/u, + ) + expect(result.spentTotal).toMatchObject({ + tokens: { input: 7, output: 2 }, + usd: 0.01, + }) + }) + + it('cascades caller cancellation into an in-flight root Router request', async () => { + const controller = new AbortController() + let started!: () => void + const requestStarted = new Promise((resolve) => { + started = resolve + }) + let transportSignal: AbortSignal | undefined + const running = supervise( + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + }), + 'cancel the root call', + { + budget, + signal: controller.signal, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + router: { + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'test', + complete: async (_body, request) => { + transportSignal = request?.signal + started() + return new Promise((_resolve, reject) => { + request?.signal?.addEventListener( + 'abort', + () => reject(request.signal?.reason ?? new Error('aborted')), + { once: true }, + ) + }) + }, + }, + }, + ) + + await requestStarted + controller.abort(new Error('caller stopped')) + const result = await running + + expect(transportSignal?.aborted).toBe(true) + expect(result.kind).toBe('no-winner') + expect(result.kind === 'no-winner' && result.reason).toBe('aborted') + }) + it('allowedModels reads a canonical AgentProfile model through its resolved default id', () => { expect(() => supervise( diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index abb9fbf4..b046b859 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -11,8 +11,8 @@ import { import { afterEach, describe, expect, it } from 'vitest' import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import { supervise } from '../../src/runtime/supervise/supervise' import { createRootHandle } from '../../src/runtime/supervise/supervisor' +import { supervise } from '../helpers/runtime-with-test-brain' type BridgeRequest = { model: string diff --git a/tests/kernel/supervise-global-concurrency.test.ts b/tests/kernel/supervise-global-concurrency.test.ts index f061c239..22bc291c 100644 --- a/tests/kernel/supervise-global-concurrency.test.ts +++ b/tests/kernel/supervise-global-concurrency.test.ts @@ -4,9 +4,7 @@ import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable import type { MakeWorkerAgent } from '../../src/mcp/tools/coordination' import { driverChild } from '../../src/runtime/supervise/driver-executor' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' -import { supervise } from '../../src/runtime/supervise/supervise' import { createSupervisor } from '../../src/runtime/supervise/supervisor' -import { supervisorAgent } from '../../src/runtime/supervise/supervisor-agent' import type { Agent, AgentSpec, @@ -14,6 +12,7 @@ import type { ExecutorResult, Scope, } from '../../src/runtime/supervise/types' +import { supervise, supervisorAgent } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/kernel/supervise-otel-spans.test.ts b/tests/kernel/supervise-otel-spans.test.ts index 8a188abf..b9d15914 100644 --- a/tests/kernel/supervise-otel-spans.test.ts +++ b/tests/kernel/supervise-otel-spans.test.ts @@ -14,7 +14,7 @@ import type { OtelExporter, OtelSpan } from '../../src/otel-export' import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' import { createSupervisorSpanRecorder } from '../../src/runtime/supervise/otel-spans' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' -import { type SuperviseOptions, supervise } from '../../src/runtime/supervise/supervise' +import type { SuperviseOptions } from '../../src/runtime/supervise/supervise' import { createSupervisor } from '../../src/runtime/supervise/supervisor' import type { Agent, @@ -28,6 +28,7 @@ import type { SupervisorOpts, UsageEvent, } from '../../src/runtime/supervise/types' +import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/kernel/supervise-worker-trace.test.ts b/tests/kernel/supervise-worker-trace.test.ts index 2cba7479..544e69e0 100644 --- a/tests/kernel/supervise-worker-trace.test.ts +++ b/tests/kernel/supervise-worker-trace.test.ts @@ -27,7 +27,6 @@ import { type SupervisorSpanRecorder, } from '../../src/runtime/supervise/otel-spans' import { createExecutor } from '../../src/runtime/supervise/runtime' -import { supervise } from '../../src/runtime/supervise/supervise' import { createSupervisor } from '../../src/runtime/supervise/supervisor' import type { Agent, @@ -37,6 +36,7 @@ import type { Scope, SupervisorOpts, } from '../../src/runtime/supervise/types' +import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 66191991..ea993b18 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -12,7 +12,6 @@ import { type ResolveSupervisorTools, resolveSupervisorProfile, type SupervisorProfile, - supervisorAgent, } from '../../src/runtime/supervise/supervisor-agent' import type { Agent, @@ -23,6 +22,7 @@ import type { UsageEvent, } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' +import { supervisorAgent } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { testAgentProfile } from './test-agent-profile' diff --git a/tests/runtime/mid-flight-steering.test.ts b/tests/runtime/mid-flight-steering.test.ts index 771e8ec0..83ccc787 100644 --- a/tests/runtime/mid-flight-steering.test.ts +++ b/tests/runtime/mid-flight-steering.test.ts @@ -27,10 +27,10 @@ import { join } from 'node:path' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import { supervise } from '../../src/runtime/supervise/supervise' import type { Budget } from '../../src/runtime/supervise/types' import type { ToolLoopChat } from '../../src/runtime/tool-loop' import type { SandboxClient } from '../../src/runtime/types' +import { supervise } from '../helpers/runtime-with-test-brain' const WRONG = 'legacy/wrong.ts' const RIGHT = 'core/right.ts' diff --git a/tests/runtime/supervisor-finalizer.test.ts b/tests/runtime/supervisor-finalizer.test.ts index e7160d99..493fff64 100644 --- a/tests/runtime/supervisor-finalizer.test.ts +++ b/tests/runtime/supervisor-finalizer.test.ts @@ -22,7 +22,6 @@ import { runFinalizer, type SupervisorFinalizer, } from '../../src/runtime/supervise/finalizer' -import { supervise } from '../../src/runtime/supervise/supervise' import type { Agent, AgentSpec, @@ -33,6 +32,7 @@ import type { TreeView, UsageEvent, } from '../../src/runtime/supervise/types' +import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from '../kernel/scripted-brain' const budget: Budget = { maxIterations: 100, maxTokens: 100_000 } diff --git a/tests/supervisor-loop-example.test.ts b/tests/supervisor-loop-example.test.ts index 0572c3d2..2fa020bc 100644 --- a/tests/supervisor-loop-example.test.ts +++ b/tests/supervisor-loop-example.test.ts @@ -7,7 +7,6 @@ import { scriptedSupervisorChat, } from '../examples/supervisor-loop/shared' import { gateOnDeliverable } from '../src/runtime/supervise/completion-gate' -import { supervise } from '../src/runtime/supervise/supervise' import type { Agent, AgentSpec, @@ -15,6 +14,7 @@ import type { ExecutorResult, UsageEvent, } from '../src/runtime/supervise/types' +import { supervise } from './helpers/runtime-with-test-brain' // ── An offline worker leaf — returns the ANSWER=42 marker, no network/LLM ───────── // The example's runners build this leaf from a real backend (`workerFromBackend`); here From 1fbd4050e7151381961ef48cfc8cb965a961f83f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 00:19:35 -0600 Subject: [PATCH 33/39] chore(release): align latest shared SDK cohort --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 2 +- pnpm-lock.yaml | 37 ++++++++++++++++++++++------------- pnpm-workspace.yaml | 4 ++-- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e17d91c2..08440cb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-sdk - ref: fc00613985c86f917a679c9bf957e6af8cbedb7c # @tangle-network/agent-interface@0.43.0 + ref: 36b8e3ecc57e9a62130eb2b0fdc1c3b5be01b338 # @tangle-network/agent-interface@0.43.1 path: .cohort/agent-sdk persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d4de9792..bbf8d962 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -135,7 +135,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-sdk - ref: fc00613985c86f917a679c9bf957e6af8cbedb7c # @tangle-network/agent-interface@0.43.0 + ref: 36b8e3ecc57e9a62130eb2b0fdc1c3b5be01b338 # @tangle-network/agent-interface@0.43.1 path: .cohort/agent-sdk persist-credentials: false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aad70079..e2de575d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,14 +10,14 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-core': - specifier: 0.4.35 - version: 0.4.35 + specifier: 0.5.0 + version: 0.5.0 '@tangle-network/agent-eval': specifier: 0.144.3 version: 0.144.3 '@tangle-network/agent-interface': - specifier: 0.43.0 - version: 0.43.0 + specifier: 0.43.1 + version: 0.43.1 '@tangle-network/agent-knowledge': specifier: 7.0.10 version: 7.0.10 @@ -52,13 +52,13 @@ importers: dependencies: '@tangle-network/agent-core': specifier: 'catalog:' - version: 0.4.35 + version: 0.5.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' version: 7.0.10 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' - version: 0.10.2(@tangle-network/agent-interface@0.43.0) + version: 0.10.2(@tangle-network/agent-interface@0.43.1) '@tangle-network/agent-trace-contract': specifier: 'catalog:' version: 1.0.2 @@ -77,7 +77,7 @@ importers: version: 0.144.3 '@tangle-network/agent-interface': specifier: 'catalog:' - version: 0.43.0 + version: 0.43.1 '@tangle-network/sandbox': specifier: 'catalog:' version: 0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) @@ -125,7 +125,7 @@ importers: version: 0.144.3 '@tangle-network/agent-interface': specifier: 'catalog:' - version: 0.43.0 + version: 0.43.1 '@tangle-network/agent-knowledge': specifier: 'catalog:' version: 7.0.10 @@ -1111,8 +1111,8 @@ packages: '@tangle-network/agent-core@0.4.33': resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-core@0.4.35': - resolution: {integrity: sha512-qLPdWUO7wCPQ86m+DE3jjk1hV0WGbmO/DkSfNcxZ0Shxx7vsNIgp8vasOsDOwfZzHlugZb9PemL0JZ5ulc8R4g==} + '@tangle-network/agent-core@0.5.0': + resolution: {integrity: sha512-xjsTAZbtF0eXiUxzzNib4j49gYUhEDF7SCYhD8bkUeIJ2O8rfa0GGl7y3egi8SvsbKRhjbIojL+4mcmERbgUzw==} '@tangle-network/agent-eval@0.144.3': resolution: {integrity: sha512-gkyeie5CCzUuEzTfuQKe6iikSsu0S9h6w8yRtfnvjDZopkJJ7Nn+VDzkWZrsLDQ31LtLaHqJ7I2FgYbPNvjW0Q==} @@ -1122,6 +1122,9 @@ packages: '@tangle-network/agent-interface@0.43.0': resolution: {integrity: sha512-t35nGOJ3sWouHoUM/5A8eUmsR+8JcuRF6VFQbT9cPzxIMUV3NsFMJgYXPGmyIj65vtvs73O6Mvijp47FAAGbuQ==} + '@tangle-network/agent-interface@0.43.1': + resolution: {integrity: sha512-nrJol7sGmLLp60S0dBk+QejIB/TXvOOVNjq/OCb23K0oAWURCFyaTWA0J8q2V/pJwxalOPDWovqptMgkcF6q1A==} + '@tangle-network/agent-knowledge@7.0.10': resolution: {integrity: sha512-igVEo0a6nnCZWmemvu9aLdIRMznAPKZpHcNWMdRJ5EyYXc5TnX0kwKTgTCzBHvg9Fqp8gfMttNHJzA4T+V3Yog==} engines: {node: '>=20.19.0'} @@ -2789,9 +2792,9 @@ snapshots: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 - '@tangle-network/agent-core@0.4.35': + '@tangle-network/agent-core@0.5.0': dependencies: - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-interface': 0.43.1 zod: 4.4.3 '@tangle-network/agent-eval@0.144.3': @@ -2812,6 +2815,12 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 + '@tangle-network/agent-interface@0.43.1': + dependencies: + '@noble/hashes': 1.8.0 + spdx-expression-parse: 5.0.0 + zod: 4.4.3 + '@tangle-network/agent-knowledge@7.0.10': dependencies: '@tangle-network/agent-eval': 0.144.3 @@ -2819,9 +2828,9 @@ snapshots: proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.10.2(@tangle-network/agent-interface@0.43.0)': + '@tangle-network/agent-profile-materialize@0.10.2(@tangle-network/agent-interface@0.43.1)': dependencies: - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-interface': 0.43.1 '@tangle-network/agent-trace-contract@1.0.2': {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 41fea6e0..30cdd01e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,10 +17,10 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 - '@tangle-network/agent-core': 0.4.35 + '@tangle-network/agent-core': 0.5.0 '@types/node': 26.1.1 '@tangle-network/agent-eval': 0.144.3 - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-interface': 0.43.1 '@tangle-network/agent-knowledge': 7.0.10 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 From 5ccca6fbf0356e2b063f493a5e2f0dab0018a6db Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:14:02 -0600 Subject: [PATCH 34/39] fix(supervise): account interrupted router turns --- src/runtime/router-client.ts | 104 +++++++++++++----- src/runtime/supervise/bridge-executor.test.ts | 92 +++++++++++++++- src/runtime/supervise/runtime.ts | 17 ++- tests/kernel/inbox.test.ts | 24 +++- tests/kernel/supervise-convenience.test.ts | 59 ++++++++++ 5 files changed, 255 insertions(+), 41 deletions(-) diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 747f2657..29dd8f7c 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -199,7 +199,7 @@ export async function routerChatWithUsage( opts?.signal, retry, ) - return parseChatResult(await response.json(), cfg.model, attempts) + return parseChatResult(await routerResponseJson(response, attempts), cfg.model, attempts) } function parseChatResult( @@ -366,7 +366,7 @@ export async function routerChatWithTools( retry, ) transportAttempts = result.attempts - return result.response.json() + return routerResponseJson(result.response, result.attempts) })() const data = raw as { model?: unknown @@ -698,7 +698,7 @@ export async function streamRouterChatWithTools( const calls = new Map() let lastCallIndex = -1 - for await (const chunk of readChatCompletionChunks(res.body)) { + for await (const chunk of readChatCompletionChunksWithAttempts(res.body, transportAttempts)) { if (chunk.error) { throw new ValidationError( `router stream error: ${chunk.error.message ?? chunk.error.type ?? 'unknown'}`, @@ -839,34 +839,67 @@ async function retryRouterOperation( operation: (signal: AbortSignal) => Promise, ): Promise<{ value: T; attempts: number }> { let attempts = 0 - const value = await withRetry( - async (attempt) => { - attempts = attempt + 1 - const attemptSignal = withRouterRequestTimeout(callerSignal, retry.requestTimeoutMs) - try { - return await operation(attemptSignal.signal) - } catch (error) { - if (callerSignal?.aborted) throw callerSignal.reason ?? error - if (attemptSignal.signal.aborted) { - throw new SDKError(`router request timeout after ${retry.requestTimeoutMs}ms`, { - code: 'TIMEOUT', - cause: error instanceof Error ? error : new Error(String(error)), - }) - } - if (error instanceof TypeError && error.message.includes('fetch')) { - throw new SDKError(`router network failure: ${error.message}`, { - code: 'NETWORK', - cause: error, - }) + try { + const value = await withRetry( + async (attempt) => { + attempts = attempt + 1 + const attemptSignal = withRouterRequestTimeout(callerSignal, retry.requestTimeoutMs) + try { + return await operation(attemptSignal.signal) + } catch (error) { + if (callerSignal?.aborted) throw callerSignal.reason ?? error + if (attemptSignal.signal.aborted) { + throw new SDKError(`router request timeout after ${retry.requestTimeoutMs}ms`, { + code: 'TIMEOUT', + cause: error instanceof Error ? error : new Error(String(error)), + }) + } + if (error instanceof TypeError && error.message.includes('fetch')) { + throw new SDKError(`router network failure: ${error.message}`, { + code: 'NETWORK', + cause: error, + }) + } + throw error + } finally { + attemptSignal.dispose() } - throw error - } finally { - attemptSignal.dispose() - } - }, - agentCoreRetryConfig(retry, callerSignal), - ) - return { value, attempts } + }, + agentCoreRetryConfig(retry, callerSignal), + ) + return { value, attempts } + } catch (error) { + throwRouterFailureWithAttempts(error, attempts) + } +} + +const routerFailureAttempts = new WeakMap() + +/** Internal evidence carried alongside a failed Router call without replacing the original error. + * Runtime uses it when an intentional interrupt resumes the worker instead of surfacing the error. */ +export function routerTransportAttemptsFromError(error: unknown): number | undefined { + const identity = routerErrorIdentity(error) + return identity === undefined ? undefined : routerFailureAttempts.get(identity) +} + +function throwRouterFailureWithAttempts(error: unknown, attempts: number): never { + const identity = routerErrorIdentity(error) + if (identity !== undefined) routerFailureAttempts.set(identity, attempts) + throw error +} + +function routerErrorIdentity(error: unknown): object | undefined { + return (typeof error === 'object' && error !== null) || typeof error === 'function' + ? (error as object) + : undefined +} + +async function routerResponseJson(response: Response, attempts: number): Promise { + try { + return await response.json() + } catch (error) { + throwRouterFailureWithAttempts(error, attempts) + } } function agentCoreRetryConfig( @@ -974,6 +1007,17 @@ async function* readChatCompletionChunks( } } +async function* readChatCompletionChunksWithAttempts( + body: ReadableStream, + attempts: number, +): AsyncIterable { + try { + yield* readChatCompletionChunks(body) + } catch (error) { + throwRouterFailureWithAttempts(error, attempts) + } +} + /** * Normalize an SSE body's line terminators to `\n`. The SSE grammar accepts CRLF, LF, AND a lone * CR, so a `\r`-separated body is legal — and left unnormalized it decodes to zero frames, which diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index a3a72e32..48933828 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -5,9 +5,9 @@ import { canonicalAgentProfileDigest, type ReasoningEffort, } from '@tangle-network/agent-interface' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { spendFromUsageEvents } from './budget' -import { bridgeExecutor } from './runtime' +import { bridgeExecutor, createExecutor } from './runtime' import type { UsageEvent } from './types' const TEST_RUN_DIGEST = `sha256:${'b'.repeat(64)}` @@ -197,9 +197,57 @@ async function drain(stream: AsyncIterable): Promise { return events } +function terminalOpenAiReceipt(input: number, output: number, finishReason: string): string { + return [ + `data: ${JSON.stringify({ + choices: [{ index: 0, delta: {}, finish_reason: finishReason }], + usage: { prompt_tokens: input, completion_tokens: output, total_tokens: input + output }, + })}`, + 'data: [DONE]', + '', + ].join('\n\n') +} + +async function consumeDirectRouterReceipt(body: string) { + vi.stubGlobal( + 'fetch', + vi.fn(async () => + Promise.resolve(new Response(body, { headers: { 'content-type': 'text/event-stream' } })), + ), + ) + try { + const profile: AgentProfile = { + name: 'direct-router-receipt-test', + harness: 'cli-base', + model: { + provider: 'tangle-router', + default: 'glm-5.2', + metadata: { stream: true }, + }, + } + const executor = createExecutor({ + backend: 'router', + routerBaseUrl: 'https://router.example.test/v1', + routerKey: 'test-key', + tools: [], + })( + { + profile, + harness: null, + }, + { signal: new AbortController().signal, seams: {} }, + ) + await executor.execute('do the task', new AbortController().signal) + return executor.resultArtifact() + } finally { + vi.unstubAllGlobals() + } +} + describe('bridgeExecutor upstream-error propagation', () => { let server: Server | undefined afterEach(async () => { + vi.unstubAllGlobals() if (server) await new Promise((resolve) => server?.close(resolve)) server = undefined }) @@ -331,6 +379,46 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(artifact.spent.tokens).toEqual({ input: 10, output: 4 }) }) + it('meters the same terminal OpenAI receipt through direct Router and bridge paths', async () => { + const receipt = terminalOpenAiReceipt(3_020, 55, 'stop') + const direct = await consumeDirectRouterReceipt(receipt) + const stub = await startBridgeStub(receipt) + server = stub.server + const executor = makeExecutor(stub.url) + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(direct).toMatchObject({ + out: { finishReason: 'stop' }, + spent: { tokens: { input: 3_020, output: 55 } }, + }) + expect(events.filter((event) => event.kind === 'tokens')).toEqual([ + { kind: 'tokens', input: 3_020, output: 55 }, + ]) + expect(executor.resultArtifact().spent.tokens).toEqual(direct.spent.tokens) + }) + + it('meters an error-shaped terminal OpenAI receipt instead of treating it as free', async () => { + const receipt = terminalOpenAiReceipt(408, 12, 'error') + const direct = await consumeDirectRouterReceipt(receipt) + const stub = await startBridgeStub(receipt) + server = stub.server + const executor = makeExecutor(stub.url) + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(direct).toMatchObject({ + out: { finishReason: 'error' }, + spent: { tokens: { input: 408, output: 12 } }, + }) + expect(events.filter((event) => event.kind === 'tokens')).toEqual([ + { kind: 'tokens', input: 408, output: 12 }, + ]) + expect(executor.resultArtifact().spent.tokens).toEqual(direct.spent.tokens) + }) + it('meters one iteration per bridge turn instead of one per content chunk', async () => { const chunks = [ `data: ${JSON.stringify({ choices: [{ delta: { content: 'first ' } }] })}`, diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 8e84ce75..b60de7e3 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -74,6 +74,7 @@ import { type RouterConfig, routerChatWithTools, routerChatWithUsage, + routerTransportAttemptsFromError, streamRouterChatWithTools, type ToolSpec, } from '../router-client' @@ -796,8 +797,9 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = } catch (e) { cleanup() // Re-plan ONLY when a forceful inbox message aborted this turn (a real AbortError, with the - // interrupt — not the external teardown/budget signal). The re-planned turn still consumes a - // loop slot when the caller configured a finite maxTurns, but does not bill a turn. + // interrupt — not the external teardown/budget signal). The request reached the transport, + // so count its iteration and exact physical attempts. Without a terminal receipt its token, + // reasoning, and dollar totals are unknown; treating it as free can overspend the pool. // Any other error — incl. a network fault coincident with an interrupt — is fatal: rethrow. const interruptAbort = e instanceof DOMException && @@ -805,11 +807,18 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = interruptSig.aborted && !signal.aborted && !controller.signal.aborted - if (interruptAbort) continue + if (interruptAbort) { + turns += 1 + transportAttempts += routerTransportAttemptsFromError(e) ?? 1 + tokensKnown = false + usdKnown = false + reasoningKnown = false + continue + } throwRouterTransportFailure('routerToolsInlineExecutor', e) } cleanup() - // The inference completed — count the turn now (an interrupted, re-planned turn doesn't bill). + // The inference completed — count the turn and merge its terminal receipt. turns += 1 transportAttempts += res.transportAttempts assertObservedRouterModel(res.model, model, 'routerToolsInlineExecutor') diff --git a/tests/kernel/inbox.test.ts b/tests/kernel/inbox.test.ts index b8ffeb08..83954e13 100644 --- a/tests/kernel/inbox.test.ts +++ b/tests/kernel/inbox.test.ts @@ -104,7 +104,7 @@ describe('router-tools executor drains the inbox', () => { expect(turn2.some((m) => m.content?.includes('also handle the wide-char edge case'))).toBe(true) }) - it('a FORCEFUL steer aborts the in-flight turn; the worker re-plans and the aborted turn is free', async () => { + it('a FORCEFUL steer aborts the in-flight turn and records its unknown spend before re-planning', async () => { const bodies: Array<{ messages: Array<{ role: string; content: string }> }> = [] let calls = 0 let deliver: (m: unknown) => void = () => {} @@ -118,7 +118,13 @@ describe('router-tools executor drains the inbox', () => { throw new DOMException('aborted', 'AbortError') } bodies.push(JSON.parse(init?.body ?? '{}')) - return noToolReply() + return new Response( + JSON.stringify({ + choices: [{ message: { content: 'done', tool_calls: [] } }], + usage: { prompt_tokens: 1, completion_tokens: 1, cost_usd: 0.01 }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ) }), ) @@ -143,13 +149,21 @@ describe('router-tools executor drains the inbox', () => { const result = await exec.execute('edit the file', new AbortController().signal) - // The aborted turn was discarded and the worker re-planned on turn 2... + // The aborted response was discarded and the worker re-planned on turn 2... expect(calls).toBe(2) - // ...which carries the forceful steer, and the aborted turn did NOT count toward iterations. + // ...which carries the forceful steer. The accepted first request still consumed an iteration + // and one transport attempt; absent a terminal receipt its token and dollar totals are unknown. expect( bodies[0]?.messages.some((m) => m.content?.includes('wrong file, edit src/core.ts')), ).toBe(true) - expect(result.spent.iterations).toBe(1) + expect(result.spent).toMatchObject({ + iterations: 2, + tokens: { input: 1, output: 1 }, + tokensKnown: false, + usd: 0.01, + usdKnown: false, + }) + expect((result.out as { transportAttempts: number }).transportAttempts).toBe(2) }) it('marks dollar cost unknown for an unpriced model even when token usage is complete', async () => { diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index 4c9d162d..6696a2c6 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -809,6 +809,65 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(result.kind).toBeDefined() }) + it('allowedModels rejects a dynamically authored child before spawn, execution, or metering', async () => { + const journal = new InMemorySpawnJournal() + let childTransportCalls = 0 + const runId = 'forbidden-dynamic-child' + const result = await supervise( + testAgentProfile('root', { + harness: 'cli-base', + model: { provider: 'tangle-router', default: 'glm-5.2' }, + }), + 'keep every descendant on glm-5.2', + { + budget, + allowedModels: ['glm-5.2'], + backend: { + backend: 'router-tools', + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'test', + tools: [], + executeToolCall: async () => '', + complete: async () => { + childTransportCalls += 1 + return { + model: 'sonnet', + choices: [{ message: { content: 'must not execute', tool_calls: [] } }], + usage: { prompt_tokens: 1, completion_tokens: 1, cost_usd: 0.01 }, + } + }, + }, + brain: scriptedBrain([ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { + profile: testAgentProfile('forbidden', { + harness: 'pi', + model: { provider: 'amazon-bedrock', default: 'sonnet' }, + }), + task: 'must be refused', + }, + }, + ], + }, + { content: 'stop after refusal' }, + ]), + journal, + runId, + }, + ) + + expect(result.kind).toBe('no-winner') + expect(childTransportCalls).toBe(0) + const events = await journal.loadTree(runId) + expect(events?.filter((event) => event.kind === 'spawned').map((event) => event.id)).toEqual([ + runId, + ]) + expect(events?.some((event) => event.kind === 'metered' && event.id !== runId)).toBe(false) + }) + it('allowedModels unset is unrestricted (any model passes)', async () => { const result = await supervise( testAgentProfile('root', { From aafd2a1bafbb56c6c30b0b23068b70a8abe87fb5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:15:59 -0600 Subject: [PATCH 35/39] fix(supervise): remove guessed worker token floor --- bench/src/agent-graphs-gen2.mts | 19 ++--- bench/src/agent-graphs-gen3.mts | 5 +- bench/src/agent-graphs-improve.mts | 35 ---------- docs/api/runtime.md | 18 ++--- skills/agent-graphs/IMPROVE.md | 6 +- skills/agent-graphs/SKILL.md | 9 ++- skills/agent-graphs/cases/floor-trap-pi.json | 11 --- .../agent-graphs/cases/review-pipeline.json | 3 +- .../cases/unmeasured-harness.json | 6 +- src/mcp/tools/coordination.ts | 49 +------------ src/runtime/supervise/budget-floor.ts | 59 ---------------- src/runtime/supervise/budget.ts | 8 +-- src/runtime/supervise/scope.ts | 19 ----- src/runtime/supervise/types.ts | 9 +-- tests/kernel/budget-floor.test.ts | 69 ------------------- tests/kernel/coordination.test.ts | 53 +------------- 16 files changed, 32 insertions(+), 346 deletions(-) delete mode 100644 skills/agent-graphs/cases/floor-trap-pi.json delete mode 100644 src/runtime/supervise/budget-floor.ts delete mode 100644 tests/kernel/budget-floor.test.ts diff --git a/bench/src/agent-graphs-gen2.mts b/bench/src/agent-graphs-gen2.mts index 3e1f674c..dffbb252 100644 --- a/bench/src/agent-graphs-gen2.mts +++ b/bench/src/agent-graphs-gen2.mts @@ -13,8 +13,8 @@ * and v2 train mean >= v1 train mean - 0.05 * * Split (declared here, enforced by runImprovementLoop's overlap check): - * TRAIN = floor-trap-pi, review-pipeline, single-agent-suffices, - * cap-as-stop-mistake, runtime-discovered-fanout + * TRAIN = review-pipeline, single-agent-suffices, cap-as-stop-mistake, + * runtime-discovered-fanout * HOLDOUT = mission-in-deliverable, steer-heavy-drafting, unmeasured-harness * * Holdout hygiene: the revision prompt is built ONLY from TRAIN-case records and the @@ -64,7 +64,6 @@ const SMOKE = process.env.GEN2_SMOKE === '1' const K = 3 const SEED = 42 const TRAIN_IDS = [ - 'floor-trap-pi', 'review-pipeline', 'single-agent-suffices', 'cap-as-stop-mistake', @@ -166,14 +165,10 @@ function buildRevisionPrompt(v1Surface: string, trainCases: GraphScenario[]): st '', ...caseBlocks, '', - 'The three failure clusters your revision must target:', - '1. Under-graphing on cheap-sounding briefs: when a brief sounds cheap or trivial the author', - ' answers "single-agent" even when the case genuinely warrants a graph (e.g. a floor-trap', - ' brief whose tight budget must be overridden upward, not dodged; a mission whose deliverable', - ' must carry the concrete mission text). Cheapness of wording is not the dialect test.', - '2. Analysts never authored when warranted: when independent post-settle findings are required,', + 'The two failure clusters your revision must target:', + '1. Analysts never authored when warranted: when independent post-settle findings are required,', ' the author omits analyzes edges entirely and merges review into the root.', - '3. Identical-role parallelism collapsed: when the work is N parallel instances of the same', + '2. Identical-role parallelism collapsed: when the work is N parallel instances of the same', ' role, the author collapses them into one worker node instead of N nodes (one delegation', ' edge each), losing the parallelism the brief asked for.', '', @@ -184,8 +179,8 @@ function buildRevisionPrompt(v1Surface: string, trainCases: GraphScenario[]): st '- Keep the decision honest: "single-agent" and "dynamic-workflow" remain the CORRECT answers', ' when one profile suffices or when topology is discovered mid-run. Do not teach "always', ' graph" — fixing under-graphing must not create over-graphing.', - '- Keep the existing correct doctrine (traversal caps, analyzes-cap-is-not-a-stop, budget', - ' floors, deliverable-carries-mission, offline proving) — sharpen it, do not delete it.', + '- Keep the existing correct doctrine (traversal caps, analyzes-cap-is-not-a-stop,', + ' deliverable-carries-mission, offline proving) — sharpen it, do not delete it.', '- The skill is consumed by a model that must output a strict JSON graph spec; keep the text', ' operational, not narrative.', '', diff --git a/bench/src/agent-graphs-gen3.mts b/bench/src/agent-graphs-gen3.mts index 5543c35f..8fc9f24c 100644 --- a/bench/src/agent-graphs-gen3.mts +++ b/bench/src/agent-graphs-gen3.mts @@ -68,7 +68,6 @@ const EXPECTED_V2_SHA = '4c6615b6164f6c5a86efb2596556bdf325d33f08a4e1715cae9d71c const K = 5 const SEED = 42 const TRAIN_IDS = [ - 'floor-trap-pi', 'review-pipeline', 'single-agent-suffices', 'cap-as-stop-mistake', @@ -267,8 +266,8 @@ function buildRevisionPrompt(v2Surface: string, trainCases: GraphScenario[]): st '- Keep the decision honest: "single-agent" and "dynamic-workflow" remain the CORRECT answers', ' when one profile suffices or when topology is discovered mid-run. Do not teach "always', ' graph" — fixing under-graphing must not create over-graphing.', - '- Keep the existing correct doctrine (traversal caps, analyzes-cap-is-not-a-stop, budget', - ' floors, deliverable-carries-mission, offline proving) — sharpen it, do not delete it.', + '- Keep the existing correct doctrine (traversal caps, analyzes-cap-is-not-a-stop,', + ' deliverable-carries-mission, offline proving) — sharpen it, do not delete it.', '- The skill is consumed by a model that must output a strict JSON graph spec; keep the text', ' operational, not narrative.', '', diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index 37e2bf24..f6f96295 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -84,12 +84,6 @@ export function loadInputs(): { surface: string; cases: CaseSpec[]; source: stri return { surface, cases, source: `git:${SKILL_REF}` } } -// The measured pi floor the floor-trap case scores against (src/runtime/supervise/budget-floor.ts). -const PI_TOKEN_FLOOR = 31_211 -// "Budget generously" proxy for unmeasured harnesses: floor-unknown means per-child headroom -// well above the one measured floor; 50k is the scorer's line, documented not tuned. -const GENEROUS_PER_CHILD_TOKENS = 50_000 - // ── Case + artifact shapes ───────────────────────────────────────────────────── export interface CaseExpect { @@ -98,16 +92,12 @@ export interface CaseExpect { correctAnswerIsGraph?: boolean nodes?: number analyzesWarranted?: boolean - floorTrap?: boolean - mustBudgetAtLeast?: number - correctAuthorOverridesBrief?: boolean maxTraversalsAtLeast?: number deliverableDescribeCarriesMission?: boolean checkIsMechanical?: boolean trapIsAnalyzesCapAsStop?: boolean correctStopIsDelegatesCapOrDeliverable?: boolean wrongIfAnalystIsNode?: boolean - generousBudgetsBecauseFloorUnknown?: boolean edges?: string[] reason?: string } @@ -461,11 +451,6 @@ function domainWords(brief: string): string[] { ] } -function perChildTokens(spec: AuthoredGraphSpec): number { - // The skill's documented default: perWorker unset means a quarter of the pool. - return spec.perWorker?.maxTokens ?? Math.floor(spec.budget.maxTokens / 4) -} - function delegatesEdges(spec: AuthoredGraphSpec): Array> { return spec.edges.filter((e): e is Extract => e.kind === 'delegates') } @@ -510,16 +495,6 @@ export function judgeArtifact(artifact: AuthoredArtifact, kase: CaseSpec): { sco note: `decision=${artifact.decision}${artifact.decision === 'graph' && g === undefined ? ' (no graph payload)' : ''}`, }) } - if (e.mustBudgetAtLeast !== undefined) { - const perChild = graphOk ? perChildTokens(g) : 0 - checks.push({ - key: 'mustBudgetAtLeast', - pass: graphOk && perChild >= e.mustBudgetAtLeast, - note: graphOk - ? `per-child tokens ${perChild} vs floor ${e.mustBudgetAtLeast} (pi floor ${PI_TOKEN_FLOOR})` - : `no graph authored (decision=${artifact.decision})`, - }) - } if (e.nodes !== undefined) { const total = graphOk ? g.nodes.length : 0 const workers = graphOk ? Math.max(total - 1, 0) : 0 @@ -600,16 +575,6 @@ export function judgeArtifact(artifact: AuthoredArtifact, kase: CaseSpec): { sco : 'no graph authored', }) } - if (e.generousBudgetsBecauseFloorUnknown !== undefined) { - const perChild = graphOk ? perChildTokens(g) : 0 - checks.push({ - key: 'generousBudgetsBecauseFloorUnknown', - pass: graphOk && perChild >= GENEROUS_PER_CHILD_TOKENS, - note: graphOk - ? `per-child tokens ${perChild} vs generous line ${GENEROUS_PER_CHILD_TOKENS}` - : 'no graph authored', - }) - } if (e.edges !== undefined) { for (const want of e.edges) { if (/delegates/i.test(want)) { diff --git a/docs/api/runtime.md b/docs/api/runtime.md index a0782b33..a6149872 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -18609,13 +18609,11 @@ Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft` ### ReservationRejection -> **ReservationRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"below-runtime-floor"` +> **ReservationRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` Why a reservation was refused. `budget-exhausted` means the pool ran out of a channel it - budgets; `below-runtime-floor` means the request is under the amount that harness needs before - it does any work at all, so it is unsatisfiable at that size and the fix is to RAISE it; - `usd-unbudgeted` means the root declared no dollar ceiling, so a dollar request is - unsatisfiable at any amount and the fix is to budget the root, not to ask for less. +budgets; `usd-unbudgeted` means the root declared no dollar ceiling, so a dollar request is +unsatisfiable at any amount and the fix is to budget the root, not to ask for less. *** @@ -19121,19 +19119,15 @@ Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall- ### SpawnRejection -> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"below-runtime-floor"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"` +> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"` Fail-closed spawn rejections: an exhausted pool, a dollar request against a root that budgets no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, or a `key` that is still LIVE in this scope (the same assignment may not run twice concurrently). - `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite +`usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite responses: an exhausted pool may admit a smaller request, while an unbudgeted dollar channel - refuses every amount until the ROOT budget names a `maxUsd`. - - `below-runtime-floor` is separate for the same reason and points the opposite way from - `budget-exhausted`: the request is under what that harness spends before it reads its task, so - it is unsatisfiable at that SIZE and the fix is to RAISE it, never to retry smaller. +refuses every amount until the ROOT budget names a `maxUsd`. *** diff --git a/skills/agent-graphs/IMPROVE.md b/skills/agent-graphs/IMPROVE.md index 29a08090..e0cb46b4 100644 --- a/skills/agent-graphs/IMPROVE.md +++ b/skills/agent-graphs/IMPROVE.md @@ -24,10 +24,10 @@ case (idea brief) ──► author agent + skill-vN ──► graph ──► ru ## Cases -`cases/` seeds eight idea-briefs, each with `expect`: the edges a correct graph must have, ledger outcomes, whether analysts are warranted, and a floor-trap flag (the case is under-budgetable and a correct author must budget above the floor). +`cases/` seeds seven idea-briefs, each with `expect`: the edges a correct graph must have, ledger outcomes, and whether analysts are warranted. Case briefs are deliberately loose — "loose context in, correct graph out" is the skill's whole claim, so tidy specs would test the wrong thing. -Holdout discipline: at least 3 of the 8 held out, never trained on; `runImprovementLoop` throws on overlap. +Holdout discipline: 3 of the 7 are held out, never trained on; `runImprovementLoop` throws on overlap. ## What is deliberately NOT built @@ -40,7 +40,7 @@ Holdout discipline: at least 3 of the 8 held out, never trained on; `runImprovem Two layers, different jobs. Foundation harnesses ship trained orchestration — Claude Code subagents, codex goal-mode, pi extensions — and prose is that layer's native API: instructing a claude-code node to "fan out subagents over these files" invokes an in-distribution capability, not vibes. The graph/script layer exists for what no single harness provides: cross-harness composition, one conserved budget across the whole tree, durable ledger evidence, resume, and heterogeneous model placement. -Rule: outer layer coarse, inner layer maximal — one harness-sized node told to use its native fan-out beats N externally-choreographed thin nodes that each pay the harness floor while suppressing the orchestration the model was trained for. +Rule: outer layer coarse, inner layer maximal — one harness-sized node told to use its native fan-out beats N externally-choreographed thin nodes that duplicate setup and context while suppressing the orchestration the model was trained for. Which harnesses qualify for native fan-out is a supervisor-lab harness-KB row, not a guess. ## Version history diff --git a/skills/agent-graphs/SKILL.md b/skills/agent-graphs/SKILL.md index 6d073bb6..27b2482f 100644 --- a/skills/agent-graphs/SKILL.md +++ b/skills/agent-graphs/SKILL.md @@ -23,7 +23,6 @@ Do not use a graph when a smaller shipped primitive already expresses the work. ### Strict authoring decisions (Do not under-graph) - **Cheapness is not the dialect test:** Do not bail to `single-agent` just because a brief sounds trivial (e.g., "write a one-line file"). If the brief implies roles, observers, or a specific tight budget, author the graph. -- **Budget Floor Traps:** If a brief demands an impossibly "tight" budget (e.g., a few thousand tokens), do not dodge it by dropping to `single-agent`. Author the graph and explicitly set `budget` to the valid measured executor floor. - **Identical-Role Parallelism:** If a brief requests N parallel instances of the same role, you MUST create N distinct worker nodes and N `delegates` edges. Do not collapse identical parallel workers into a single node. - **Mandatory Analysts:** If a brief requires independent observation, review, or post-settle findings (e.g., "neutral decider", "review by two perspectives", "watch the worker"), you MUST author `analyzes` edges. Do not omit analysts and attempt to merge their logic into the root's prompt. - **Caps are not stops:** Do not use an analysis edge `maxTraversals` cap as a global stop condition. To stop after N findings, use `deliverable.check` or `maxTraversals` on a `delegates` edge. @@ -74,8 +73,8 @@ Put the concrete mission in `deliverable.describe`; Runtime uses that text as th `budget` is one conserved pool for the full graph. Set `options.perWorker` explicitly from the actual executor cost. -For Pi, `WORKER_TOKEN_FLOOR.pi` is 31,211 input tokens before useful work, so a worker allocation below that value is refused. If a brief asks for a budget lower than the floor, do not switch to `single-agent`; output the graph with the floor allocation. -Treat an unmeasured executor floor as unknown rather than zero. +Size each allocation from measurements of the actual profile, mounted context, tools, and task shape when those measurements exist. +Do not turn one run's cumulative spend into a universal harness minimum; Runtime enforces the caller's conserved pool, not guessed per-harness floors. Analyst nodes spend from the same pool and need the same honest accounting as ordinary workers. ## Authoring procedure @@ -87,7 +86,7 @@ Analyst nodes spend from the same pool and need the same honest accounting as or 5. **Register directives:** Register a versioned directive for every edge. 6. **Delegate work:** Add one delegation edge per ordinary worker from the root. 7. **Attach analysts:** Add `analyzes` edges only when findings must be produced independently after a worker settles. Do not skip this if the brief asked for a watcher/reviewer. -8. **Size the pool:** Set budget, per-worker allocation, traversal caps, time, and concurrency from measured executor behavior. Ensure budgets meet the executor floor. +8. **Size the pool:** Set budget, per-worker allocation, traversal caps, time, and concurrency from comparable measured runs when available. 9. **Prove and inspect:** Run the structure offline, then run the real backend and inspect its result. ## Prove the graph before spending @@ -115,7 +114,7 @@ A passing completion test proves only what that test checks. ## Common mistakes -- Bailing to `single-agent` because a brief sounds trivial, instead of respecting requested roles or applying budget floors. +- Bailing to `single-agent` because a brief sounds trivial, instead of respecting requested roles. - Collapsing N requested parallel identical roles into a single worker node. - Skipping `analyzes` edges when an observer or reviewer is explicitly requested. - Putting the task only in a spawn prompt instead of `deliverable.describe`. diff --git a/skills/agent-graphs/cases/floor-trap-pi.json b/skills/agent-graphs/cases/floor-trap-pi.json deleted file mode 100644 index caffd4f1..00000000 --- a/skills/agent-graphs/cases/floor-trap-pi.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id": "floor-trap-pi", - "brief": "Cheap little task: have a pi agent write a one-line file. Keep the budget tight, like a few thousand tokens.", - "expect": { - "nodes": 1, - "analyzesWarranted": false, - "floorTrap": true, - "mustBudgetAtLeast": 31211, - "correctAuthorOverridesBrief": true - } -} \ No newline at end of file diff --git a/skills/agent-graphs/cases/review-pipeline.json b/skills/agent-graphs/cases/review-pipeline.json index c5682f08..f7342911 100644 --- a/skills/agent-graphs/cases/review-pipeline.json +++ b/skills/agent-graphs/cases/review-pipeline.json @@ -4,11 +4,10 @@ "expect": { "nodes": 3, "analyzesWarranted": true, - "floorTrap": false, "edges": [ "delegates to each reviewer", "analyzes routing findings to root" ], "wrongIfAnalystIsNode": true } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/unmeasured-harness.json b/skills/agent-graphs/cases/unmeasured-harness.json index c68ec976..88c6cb8b 100644 --- a/skills/agent-graphs/cases/unmeasured-harness.json +++ b/skills/agent-graphs/cases/unmeasured-harness.json @@ -2,8 +2,6 @@ "id": "unmeasured-harness", "brief": "Run three probes on claude-code workers and collect what they output.", "expect": { - "nodes": 3, - "floorTrap": false, - "generousBudgetsBecauseFloorUnknown": true + "nodes": 3 } -} \ No newline at end of file +} diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index 1e0c49c8..c38f6a8b 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -29,7 +29,6 @@ import type { WorkerTraceEvidence, } from '../../runtime' import { assertValidBudget } from '../../runtime/supervise/budget' -import { WORKER_TOKEN_FLOOR, workerTokenFloor } from '../../runtime/supervise/budget-floor' import type { DeliverableSpec } from '../../runtime/supervise/completion-gate' import { type WatchTraceOptions, watchTrace } from '../../runtime/supervise/detector-monitor' import { freeSlots } from '../../runtime/supervise/dispatch' @@ -41,44 +40,6 @@ import { } from '../../runtime/supervise/trace-evidence' import type { McpToolDescriptor } from '../server' -// The floors a root must know BEFORE it authors a child budget, generated from the measured -// census so a newly measured harness appears in the published schema without a prose edit. -// Across a live n=8 sample, 5 of 6 authored child budgets were below the pi floor — the guard -// in scope.spawn defends, but only this description TELLS the root at tool-discovery time. -const measuredFloors: ReadonlyArray = Object.entries( - WORKER_TOKEN_FLOOR, -).flatMap(([harness, floor]) => (floor === null ? [] : [[harness, floor] as const])) -const measuredFloorSentence = - measuredFloors.length === 0 - ? '' - : ' A harness child spends a measured minimum before any work — a maxTokens under that ' + - 'minimum is refused (`error: "below-runtime-floor"`). Measured floors (input tokens): ' + - measuredFloors.map(([harness, floor]) => `${harness}=${floor}`).join(', ') + - '. Unmeasured harnesses have no floor and are admitted.' - -/** - * The actionable fact behind a `below-runtime-floor` refusal: the floor for the harness the - * root declared, and that the only fix is to raise maxTokens to at least that floor. Falls back - * to the whole measured census if the declared harness resolves to no floor (a guard fired on a - * spec-level harness this profile did not name). - */ -function belowFloorHint(harness: string | undefined): string { - const floor = workerTokenFloor(harness ?? null) - if (floor !== null) - return ( - `The ${harness} harness spends a measured minimum of ${floor} input tokens before any ` + - `work, so this budget's maxTokens can never be satisfied. Raise maxTokens to at least ` + - `${floor}; retrying with a smaller budget will fail identically.` - ) - return ( - "This budget's maxTokens is below the measured minimum a harness child spends before any " + - 'work, so it can never be satisfied. Raise maxTokens to at least the floor for the child ' + - 'harness — measured floors (input tokens): ' + - measuredFloors.map(([h, f]) => `${h}=${f}`).join(', ') + - ' — retrying with a smaller budget will fail identically.' - ) -} - /** A worker the driver has drained via `await_event`. */ export interface SettledWorker { readonly id: string @@ -1815,8 +1776,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin type: 'object', description: 'Optional per-spawn budget that merges over the per-worker default (per field). ' + - 'Only set the ceilings this sub-task needs raised; the conserved pool still fences.' + - measuredFloorSentence, + 'Only set the ceilings this sub-task needs raised; the conserved pool still fences.', properties: { maxIterations: { type: 'number', minimum: 0 }, maxTokens: { type: 'number', minimum: 0 }, @@ -1966,13 +1926,6 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin 'to give the run a root maxUsd.', } : {}), - // Same purpose as the `usd-unbudgeted` hint: `below-runtime-floor` reads as - // "budget" and invites a SMALLER retry, which is the exact wrong move — the - // ceiling is unsatisfiable at that size, not tight. The profile the root authored - // is in scope here, so name that harness's measured floor directly. - ...(res.reason === 'below-runtime-floor' - ? { hint: belowFloorHint(profile.harness) } - : {}), live: liveWorkerCount(), freeSlots: freeWorkerSlots(), }, diff --git a/src/runtime/supervise/budget-floor.ts b/src/runtime/supervise/budget-floor.ts deleted file mode 100644 index 208b24aa..00000000 --- a/src/runtime/supervise/budget-floor.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * The token budget below which a harness child cannot do ANY useful work. - * - * A coding-agent harness spends a fixed amount before it reads its task: system prompt, tool - * definitions, workspace context — re-sent every turn. A parent that authors a child budget under - * that amount has not set a tight ceiling, it has set an impossible one, and the child dies partway - * through with no signal that the number was never satisfiable. - * - * Measured, not guessed. Six live `pi` children on `tangle-router/glm-5.2` settled at 31,211 / - * 32,296 / 47,365 / 47,965 / 48,253 / 62,616 input tokens for 1–4 iterations, against 515–1,128 - * output. Four separate roots then authored child budgets of 2,000 / 8,000 / 12,000 / 20,000 — - * every one below the observed minimum. One of those children wrote its deliverable correctly and - * was killed before it could report, which the run recorded as a failure. - * - * A floor is `null` where nobody has measured one. That is deliberate: an unmeasured floor is - * UNKNOWN, never zero, and enforcing a made-up number would refuse work that would have succeeded. - * `null` means "admit anything and let the pool decide", the behavior before this table existed. - * - * The floor is a property of a harness/model pairing, not a universal constant, so this is a - * conservative lower bound: it uses the smallest spend ever observed, not the median, so a refusal - * means the budget is below what has EVER worked rather than below what is typical. - */ -import type { BackendType } from '@tangle-network/sandbox' - -// Already the target shape: one exhaustive row per harness, `satisfies` pinned so a new -// `BackendType` is a compile error rather than a silent gap. Add a harness by adding a row. -export const WORKER_TOKEN_FLOOR = { - // 31,211 was the lowest of six measured settlements; nothing rounds it up. - pi: 31_211, - // Unmeasured. Add a number here only with runs behind it. - opencode: null, - 'claude-code': null, - 'kimi-code': null, - codex: null, - amp: null, - 'factory-droids': null, - hermes: null, - forge: null, - openclaw: null, - nanoclaw: null, - acp: null, - cursor: null, - 'cli-base': null, -} as const satisfies Record - -/** - * The floor for a spawn's harness, or `null` when there is none to enforce. - * - * A `null` harness is the router/inline arm: it holds only coordination verbs and re-sends no - * harness scaffolding, so it has no floor of this kind. - * - * Accepts a bare `string` because `AgentProfile.harness` is not exactly `BackendType` — an - * unrecognised name resolves to `null` (no floor) rather than throwing, which keeps an unknown - * harness admissible instead of refusing work on a name this table has not heard of. - */ -export function workerTokenFloor(harness: BackendType | string | null | undefined): number | null { - if (harness === null || harness === undefined) return null - return (WORKER_TOKEN_FLOOR as Record)[harness] ?? null -} diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index 6ac5f9e8..f512bd08 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -86,11 +86,9 @@ export type BudgetReadout = Readonly<{ reservedTokens: number }> /** Why a reservation was refused. `budget-exhausted` means the pool ran out of a channel it - * budgets; `below-runtime-floor` means the request is under the amount that harness needs before - * it does any work at all, so it is unsatisfiable at that size and the fix is to RAISE it; - * `usd-unbudgeted` means the root declared no dollar ceiling, so a dollar request is - * unsatisfiable at any amount and the fix is to budget the root, not to ask for less. */ -export type ReservationRejection = 'budget-exhausted' | 'usd-unbudgeted' | 'below-runtime-floor' + * budgets; `usd-unbudgeted` means the root declared no dollar ceiling, so a dollar request is + * unsatisfiable at any amount and the fix is to budget the root, not to ask for less. */ +export type ReservationRejection = 'budget-exhausted' | 'usd-unbudgeted' /** State recovered from a prior process before new work is admitted. `committed` is measured spend * already present in the durable journal. Each `uncertainReservation` is a child that was recorded diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 51cb25a8..01814153 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -36,7 +36,6 @@ import { ValidationError } from '../../errors' import { notifyRuntimeHookEvent, type RuntimeHooks } from '../../runtime-hooks' import type { Iteration } from '../types' import { type BudgetPool, createBudgetPool, type ReservationTicket } from './budget' -import { workerTokenFloor } from './budget-floor' import { armDeadlineTimer, boundedChildDeadlineAt, @@ -582,24 +581,6 @@ export function createScope(args: ScopeArgs): Scope { permit.release() return { ok: false, reason: 'invalid-identity' } } - // A budget under the harness's measured floor is UNSATISFIABLE at that size, not merely - // tight — the child burns the whole ceiling on scaffolding it re-sends every turn and dies - // before reaching the task. Its own reason, for the same purpose `usd-unbudgeted` has one: - // `budget-exhausted` invites a caller to retry SMALLER, which here is the exact wrong move. - // Checked after resolution because that is where the harness is known, and refunded the way - // every sibling failure on this path is. - // `spec.harness` is null whenever the harness rides in the backend config instead of the - // spec — which is the path that actually spawns workers through the bridge. Reading only it - // made this guard miss the case it was written for: a root authored a 6,000-token child - // against a measured 31,211 floor, the guard never fired, and the child died having produced - // nothing (discovery-lab run proof-bridge-20260801f). The AgentProfile is where a root - // declares its child's harness, so it is the more reliable of the two. - const floor = workerTokenFloor(spec.harness ?? spec.profile.harness ?? null) - if (floor !== null && opts.budget.maxTokens < floor) { - args.pool.reconcile(reservation.ticket, zeroSpend()) - permit.release() - return { ok: false, reason: 'below-runtime-floor' } - } const outcome = args.executors.resolve(spec) if (!outcome.succeeded) throw new ValidationError(`scope.spawn: ${outcome.error}`) resolved = outcome diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 3d0e0bb9..da30a495 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -516,17 +516,12 @@ export interface SpawnOpts { * no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, or a `key` that * is still LIVE in this scope (the same assignment may not run twice concurrently). * - * `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite + * `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite * responses: an exhausted pool may admit a smaller request, while an unbudgeted dollar channel - * refuses every amount until the ROOT budget names a `maxUsd`. - * - * `below-runtime-floor` is separate for the same reason and points the opposite way from - * `budget-exhausted`: the request is under what that harness spends before it reads its task, so - * it is unsatisfiable at that SIZE and the fix is to RAISE it, never to retry smaller. */ + * refuses every amount until the ROOT budget names a `maxUsd`. */ export type SpawnRejection = | 'budget-exhausted' | 'usd-unbudgeted' - | 'below-runtime-floor' | 'depth-exceeded' | 'duplicate-key' | 'invalid-identity' diff --git a/tests/kernel/budget-floor.test.ts b/tests/kernel/budget-floor.test.ts deleted file mode 100644 index b1ac52d6..00000000 --- a/tests/kernel/budget-floor.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * A parent authors its child's budget. Nothing stopped it authoring one no child of that harness - * could survive, and four separate roots did exactly that across six measured runs — 2,000 / 8,000 - * / 12,000 / 20,000 tokens against a `pi` floor whose lowest observed settlement was 31,211. One of - * those children wrote its deliverable correctly and was killed before reporting. - */ -import { describe, expect, it } from 'vitest' -import { WORKER_TOKEN_FLOOR, workerTokenFloor } from '../../src/runtime/supervise/budget-floor' - -describe('worker token floor', () => { - it('reports the measured pi floor', () => { - // The lowest of six live settlements, not the median: a refusal must mean "below what has EVER - // worked", never "below what is typical". - expect(workerTokenFloor('pi')).toBe(31_211) - }) - - it('has NO floor for the router arm, which re-sends no harness scaffolding', () => { - expect(workerTokenFloor(null)).toBeNull() - }) - - it('reports null — not zero — for a harness nobody has measured', () => { - // An unmeasured floor is UNKNOWN. Zero would read as "measured, and it is free". - expect(workerTokenFloor('codex')).toBeNull() - expect(workerTokenFloor('claude-code')).toBeNull() - }) - - it('refuses every budget the measured roots actually authored', () => { - const floor = workerTokenFloor('pi')! - for (const authored of [2_000, 8_000, 12_000, 20_000]) { - expect(authored).toBeLessThan(floor) - } - }) - - it('admits a budget at or above the floor', () => { - const floor = workerTokenFloor('pi')! - expect(floor).toBeLessThanOrEqual(31_211) - expect(120_000).toBeGreaterThan(floor) - }) - - it('carries only measured numbers, so the table cannot drift into invention', () => { - for (const [harness, value] of Object.entries(WORKER_TOKEN_FLOOR)) { - if (value === null) continue - expect(harness).toBe('pi') - expect(value).toBeGreaterThan(0) - } - }) -}) - -describe('the floor reads the harness a root actually declared', () => { - // The guard originally read only `spec.harness`, which is null whenever the harness rides in the - // backend config rather than the spec — the path that actually spawns workers through the - // bridge. So it missed the case it exists for: run proof-bridge-20260801f authored a - // 6,000-token child against a 31,211 floor, admitted it, and the child produced nothing. - it('resolves a floor from an AgentProfile harness when the spec carries none', () => { - const specHarness = null - const profileHarness = 'pi' as const - expect(workerTokenFloor(specHarness ?? profileHarness)).toBe(31_211) - }) - - it('still has no floor when neither names a harness (router/inline arm)', () => { - const specHarness = null - const profileHarness = null - expect(workerTokenFloor(specHarness ?? profileHarness)).toBeNull() - }) - - it('refuses the exact budget that slipped through', () => { - expect(6_000).toBeLessThan(workerTokenFloor('pi')!) - }) -}) diff --git a/tests/kernel/coordination.test.ts b/tests/kernel/coordination.test.ts index 152e0e15..3a18c0a3 100644 --- a/tests/kernel/coordination.test.ts +++ b/tests/kernel/coordination.test.ts @@ -16,7 +16,6 @@ import { WORKER_TOOL_TRACE_SCHEMA_VERSION, watchTrace, } from '../../src/runtime' -import { WORKER_TOKEN_FLOOR } from '../../src/runtime/supervise/budget-floor' const zeroSpend = (): Spend => ({ iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 }) @@ -96,7 +95,7 @@ function mockScope() { }, ] let admit = true - let refusal: 'budget-exhausted' | 'below-runtime-floor' = 'budget-exhausted' + let refusal: 'budget-exhausted' | 'usd-unbudgeted' = 'budget-exhausted' const scope = { spawn: (_agent: unknown, task: unknown, opts: { budget: unknown; label: string }) => { spawns.push({ task, opts }) @@ -524,56 +523,6 @@ describe('coordination tools', () => { ).toThrow(/"budget.maxTokens" must be a finite number/) }) - it('publishes every measured runtime floor in the budget description, from the census', () => { - const { scope } = mockScope() - const tb = createCoordinationTools({ - scope, - blobs, - makeWorkerAgent, - perWorker: { maxIterations: 1, maxTokens: 10 }, - }) - const input = tool(tb, 'spawn_agent').inputSchema as { - properties: { budget: { description: string } } - } - const description = input.properties.budget.description - // Generated FROM the census, never transcribed: a harness measured tomorrow appears here - // with no edit to the tool text. `pi` is the one floor measured today. - expect(description).toContain('below-runtime-floor') - for (const [harness, floor] of Object.entries(WORKER_TOKEN_FLOOR)) - if (floor !== null) expect(description).toContain(`${harness}=${floor}`) - expect(description).toContain(`pi=${WORKER_TOKEN_FLOOR.pi}`) - }) - - it('spawn_agent tells a below-runtime-floor refusal the floor and to RAISE, never retry smaller', async () => { - const { scope, setAdmit, setRefusal } = mockScope() - setAdmit(false) - setRefusal('below-runtime-floor') - const tb = createCoordinationTools({ - scope, - blobs, - makeWorkerAgent, - perWorker: { maxIterations: 1, maxTokens: 4000 }, - }) - // The root declared the harness on the child profile, so the hint names THAT floor. - const refused = await tool(tb, 'spawn_agent').handler({ - profile: { harness: 'pi' }, - task: 'go', - }) - expect(refused).toMatchObject({ error: 'below-runtime-floor', hint: expect.any(String) }) - const hint = (refused as { hint: string }).hint - expect(hint).toContain(String(WORKER_TOKEN_FLOOR.pi)) - expect(hint).toMatch(/raise/i) - // The whole point of a distinct reason: the driver must not walk the budget DOWN. - expect(hint).toMatch(/fail identically/i) - - // No harness on the profile (the guard fired on a spec-level harness): the hint still - // carries every measured floor rather than an unactionable bare reason. - const bare = await tool(tb, 'spawn_agent').handler({ profile: {}, task: 'go' }) - const bareHint = (bare as { hint: string }).hint - expect(bareHint).toContain(String(WORKER_TOKEN_FLOOR.pi)) - expect(bareHint).toMatch(/raise/i) - }) - it('observe_agent returns live status and settled output', async () => { const { scope } = mockScope() const tb = createCoordinationTools({ From ae0e63661ba9ea07750ab87828f8aab7e7e5cbdd Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:16:07 -0600 Subject: [PATCH 36/39] fix(bench): import scripted brain from testing entry --- bench/src/agent-graphs-improve/offline-seams.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/agent-graphs-improve/offline-seams.mts b/bench/src/agent-graphs-improve/offline-seams.mts index a0b73585..fb90d3c0 100644 --- a/bench/src/agent-graphs-improve/offline-seams.mts +++ b/bench/src/agent-graphs-improve/offline-seams.mts @@ -19,9 +19,9 @@ import { type Executor, type ExecutorResult, type MakeWorkerAgent, - type ToolLoopChat, type TraceSource, } from '../../../src/runtime/index.ts' +import type { ToolLoopChat } from '../../../src/testing/index.ts' // ── The scripted driver brain ────────────────────────────────────────────────── From 9b383551609f639b2a4d0a4e6990f8988ad02b64 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:22:06 -0600 Subject: [PATCH 37/39] chore(release): advance published agent cohort --- .github/workflows/ci.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- pnpm-lock.yaml | 36 +++++++++++++++++------------------ pnpm-workspace.yaml | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08440cb4..799f5763 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 3f78bc880d7b2d3d477069ade50bb78f5c34bf72 # v0.144.3 + ref: cd4c3195fec79c79976ce2fbe43f307107544798 # v0.144.4 path: .cohort/agent-eval persist-credentials: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 6dc011ba7452638fbdbf89f454fb21f30e397d54 # v7.0.10 + ref: b5c82eb9a508b13a32e14353a867c197cda517b8 # v7.0.11 path: .cohort/agent-knowledge persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bbf8d962..1704deb8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -143,7 +143,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 3f78bc880d7b2d3d477069ade50bb78f5c34bf72 # v0.144.3 + ref: cd4c3195fec79c79976ce2fbe43f307107544798 # v0.144.4 path: .cohort/agent-eval persist-credentials: false @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 6dc011ba7452638fbdbf89f454fb21f30e397d54 # v7.0.10 + ref: b5c82eb9a508b13a32e14353a867c197cda517b8 # v7.0.11 path: .cohort/agent-knowledge persist-credentials: false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2de575d..d6523413 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,14 +13,14 @@ catalogs: specifier: 0.5.0 version: 0.5.0 '@tangle-network/agent-eval': - specifier: 0.144.3 - version: 0.144.3 + specifier: 0.144.4 + version: 0.144.4 '@tangle-network/agent-interface': specifier: 0.43.1 version: 0.43.1 '@tangle-network/agent-knowledge': - specifier: 7.0.10 - version: 7.0.10 + specifier: 7.0.11 + version: 7.0.11 '@tangle-network/agent-profile-materialize': specifier: 0.10.2 version: 0.10.2 @@ -55,7 +55,7 @@ importers: version: 0.5.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.10 + version: 7.0.11 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.10.2(@tangle-network/agent-interface@0.43.1) @@ -74,7 +74,7 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.3 + version: 0.144.4 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.1 @@ -122,13 +122,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.3 + version: 0.144.4 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.43.1 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.0.10 + version: 7.0.11 '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1114,8 +1114,8 @@ packages: '@tangle-network/agent-core@0.5.0': resolution: {integrity: sha512-xjsTAZbtF0eXiUxzzNib4j49gYUhEDF7SCYhD8bkUeIJ2O8rfa0GGl7y3egi8SvsbKRhjbIojL+4mcmERbgUzw==} - '@tangle-network/agent-eval@0.144.3': - resolution: {integrity: sha512-gkyeie5CCzUuEzTfuQKe6iikSsu0S9h6w8yRtfnvjDZopkJJ7Nn+VDzkWZrsLDQ31LtLaHqJ7I2FgYbPNvjW0Q==} + '@tangle-network/agent-eval@0.144.4': + resolution: {integrity: sha512-tR9U3K+5sBp8cCP/YllnSJ3WCQkkdgJhAlvC06DxrxgJVjVIT0HDjJaMXWcTYXKi+2Fk0Pv0f2PW5NSD171gzQ==} engines: {node: '>=20'} hasBin: true @@ -1125,8 +1125,8 @@ packages: '@tangle-network/agent-interface@0.43.1': resolution: {integrity: sha512-nrJol7sGmLLp60S0dBk+QejIB/TXvOOVNjq/OCb23K0oAWURCFyaTWA0J8q2V/pJwxalOPDWovqptMgkcF6q1A==} - '@tangle-network/agent-knowledge@7.0.10': - resolution: {integrity: sha512-igVEo0a6nnCZWmemvu9aLdIRMznAPKZpHcNWMdRJ5EyYXc5TnX0kwKTgTCzBHvg9Fqp8gfMttNHJzA4T+V3Yog==} + '@tangle-network/agent-knowledge@7.0.11': + resolution: {integrity: sha512-/vJk5aSFaUo18s3ucprmhtZWgmHA/d/LzoQi6+iulov6f/TOEoo/elK898dhwLkmK1AyzxwVHvjueq/xqiZrsg==} engines: {node: '>=20.19.0'} hasBin: true @@ -2797,12 +2797,12 @@ snapshots: '@tangle-network/agent-interface': 0.43.1 zod: 4.4.3 - '@tangle-network/agent-eval@0.144.3': + '@tangle-network/agent-eval@0.144.4': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) - '@tangle-network/agent-core': 0.4.33 - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-core': 0.5.0 + '@tangle-network/agent-interface': 0.43.1 '@tangle-network/agent-trace-contract': 1.0.2 hono: 4.12.32 linear-sum-assignment: 1.0.9 @@ -2821,10 +2821,10 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@7.0.10': + '@tangle-network/agent-knowledge@7.0.11': dependencies: - '@tangle-network/agent-eval': 0.144.3 - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-eval': 0.144.4 + '@tangle-network/agent-interface': 0.43.1 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 30cdd01e..0131761f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,9 +19,9 @@ catalog: '@arethetypeswrong/cli': 0.18.5 '@tangle-network/agent-core': 0.5.0 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.144.3 + '@tangle-network/agent-eval': 0.144.4 '@tangle-network/agent-interface': 0.43.1 - '@tangle-network/agent-knowledge': 7.0.10 + '@tangle-network/agent-knowledge': 7.0.11 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': 0.17.3 From 979906a3588074066b8191ba1d461f423645ee9e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:36:10 -0600 Subject: [PATCH 38/39] fix(release): align eval cohort metadata --- CHANGELOG.md | 2 +- README.md | 4 ++-- bench/CHANGELOG.md | 2 +- bench/src/swe-arena/gepa-seat.mts | 2 +- docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 2 +- package.json | 2 +- src/improvement/official-optimizers.ts | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69eae1c..2214d7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.129.0 -- Require Agent Eval 0.144.3 and Agent Knowledge 7.0.10, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. +- Require Agent Eval 0.144.4 and Agent Knowledge 7.0.11, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. - Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. - Require `defineLeaderboard` callers to supply an exact `baseProfile`; remove its `modelBackend` override so each matrix cell's profile remains the only model authority. - Require generic coder, researcher, and supervised-knowledge paths to receive complete profiles; remove harness/model overlays and MCP environment alias ladders. diff --git a/README.md b/README.md index 9e59a4f6..e191a72d 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.144.3" +python -m pip install "agent-eval-rpc==0.144.4" python -m pip install "gepa[full]==0.1.4" ``` @@ -256,7 +256,7 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.144.3" +python -m pip install "agent-eval-rpc==0.144.4" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index d80d05a9..9b12a731 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.7.2 -- Consume Runtime 0.129.0, Eval 0.144.3, Knowledge 7.0.10, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. +- Consume Runtime 0.129.0, Eval 0.144.4, Knowledge 7.0.11, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. ## 0.7.1 diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index a4e002d9..4e99bc2c 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -234,7 +234,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.144.3`, then install ' + + 'install `agent-eval-rpc==0.144.4`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 3c88af52..1e043e88 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.129.0` and `@tangle-network/agent-eval@0.144.3` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.129.0` and `@tangle-network/agent-eval@0.144.4` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index b37866bf..87d8751d 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -6,7 +6,7 @@ Run pnpm docs:freshness after editing this file. --> > **Version 0.129.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.144.3 <0.145.0`. +> `agent-eval` must satisfy `>=0.144.4 <0.145.0`. > `sandbox` must satisfy `>=0.17.2 <0.18.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.0 <0.44.0`. > diff --git a/package.json b/package.json index 4d10a660..c69053fb 100644 --- a/package.json +++ b/package.json @@ -169,7 +169,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.144.3 <0.145.0", + "@tangle-network/agent-eval": ">=0.144.4 <0.145.0", "@tangle-network/agent-interface": ">=0.43.0 <0.44.0", "@tangle-network/sandbox": ">=0.17.2 <0.18.0" }, diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index 025e95e7..3c33a79b 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -25,7 +25,7 @@ import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.3"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.4"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' From 37d83656a121a0ce28baf387b505edea51a998f3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 02:02:14 -0600 Subject: [PATCH 39/39] fix(release): enforce exact shared package cohort --- CHANGELOG.md | 2 +- bench/CHANGELOG.md | 2 +- docs/canonical-api.md | 4 +- package.json | 4 +- pnpm-lock.yaml | 35 ++++--------- pnpm-workspace.yaml | 2 +- scripts/lib/packed-package-test.mjs | 19 +++++++ scripts/verify-official-optimizers.mjs | 25 ++++++--- scripts/verify-package-exports.mjs | 70 +++++++++++++++++++++++++- scripts/verify-packed-cohort.mjs | 7 +++ 10 files changed, 130 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2214d7ea..a5505988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.129.0 -- Require Agent Eval 0.144.4 and Agent Knowledge 7.0.11, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. +- Require Agent Eval 0.144.4, Agent Interface 0.43.1, Agent Knowledge 7.0.11, and Sandbox 0.19.1 as one dependency set, and route the official-optimizer callback through Runtime's exact `AgentProfile` execution path. - Reject model, provider, reasoning, prompt, tool, resource, harness, and generation-setting conflicts before transport; consumers must declare those fields in the profile. - Require `defineLeaderboard` callers to supply an exact `baseProfile`; remove its `modelBackend` override so each matrix cell's profile remains the only model authority. - Require generic coder, researcher, and supervised-knowledge paths to receive complete profiles; remove harness/model overlays and MCP environment alias ladders. diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 9b12a731..ddfba723 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.7.2 -- Consume Runtime 0.129.0, Eval 0.144.4, Knowledge 7.0.11, and Sandbox 0.17.3 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract. +- Consume Runtime 0.129.0, Eval 0.144.4, Interface 0.43.1, Knowledge 7.0.11, and Sandbox 0.19.1 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract without loading duplicate agent contracts. ## 0.7.1 diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 87d8751d..01a7f2c3 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -7,8 +7,8 @@ Run pnpm docs:freshness after editing this file. --> > **Version 0.129.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.144.4 <0.145.0`. -> `sandbox` must satisfy `>=0.17.2 <0.18.0`. -> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.0 <0.44.0`. +> `sandbox` must satisfy `>=0.19.1 <0.20.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.1 <0.44.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. > diff --git a/package.json b/package.json index c69053fb..e1ffc3bc 100644 --- a/package.json +++ b/package.json @@ -170,8 +170,8 @@ "packageManager": "pnpm@11.17.0", "peerDependencies": { "@tangle-network/agent-eval": ">=0.144.4 <0.145.0", - "@tangle-network/agent-interface": ">=0.43.0 <0.44.0", - "@tangle-network/sandbox": ">=0.17.2 <0.18.0" + "@tangle-network/agent-interface": ">=0.43.1 <0.44.0", + "@tangle-network/sandbox": ">=0.19.1 <0.20.0" }, "peerDependenciesMeta": { "@tangle-network/sandbox": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6523413..831a8987 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,8 +28,8 @@ catalogs: specifier: ^1.0.2 version: 1.0.2 '@tangle-network/sandbox': - specifier: 0.17.3 - version: 0.17.3 + specifier: 0.19.1 + version: 0.19.1 '@types/node': specifier: 26.1.1 version: 26.1.1 @@ -80,7 +80,7 @@ importers: version: 0.43.1 '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.19.1(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) '@types/node': specifier: 'catalog:' version: 26.1.1 @@ -134,7 +134,7 @@ importers: version: link:.. '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.19.1(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) devDependencies: '@arethetypeswrong/cli': specifier: 'catalog:' @@ -1108,9 +1108,6 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - '@tangle-network/agent-core@0.4.33': - resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-core@0.5.0': resolution: {integrity: sha512-xjsTAZbtF0eXiUxzzNib4j49gYUhEDF7SCYhD8bkUeIJ2O8rfa0GGl7y3egi8SvsbKRhjbIojL+4mcmERbgUzw==} @@ -1119,9 +1116,6 @@ packages: engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@0.43.0': - resolution: {integrity: sha512-t35nGOJ3sWouHoUM/5A8eUmsR+8JcuRF6VFQbT9cPzxIMUV3NsFMJgYXPGmyIj65vtvs73O6Mvijp47FAAGbuQ==} - '@tangle-network/agent-interface@0.43.1': resolution: {integrity: sha512-nrJol7sGmLLp60S0dBk+QejIB/TXvOOVNjq/OCb23K0oAWURCFyaTWA0J8q2V/pJwxalOPDWovqptMgkcF6q1A==} @@ -1138,8 +1132,8 @@ packages: '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} - '@tangle-network/sandbox@0.17.3': - resolution: {integrity: sha512-Cfq7egwLEbFGCbLVVe8L7lFoimKiKXFsZkPZSVLvvbU1dDTj2v7fNV7+TJzPiEL1HGIEvvLbHvsZvUAPk7IUTg==} + '@tangle-network/sandbox@0.19.1': + resolution: {integrity: sha512-SuSl+HbX8B5aX0TF0QYDNSAN+huFwl04Aypt5+pYay4TMUEqvDGrNOhuSFLUatAvPQQi1Z35oZcboH+PlOatsw==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -2787,11 +2781,6 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@tangle-network/agent-core@0.4.33': - dependencies: - '@tangle-network/agent-interface': 0.43.0 - zod: 4.4.3 - '@tangle-network/agent-core@0.5.0': dependencies: '@tangle-network/agent-interface': 0.43.1 @@ -2809,12 +2798,6 @@ snapshots: re2js: 2.8.6 zod: 4.4.3 - '@tangle-network/agent-interface@0.43.0': - dependencies: - '@noble/hashes': 1.8.0 - spdx-expression-parse: 5.0.0 - zod: 4.4.3 - '@tangle-network/agent-interface@0.43.1': dependencies: '@noble/hashes': 1.8.0 @@ -2834,10 +2817,10 @@ snapshots: '@tangle-network/agent-trace-contract@1.0.2': {} - '@tangle-network/sandbox@0.17.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.19.1(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': dependencies: - '@tangle-network/agent-core': 0.4.33 - '@tangle-network/agent-interface': 0.43.0 + '@tangle-network/agent-core': 0.5.0 + '@tangle-network/agent-interface': 0.43.1 zod: 4.4.3 optionalDependencies: viem: 2.54.6(typescript@6.0.3)(zod@4.4.3) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0131761f..4db9433f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -24,7 +24,7 @@ catalog: '@tangle-network/agent-knowledge': 7.0.11 '@tangle-network/agent-profile-materialize': 0.10.2 '@tangle-network/agent-trace-contract': ^1.0.2 - '@tangle-network/sandbox': 0.17.3 + '@tangle-network/sandbox': 0.19.1 publint: 0.3.22 tsdown: 0.22.14 tsx: 4.23.1 diff --git a/scripts/lib/packed-package-test.mjs b/scripts/lib/packed-package-test.mjs index da65e52d..7644e8de 100644 --- a/scripts/lib/packed-package-test.mjs +++ b/scripts/lib/packed-package-test.mjs @@ -34,6 +34,25 @@ export function requiredPackedPackageVersion(version, name, owner) { return version } +export function currentMinorPeerRange(version) { + const match = /^(\d+)\.(\d+)\.\d+(?:-.+)?$/.exec(version) + if (!match) throw new Error(`cannot derive peer range from version ${version}`) + return `>=${version} <${match[1]}.${Number(match[2]) + 1}.0` +} + +export function assertPeerMatchesDevelopmentDependency(packageJson, name) { + const version = requiredPackedDevelopmentDependency(packageJson, name) + const expected = currentMinorPeerRange(version) + const actual = packageJson.peerDependencies?.[name] + if (actual !== expected) { + const packageName = + typeof packageJson.name === 'string' ? packageJson.name : 'packed package' + throw new Error( + `${packageName} peerDependencies.${name} must match its resolved development dependency: expected ${expected}, found ${String(actual)}`, + ) + } +} + export function createStrictNodeConsumerTsconfig(options = {}) { return { compilerOptions: { diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs index 920284ee..fbe47e76 100644 --- a/scripts/verify-official-optimizers.mjs +++ b/scripts/verify-official-optimizers.mjs @@ -12,6 +12,8 @@ import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { + assertPeerMatchesDevelopmentDependency, + currentMinorPeerRange, requiredPackedDevelopmentDependency, requiredPackedPackageVersion, } from './lib/packed-package-test.mjs' @@ -45,6 +47,16 @@ assertVersion( currentMinorPeerRange(agentEvalVersion), '@tangle-network/agent-eval peer dependency', ) +assertVersion( + packageJson.peerDependencies?.['@tangle-network/agent-interface'], + currentMinorPeerRange(workspaceAgentInterfaceVersion), + '@tangle-network/agent-interface peer dependency', +) +assertVersion( + packageJson.peerDependencies?.['@tangle-network/sandbox'], + currentMinorPeerRange(workspaceSandboxVersion), + '@tangle-network/sandbox peer dependency', +) try { run('pnpm', ['build'], repoRoot) @@ -73,6 +85,13 @@ try { mkdirSync(unpackDir, { recursive: true }) run('tar', ['-xzf', runtimeTarball, '-C', unpackDir], repoRoot) const packedPackageJson = readJson(join(unpackDir, 'package', 'package.json')) + for (const name of [ + '@tangle-network/agent-eval', + '@tangle-network/agent-interface', + '@tangle-network/sandbox', + ]) { + assertPeerMatchesDevelopmentDependency(packedPackageJson, name) + } const packedAgentEvalVersion = requiredPackedDevelopmentDependency( packedPackageJson, '@tangle-network/agent-eval', @@ -301,12 +320,6 @@ function assertVersion(actual, expected, label) { } } -function currentMinorPeerRange(version) { - const match = /^(\d+)\.(\d+)\.\d+(?:-.+)?$/.exec(version) - if (!match) throw new Error(`cannot derive peer range from version ${version}`) - return `>=${version} <${match[1]}.${Number(match[2]) + 1}.0` -} - function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')) } diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 6a1638ba..91de1bbc 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -12,9 +12,11 @@ import { tmpdir } from 'node:os' import { dirname, isAbsolute, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { + assertPeerMatchesDevelopmentDependency, assertPublishableDependencySpecs, createStrictNodeConsumerTsconfig, requiredPackedDevelopmentDependency, + requiredPackedPackageVersion, } from './lib/packed-package-test.mjs' import { findLiteralModuleSpecifiers, @@ -62,6 +64,13 @@ try { const packageDir = join(unpackDir, 'package') const packageJson = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8')) assertPublishableDependencySpecs(packageJson) + for (const name of [ + '@tangle-network/agent-eval', + '@tangle-network/agent-interface', + '@tangle-network/sandbox', + ]) { + assertPeerMatchesDevelopmentDependency(packageJson, name) + } if (packageJson.peerDependenciesMeta?.['@tangle-network/agent-eval']?.optional) { throw new Error('@tangle-network/agent-eval must stay required: root and ./kernel import it at runtime') } @@ -343,7 +352,37 @@ try { // This fixture type-checks with its declared dev toolchain; ambient production // install settings must not silently omit TypeScript or the Node declarations. run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund'], appDir) - assertNoEdgeUnsafeStaticImports(join(appDir, 'node_modules', '@tangle-network')) + const installedScope = join(appDir, 'node_modules', '@tangle-network') + const expectedFirstPartyVersions = { + '@tangle-network/agent-core': requiredPackedPackageVersion( + packageJson.dependencies?.['@tangle-network/agent-core'], + '@tangle-network/agent-core', + packageJson.name, + ), + '@tangle-network/agent-eval': requiredPackedDevelopmentDependency( + packageJson, + '@tangle-network/agent-eval', + ), + '@tangle-network/agent-interface': requiredPackedDevelopmentDependency( + packageJson, + '@tangle-network/agent-interface', + ), + '@tangle-network/agent-knowledge': requiredPackedPackageVersion( + packageJson.dependencies?.['@tangle-network/agent-knowledge'], + '@tangle-network/agent-knowledge', + packageJson.name, + ), + '@tangle-network/sandbox': requiredPackedDevelopmentDependency( + packageJson, + '@tangle-network/sandbox', + ), + } + for (const [packageName, expectedVersion] of Object.entries( + expectedFirstPartyVersions, + )) { + assertSingleFirstPartyPackageVersion(installedScope, packageName, expectedVersion) + } + assertNoEdgeUnsafeStaticImports(installedScope) run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], appDir) run( @@ -751,6 +790,35 @@ function assertNoEdgeUnsafeStaticImports(scopeDir) { ) } +function assertSingleFirstPartyPackageVersion(scopeDir, packageName, expectedVersion) { + const installations = firstPartyPackageDirs(scopeDir) + .map((packageDir) => ({ + packageDir, + packageJson: JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8')), + })) + .filter(({ packageJson }) => packageJson.name === packageName) + if (installations.length === 0) { + throw new Error(`packed consumer did not install ${packageName}`) + } + if ( + installations.length !== 1 || + installations[0].packageJson.version !== expectedVersion + ) { + throw new Error( + [ + `packed consumer must load exactly one ${packageName}@${expectedVersion}; found ${installations.length} installed path(s)`, + ...installations.map( + ({ packageDir, packageJson }) => + ` - ${packageJson.version} at ${relative(scopeDir, packageDir)}`, + ), + ].join('\n'), + ) + } + process.stdout.write( + `${packageName}: one ${expectedVersion} installation.\n`, + ) +} + function staticExternalImportClosure(entryPath, packageDir) { const queue = [resolve(entryPath)] const visited = new Set() diff --git a/scripts/verify-packed-cohort.mjs b/scripts/verify-packed-cohort.mjs index dd862bb6..2f2d03c5 100644 --- a/scripts/verify-packed-cohort.mjs +++ b/scripts/verify-packed-cohort.mjs @@ -19,6 +19,7 @@ import { basename, delimiter, dirname, join, relative, resolve, sep } from 'node import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' import { + assertPeerMatchesDevelopmentDependency, assertPublishableDependencySpecs, createStrictNodeConsumerTsconfig, requiredPackedDevelopmentDependency, @@ -529,6 +530,12 @@ function assertCohortPackageContracts({ assertExactDependency(agentKnowledge, agentInterface) assertExactDependency(agentKnowledge, agentEval) assertExactDependency(agentRuntime, agentKnowledge) + assertPeerMatchesDevelopmentDependency(agentRuntime.packageJson, agentInterface.name) + assertPeerMatchesDevelopmentDependency(agentRuntime.packageJson, agentEval.name) + assertPeerMatchesDevelopmentDependency( + agentRuntime.packageJson, + '@tangle-network/sandbox', + ) assertRequiredPeer(agentRuntime, agentInterface) assertRequiredPeer(agentRuntime, agentEval) }