From f52f878392634034c3fb5ee4ac0e36b577d9953a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:25:51 -0600 Subject: [PATCH 1/7] fix(compatibility): preserve native profile prompt delivery --- README.md | 4 +- bench/src/swe-arena/gepa-seat.mts | 2 +- docs/canonical-api.md | 6 +- .../coding-benchmark/coding-benchmark.test.ts | 7 +- examples/coding-benchmark/profiles.ts | 21 +- src/candidate-execution/prepare.ts | 16 +- src/candidate-execution/system-prompt.ts | 264 +++++++++++++----- src/improvement/official-optimizers.ts | 2 +- src/runtime/environment-provider.test.ts | 74 ++++- src/runtime/environment-provider.ts | 16 +- ...gle-sandbox-exact-process-provider.test.ts | 1 + tests/candidate-execution-prepare.test.ts | 208 ++++++++++++-- 12 files changed, 495 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 6240c92f..9a24d643 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.144.4" +python -m pip install "agent-eval-rpc==0.144.6" python -m pip install "gepa[full]==0.1.4" ``` @@ -256,7 +256,7 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.144.4" +python -m pip install "agent-eval-rpc==0.144.6" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 4e99bc2c..fefb363e 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -234,7 +234,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.144.4`, then install ' + + 'install `agent-eval-rpc==0.144.6`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/docs/canonical-api.md b/docs/canonical-api.md index fb640e0b..2ec39ed1 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -6,9 +6,9 @@ Run pnpm docs:freshness after editing this file. --> > **Version 0.130.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.144.4 <0.145.0`. -> `sandbox` must satisfy `>=0.19.1 <0.20.0`. -> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.43.1 <0.44.0`. +> `agent-eval` must satisfy `>=0.144.6 <0.145.0`. +> `sandbox` must satisfy `>=0.19.3 <0.20.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.46.1 <0.47.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. > diff --git a/examples/coding-benchmark/coding-benchmark.test.ts b/examples/coding-benchmark/coding-benchmark.test.ts index d4324155..e281ce30 100644 --- a/examples/coding-benchmark/coding-benchmark.test.ts +++ b/examples/coding-benchmark/coding-benchmark.test.ts @@ -24,7 +24,7 @@ import { leaderboard } from '@tangle-network/agent-runtime/kernel' import { describe, expect, it } from 'vitest' import { main, offlineAgentScripts } from './benchmark' import { type CheckBox, composeScore, runChecks, runHeldout } from './eval' -import { harnessProfiles } from './profiles' +import { harnesses, harnessOf, harnessProfiles } from './profiles' import { type CodingScenario, checkCmds, routeCodingFields, scenarios } from './scenarios' const execAsync = promisify(execCb) @@ -95,6 +95,11 @@ async function gradeSolution( } describe('coding-benchmark (offline)', () => { + it('uses the canonical AgentProfile harness field', () => { + expect(harnessProfiles.map((profile) => profile.harness)).toEqual(harnesses) + expect(harnessProfiles.map(harnessOf)).toEqual(harnesses) + }) + // Integration smoke: runs the real matrix end-to-end (real box.exec on the offline // toolchain, all refine rounds since the dev checks can't pass without tsc). it('runs the full matrix and returns a defined leaderboard', async () => { diff --git a/examples/coding-benchmark/profiles.ts b/examples/coding-benchmark/profiles.ts index 2650db66..b85bc742 100644 --- a/examples/coding-benchmark/profiles.ts +++ b/examples/coding-benchmark/profiles.ts @@ -4,14 +4,13 @@ * Each profile is deliberately bare (name + model, no skills, no injected prompt) so we * measure the HARNESS, not our scaffolding; the tool surface is a separate orthogonal knob * (`withTools`), making harness × tool a clean cartesian. Two non-obvious facts about the - * shape: `AgentProfile` (`@tangle-network/agent-interface`) has no `harness` field (harness - * is a SANDBOX concept), so the harness selector rides `metadata.harness` (`harnessOf()` is - * the one reader); and `runProfileMatrix` REQUIRES a snapshot-dated `model.default` — see - * `harnessModel` below. + * shape: `AgentProfile.harness` is the canonical selector, while Sandbox owns validation + * that the selected harness is an executable backend; and `runProfileMatrix` REQUIRES a + * snapshot-dated `model.default` — see `harnessModel` below. */ import type { AgentProfile, AgentProfileMcpServer } from '@tangle-network/agent-interface' -import type { BackendType } from '@tangle-network/sandbox' +import { type BackendType, parseBackendType } from '@tangle-network/sandbox' /** The harnesses we sweep. `cli-base` is the plain-CLI baseline (no agent harness). */ export const harnesses = [ @@ -21,13 +20,12 @@ export const harnesses = [ 'cli-base', ] as const satisfies readonly BackendType[] -/** Read the harness a profile targets. The ONE place metadata.harness is decoded. */ +/** Read and validate the executable harness a profile targets. */ export function harnessOf(profile: AgentProfile): BackendType { - const h = profile.metadata?.harness - if (typeof h !== 'string') { - throw new Error(`profile "${profile.name}" is missing metadata.harness — see profiles.ts`) + if (profile.harness === undefined) { + throw new Error(`profile "${profile.name}" is missing harness — see profiles.ts`) } - return h as BackendType + return parseBackendType(profile.harness) } /** The default model each harness runs (override per-harness via env). The model id MUST @@ -46,6 +44,7 @@ const harnessModel: Record = { amp: 'anthropic/claude-sonnet-4-5-2025-09-29', 'factory-droids': 'anthropic/claude-sonnet-4-5-2025-09-29', pi: 'openai/gpt-4.1-2025-04-14', + prime: 'openai/gpt-4.1-2025-04-14', hermes: 'openai/gpt-4.1-2025-04-14', forge: 'openai/gpt-4.1-2025-04-14', openclaw: 'anthropic/claude-sonnet-4-5-2025-09-29', @@ -57,8 +56,8 @@ const harnessModel: Record = { /** One bare baseline profile per harness — the harness's out-of-the-box behavior. */ export const harnessProfiles: AgentProfile[] = harnesses.map((harness) => ({ name: `${harness}-baseline`, + harness, model: { default: harnessModel[harness] }, - metadata: { harness }, })) // ── the tool knob ───────────────────────────────────────────────────────────── diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts index 688961f1..2ca116c9 100644 --- a/src/candidate-execution/prepare.ts +++ b/src/candidate-execution/prepare.ts @@ -64,7 +64,7 @@ import { import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' import { createPreparedCandidateExecution } from './prepared-state' import { candidateMaterializerHarness } from './profile' -import { projectCandidateSystemPrompt } from './system-prompt' +import { projectCandidatePromptIntents } from './system-prompt' import { type AgentCandidateExecutionPorts, type AgentCandidateTaskExecution, @@ -178,12 +178,18 @@ export async function prepareAgentCandidateExecution( } await assertEmptyDirectory(task.stagingRoots.profileRoot) - const profileWorkspacePlan = projectCandidateSystemPrompt( - materializeCandidateProfile(bundle.profile, harness, { - resolvedResources: verifiedResourceTextByDigest(candidate), - }), + // This adapter owns the exact spawn and forwards every materializer flag, including OpenCode's + // selected primary agent; a plan-forwarding caller that does not own the spawn cannot claim it. + const candidateProfileMaterialization = { + resolvedResources: verifiedResourceTextByDigest(candidate), + binds: ['systemPrompt'] as const, + } + const profileWorkspacePlan = projectCandidatePromptIntents( + materializeCandidateProfile(bundle.profile, harness, candidateProfileMaterialization), bundle.execution.launch, profileSystemPromptExecutionPath(bundle.execution.cwd.workspace, task.executionRoots), + bundle.profile.prompt?.systemPrompt, + bundle.profile.prompt?.appendSystemPrompt, ) const profileApplication = applyAgentCandidateWorkspacePlan( profileWorkspacePlan, diff --git a/src/candidate-execution/system-prompt.ts b/src/candidate-execution/system-prompt.ts index 1422c217..3440ace3 100644 --- a/src/candidate-execution/system-prompt.ts +++ b/src/candidate-execution/system-prompt.ts @@ -10,47 +10,55 @@ import type { } from '@tangle-network/agent-profile-materialize' const SYSTEM_PROMPT_FILE = '.tangle/system-prompt.md' +const APPEND_SYSTEM_PROMPT_FILE = '.tangle/append-system-prompt.md' /** - * How ONE harness expresses a replacement system prompt natively, for the harnesses whose plans - * still arrive with `plan.systemPrompt` set — the materializer delegates the lowering to the - * launcher for these (claude-code and pi both take a prompt-file flag on their own argv). - * - * codex and opencode are deliberately NOT rows here anymore. agent-profile-materialize 0.12 lowers - * codex's prompt itself — the bytes land in the plan at `.codex/system-prompt.md` and the flags - * carry `-c model_instructions_file=…` — and refuses opencode outright (its only replacement - * control binds to the agent selected at launch, which a workspace plan cannot guarantee). A plan - * whose `systemPrompt` is set for a harness with no row is refused rather than launched with an - * unprojected prompt. + * One row describes the native process control for each prompt intent this adapter owns. + * A row with no executor projection means the shared materializer already lowered the intent into + * the signed workspace plan; this adapter still proves that the launch reaches the native binary. */ interface HarnessSystemPrompt { /** The native binary the launch must run for this projection to be provable. */ readonly executable: string - /** Apply the prompt to the harness's own control, returning the projected plan. */ + /** Apply a replacement prompt to the harness's own control. */ readonly project: ( plan: AgentCandidateWorkspacePlan, systemPrompt: string, systemPromptFilePath: string, ) => AgentCandidateWorkspacePlan - /** Does the caller's argv already set a system prompt this projection would silently shadow? */ + /** Apply an addition without replacing the harness's own prompt. */ + readonly projectAppend?: ( + plan: AgentCandidateWorkspacePlan, + appendSystemPrompt: string, + appendSystemPromptFilePath: string, + ) => AgentCandidateWorkspacePlan + /** The materializer already projected replacement text into native plan files and flags. */ + readonly retainsProjectedSystemPrompt?: boolean + /** Does caller argv already set a replacement control this projection would shadow? */ readonly conflictsWithArgs: (values: readonly string[]) => boolean + /** Does caller argv already set an additive control this projection would shadow? */ + readonly conflictsWithAppendArgs: (values: readonly string[]) => boolean } const SYSTEM_PROMPT_FLAGS = ['--system-prompt', '--system-prompt-file'] as const +const APPEND_SYSTEM_PROMPT_FLAGS = [ + '--append-system-prompt', + '--append-system-prompt-file', +] as const -/** Shared by the harnesses whose native control IS a `--system-prompt*` flag. */ function argsSetSystemPromptFlag(values: readonly string[]): boolean { return values.some((value) => SYSTEM_PROMPT_FLAGS.some((flag) => value === flag || value.startsWith(`${flag}=`)), ) } -/** - * Codex config keys that decide the request's instructions. `model_instructions_file` replaces the - * whole instructions field (the materializer's own delivery key); `developer_instructions` injects - * developer-channel text beside it. A caller argv setting either would shadow or contaminate the - * profile's sealed prompt, so both refuse. - */ +function argsSetAppendSystemPromptFlag(values: readonly string[]): boolean { + return values.some((value) => + APPEND_SYSTEM_PROMPT_FLAGS.some((flag) => value === flag || value.startsWith(`${flag}=`)), + ) +} + +/** Codex keys that can replace or contaminate the materializer's sealed instructions file. */ const CODEX_INSTRUCTION_KEYS = ['model_instructions_file', 'developer_instructions'] as const function argsSetCodexInstructionOverride(values: readonly string[]): boolean { @@ -76,60 +84,114 @@ const HARNESS_SYSTEM_PROMPTS = { 'claude-code': { executable: 'claude', project: (plan, systemPrompt, path) => - appendFlags(addSystemPromptFile(plan, systemPrompt), '--system-prompt-file', path), + appendFlags( + addPromptFile(plan, SYSTEM_PROMPT_FILE, systemPrompt), + '--system-prompt-file', + path, + ), + projectAppend: (plan, appendSystemPrompt, path) => + appendFlags( + addPromptFile(plan, APPEND_SYSTEM_PROMPT_FILE, appendSystemPrompt), + '--append-system-prompt-file', + path, + ), conflictsWithArgs: argsSetSystemPromptFlag, + conflictsWithAppendArgs: argsSetAppendSystemPromptFlag, + }, + /** OpenCode's bound primary agent is already selected by the materializer's --agent flag. */ + opencode: { + executable: 'opencode', + project: (plan) => plan, + retainsProjectedSystemPrompt: true, + conflictsWithArgs: (values) => + values.some((value) => value === '--agent' || value.startsWith('--agent=')), + conflictsWithAppendArgs: argsSetAppendSystemPromptFlag, }, pi: { executable: 'pi', project: (plan, systemPrompt, path) => - appendFlags(addSystemPromptFile(plan, systemPrompt), '--system-prompt', path), + appendFlags(addPromptFile(plan, SYSTEM_PROMPT_FILE, systemPrompt), '--system-prompt', path), + projectAppend: (plan, appendSystemPrompt, path) => + appendFlags( + addPromptFile(plan, APPEND_SYSTEM_PROMPT_FILE, appendSystemPrompt), + '--append-system-prompt', + path, + ), + conflictsWithArgs: argsSetSystemPromptFlag, + conflictsWithAppendArgs: argsSetAppendSystemPromptFlag, + }, + prime: { + executable: 'prime-agent', + project: (plan, systemPrompt, path) => + appendFlags(addPromptFile(plan, SYSTEM_PROMPT_FILE, systemPrompt), '--system-prompt', path), + projectAppend: (plan, appendSystemPrompt, path) => + appendFlags( + addPromptFile(plan, APPEND_SYSTEM_PROMPT_FILE, appendSystemPrompt), + '--append-system-prompt', + path, + ), + conflictsWithArgs: argsSetSystemPromptFlag, + conflictsWithAppendArgs: argsSetAppendSystemPromptFlag, + }, + /** Gemini's file and env lowering is materializer-owned; this row supplies the launch proof. */ + gemini: { + executable: 'gemini', + project: (plan) => plan, conflictsWithArgs: argsSetSystemPromptFlag, + conflictsWithAppendArgs: argsSetAppendSystemPromptFlag, }, } as const satisfies Partial> -/** - * Deliveries the MATERIALIZER already lowered into the plan, whose effect still rides argv. The - * prompt bytes are in the digested plan files, but the flag that makes the harness read them is - * only meaningful to the native binary's own argument parser — handed to any other executable it - * is inert argv, and the sealed candidate would claim an active prompt that never applied. - * - * That is the exact hazard the launch guard exists for, and it is how the guard was once silently - * disabled: when agent-profile-materialize 0.12 moved codex from an inline flag this module - * projected to file+flag it lowers itself, `plan.systemPrompt` stopped being set for codex, the - * old `systemPrompt === undefined` early-return skipped every check, and two refusal tests began - * resolving. Delivery detection therefore keys on the LOWERED FLAG, not on the field. - */ -interface MaterializedFlagDelivery { +interface MaterializedPromptDelivery { readonly executable: string readonly delivered: (plan: AgentCandidateWorkspacePlan) => boolean readonly conflictsWithArgs: (values: readonly string[]) => boolean } -const MATERIALIZED_FLAG_DELIVERIES = { +/** + * These controls are lowered by agent-profile-materialize into files plus launch inputs. The + * candidate adapter does not own the spawn, so it must reject an arbitrary entrypoint even when + * the plan has the right bytes: only the native binary can make those inputs effective. + */ +const MATERIALIZED_PROMPT_DELIVERIES = { codex: { executable: 'codex', delivered: (plan) => - plan.flags.some((flag) => flag.value.startsWith('model_instructions_file=')), + plan.files.some((file) => file.relPath === '.codex/system-prompt.md') && + plan.flags.some((flag) => flag.value === 'model_instructions_file=.codex/system-prompt.md'), conflictsWithArgs: argsSetCodexInstructionOverride, }, -} as const satisfies Partial> + gemini: { + executable: 'gemini', + delivered: (plan) => + plan.files.some((file) => file.relPath === '.gemini/system.md') && + plan.env.GEMINI_SYSTEM_MD?.value === '1', + conflictsWithArgs: argsSetSystemPromptFlag, + }, +} as const satisfies Partial> -/** Project a replacement system prompt onto the exact native process control. */ -export function projectCandidateSystemPrompt( +/** Project both prompt intents onto their distinct native process controls. */ +export function projectCandidatePromptIntents( plan: AgentCandidateWorkspacePlan, launch: AgentCandidateLaunch, systemPromptFilePath: string, + requestedSystemPrompt: string | undefined = plan.systemPrompt?.value, + requestedAppendSystemPrompt: string | undefined = plan.appendSystemPrompt?.value, ): AgentCandidateWorkspacePlan { const systemPrompt = plan.systemPrompt - if (systemPrompt === undefined) { - // No launcher-delegated prompt — but the materializer may have lowered one into the plan - // whose flag only the native binary can honor. Same guards, no projection to apply. - const delivery = MATERIALIZED_FLAG_DELIVERIES[ - plan.harness as keyof typeof MATERIALIZED_FLAG_DELIVERIES - ] as MaterializedFlagDelivery | undefined - if (!delivery || !delivery.delivered(plan)) return plan - assertProvableNativeLaunch(plan.harness, delivery.executable, launch) - assertNoShadowingArgs(plan.harness, delivery.conflictsWithArgs, launch) + const appendSystemPrompt = plan.appendSystemPrompt + const delivery = MATERIALIZED_PROMPT_DELIVERIES[ + plan.harness as keyof typeof MATERIALIZED_PROMPT_DELIVERIES + ] as MaterializedPromptDelivery | undefined + + if (systemPrompt === undefined && delivery?.delivered(plan)) { + assertProvableNativeLaunch(plan.harness, delivery.executable, launch, 'replacement') + assertNoShadowingArgs(plan.harness, delivery.conflictsWithArgs, launch, 'replacement') + if (requestedSystemPrompt === undefined) return plan + return plan + } + + if (requestedSystemPrompt === undefined && requestedAppendSystemPrompt === undefined) { return plan } @@ -139,25 +201,77 @@ export function projectCandidateSystemPrompt( if (!projection) { throw new Error(`candidate system prompt has no native launch projection for ${plan.harness}`) } - assertProvableNativeLaunch(plan.harness, projection.executable, launch) - assertNoShadowingArgs(plan.harness, projection.conflictsWithArgs, launch) - // The source-profile digest already binds the authored value. Sign only the - // native projection here so an inert systemPrompt field cannot look active. - return projection.project( - omitUnappliedSystemPrompt(plan), - systemPrompt.value, - systemPromptFilePath, - ) + + const requestedReplacement = requestedSystemPrompt !== undefined + if (requestedReplacement && systemPrompt === undefined) { + throw new Error( + `profile materializer did not deliver the candidate system prompt for ${plan.harness}`, + ) + } + if (systemPrompt !== undefined && requestedSystemPrompt === undefined) { + throw new Error( + `profile materializer added an unexpected candidate system prompt for ${plan.harness}`, + ) + } + if (systemPrompt !== undefined && systemPrompt.value !== requestedSystemPrompt) { + throw new Error('profile materializer changed the candidate system prompt') + } + if ( + appendSystemPrompt !== undefined && + appendSystemPrompt.value !== requestedAppendSystemPrompt + ) { + throw new Error('profile materializer changed the candidate append system prompt') + } + + const intent = requestedReplacement ? 'replacement' : 'addition' + assertProvableNativeLaunch(plan.harness, projection.executable, launch, intent) + const launchArgs = (launch.args ?? []).map((value) => value.value) + if (requestedReplacement && projection.conflictsWithArgs(launchArgs)) { + throw new Error( + `${plan.harness} launch arguments conflict with the candidate profile system prompt`, + ) + } + if (requestedAppendSystemPrompt !== undefined && projection.conflictsWithAppendArgs(launchArgs)) { + throw new Error( + `${plan.harness} launch arguments conflict with the candidate profile append system prompt`, + ) + } + + let projected = plan + if (systemPrompt !== undefined) { + projected = projection.project( + projection.retainsProjectedSystemPrompt + ? projected + : omitPromptIntent(projected, 'systemPrompt'), + systemPrompt.value, + systemPromptFilePath, + ) + } + if (appendSystemPrompt !== undefined) { + if (!projection.projectAppend) { + throw new Error( + `candidate append system prompt has no native launch projection for ${plan.harness}`, + ) + } + projected = projection.projectAppend( + omitPromptIntent(projected, 'appendSystemPrompt'), + appendSystemPrompt.value, + posix.join(posix.dirname(systemPromptFilePath), 'append-system-prompt.md'), + ) + } + // The source-profile digest binds authored values. Remove executor handoff fields only after + // converting them into native controls; materializer-owned controls remain visible in the plan. + return projected } -/** A prompt whose delivery rides argv is provable only on the native binary's own command line. */ function assertProvableNativeLaunch( harness: string, expectedExecutable: string, launch: AgentCandidateLaunch, + intent: 'replacement' | 'addition', ): asserts launch is AgentCandidateLaunch & { kind: 'container-command' } { if (launch.kind !== 'container-command') { - throw new Error(`candidate-entrypoint launch cannot prove ${harness} system-prompt replacement`) + throw new Error(`candidate-entrypoint launch cannot prove ${harness} system-prompt ${intent}`) } if ( launch.executable !== expectedExecutable && @@ -167,7 +281,7 @@ function assertProvableNativeLaunch( ) ) { throw new Error( - `${harness} system-prompt replacement requires the native ${expectedExecutable} executable`, + `${harness} system-prompt ${intent} requires the native ${expectedExecutable} executable`, ) } } @@ -176,14 +290,21 @@ function assertNoShadowingArgs( harness: string, conflictsWithArgs: (values: readonly string[]) => boolean, launch: Extract, + intent: 'replacement' | 'addition', ): void { if (conflictsWithArgs((launch.args ?? []).map((value) => value.value))) { - throw new Error(`${harness} launch arguments conflict with the candidate profile system prompt`) + throw new Error( + `${harness} launch arguments conflict with the candidate profile ${intent === 'replacement' ? 'system' : 'append system'} prompt`, + ) } } -function omitUnappliedSystemPrompt(plan: AgentCandidateWorkspacePlan): AgentCandidateWorkspacePlan { - const { systemPrompt: _systemPrompt, ...projected } = plan +function omitPromptIntent( + plan: AgentCandidateWorkspacePlan, + intent: 'systemPrompt' | 'appendSystemPrompt', +): AgentCandidateWorkspacePlan { + const projected = { ...plan } + delete projected[intent] return projected } @@ -201,22 +322,23 @@ function publicValue(value: string): AgentCandidateConfigValue { return { kind: 'public', value } } -function addSystemPromptFile( +function addPromptFile( plan: AgentCandidateWorkspacePlan, - systemPrompt: string, + relPath: string, + prompt: string, ): AgentCandidateWorkspacePlan { - // Candidate argv rejects control characters. Native prompt-file loading - // preserves multiline bytes without weakening the shared process schema. - if (plan.files.some((file) => file.relPath === SYSTEM_PROMPT_FILE)) { - throw new Error(`candidate profile conflicts with reserved ${SYSTEM_PROMPT_FILE}`) + // Candidate argv rejects control characters. Native prompt-file loading preserves multiline + // bytes without weakening the shared process schema. + if (plan.files.some((file) => file.relPath === relPath)) { + throw new Error(`candidate profile conflicts with reserved ${relPath}`) } return { ...plan, files: [ ...plan.files, { - relPath: SYSTEM_PROMPT_FILE, - content: systemPrompt, + relPath, + content: prompt, source: 'generated', }, ], diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index 3c33a79b..d3eec005 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -25,7 +25,7 @@ import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.4"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.6"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index f1c02b03..39160fcb 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -238,6 +238,70 @@ describe('environment provider adapters', () => { expect(await environment.placement?.()).toMatchObject({ kind: 'sandbox', sandboxId: 'sbx-1' }) }) + it('preserves sandbox routing coordinates without treating them as output or usage', async () => { + const box = { + id: 'sandbox-routing-proof', + status: 'running', + async *streamPrompt(): AsyncIterable { + yield { + type: 'result', + data: { + finalText: 'routing-safe result', + runtimeSessionId: 'runtime-session-7', + sandboxId: 'sandbox-routing-proof', + usage: { + inputTokens: 7, + outputTokens: 11, + reasoningTokens: 5, + totalCostUsd: 0.03, + }, + }, + } + }, + async delete(): Promise {}, + } as unknown as SandboxInstance + const client: SandboxClient = { + async create(): Promise { + return box + }, + } + const factory = providerAsExecutor(sandboxClientAsProvider(client)) + const spec: AgentSpec = { + profile: { name: 'routing-proof' } as AgentProfile, + harness: null, + } + const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } + const executor = factory(spec, ctx) + + const usage = await collect(executor.execute('task', ctx.signal) as AsyncIterable) + const artifact = executor.resultArtifact() + + expect(usage).toEqual([ + { kind: 'tokens', input: 7, output: 16 }, + { kind: 'cost', usd: 0.03 }, + { kind: 'iteration' }, + ]) + expect(artifact).toMatchObject({ + out: { + content: 'routing-safe result', + events: [ + { + providerEvent: { + data: { + runtimeSessionId: 'runtime-session-7', + sandboxId: 'sandbox-routing-proof', + }, + }, + }, + ], + }, + spent: { + tokens: { input: 7, output: 16 }, + usd: 0.03, + }, + }) + }) + it('requires explicit resolution for named profiles before calling current Sandbox', async () => { let createCalls = 0 let createOptions: CreateSandboxOptions | undefined @@ -258,7 +322,10 @@ describe('environment provider adapters', () => { const unresolved = sandboxClientAsProvider(client) await expect(unresolved.capabilities()).resolves.toMatchObject({ - profile: { namedProfiles: false }, + profile: { + namedProfiles: false, + systemPrompt: { replace: false, append: false }, + }, }) await expect(unresolved.create({ profile: 'catalog/researcher' })).rejects.toThrow( /requires an inline AgentProfile/, @@ -269,7 +336,10 @@ describe('environment provider adapters', () => { resolveProfile: async (profileId) => ({ name: `resolved:${profileId}` }), }) await expect(resolved.capabilities()).resolves.toMatchObject({ - profile: { namedProfiles: true }, + profile: { + namedProfiles: true, + systemPrompt: { replace: false, append: false }, + }, }) await resolved.create({ profile: 'catalog/researcher' }) diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 42eaf8d5..29f109c5 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -1,8 +1,9 @@ -import type { - AgentProfile, - AgentProfileValidationResult, - InputPart, - TokenUsage, +import { + type AgentProfile, + type AgentProfileValidationResult, + harnessSystemPromptIntents, + type InputPart, + type TokenUsage, } from '@tangle-network/agent-interface' import type { AgentEnvironment, @@ -1329,10 +1330,7 @@ function defaultTangleSandboxCapabilities(options: { return { profile: { namedProfiles: options.namedProfiles, - // Interface 0.44 split this from one boolean into the two things a backend can actually do - // with a caller's system prompt: REPLACE the harness's own, or APPEND to it. The sandbox - // materializes the whole profile, so it honors both. - systemPrompt: { replace: true, append: true }, + systemPrompt: { ...harnessSystemPromptIntents(undefined) }, instructions: true, tools: true, permissions: true, diff --git a/src/runtime/tangle-sandbox-exact-process-provider.test.ts b/src/runtime/tangle-sandbox-exact-process-provider.test.ts index 14a9feba..edafb2de 100644 --- a/src/runtime/tangle-sandbox-exact-process-provider.test.ts +++ b/src/runtime/tangle-sandbox-exact-process-provider.test.ts @@ -60,6 +60,7 @@ describe('Tangle Sandbox exact-process provider', () => { metadata: expect.objectContaining({ run: 'candidate-1' }), }) expect(provider.capabilities()).toMatchObject({ + profile: { systemPrompt: { replace: false, append: false } }, exactProcess: { egress: ['blocked', 'strict'] }, workspace: { read: false, write: false, exec: false }, }) diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index dd528887..46c24c7e 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -48,11 +48,26 @@ describe('candidate execution preparation', () => { executable: 'claude', flags: ['--system-prompt-file', '/workspace/task/.tangle/system-prompt.md'], }, + { + harness: 'opencode', + executable: 'opencode', + flags: ['--agent', expect.stringMatching(/^tangle-profile-[a-f0-9]{16}$/)], + }, { harness: 'pi', executable: 'pi', flags: ['--system-prompt', '/workspace/task/.tangle/system-prompt.md'], }, + { + harness: 'prime', + executable: 'prime-agent', + flags: ['--system-prompt', '/workspace/task/.tangle/system-prompt.md'], + }, + { + harness: 'gemini', + executable: 'gemini', + flags: [], + }, ] as const)( 'projects the replacement system prompt onto the native $harness process', async ({ harness, executable, flags }) => { @@ -78,7 +93,14 @@ describe('candidate execution preparation', () => { value.ports, ) - expect(prepared.profilePlan.value.material.systemPrompt).toBeUndefined() + if (harness === 'opencode') { + expect(prepared.profilePlan.value.material.systemPrompt).toEqual({ + kind: 'public', + value: systemPrompt, + }) + } else { + expect(prepared.profilePlan.value.material.systemPrompt).toBeUndefined() + } expect(prepared.profilePlan.value.material.sourceProfileDigest).toBe( canonicalCandidateDigest(value.bundle.profile), ) @@ -98,54 +120,154 @@ describe('candidate execution preparation', () => { const codexPromptFile = prepared.profileActivation.files.find( (file) => file.path === '.codex/system-prompt.md', ) - if (harness === 'claude-code' || harness === 'pi') { + const geminiPromptFile = prepared.profileActivation.files.find( + (file) => file.path === '.gemini/system.md', + ) + const openCodeConfig = prepared.profileActivation.files.find( + (file) => file.path === 'opencode.json', + ) + if (harness === 'opencode') { + expect(JSON.parse(openCodeConfig?.content ?? '')).toMatchObject({ + instructions: ['.opencode/profile-instructions.md'], + }) + const primaryAgent = prepared.profileActivation.files.find((file) => + file.path.startsWith('.opencode/agents/tangle-profile-'), + ) + expect(primaryAgent?.content).toContain(systemPrompt) + expect(systemPromptFile).toBeUndefined() + } else if (harness === 'claude-code' || harness === 'pi' || harness === 'prime') { + expect(openCodeConfig).toBeUndefined() expect(codexPromptFile).toBeUndefined() expect(systemPromptFile?.content).toBe(systemPrompt) + expect(geminiPromptFile).toBeUndefined() + } else if (harness === 'codex') { + expect(openCodeConfig).toBeUndefined() + expect(systemPromptFile).toBeUndefined() + expect(codexPromptFile?.content).toBe(systemPrompt) + expect(geminiPromptFile).toBeUndefined() } else { // The flag is only half the delivery; the digested plan must carry the exact bytes the // flag points at, or the launch would reference a file that does not exist. + expect(openCodeConfig).toBeUndefined() expect(systemPromptFile).toBeUndefined() - expect(codexPromptFile?.content).toBe(systemPrompt) + expect(codexPromptFile).toBeUndefined() + expect(geminiPromptFile?.content).toBe(systemPrompt) } }, ) - it('refuses an opencode candidate system prompt, because delivery cannot be guaranteed', async () => { - // agent-profile-materialize 0.12 refuses this outright: opencode's only replacement control - // is per-agent (`agent..prompt`), bound to whichever agent the launcher selects — a - // guarantee a sealed workspace plan cannot make. The refusal replaced this repo's earlier - // opencode.json mutation, which patched both built-in agents and hoped one was selected. - // Fail-closed is correct: no silent drop, an actionable reason, and the capability returns - // upstream via a binds-aware candidate materializer rather than a local workaround. + it('keeps replacement and additive prompts on distinct Claude controls', async () => { const value = fixture() + const systemPrompt = 'Replace the native prompt.' + const appendSystemPrompt = 'Add this after the replacement.' value.bundle = redigestBundle(value.bundle, { profile: { ...value.bundle.profile, - harness: 'opencode', - prompt: { ...value.bundle.profile.prompt, systemPrompt: 'Must be active.' }, + harness: 'claude-code', + prompt: { ...value.bundle.profile.prompt, systemPrompt, appendSystemPrompt }, }, execution: { ...value.bundle.execution, - harness: 'opencode', - launch: { kind: 'container-command', executable: 'opencode' }, + harness: 'claude-code', + launch: { kind: 'container-command', executable: 'claude' }, }, }) bindCandidateFixtureBundle(value) - await expect( - prepareAgentCandidateExecution( + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + + expect(prepared.launch.flags).toEqual([ + '--system-prompt-file', + '/workspace/task/.tangle/system-prompt.md', + '--append-system-prompt-file', + '/workspace/task/.tangle/append-system-prompt.md', + ]) + expect(prepared.profilePlan.value.material).not.toHaveProperty('systemPrompt') + expect(prepared.profilePlan.value.material).not.toHaveProperty('appendSystemPrompt') + expect( + prepared.profileActivation.files.find((entry) => entry.path === '.tangle/system-prompt.md') + ?.content, + ).toBe(systemPrompt) + expect( + prepared.profileActivation.files.find( + (entry) => entry.path === '.tangle/append-system-prompt.md', + )?.content, + ).toBe(appendSystemPrompt) + }) + + it.each([ + { + harness: 'claude-code', + executable: 'claude', + flags: ['--append-system-prompt-file', '/workspace/task/.tangle/append-system-prompt.md'], + file: '.tangle/append-system-prompt.md', + }, + { + harness: 'opencode', + executable: 'opencode', + flags: [], + file: '.opencode/agent-system-prompt.md', + }, + { + harness: 'pi', + executable: 'pi', + flags: ['--append-system-prompt', '/workspace/task/.tangle/append-system-prompt.md'], + file: '.tangle/append-system-prompt.md', + }, + { + harness: 'prime', + executable: 'prime-agent', + flags: ['--append-system-prompt', '/workspace/task/.tangle/append-system-prompt.md'], + file: '.tangle/append-system-prompt.md', + }, + ] as const)( + 'projects the additive system prompt onto the native $harness process', + async ({ harness, executable, flags, file }) => { + const value = fixture() + const appendSystemPrompt = 'Keep the native prompt and add this.\nSecond line.' + value.bundle = redigestBundle(value.bundle, { + profile: { + ...value.bundle.profile, + harness, + prompt: { ...value.bundle.profile.prompt, appendSystemPrompt }, + }, + execution: { + ...value.bundle.execution, + harness, + launch: { kind: 'container-command', executable }, + }, + }) + bindCandidateFixtureBundle(value) + + const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(value.bundle, value.ports), value.task, value.ports, - ), - ).rejects.toThrow(/only system-prompt replacement is per-agent/) - }) + ) + + expect(prepared.profilePlan.value.material.appendSystemPrompt).toBeUndefined() + expect(prepared.launch.flags).toEqual(flags) + expect(prepared.profileActivation.files.find((entry) => entry.path === file)?.content).toBe( + harness === 'opencode' ? `${appendSystemPrompt}\n` : appendSystemPrompt, + ) + if (harness === 'opencode') { + const config = prepared.profileActivation.files.find( + (entry) => entry.path === 'opencode.json', + ) + expect(JSON.parse(config?.content ?? '').instructions[0]).toBe(file) + } + }, + ) it.each([ { harness: 'codex', executable: 'codex', - args: ['-c', 'developer_instructions="already set"'], + args: ['-c', 'model_instructions_file=/elsewhere'], }, { // The materializer's own delivery key: a caller argv setting it would re-point codex's @@ -155,9 +277,15 @@ describe('candidate execution preparation', () => { executable: 'codex', args: ['--config=model_instructions_file=/somewhere/else.md'], }, + { + harness: 'opencode', + executable: 'opencode', + args: ['--agent', 'caller-selected-agent'], + }, { harness: 'claude-code', executable: 'claude', args: ['--system-prompt-file', '/elsewhere'] }, { harness: 'claude-code', executable: 'claude', args: ['--system-prompt=inline'] }, { harness: 'pi', executable: 'pi', args: ['--system-prompt', '/elsewhere'] }, + { harness: 'prime', executable: 'prime-agent', args: ['--system-prompt', '/elsewhere'] }, ] as const)( 'refuses $harness launch args that would shadow the profile system prompt', async ({ harness, executable, args }) => { @@ -192,6 +320,46 @@ describe('candidate execution preparation', () => { }, ) + it.each([ + { harness: 'claude-code', executable: 'claude' }, + { harness: 'pi', executable: 'pi' }, + { harness: 'prime', executable: 'prime-agent' }, + ] as const)( + 'refuses $harness launch args that shadow the profile append system prompt', + async ({ harness, executable }) => { + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + profile: { + ...value.bundle.profile, + harness, + prompt: { ...value.bundle.profile.prompt, appendSystemPrompt: 'Must be additive.' }, + }, + execution: { + ...value.bundle.execution, + harness, + launch: { + kind: 'container-command', + executable, + args: [{ kind: 'public', value: '--append-system-prompt=elsewhere' }], + }, + }, + }) + bindCandidateFixtureBundle(value) + + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow( + new RegExp( + `${harness} launch arguments conflict with the candidate profile append system prompt`, + ), + ) + }, + ) + it('rejects a system prompt when an arbitrary candidate entrypoint cannot apply it', async () => { const value = fixture(true) value.bundle = redigestBundle(value.bundle, { From d93cbab4531e25e3513470c2fdfb93b36283ab2e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:25:51 -0600 Subject: [PATCH 2/7] fix(compatibility): preserve native profile prompt delivery --- src/candidate-execution/prepare.ts | 3 -- tests/candidate-execution-prepare.test.ts | 57 ++++++++++++----------- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts index 2ca116c9..9834a00a 100644 --- a/src/candidate-execution/prepare.ts +++ b/src/candidate-execution/prepare.ts @@ -178,11 +178,8 @@ export async function prepareAgentCandidateExecution( } await assertEmptyDirectory(task.stagingRoots.profileRoot) - // This adapter owns the exact spawn and forwards every materializer flag, including OpenCode's - // selected primary agent; a plan-forwarding caller that does not own the spawn cannot claim it. const candidateProfileMaterialization = { resolvedResources: verifiedResourceTextByDigest(candidate), - binds: ['systemPrompt'] as const, } const profileWorkspacePlan = projectCandidatePromptIntents( materializeCandidateProfile(bundle.profile, harness, candidateProfileMaterialization), diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index 46c24c7e..af1ba5e6 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -48,11 +48,6 @@ describe('candidate execution preparation', () => { executable: 'claude', flags: ['--system-prompt-file', '/workspace/task/.tangle/system-prompt.md'], }, - { - harness: 'opencode', - executable: 'opencode', - flags: ['--agent', expect.stringMatching(/^tangle-profile-[a-f0-9]{16}$/)], - }, { harness: 'pi', executable: 'pi', @@ -93,14 +88,7 @@ describe('candidate execution preparation', () => { value.ports, ) - if (harness === 'opencode') { - expect(prepared.profilePlan.value.material.systemPrompt).toEqual({ - kind: 'public', - value: systemPrompt, - }) - } else { - expect(prepared.profilePlan.value.material.systemPrompt).toBeUndefined() - } + expect(prepared.profilePlan.value.material.systemPrompt).toBeUndefined() expect(prepared.profilePlan.value.material.sourceProfileDigest).toBe( canonicalCandidateDigest(value.bundle.profile), ) @@ -126,16 +114,7 @@ describe('candidate execution preparation', () => { const openCodeConfig = prepared.profileActivation.files.find( (file) => file.path === 'opencode.json', ) - if (harness === 'opencode') { - expect(JSON.parse(openCodeConfig?.content ?? '')).toMatchObject({ - instructions: ['.opencode/profile-instructions.md'], - }) - const primaryAgent = prepared.profileActivation.files.find((file) => - file.path.startsWith('.opencode/agents/tangle-profile-'), - ) - expect(primaryAgent?.content).toContain(systemPrompt) - expect(systemPromptFile).toBeUndefined() - } else if (harness === 'claude-code' || harness === 'pi' || harness === 'prime') { + if (harness === 'claude-code' || harness === 'pi' || harness === 'prime') { expect(openCodeConfig).toBeUndefined() expect(codexPromptFile).toBeUndefined() expect(systemPromptFile?.content).toBe(systemPrompt) @@ -156,6 +135,33 @@ describe('candidate execution preparation', () => { }, ) + it('refuses an opencode candidate system prompt until the materializer can bind its selected agent', async () => { + // The published candidate materializer does not forward launcher bindings yet. OpenCode's + // replacement control is per-agent, so this path must remain an explicit upstream refusal. + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + profile: { + ...value.bundle.profile, + harness: 'opencode', + prompt: { ...value.bundle.profile.prompt, systemPrompt: 'Must be active.' }, + }, + execution: { + ...value.bundle.execution, + harness: 'opencode', + launch: { kind: 'container-command', executable: 'opencode' }, + }, + }) + bindCandidateFixtureBundle(value) + + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/only system-prompt replacement is per-agent/) + }) + it('keeps replacement and additive prompts on distinct Claude controls', async () => { const value = fixture() const systemPrompt = 'Replace the native prompt.' @@ -277,11 +283,6 @@ describe('candidate execution preparation', () => { executable: 'codex', args: ['--config=model_instructions_file=/somewhere/else.md'], }, - { - harness: 'opencode', - executable: 'opencode', - args: ['--agent', 'caller-selected-agent'], - }, { harness: 'claude-code', executable: 'claude', args: ['--system-prompt-file', '/elsewhere'] }, { harness: 'claude-code', executable: 'claude', args: ['--system-prompt=inline'] }, { harness: 'pi', executable: 'pi', args: ['--system-prompt', '/elsewhere'] }, From f2bab68dc42db03cfc8c8ce6bb13815c9e492f2d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:29:57 -0600 Subject: [PATCH 3/7] fix(bench): remove duplicate prime model mapping --- examples/coding-benchmark/profiles.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/coding-benchmark/profiles.ts b/examples/coding-benchmark/profiles.ts index b85bc742..243f556e 100644 --- a/examples/coding-benchmark/profiles.ts +++ b/examples/coding-benchmark/profiles.ts @@ -44,7 +44,6 @@ const harnessModel: Record = { amp: 'anthropic/claude-sonnet-4-5-2025-09-29', 'factory-droids': 'anthropic/claude-sonnet-4-5-2025-09-29', pi: 'openai/gpt-4.1-2025-04-14', - prime: 'openai/gpt-4.1-2025-04-14', hermes: 'openai/gpt-4.1-2025-04-14', forge: 'openai/gpt-4.1-2025-04-14', openclaw: 'anthropic/claude-sonnet-4-5-2025-09-29', From 67729cc95e05e974bdabb84945b2ac93b6bc5c19 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:30:36 -0600 Subject: [PATCH 4/7] test(candidate): retain codex instruction shadow guard --- tests/candidate-execution-prepare.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index af1ba5e6..fecbedfa 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -270,6 +270,11 @@ describe('candidate execution preparation', () => { ) it.each([ + { + harness: 'codex', + executable: 'codex', + args: ['-c', 'developer_instructions="already set"'], + }, { harness: 'codex', executable: 'codex', From 94704d4529b6ff9b0a68c6216d5072a20205a8cd Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:35:01 -0600 Subject: [PATCH 5/7] fix(compatibility): bind candidate prompt launch controls --- pnpm-lock.yaml | 14 +++++------ pnpm-workspace.yaml | 2 +- src/candidate-execution/prepare.ts | 3 ++- src/candidate-execution/profile.ts | 3 +++ src/intelligence/improvement-cycle.ts | 6 ++++- tests/candidate-execution-prepare.test.ts | 29 +++++++++++++---------- 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f8120bb..be4d5d8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ catalogs: specifier: 7.1.2 version: 7.1.2 '@tangle-network/agent-profile-materialize': - specifier: 0.12.0 - version: 0.12.0 + specifier: 0.13.1 + version: 0.13.1 '@tangle-network/agent-trace-contract': specifier: ^1.0.2 version: 1.0.2 @@ -58,7 +58,7 @@ importers: version: 7.1.2 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' - version: 0.12.0(@tangle-network/agent-interface@0.46.1) + version: 0.13.1(@tangle-network/agent-interface@0.46.1) '@tangle-network/agent-trace-contract': specifier: 'catalog:' version: 1.0.2 @@ -1130,10 +1130,10 @@ packages: engines: {node: '>=20.19.0'} hasBin: true - '@tangle-network/agent-profile-materialize@0.12.0': - resolution: {integrity: sha512-SfTgqm4Q4HnuHcnjRJ//IrBTuiB8Hbk/Jx4sEKa9PxYL9v0Pc5yCKiSuc5YKEFEcM3NKckZbuc5L6zaq9pFELg==} + '@tangle-network/agent-profile-materialize@0.13.1': + resolution: {integrity: sha512-2g/F8ABiJ6gB8lifTKKIMGPBgYjCyPHQF93HF3g7c5d0D5Rh8cEZ9WGRvggYjOm4HPi3/4PViG6VKEmr5sXwIg==} peerDependencies: - '@tangle-network/agent-interface': '>=0.45.0 <0.46.0' + '@tangle-network/agent-interface': '>=0.46.0 <0.47.0' '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} @@ -2828,7 +2828,7 @@ snapshots: proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.12.0(@tangle-network/agent-interface@0.46.1)': + '@tangle-network/agent-profile-materialize@0.13.1(@tangle-network/agent-interface@0.46.1)': dependencies: '@tangle-network/agent-interface': 0.46.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f1611c6d..fc1ae552 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -22,7 +22,7 @@ catalog: '@tangle-network/agent-eval': 0.144.6 '@tangle-network/agent-interface': 0.46.1 '@tangle-network/agent-knowledge': 7.1.2 - '@tangle-network/agent-profile-materialize': 0.12.0 + '@tangle-network/agent-profile-materialize': 0.13.1 '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': 0.19.3 publint: 0.3.22 diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts index 9834a00a..8876eb91 100644 --- a/src/candidate-execution/prepare.ts +++ b/src/candidate-execution/prepare.ts @@ -63,7 +63,7 @@ import { } from './knowledge' import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' import { createPreparedCandidateExecution } from './prepared-state' -import { candidateMaterializerHarness } from './profile' +import { CANDIDATE_PROFILE_MATERIALIZER_BINDS, candidateMaterializerHarness } from './profile' import { projectCandidatePromptIntents } from './system-prompt' import { type AgentCandidateExecutionPorts, @@ -179,6 +179,7 @@ export async function prepareAgentCandidateExecution( await assertEmptyDirectory(task.stagingRoots.profileRoot) const candidateProfileMaterialization = { + binds: CANDIDATE_PROFILE_MATERIALIZER_BINDS, resolvedResources: verifiedResourceTextByDigest(candidate), } const profileWorkspacePlan = projectCandidatePromptIntents( diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index 04939fc9..b2ecbe9e 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -43,6 +43,9 @@ export function candidateMaterializerHarness(harness: HarnessType): HarnessId { return harness } +/** Runtime applies the materializer's launch flags to the candidate process. */ +export const CANDIDATE_PROFILE_MATERIALIZER_BINDS = ['systemPrompt'] as const + /** Bind exact native profile text to the canonical plan captured during preparation. */ export function createAgentCandidateProfileActivation( plan: AgentCandidateWorkspacePlan, diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index fc4c3f80..09f9ba6a 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -87,6 +87,7 @@ import { } from '../candidate-execution/prepare' import { assertCandidateProfileBinding, + CANDIDATE_PROFILE_MATERIALIZER_BINDS, candidateMaterializerHarness, createAgentCandidateProfileActivation, parseAgentCandidateProfileActivation, @@ -1327,7 +1328,10 @@ export function verifyCandidateExecutionEvidence( const expectedProfilePlan = materializeCandidateProfile( bundle.profile, candidateMaterializerHarness(materialization.harness), - { resolvedResources: options.resolvedResources }, + { + binds: CANDIDATE_PROFILE_MATERIALIZER_BINDS, + resolvedResources: options.resolvedResources, + }, ) const activation = parseAgentCandidateProfileActivation( materialization.profileActivation, diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index fecbedfa..08adf118 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -36,7 +36,7 @@ afterEach(() => { describe('candidate execution preparation', () => { it.each([ { - // codex delivery is materializer-lowered file+flag since agent-profile-materialize 0.12: + // codex delivery is materializer-lowered file+flag since agent-profile-materialize 0.13.1: // the prompt bytes live in the digested plan at .codex/system-prompt.md and the flag makes // codex read them. The path is cwd-relative on purpose (survives docker/jail remapping). harness: 'codex', @@ -135,15 +135,14 @@ describe('candidate execution preparation', () => { }, ) - it('refuses an opencode candidate system prompt until the materializer can bind its selected agent', async () => { - // The published candidate materializer does not forward launcher bindings yet. OpenCode's - // replacement control is per-agent, so this path must remain an explicit upstream refusal. + it('binds an OpenCode candidate replacement to its selected primary agent', async () => { const value = fixture() + const systemPrompt = 'Must be active.' value.bundle = redigestBundle(value.bundle, { profile: { ...value.bundle.profile, harness: 'opencode', - prompt: { ...value.bundle.profile.prompt, systemPrompt: 'Must be active.' }, + prompt: { ...value.bundle.profile.prompt, systemPrompt }, }, execution: { ...value.bundle.execution, @@ -153,13 +152,19 @@ describe('candidate execution preparation', () => { }) bindCandidateFixtureBundle(value) - await expect( - prepareAgentCandidateExecution( - await verifyAgentCandidateBundle(value.bundle, value.ports), - value.task, - value.ports, - ), - ).rejects.toThrow(/only system-prompt replacement is per-agent/) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + + expect(prepared.profilePlan.value.material.systemPrompt?.value).toBe(systemPrompt) + expect(prepared.profilePlan.value.material.unsupported).toEqual([]) + expect(prepared.launch.flags).toEqual(['--agent', expect.any(String)]) + const agentFile = prepared.profileActivation.files.find((file) => + file.path.startsWith('.opencode/agents/'), + ) + expect(agentFile?.content).toBe(`---\nmode: primary\n---\n${systemPrompt}\n`) }) it('keeps replacement and additive prompts on distinct Claude controls', async () => { From c21f6e3489ac7daa7b1995a622d1351ae5f25203 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:38:44 -0600 Subject: [PATCH 6/7] chore(release): prepare runtime 0.131.0 --- CHANGELOG.md | 6 ++++ bench/CHANGELOG.md | 4 +++ bench/package.json | 2 +- docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 4 +-- package.json | 4 +-- pnpm-lock.yaml | 35 +++++-------------- pnpm-workspace.yaml | 2 +- .../fixtures/agent-improvement-proposal.json | 10 +++--- .../agent-profile-improvement-proposal.json | 6 ++-- 10 files changed, 34 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbd5242a..61922e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.131.0 + +- Consume agent-profile-materialize 0.13.1 and Sandbox 0.19.4 with Interface 0.46.1, Eval 0.144.6, and Knowledge 7.1.2. +- Bind candidate system-prompt launch controls through the native harness plan, including OpenCode's generated primary agent. +- Keep Codex and Gemini prompt delivery fail-closed, and preserve separate replacement and additive controls for Claude Code, Pi, and Prime. + ## 0.130.0 ### Stability contract + first graduation diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index ddfba723..a7ae53c1 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.8.0 + +- Consume Runtime 0.131.0, Eval 0.144.6, Interface 0.46.1, Knowledge 7.1.2, and Sandbox 0.19.4 as one compatible dependency set. + ## 0.7.2 - Consume Runtime 0.129.0, Eval 0.144.4, Interface 0.43.1, Knowledge 7.0.11, and Sandbox 0.19.1 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract without loading duplicate agent contracts. diff --git a/bench/package.json b/bench/package.json index f8b9fc4b..69047fdf 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.7.2", + "version": "0.8.0", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index ac51d06b..6d5e0166 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.130.0` and `@tangle-network/agent-eval@0.144.6` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.131.0` and `@tangle-network/agent-eval@0.144.6` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 2ec39ed1..2cdf378c 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,10 +4,10 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.130.0.** +> **Version 0.131.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.144.6 <0.145.0`. -> `sandbox` must satisfy `>=0.19.3 <0.20.0`. +> `sandbox` must satisfy `>=0.19.4 <0.20.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.46.1 <0.47.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. diff --git a/package.json b/package.json index 5b0d1d18..861f340d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.130.0", + "version": "0.131.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -171,7 +171,7 @@ "peerDependencies": { "@tangle-network/agent-eval": ">=0.144.6 <0.145.0", "@tangle-network/agent-interface": ">=0.46.1 <0.47.0", - "@tangle-network/sandbox": ">=0.19.3 <0.20.0" + "@tangle-network/sandbox": ">=0.19.4 <0.20.0" }, "peerDependenciesMeta": { "@tangle-network/sandbox": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be4d5d8c..ef073c39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,8 +28,8 @@ catalogs: specifier: ^1.0.2 version: 1.0.2 '@tangle-network/sandbox': - specifier: 0.19.3 - version: 0.19.3 + specifier: 0.19.4 + version: 0.19.4 '@types/node': specifier: 26.1.1 version: 26.1.1 @@ -80,7 +80,7 @@ importers: version: 0.46.1 '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.19.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.19.4(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) '@types/node': specifier: 'catalog:' version: 26.1.1 @@ -134,7 +134,7 @@ importers: version: link:.. '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.19.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.19.4(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) devDependencies: '@arethetypeswrong/cli': specifier: 'catalog:' @@ -1108,9 +1108,6 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - '@tangle-network/agent-core@0.5.2': - resolution: {integrity: sha512-2zfr680Ay0feUR5ZBjdk0nezqkqneXuXluL+RPZxWX+DKKVhheDXhwITOaDxFcUgpKyQxZmE5/VEfTDWER+KjQ==} - '@tangle-network/agent-core@0.5.4': resolution: {integrity: sha512-k6gYv3BlagkfuWrGyTJH6mKUBgsLY6TXxizACqt0QF8a1/5uqy0UYc6R2Wo9nqQVJuaRxDnoiRf8YtsRVqA75g==} @@ -1119,9 +1116,6 @@ packages: engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@0.45.0': - resolution: {integrity: sha512-VCDI+ta79cTzQbUaos6v2rBMMvuQ/Ojt2VPGkdvfAcrYubP0JVD8qRvx76pViBwh6VS+truPgeU4VvGRI6kipg==} - '@tangle-network/agent-interface@0.46.1': resolution: {integrity: sha512-6a3GRkDxS+r6Bmlu8y6LQpiVE20oCYPzE2opb5o+AZeDzRW5KemyxreyYIprjgKfDrvBTMo0tgvC2JB51Sl84w==} @@ -1138,8 +1132,8 @@ packages: '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} - '@tangle-network/sandbox@0.19.3': - resolution: {integrity: sha512-cUfqoGfm+EigTLFHFV2qfnXlAetmbgXWNhKUB7qOV9x5HgUKP2jpUWbKLT0+vJsFmj4eCqE9kaIkzp4/Z8T6zg==} + '@tangle-network/sandbox@0.19.4': + resolution: {integrity: sha512-rp4NGX7Em0ryNexEuoGYgX9jC5AyvYlDP/JLWBMCZRGdn+wE9HDcy9SsX8+geluyxpnW2Gg3BczefRYSnIcb2Q==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -2787,11 +2781,6 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@tangle-network/agent-core@0.5.2': - dependencies: - '@tangle-network/agent-interface': 0.45.0 - zod: 4.4.3 - '@tangle-network/agent-core@0.5.4': dependencies: '@tangle-network/agent-interface': 0.46.1 @@ -2809,12 +2798,6 @@ snapshots: re2js: 2.8.6 zod: 4.4.3 - '@tangle-network/agent-interface@0.45.0': - dependencies: - '@noble/hashes': 1.8.0 - spdx-expression-parse: 5.0.0 - zod: 4.4.3 - '@tangle-network/agent-interface@0.46.1': dependencies: '@noble/hashes': 1.8.0 @@ -2834,10 +2817,10 @@ snapshots: '@tangle-network/agent-trace-contract@1.0.2': {} - '@tangle-network/sandbox@0.19.3(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.19.4(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': dependencies: - '@tangle-network/agent-core': 0.5.2 - '@tangle-network/agent-interface': 0.45.0 + '@tangle-network/agent-core': 0.5.4 + '@tangle-network/agent-interface': 0.46.1 zod: 4.4.3 optionalDependencies: viem: 2.54.6(typescript@6.0.3)(zod@4.4.3) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fc1ae552..f87fd8d5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -24,7 +24,7 @@ catalog: '@tangle-network/agent-knowledge': 7.1.2 '@tangle-network/agent-profile-materialize': 0.13.1 '@tangle-network/agent-trace-contract': ^1.0.2 - '@tangle-network/sandbox': 0.19.3 + '@tangle-network/sandbox': 0.19.4 publint: 0.3.22 tsdown: 0.22.14 tsx: 4.23.1 diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 6aed557a..c01e655f 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:65a07b469c7234762022990daac8c74cbbddddaaf708717f550feba96448f731", + "digest": "sha256:01a9d56883f0e99ee43e54a1ec921ee80542e1b67b78659ca6335aee267609d5", "evaluation": { "decision": { "contributingChecks": [ @@ -4870,7 +4870,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.130.0" + "runtimeVersion": "0.131.0" }, "objectives": [ { @@ -4981,8 +4981,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:a0c2b55593067a7563ba3a51ab240b2cbcc8a215aff93da4106d4bf002980c45", - "runId": "agent-runtime-0.130.0-proposal-fixture", + "recordDigest": "sha256:1be3dab07f5e01c97fb437eec6772825ac7b32a78e875419ecdceb32af1a198d", + "runId": "agent-runtime-0.131.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5009,5 +5009,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.130.0-proposal-fixture" + "runId": "agent-runtime-0.131.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 0025cac3..7154b8fa 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:dff67627a77309e9c0abe5173e4ff4c581e289049be4c8ccb9999bf429430482", + "digest": "sha256:97ab9bd1ba910859a42954c9b0dc85a377db2dd454516ac4e76bd118c15e69ca", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.130.0" + "runtimeVersion": "0.131.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:732444d0214da643c8acb3ac9c76823b0ab5c9751f4497638d4c8db016a2bf86", + "recordDigest": "sha256:30fa9b74d5682ada3d9c40eb90298cda4c3e17c0a45aaf8f821fd422e3bafbf8", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From dc00bb6bf8e50a7f1d4fef678cc743c4083f5732 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 19:53:42 -0600 Subject: [PATCH 7/7] fix(candidate): verify projected prompt plans --- src/candidate-execution/prepare.ts | 35 ++++---------- src/candidate-execution/profile.ts | 46 +++++++++++++++++- src/intelligence/improvement-cycle.ts | 21 ++++----- tests/improvement-cycle.test.ts | 68 +++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 39 deletions(-) diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts index 8876eb91..7bc1fd6b 100644 --- a/src/candidate-execution/prepare.ts +++ b/src/candidate-execution/prepare.ts @@ -26,10 +26,7 @@ import { agentCandidateWorkspaceSnapshotEvidenceSchema, sha256DigestSchema, } from '@tangle-network/agent-interface' -import { - applyAgentCandidateWorkspacePlan, - materializeCandidateProfile, -} from '@tangle-network/agent-profile-materialize' +import { applyAgentCandidateWorkspacePlan } from '@tangle-network/agent-profile-materialize' import { readMaterializedWorkspaceFiles, @@ -63,8 +60,7 @@ import { } from './knowledge' import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' import { createPreparedCandidateExecution } from './prepared-state' -import { CANDIDATE_PROFILE_MATERIALIZER_BINDS, candidateMaterializerHarness } from './profile' -import { projectCandidatePromptIntents } from './system-prompt' +import { materializeAgentCandidateProfilePlan } from './profile' import { type AgentCandidateExecutionPorts, type AgentCandidateTaskExecution, @@ -109,7 +105,6 @@ export async function prepareAgentCandidateExecution( maxAttempts: benchmarkTask.attempt.maxAttempts, retryPolicy: benchmarkTask.attempt.retryPolicy, } as const - const harness = candidateMaterializerHarness(bundle.execution.harness) assertTaskInput(task, bundle.execution.instructionDelivery) const resultTimeoutMs = candidateResultTimeout( options.resultTimeoutMs, @@ -178,17 +173,14 @@ export async function prepareAgentCandidateExecution( } await assertEmptyDirectory(task.stagingRoots.profileRoot) - const candidateProfileMaterialization = { - binds: CANDIDATE_PROFILE_MATERIALIZER_BINDS, + const profileWorkspacePlan = materializeAgentCandidateProfilePlan({ + profile: bundle.profile, + harness: bundle.execution.harness, + launch: bundle.execution.launch, + workspace: bundle.execution.cwd.workspace, + workspaces: task.executionRoots, resolvedResources: verifiedResourceTextByDigest(candidate), - } - const profileWorkspacePlan = projectCandidatePromptIntents( - materializeCandidateProfile(bundle.profile, harness, candidateProfileMaterialization), - bundle.execution.launch, - profileSystemPromptExecutionPath(bundle.execution.cwd.workspace, task.executionRoots), - bundle.profile.prompt?.systemPrompt, - bundle.profile.prompt?.appendSystemPrompt, - ) + }) const profileApplication = applyAgentCandidateWorkspacePlan( profileWorkspacePlan, task.stagingRoots.profileRoot, @@ -939,15 +931,6 @@ function absoluteExecutionCwd( return absolute } -function profileSystemPromptExecutionPath( - workspace: VerifiedAgentCandidate['bundle']['execution']['cwd']['workspace'], - roots: AgentCandidateTaskExecution['executionRoots'], -): string { - const root = workspace === 'task' ? roots.taskRoot : roots.candidateRoot - if (!root) throw new Error('candidate profile target is missing its execution workspace root') - return posix.join(root, '.tangle/system-prompt.md') -} - function validateProtectedModelReservation( reservation: { preparationId: string diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index b2ecbe9e..6c88fe23 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -1,5 +1,8 @@ +import { posix } from 'node:path' + import type { AgentCandidateConfigValue, + AgentCandidateLaunch, AgentCandidateProfile, AgentCandidateProfileActivation, AgentCandidateProfilePlanEvidence, @@ -10,6 +13,7 @@ import type { AgentProfileMcpServer, AgentProfileResourceRef, HarnessType, + Sha256Digest, } from '@tangle-network/agent-interface' import { agentCandidateProfileActivationSchema, @@ -22,7 +26,10 @@ import type { AgentCandidateWorkspacePlan, HarnessId, } from '@tangle-network/agent-profile-materialize' -import { isMaterializerHarness } from '@tangle-network/agent-profile-materialize' +import { + isMaterializerHarness, + materializeCandidateProfile, +} from '@tangle-network/agent-profile-materialize' import { canonicalCandidateBytes, @@ -33,6 +40,7 @@ import { omitTopLevelDigest, sha256Bytes, } from './digest' +import { projectCandidatePromptIntents } from './system-prompt' export function candidateMaterializerHarness(harness: HarnessType): HarnessId { if (!isMaterializerHarness(harness)) { @@ -46,6 +54,42 @@ export function candidateMaterializerHarness(harness: HarnessType): HarnessId { /** Runtime applies the materializer's launch flags to the candidate process. */ export const CANDIDATE_PROFILE_MATERIALIZER_BINDS = ['systemPrompt'] as const +interface MaterializeAgentCandidateProfilePlanOptions { + profile: AgentCandidateProfile + harness: HarnessType + launch: AgentCandidateLaunch + workspace: 'task' | 'candidate' + workspaces: { + taskRoot: string + candidateRoot?: string + } + resolvedResources?: ReadonlyMap +} + +/** Derive the exact native profile plan used by preparation and later evidence checks. */ +export function materializeAgentCandidateProfilePlan( + options: MaterializeAgentCandidateProfilePlanOptions, +): AgentCandidateWorkspacePlan { + const root = + options.workspace === 'task' ? options.workspaces.taskRoot : options.workspaces.candidateRoot + if (!root) throw new Error('candidate profile target is missing its execution workspace root') + const plan = materializeCandidateProfile( + options.profile, + candidateMaterializerHarness(options.harness), + { + binds: CANDIDATE_PROFILE_MATERIALIZER_BINDS, + resolvedResources: options.resolvedResources, + }, + ) + return projectCandidatePromptIntents( + plan, + options.launch, + posix.join(root, '.tangle/system-prompt.md'), + options.profile.prompt?.systemPrompt, + options.profile.prompt?.appendSystemPrompt, + ) +} + /** Bind exact native profile text to the canonical plan captured during preparation. */ export function createAgentCandidateProfileActivation( plan: AgentCandidateWorkspacePlan, diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index 09f9ba6a..d36ce760 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -65,8 +65,6 @@ import { candidateExecutionEvidenceSchema, numbersApproximatelyEqual, } from '@tangle-network/agent-interface' -import { materializeCandidateProfile } from '@tangle-network/agent-profile-materialize' - import { runAnalystLoop } from '../analyst-loop' import type { RunAnalystLoopOpts, RunAnalystLoopResult } from '../analyst-loop/types' import { @@ -87,9 +85,8 @@ import { } from '../candidate-execution/prepare' import { assertCandidateProfileBinding, - CANDIDATE_PROFILE_MATERIALIZER_BINDS, - candidateMaterializerHarness, createAgentCandidateProfileActivation, + materializeAgentCandidateProfilePlan, parseAgentCandidateProfileActivation, parseExactAgentProfile, } from '../candidate-execution/profile' @@ -1325,14 +1322,14 @@ export function verifyCandidateExecutionEvidence( materialization.profileActivation.profilePlan, 'candidate profile plan', ) - const expectedProfilePlan = materializeCandidateProfile( - bundle.profile, - candidateMaterializerHarness(materialization.harness), - { - binds: CANDIDATE_PROFILE_MATERIALIZER_BINDS, - resolvedResources: options.resolvedResources, - }, - ) + const expectedProfilePlan = materializeAgentCandidateProfilePlan({ + profile: bundle.profile, + harness: bundle.execution.harness, + launch: bundle.execution.launch, + workspace: bundle.execution.cwd.workspace, + workspaces: plan.material.workspaces, + resolvedResources: options.resolvedResources, + }) const activation = parseAgentCandidateProfileActivation( materialization.profileActivation, materialization.profileActivation.profilePlan.digest, diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index 01bd33a3..635ac1f1 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -34,6 +34,7 @@ import { type AgentImprovementExperimentMaterial, createAgentImprovementActivation, createAgentImprovementProposal, + executeAgentCandidateExperimentCell, proposeAgentImprovement, proposeAgentProfileImprovement, reviewAgentImprovementProposal, @@ -61,6 +62,7 @@ import { type CandidateExperimentFixture, cleanupCandidateExperimentFixtures, createCandidateExperimentFixture, + executeCandidateExperimentInput, } from './helpers/candidate-experiment-fixture' import { candidateExperimentMaterial, @@ -1639,6 +1641,72 @@ describe('agent improvement lifecycle', { timeout: 30_000 }, () => { ).toThrow(/activation targets/) }) + it.each([ + { + intent: 'replacement', + prompt: { systemPrompt: 'Use the measured Prime instructions.' }, + file: '.tangle/system-prompt.md', + }, + { + intent: 'addition', + prompt: { appendSystemPrompt: 'Keep the native prompt and add measured instructions.' }, + file: '.tangle/append-system-prompt.md', + }, + ] as const)( + 'verifies Prime $intent prompt evidence through the executed cell', + async (testCase) => { + const base = candidateBundle({ + harness: 'prime', + launch: { kind: 'container-command', executable: 'prime-agent' }, + }) + const baseline = redigestCandidateBundle(base, { + profile: { + ...base.profile, + harness: 'prime', + }, + }) + const candidate = redigestCandidateBundle(baseline, { + profile: { + ...baseline.profile, + prompt: { ...baseline.profile.prompt, ...testCase.prompt }, + }, + }) + const rig = createCandidateExperimentFixture({ baseline, candidate }) + const task = rig.experiment.benchmark.tasks[0] + if (!task) throw new Error('expected candidate benchmark task') + const benchmarkCell = { + suiteDigest: rig.experiment.benchmark.suite.digest, + taskIndex: 0, + repetition: 0, + } + const input = { + experiment: rig.experiment, + arm: 'candidate' as const, + bundle: rig.experiment.candidate, + task, + benchmarkCell, + seed: 101, + } + const evidence = await executeCandidateExperimentInput( + input, + rig.placeCell, + executeAgentCandidateExperimentCell, + ) + + expect( + evidence.materializationReceipt.profileActivation.files.map((file) => file.path), + ).toContain(testCase.file) + expect( + verifyCandidateExecutionEvidence(evidence, { + experiment: rig.experiment, + arm: 'candidate', + benchmarkCell, + seed: 101, + }), + ).toEqual(evidence) + }, + ) + it('does not create a proposal from an inconclusive comparison', async () => { const rig = createCandidateExperimentFixture({ scoreFor: () => 1 }) const result = await runAgentCandidateExperiment({