From d4eb324c59c7af2c3a32cb3ca336905f3e904580 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 7 Aug 2026 17:45:38 -0700 Subject: [PATCH 1/5] fix(agent): overly broad check for secrets protection --- .../handlers/agent/agent-handler.test.ts | 36 +++++++++++++++++++ .../executor/handlers/agent/agent-handler.ts | 1 - 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index cd9bb304b4f..2541d551ade 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -3497,6 +3497,42 @@ describe('AgentBlockHandler', () => { expect(tools[0].parameters.required).toContain('format') }) + it('resolves a secret-backed customToolId without exposing it to the provider', async () => { + const toolId = 'custom-tool-123' + mockDBForCustomTool(toolId) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'CANARY_CUSTOM_TOOL_ID', + plaintext: toolId, + encryptedValue: 'encrypted-custom-tool-id', + }, + ]) + const inputPath = ['tools', '0', 'customToolId'] as const + registry.recordResolvedAtInputPath('CANARY_CUSTOM_TOOL_ID', toolId, inputPath) + registry.recordResolvedInputProjection(inputPath, toolId, '{{CANARY_CUSTOM_TOOL_ID}}') + mockContext.resolvedSecretTraceRegistry = registry + + await handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + userPrompt: 'Format a report', + apiKey: 'test-api-key', + tools: [ + { + type: 'custom-tool', + customToolId: toolId, + usageControl: 'auto', + }, + ], + }) + + expect(mockGetCustomToolById).toHaveBeenCalledWith(expect.objectContaining({ toolId })) + const providerRequest = mockExecuteProviderRequest.mock.calls[0][1] + expect(providerRequest.tools).toHaveLength(1) + expect(providerRequest.tools[0].name).toBe('formatReport') + expect(JSON.stringify(providerRequest.tools)).not.toContain(toolId) + expect(JSON.stringify(providerRequest.tools)).not.toContain('CANARY_CUSTOM_TOOL_ID') + }) + it('should fall back to inline schema when DB fetch fails and inline exists', async () => { mockDBFailure() diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index cb5cbf32e05..19801570691 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -628,7 +628,6 @@ export class AgentBlockHandler implements BlockHandler { const root = ['tools', String(toolIndex)] as const const paths: ResolvedSecretInputPath[] = [[...root, 'type']] if (tool.operation !== undefined) paths.push([...root, 'operation']) - if (tool.customToolId !== undefined) paths.push([...root, 'customToolId']) if (tool.type === 'mcp') { paths.push([...root, 'params', 'serverId'], [...root, 'params', 'toolName']) } From c1827237c0a951785ed3176805c834cc6a6340ed Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 7 Aug 2026 19:08:39 -0700 Subject: [PATCH 2/5] remove opaque input processing --- .agents/skills/add-integration/SKILL.md | 43 +- .agents/skills/add-tools/SKILL.md | 16 +- .agents/skills/validate-integration/SKILL.md | 30 +- .claude/commands/add-integration.md | 43 +- .claude/commands/add-tools.md | 16 +- .claude/commands/validate-integration.md | 30 +- .cursor/commands/add-integration.md | 43 +- .cursor/commands/add-tools.md | 16 +- .cursor/commands/validate-integration.md | 30 +- .../executor/execution/block-executor.test.ts | 184 ++++++ apps/sim/executor/execution/block-executor.ts | 25 +- .../handlers/agent/agent-handler.test.ts | 405 +++++++++++- .../executor/handlers/agent/agent-handler.ts | 525 ++++++++++------ .../handlers/generic/generic-handler.ts | 5 - .../mothership/mothership-handler.test.ts | 123 ++++ .../handlers/mothership/mothership-handler.ts | 128 +++- .../handlers/pi/babysit-backend.test.ts | 1 - .../executor/handlers/pi/babysit-backend.ts | 65 +- .../executor/handlers/pi/babysit-github.ts | 12 +- .../handlers/pi/babysit-round.test.ts | 34 +- .../sim/executor/handlers/pi/babysit-round.ts | 6 +- .../handlers/pi/cloud-backend.test.ts | 13 +- .../sim/executor/handlers/pi/cloud-backend.ts | 65 +- .../handlers/pi/cloud-review-backend.test.ts | 27 +- .../handlers/pi/cloud-review-backend.ts | 22 +- .../handlers/pi/cloud-review-tools.test.ts | 23 +- .../handlers/pi/cloud-review-tools.ts | 2 +- apps/sim/executor/handlers/pi/cloud-shared.ts | 2 +- .../handlers/pi/local-backend.test.ts | 49 +- .../sim/executor/handlers/pi/local-backend.ts | 33 +- apps/sim/executor/handlers/pi/pi-sdk.ts | 11 +- .../executor/handlers/pi/redaction.test.ts | 16 +- apps/sim/executor/handlers/pi/redaction.ts | 23 +- .../lib/copilot/chat/process-contents.test.ts | 58 +- apps/sim/lib/copilot/chat/process-contents.ts | 7 +- .../sim/lib/uploads/utils/model-input.test.ts | 165 ++++- apps/sim/lib/uploads/utils/model-input.ts | 88 ++- apps/sim/tools/a2a/send_message.ts | 28 +- apps/sim/tools/browser_use/run_task.ts | 4 - apps/sim/tools/context_dev/extract.ts | 4 - apps/sim/tools/context_dev/extract_product.ts | 4 - .../sim/tools/context_dev/extract_products.ts | 4 - apps/sim/tools/cursor/add_followup.ts | 12 +- apps/sim/tools/cursor/launch_agent.ts | 8 +- apps/sim/tools/cursor/model-input.ts | 44 +- apps/sim/tools/elevenlabs/audio-isolation.ts | 10 +- apps/sim/tools/elevenlabs/model-input.ts | 28 +- apps/sim/tools/elevenlabs/speech-to-speech.ts | 10 +- apps/sim/tools/exa/find_similar_links.ts | 4 - apps/sim/tools/exa/get_contents.ts | 4 - apps/sim/tools/extend/parser.ts | 19 - apps/sim/tools/firecrawl/agent.ts | 4 - apps/sim/tools/firecrawl/batch-scrape.ts | 8 - apps/sim/tools/firecrawl/crawl.ts | 11 - apps/sim/tools/firecrawl/extract.ts | 4 - apps/sim/tools/firecrawl/parse.ts | 26 +- apps/sim/tools/firecrawl/scrape.ts | 8 - apps/sim/tools/fireflies/upload_audio.ts | 9 - apps/sim/tools/index.test.ts | 202 +----- apps/sim/tools/index.ts | 39 -- apps/sim/tools/jina/read_url.ts | 5 - .../tools/nested-model-input-adapters.test.ts | 121 +++- .../opaque-model-input-selectors.test.ts | 578 +++++------------- apps/sim/tools/pulse/parser.ts | 19 - apps/sim/tools/reducto/parser.ts | 19 - apps/sim/tools/stt/assemblyai.ts | 2 - apps/sim/tools/stt/deepgram.ts | 2 - apps/sim/tools/stt/elevenlabs.ts | 2 - apps/sim/tools/stt/gemini.ts | 2 - apps/sim/tools/stt/model-input.test.ts | 56 +- apps/sim/tools/stt/model-input.ts | 55 +- apps/sim/tools/stt/whisper.ts | 12 +- apps/sim/tools/tavily/crawl.ts | 4 - apps/sim/tools/tavily/map.ts | 4 - apps/sim/tools/textract/analyze-expense.ts | 17 - apps/sim/tools/textract/analyze-id.ts | 21 - apps/sim/tools/textract/parser.ts | 14 - apps/sim/tools/types.ts | 16 +- apps/sim/tools/video/runway.ts | 3 - 79 files changed, 2283 insertions(+), 1547 deletions(-) diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index 5b82f1070de..e72dd878bbe 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -131,20 +131,24 @@ service's official documentation or an unambiguous local execution path proves t field is consumed by an AI model. If that cannot be established, preserve existing tool behavior and leave the field unannotated. -- **Ordinary provider/API input:** leave it unchanged. Do not add blanket result sanitization. +- **Ordinary provider/API input:** leave it unchanged. Explicit `{{...}}` references resolve and are + sent with their normal request semantics. A URL, domain, resource ID, control field, or opaque + payload is not model-visible merely because the provider is AI-backed or may process the + referenced resource later. - **Text or structured content consumed by an AI model:** declare `request.modelInput` with `mode: 'project'` and select only the exact model-visible fields. The shared executor replaces activated Sim secrets with canonical `{{NAME}}` labels before request formatting. For nested or JSON-string fields, use a small shared selector plus `applyProjected`; verify that selecting the rebuilt params reproduces the projected selection. -- **Opaque model input sent directly to an external provider** such as a model-read URL or image - payload: declare `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and select only - the exact effective value. The shared `executeTool` preflight rejects incomplete or secret-bearing - committed provenance before URL/body formatting or network I/O, preserves safe request bytes, - and sends no provenance metadata to the provider. -- **Opaque model input owned by an authenticated internal route** such as uploaded audio, image, - video, file bytes, or signed URLs: add `privateProvenance` to a projected request, or use - `mode: 'private-provenance'` when there is no textual projection. The route must call +- **Serialized model content sent directly to an external provider:** include the serialized + top-level param in `request.modelInput`. Project the private copy before the existing request + formatter parses it; keep formatter behavior deterministic when a whole-value placeholder is not + valid in the serialized grammar. Do not introduce a second hard-rejection path. +- **Opaque model input owned by an authenticated internal route** such as inline audio, image, + video, or document bytes: add `privateProvenance` to a projected request, or use + `mode: 'private-provenance'` when there is no textual projection. Do not select storage keys, + paths, signed URLs, or ordinary remote URLs as byte provenance; the owning route must authorize + stored bytes independently at model egress. The route must call `validateOpaqueModelInputProvenance` before downloading or sending content to the model and must apply the workspace-file provenance guard before reading a persisted workspace file. - **Sim-owned durable storage or internal execution handoff** that can later enter a workflow/model @@ -160,9 +164,9 @@ Hard rules: - Never substitute secret plaintext into source or serialize plaintext provenance. - Never hand-roll private provenance headers/envelopes; the shared `executeTool` boundary owns transport and strips private metadata from functional results. -- Never attach private provenance to an external URL or to `directExecution`. Use the centralized - `opaqueModelInput` rejection mode for external/direct opaque model inputs, or an authenticated - internal route when encrypted provenance must cross the boundary. +- Never attach private provenance to an external URL or to `directExecution`. Project proven + model-visible external fields with `request.modelInput`; otherwise preserve ordinary request + semantics. Use an authenticated internal route when encrypted provenance must cross the boundary. - Never sanitize arbitrary third-party tool results. Projection applies only to secrets activated by Sim's resolved-secret provenance for that execution/tool call. - Do not add provenance merely because a value is persisted, returned by a tool, or appears in a @@ -173,12 +177,11 @@ Hard rules: provider responses, filenames, URLs, and errors remain unchanged when Sim did not resolve a secret into them. -Add focused tests covering named projection, ordinary identical text without provenance, nested -shape preservation, malformed/incomplete private metadata failing closed, centralized external -opaque rejection before formatting/I/O without byte changes or metadata transport, headerless -legacy requests, and absence of private metadata in the public tool result. For durable sinks, also -cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, stale/missing sidecars, -and scope isolation. +Add focused tests covering named projection, ordinary identical text without provenance, nested and +serialized shape handling, unchanged ordinary external inputs, malformed/incomplete private metadata +failing closed, headerless legacy requests, and absence of private metadata in the public tool result. +For durable sinks, also cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, +stale/missing sidecars, and scope isolation. ## Step 3: Create Block @@ -594,8 +597,8 @@ If creating V2 versions (API-aligned outputs): - [ ] Registered all tools in `tools/registry.ts` - [ ] Ran `bun run tool-metadata:generate` and committed the regenerated artifacts - [ ] Classified every model-visible, opaque, Sim-durable, and internal-execution request field -- [ ] Added shared model-input projection, centralized opaque rejection, or private provenance only - where required +- [ ] Added shared model-input projection or private provenance only where required; ordinary + external resource locators and control inputs retain their request semantics - [ ] Confirmed ordinary third-party tool results are not generically sanitized - [ ] Added provenance compatibility and fail-closed boundary tests where applicable diff --git a/.agents/skills/add-tools/SKILL.md b/.agents/skills/add-tools/SKILL.md index 2e3afe56063..734c03bcd9c 100644 --- a/.agents/skills/add-tools/SKILL.md +++ b/.agents/skills/add-tools/SKILL.md @@ -150,12 +150,16 @@ export const {serviceName}{Action}Tool: ToolConfig< - Leave ordinary external API inputs and third-party results unchanged. Add provenance handling only when an exact field is proven to cross a Sim model, durable-storage, or internal-execution boundary. - Project AI-consumed text/structured fields with the smallest exact `request.modelInput` selector. -- Reject resolved secrets in opaque model input sent directly to an external provider with - `request.opaqueModelInput`; never attach private metadata to an external URL or `directExecution`. -- For authenticated internal routes, use `privateProvenance` for opaque model input or - `request.secretProvenance` for durable writes and execution handoffs. Authenticate first, validate - the exact selection and scope, strip the private envelope, then import or propagate provenance at - the receiving boundary. Preserve documented headerless legacy behavior. +- Treat URLs, domains, resource IDs, and control fields as ordinary request values unless the exact + field is proven model-visible. For serialized external model content, project the serialized + top-level param through `request.modelInput` before the existing formatter parses it; do not add a + separate hard-rejection mechanism. +- For authenticated internal routes, use `privateProvenance` for actual inline/raw model bytes or + `request.secretProvenance` for durable writes and execution handoffs. Do not treat a storage key, + path, signed URL, or remote URL as provenance for fetched bytes; authorize tracked stored bytes at + the owning model-egress boundary. Authenticate first, validate the exact selection and scope, + strip the private envelope, then import or propagate provenance at the receiving boundary. + Preserve documented headerless legacy behavior. - Never substitute secret plaintext into source, serialize plaintext provenance, hand-roll private headers, or blanket-sanitize tool results. - Add focused tests for named projection, identical unproven public text, malformed/incomplete diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index 3d76c78b674..da5ac1dd984 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -141,21 +141,25 @@ search, extraction, or "AI-powered" marketing terminology. - [ ] AI-consumed text/structured fields use `request.modelInput` with `mode: 'project'` and a minimal exact selector; nested/JSON-string adapters preserve shape through `applyProjected` -- [ ] Opaque AI-consumed values sent directly to an external provider or `directExecution` use - `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and an exact effective-value - selector; the central executor rejects incomplete/secret-bearing committed provenance before - formatting or I/O, leaves safe bytes unchanged, and sends no provenance metadata externally -- [ ] Opaque AI-consumed files/bytes/URLs owned by an authenticated internal route use +- [ ] Ordinary external URLs, domains, resource IDs, and control fields retain normal request + semantics unless the exact field is proven model-visible; an AI-backed provider or later model + processing of the referenced resource is not sufficient evidence +- [ ] Serialized content proven to be sent directly to an external model is selected by + `request.modelInput`, projected before the existing formatter parses it, and has deterministic + formatter behavior when a whole-value placeholder is invalid for the serialized grammar +- [ ] Actual inline/raw AI-consumed bytes owned by an authenticated internal route use `privateProvenance` (or `mode: 'private-provenance'`), and the route validates - `validateOpaqueModelInputProvenance` before any download or model call + `validateOpaqueModelInputProvenance` before model egress; storage keys, paths, signed URLs, + and ordinary remote URLs are not treated as byte provenance, while tracked stored bytes are + authorized independently at the owning model-egress boundary - [ ] Persisted workspace-file contents are checked with the shared provenance guard only when their bytes or decoded content cross into a model/tool-result boundary; ordinary file APIs remain unchanged. Unsupported secret-bearing file paths are rejected at `file_write` - [ ] Sim-owned durable writes and internal execution handoffs that can enter workflows/models use field-scoped `request.secretProvenance`; authenticated receivers validate the exact selection and scope, strip private metadata, and persist, import, or propagate it at the owning boundary -- [ ] Private provenance is never attached to external URLs or `directExecution`; those paths use - centralized `opaqueModelInput` rejection when their opaque values are model-bound +- [ ] Private provenance is never attached to external URLs or `directExecution`; proven + model-visible external fields use projection, while other external inputs remain unchanged - [ ] No tool performs raw secret plaintext/source substitution or serializes plaintext provenance - [ ] No `transformResponse` or tool-local helper blanket-sanitizes ordinary third-party results; only execution-scoped, activated Sim provenance is projected at shared model/log boundaries @@ -166,10 +170,9 @@ search, extraction, or "AI-powered" marketing terminology. metadata, provider results, or API payloads - [ ] Diagnostic projection is applied only to values carrying execution-scoped provenance; ordinary provider responses, filenames, URLs, and errors are unchanged -- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested shape - preservation, malformed/incomplete metadata, centralized opaque rejection before formatting - or I/O with safe-byte preservation, headerless legacy requests, metadata stripping, and - durable legacy/stale/scope cases when applicable +- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested and serialized + shape handling, unchanged ordinary external inputs, malformed/incomplete metadata, headerless + legacy requests, metadata stripping, and durable legacy/stale/scope cases when applicable Treat a missing or bypassed model, durable, or internal-execution provenance boundary as **critical**. Do not fix it with a tool-specific string replacer or by sanitizing every provider @@ -348,8 +351,7 @@ Group findings by severity: - Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` -- AI-consumed request fields bypass the shared projection, centralized opaque rejection, or - private-provenance boundary +- Proven model-visible request fields bypass the shared projection or private-provenance boundary - Opaque model input is downloaded or sent before provenance and workspace-file checks - A Sim-owned durable sink or internal execution handoff drops encrypted provenance or breaks legacy headerless/`NULL` data diff --git a/.claude/commands/add-integration.md b/.claude/commands/add-integration.md index 2b8e6a4fc13..864dc9ab9b3 100644 --- a/.claude/commands/add-integration.md +++ b/.claude/commands/add-integration.md @@ -130,20 +130,24 @@ service's official documentation or an unambiguous local execution path proves t field is consumed by an AI model. If that cannot be established, preserve existing tool behavior and leave the field unannotated. -- **Ordinary provider/API input:** leave it unchanged. Do not add blanket result sanitization. +- **Ordinary provider/API input:** leave it unchanged. Explicit `{{...}}` references resolve and are + sent with their normal request semantics. A URL, domain, resource ID, control field, or opaque + payload is not model-visible merely because the provider is AI-backed or may process the + referenced resource later. - **Text or structured content consumed by an AI model:** declare `request.modelInput` with `mode: 'project'` and select only the exact model-visible fields. The shared executor replaces activated Sim secrets with canonical `{{NAME}}` labels before request formatting. For nested or JSON-string fields, use a small shared selector plus `applyProjected`; verify that selecting the rebuilt params reproduces the projected selection. -- **Opaque model input sent directly to an external provider** such as a model-read URL or image - payload: declare `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and select only - the exact effective value. The shared `executeTool` preflight rejects incomplete or secret-bearing - committed provenance before URL/body formatting or network I/O, preserves safe request bytes, - and sends no provenance metadata to the provider. -- **Opaque model input owned by an authenticated internal route** such as uploaded audio, image, - video, file bytes, or signed URLs: add `privateProvenance` to a projected request, or use - `mode: 'private-provenance'` when there is no textual projection. The route must call +- **Serialized model content sent directly to an external provider:** include the serialized + top-level param in `request.modelInput`. Project the private copy before the existing request + formatter parses it; keep formatter behavior deterministic when a whole-value placeholder is not + valid in the serialized grammar. Do not introduce a second hard-rejection path. +- **Opaque model input owned by an authenticated internal route** such as inline audio, image, + video, or document bytes: add `privateProvenance` to a projected request, or use + `mode: 'private-provenance'` when there is no textual projection. Do not select storage keys, + paths, signed URLs, or ordinary remote URLs as byte provenance; the owning route must authorize + stored bytes independently at model egress. The route must call `validateOpaqueModelInputProvenance` before downloading or sending content to the model and must apply the workspace-file provenance guard before reading a persisted workspace file. - **Sim-owned durable storage or internal execution handoff** that can later enter a workflow/model @@ -159,9 +163,9 @@ Hard rules: - Never substitute secret plaintext into source or serialize plaintext provenance. - Never hand-roll private provenance headers/envelopes; the shared `executeTool` boundary owns transport and strips private metadata from functional results. -- Never attach private provenance to an external URL or to `directExecution`. Use the centralized - `opaqueModelInput` rejection mode for external/direct opaque model inputs, or an authenticated - internal route when encrypted provenance must cross the boundary. +- Never attach private provenance to an external URL or to `directExecution`. Project proven + model-visible external fields with `request.modelInput`; otherwise preserve ordinary request + semantics. Use an authenticated internal route when encrypted provenance must cross the boundary. - Never sanitize arbitrary third-party tool results. Projection applies only to secrets activated by Sim's resolved-secret provenance for that execution/tool call. - Do not add provenance merely because a value is persisted, returned by a tool, or appears in a @@ -172,12 +176,11 @@ Hard rules: provider responses, filenames, URLs, and errors remain unchanged when Sim did not resolve a secret into them. -Add focused tests covering named projection, ordinary identical text without provenance, nested -shape preservation, malformed/incomplete private metadata failing closed, centralized external -opaque rejection before formatting/I/O without byte changes or metadata transport, headerless -legacy requests, and absence of private metadata in the public tool result. For durable sinks, also -cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, stale/missing sidecars, -and scope isolation. +Add focused tests covering named projection, ordinary identical text without provenance, nested and +serialized shape handling, unchanged ordinary external inputs, malformed/incomplete private metadata +failing closed, headerless legacy requests, and absence of private metadata in the public tool result. +For durable sinks, also cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, +stale/missing sidecars, and scope isolation. ## Step 3: Create Block @@ -593,8 +596,8 @@ If creating V2 versions (API-aligned outputs): - [ ] Registered all tools in `tools/registry.ts` - [ ] Ran `bun run tool-metadata:generate` and committed the regenerated artifacts - [ ] Classified every model-visible, opaque, Sim-durable, and internal-execution request field -- [ ] Added shared model-input projection, centralized opaque rejection, or private provenance only - where required +- [ ] Added shared model-input projection or private provenance only where required; ordinary + external resource locators and control inputs retain their request semantics - [ ] Confirmed ordinary third-party tool results are not generically sanitized - [ ] Added provenance compatibility and fail-closed boundary tests where applicable diff --git a/.claude/commands/add-tools.md b/.claude/commands/add-tools.md index e5c0da8997a..6b390520b64 100644 --- a/.claude/commands/add-tools.md +++ b/.claude/commands/add-tools.md @@ -149,12 +149,16 @@ export const {serviceName}{Action}Tool: ToolConfig< - Leave ordinary external API inputs and third-party results unchanged. Add provenance handling only when an exact field is proven to cross a Sim model, durable-storage, or internal-execution boundary. - Project AI-consumed text/structured fields with the smallest exact `request.modelInput` selector. -- Reject resolved secrets in opaque model input sent directly to an external provider with - `request.opaqueModelInput`; never attach private metadata to an external URL or `directExecution`. -- For authenticated internal routes, use `privateProvenance` for opaque model input or - `request.secretProvenance` for durable writes and execution handoffs. Authenticate first, validate - the exact selection and scope, strip the private envelope, then import or propagate provenance at - the receiving boundary. Preserve documented headerless legacy behavior. +- Treat URLs, domains, resource IDs, and control fields as ordinary request values unless the exact + field is proven model-visible. For serialized external model content, project the serialized + top-level param through `request.modelInput` before the existing formatter parses it; do not add a + separate hard-rejection mechanism. +- For authenticated internal routes, use `privateProvenance` for actual inline/raw model bytes or + `request.secretProvenance` for durable writes and execution handoffs. Do not treat a storage key, + path, signed URL, or remote URL as provenance for fetched bytes; authorize tracked stored bytes at + the owning model-egress boundary. Authenticate first, validate the exact selection and scope, + strip the private envelope, then import or propagate provenance at the receiving boundary. + Preserve documented headerless legacy behavior. - Never substitute secret plaintext into source, serialize plaintext provenance, hand-roll private headers, or blanket-sanitize tool results. - Add focused tests for named projection, identical unproven public text, malformed/incomplete diff --git a/.claude/commands/validate-integration.md b/.claude/commands/validate-integration.md index 2c343a8fb65..79276796280 100644 --- a/.claude/commands/validate-integration.md +++ b/.claude/commands/validate-integration.md @@ -140,21 +140,25 @@ search, extraction, or "AI-powered" marketing terminology. - [ ] AI-consumed text/structured fields use `request.modelInput` with `mode: 'project'` and a minimal exact selector; nested/JSON-string adapters preserve shape through `applyProjected` -- [ ] Opaque AI-consumed values sent directly to an external provider or `directExecution` use - `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and an exact effective-value - selector; the central executor rejects incomplete/secret-bearing committed provenance before - formatting or I/O, leaves safe bytes unchanged, and sends no provenance metadata externally -- [ ] Opaque AI-consumed files/bytes/URLs owned by an authenticated internal route use +- [ ] Ordinary external URLs, domains, resource IDs, and control fields retain normal request + semantics unless the exact field is proven model-visible; an AI-backed provider or later model + processing of the referenced resource is not sufficient evidence +- [ ] Serialized content proven to be sent directly to an external model is selected by + `request.modelInput`, projected before the existing formatter parses it, and has deterministic + formatter behavior when a whole-value placeholder is invalid for the serialized grammar +- [ ] Actual inline/raw AI-consumed bytes owned by an authenticated internal route use `privateProvenance` (or `mode: 'private-provenance'`), and the route validates - `validateOpaqueModelInputProvenance` before any download or model call + `validateOpaqueModelInputProvenance` before model egress; storage keys, paths, signed URLs, + and ordinary remote URLs are not treated as byte provenance, while tracked stored bytes are + authorized independently at the owning model-egress boundary - [ ] Persisted workspace-file contents are checked with the shared provenance guard only when their bytes or decoded content cross into a model/tool-result boundary; ordinary file APIs remain unchanged. Unsupported secret-bearing file paths are rejected at `file_write` - [ ] Sim-owned durable writes and internal execution handoffs that can enter workflows/models use field-scoped `request.secretProvenance`; authenticated receivers validate the exact selection and scope, strip private metadata, and persist, import, or propagate it at the owning boundary -- [ ] Private provenance is never attached to external URLs or `directExecution`; those paths use - centralized `opaqueModelInput` rejection when their opaque values are model-bound +- [ ] Private provenance is never attached to external URLs or `directExecution`; proven + model-visible external fields use projection, while other external inputs remain unchanged - [ ] No tool performs raw secret plaintext/source substitution or serializes plaintext provenance - [ ] No `transformResponse` or tool-local helper blanket-sanitizes ordinary third-party results; only execution-scoped, activated Sim provenance is projected at shared model/log boundaries @@ -165,10 +169,9 @@ search, extraction, or "AI-powered" marketing terminology. metadata, provider results, or API payloads - [ ] Diagnostic projection is applied only to values carrying execution-scoped provenance; ordinary provider responses, filenames, URLs, and errors are unchanged -- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested shape - preservation, malformed/incomplete metadata, centralized opaque rejection before formatting - or I/O with safe-byte preservation, headerless legacy requests, metadata stripping, and - durable legacy/stale/scope cases when applicable +- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested and serialized + shape handling, unchanged ordinary external inputs, malformed/incomplete metadata, headerless + legacy requests, metadata stripping, and durable legacy/stale/scope cases when applicable Treat a missing or bypassed model, durable, or internal-execution provenance boundary as **critical**. Do not fix it with a tool-specific string replacer or by sanitizing every provider @@ -347,8 +350,7 @@ Group findings by severity: - Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` -- AI-consumed request fields bypass the shared projection, centralized opaque rejection, or - private-provenance boundary +- Proven model-visible request fields bypass the shared projection or private-provenance boundary - Opaque model input is downloaded or sent before provenance and workspace-file checks - A Sim-owned durable sink or internal execution handoff drops encrypted provenance or breaks legacy headerless/`NULL` data diff --git a/.cursor/commands/add-integration.md b/.cursor/commands/add-integration.md index 9c5498257b6..40cc28d8b8f 100644 --- a/.cursor/commands/add-integration.md +++ b/.cursor/commands/add-integration.md @@ -125,20 +125,24 @@ service's official documentation or an unambiguous local execution path proves t field is consumed by an AI model. If that cannot be established, preserve existing tool behavior and leave the field unannotated. -- **Ordinary provider/API input:** leave it unchanged. Do not add blanket result sanitization. +- **Ordinary provider/API input:** leave it unchanged. Explicit `{{...}}` references resolve and are + sent with their normal request semantics. A URL, domain, resource ID, control field, or opaque + payload is not model-visible merely because the provider is AI-backed or may process the + referenced resource later. - **Text or structured content consumed by an AI model:** declare `request.modelInput` with `mode: 'project'` and select only the exact model-visible fields. The shared executor replaces activated Sim secrets with canonical `{{NAME}}` labels before request formatting. For nested or JSON-string fields, use a small shared selector plus `applyProjected`; verify that selecting the rebuilt params reproduces the projected selection. -- **Opaque model input sent directly to an external provider** such as a model-read URL or image - payload: declare `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and select only - the exact effective value. The shared `executeTool` preflight rejects incomplete or secret-bearing - committed provenance before URL/body formatting or network I/O, preserves safe request bytes, - and sends no provenance metadata to the provider. -- **Opaque model input owned by an authenticated internal route** such as uploaded audio, image, - video, file bytes, or signed URLs: add `privateProvenance` to a projected request, or use - `mode: 'private-provenance'` when there is no textual projection. The route must call +- **Serialized model content sent directly to an external provider:** include the serialized + top-level param in `request.modelInput`. Project the private copy before the existing request + formatter parses it; keep formatter behavior deterministic when a whole-value placeholder is not + valid in the serialized grammar. Do not introduce a second hard-rejection path. +- **Opaque model input owned by an authenticated internal route** such as inline audio, image, + video, or document bytes: add `privateProvenance` to a projected request, or use + `mode: 'private-provenance'` when there is no textual projection. Do not select storage keys, + paths, signed URLs, or ordinary remote URLs as byte provenance; the owning route must authorize + stored bytes independently at model egress. The route must call `validateOpaqueModelInputProvenance` before downloading or sending content to the model and must apply the workspace-file provenance guard before reading a persisted workspace file. - **Sim-owned durable storage or internal execution handoff** that can later enter a workflow/model @@ -154,9 +158,9 @@ Hard rules: - Never substitute secret plaintext into source or serialize plaintext provenance. - Never hand-roll private provenance headers/envelopes; the shared `executeTool` boundary owns transport and strips private metadata from functional results. -- Never attach private provenance to an external URL or to `directExecution`. Use the centralized - `opaqueModelInput` rejection mode for external/direct opaque model inputs, or an authenticated - internal route when encrypted provenance must cross the boundary. +- Never attach private provenance to an external URL or to `directExecution`. Project proven + model-visible external fields with `request.modelInput`; otherwise preserve ordinary request + semantics. Use an authenticated internal route when encrypted provenance must cross the boundary. - Never sanitize arbitrary third-party tool results. Projection applies only to secrets activated by Sim's resolved-secret provenance for that execution/tool call. - Do not add provenance merely because a value is persisted, returned by a tool, or appears in a @@ -167,12 +171,11 @@ Hard rules: provider responses, filenames, URLs, and errors remain unchanged when Sim did not resolve a secret into them. -Add focused tests covering named projection, ordinary identical text without provenance, nested -shape preservation, malformed/incomplete private metadata failing closed, centralized external -opaque rejection before formatting/I/O without byte changes or metadata transport, headerless -legacy requests, and absence of private metadata in the public tool result. For durable sinks, also -cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, stale/missing sidecars, -and scope isolation. +Add focused tests covering named projection, ordinary identical text without provenance, nested and +serialized shape handling, unchanged ordinary external inputs, malformed/incomplete private metadata +failing closed, headerless legacy requests, and absence of private metadata in the public tool result. +For durable sinks, also cover legacy `NULL` markers, exact-empty new writes, tracked secret writes, +stale/missing sidecars, and scope isolation. ## Step 3: Create Block @@ -588,8 +591,8 @@ If creating V2 versions (API-aligned outputs): - [ ] Registered all tools in `tools/registry.ts` - [ ] Ran `bun run tool-metadata:generate` and committed the regenerated artifacts - [ ] Classified every model-visible, opaque, Sim-durable, and internal-execution request field -- [ ] Added shared model-input projection, centralized opaque rejection, or private provenance only - where required +- [ ] Added shared model-input projection or private provenance only where required; ordinary + external resource locators and control inputs retain their request semantics - [ ] Confirmed ordinary third-party tool results are not generically sanitized - [ ] Added provenance compatibility and fail-closed boundary tests where applicable diff --git a/.cursor/commands/add-tools.md b/.cursor/commands/add-tools.md index 45399b10698..c8611887dd8 100644 --- a/.cursor/commands/add-tools.md +++ b/.cursor/commands/add-tools.md @@ -144,12 +144,16 @@ export const {serviceName}{Action}Tool: ToolConfig< - Leave ordinary external API inputs and third-party results unchanged. Add provenance handling only when an exact field is proven to cross a Sim model, durable-storage, or internal-execution boundary. - Project AI-consumed text/structured fields with the smallest exact `request.modelInput` selector. -- Reject resolved secrets in opaque model input sent directly to an external provider with - `request.opaqueModelInput`; never attach private metadata to an external URL or `directExecution`. -- For authenticated internal routes, use `privateProvenance` for opaque model input or - `request.secretProvenance` for durable writes and execution handoffs. Authenticate first, validate - the exact selection and scope, strip the private envelope, then import or propagate provenance at - the receiving boundary. Preserve documented headerless legacy behavior. +- Treat URLs, domains, resource IDs, and control fields as ordinary request values unless the exact + field is proven model-visible. For serialized external model content, project the serialized + top-level param through `request.modelInput` before the existing formatter parses it; do not add a + separate hard-rejection mechanism. +- For authenticated internal routes, use `privateProvenance` for actual inline/raw model bytes or + `request.secretProvenance` for durable writes and execution handoffs. Do not treat a storage key, + path, signed URL, or remote URL as provenance for fetched bytes; authorize tracked stored bytes at + the owning model-egress boundary. Authenticate first, validate the exact selection and scope, + strip the private envelope, then import or propagate provenance at the receiving boundary. + Preserve documented headerless legacy behavior. - Never substitute secret plaintext into source, serialize plaintext provenance, hand-roll private headers, or blanket-sanitize tool results. - Add focused tests for named projection, identical unproven public text, malformed/incomplete diff --git a/.cursor/commands/validate-integration.md b/.cursor/commands/validate-integration.md index 4ec7b32e9ec..0c08276a7f1 100644 --- a/.cursor/commands/validate-integration.md +++ b/.cursor/commands/validate-integration.md @@ -135,21 +135,25 @@ search, extraction, or "AI-powered" marketing terminology. - [ ] AI-consumed text/structured fields use `request.modelInput` with `mode: 'project'` and a minimal exact selector; nested/JSON-string adapters preserve shape through `applyProjected` -- [ ] Opaque AI-consumed values sent directly to an external provider or `directExecution` use - `request.opaqueModelInput` with `mode: 'reject-resolved-secrets'` and an exact effective-value - selector; the central executor rejects incomplete/secret-bearing committed provenance before - formatting or I/O, leaves safe bytes unchanged, and sends no provenance metadata externally -- [ ] Opaque AI-consumed files/bytes/URLs owned by an authenticated internal route use +- [ ] Ordinary external URLs, domains, resource IDs, and control fields retain normal request + semantics unless the exact field is proven model-visible; an AI-backed provider or later model + processing of the referenced resource is not sufficient evidence +- [ ] Serialized content proven to be sent directly to an external model is selected by + `request.modelInput`, projected before the existing formatter parses it, and has deterministic + formatter behavior when a whole-value placeholder is invalid for the serialized grammar +- [ ] Actual inline/raw AI-consumed bytes owned by an authenticated internal route use `privateProvenance` (or `mode: 'private-provenance'`), and the route validates - `validateOpaqueModelInputProvenance` before any download or model call + `validateOpaqueModelInputProvenance` before model egress; storage keys, paths, signed URLs, + and ordinary remote URLs are not treated as byte provenance, while tracked stored bytes are + authorized independently at the owning model-egress boundary - [ ] Persisted workspace-file contents are checked with the shared provenance guard only when their bytes or decoded content cross into a model/tool-result boundary; ordinary file APIs remain unchanged. Unsupported secret-bearing file paths are rejected at `file_write` - [ ] Sim-owned durable writes and internal execution handoffs that can enter workflows/models use field-scoped `request.secretProvenance`; authenticated receivers validate the exact selection and scope, strip private metadata, and persist, import, or propagate it at the owning boundary -- [ ] Private provenance is never attached to external URLs or `directExecution`; those paths use - centralized `opaqueModelInput` rejection when their opaque values are model-bound +- [ ] Private provenance is never attached to external URLs or `directExecution`; proven + model-visible external fields use projection, while other external inputs remain unchanged - [ ] No tool performs raw secret plaintext/source substitution or serializes plaintext provenance - [ ] No `transformResponse` or tool-local helper blanket-sanitizes ordinary third-party results; only execution-scoped, activated Sim provenance is projected at shared model/log boundaries @@ -160,10 +164,9 @@ search, extraction, or "AI-powered" marketing terminology. metadata, provider results, or API payloads - [ ] Diagnostic projection is applied only to values carrying execution-scoped provenance; ordinary provider responses, filenames, URLs, and errors are unchanged -- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested shape - preservation, malformed/incomplete metadata, centralized opaque rejection before formatting - or I/O with safe-byte preservation, headerless legacy requests, metadata stripping, and - durable legacy/stale/scope cases when applicable +- [ ] Tests cover named `{{NAME}}` projection, unproven identical public text, nested and serialized + shape handling, unchanged ordinary external inputs, malformed/incomplete metadata, headerless + legacy requests, metadata stripping, and durable legacy/stale/scope cases when applicable Treat a missing or bypassed model, durable, or internal-execution provenance boundary as **critical**. Do not fix it with a tool-specific string replacer or by sanitizing every provider @@ -342,8 +345,7 @@ Group findings by severity: - Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` -- AI-consumed request fields bypass the shared projection, centralized opaque rejection, or - private-provenance boundary +- Proven model-visible request fields bypass the shared projection or private-provenance boundary - Opaque model input is downloaded or sent before provenance and workspace-file checks - A Sim-owned durable sink or internal execution handoff drops encrypted provenance or breaks legacy headerless/`NULL` data diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index b6ac826c78b..fb7a7ac870f 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -556,6 +556,112 @@ describe('BlockExecutor', () => { expect(onBlockComplete.mock.calls[1]?.[3]?.resolvedSecretTraceProvenance?.entries).toEqual([]) }) + it('uses a handler-narrowed registry for output provenance and parent commit', async () => { + const block: SerializedBlock = { + ...createBlock(), + metadata: { id: BlockType.MOTHERSHIP, name: 'Sim Chat' }, + config: { tool: BlockType.MOTHERSHIP, params: { selector: 'x' } }, + } + const workflow: SerializedWorkflow = { + version: '1', + blocks: [block], + connections: [], + loops: {}, + parallels: {}, + } + const state = new ExecutionState() + const resolver = new VariableResolver(workflow, {}, state) + const onBlockComplete = vi.fn(async () => {}) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PRIVATE_SELECTOR', plaintext: 'x', encryptedValue: 'encrypted-selector' }, + ]) + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext, _block, inputs) => { + const callRegistry = blockContext.resolvedSecretTraceRegistry! + callRegistry.recordResolvedAtInputPath('PRIVATE_SELECTOR', 'x', ['selector']) + callRegistry.recordResolvedInputProjection(['selector'], 'x', '{{PRIVATE_SELECTOR}}') + inputs.selector = '{{PRIVATE_SELECTOR}}' + blockContext.resolvedSecretTraceRegistry = callRegistry.forkForInputPaths([]) + return { result: 'Box' } + }, + } + const executor = new BlockExecutor([handler], resolver, { onBlockComplete }, state) + const ctx = createContext(state) + ctx.resolvedSecretTraceRegistry = registry + + await expect(executor.execute(ctx, createNode(block), block)).resolves.toEqual({ + result: 'Box', + }) + await vi.waitFor(() => expect(onBlockComplete).toHaveBeenCalledOnce()) + + expect(state.getBlockOutput(block.id)).toEqual({ result: 'Box' }) + expect(ctx.blockLogs[0]?.input).toEqual({ selector: '{{PRIVATE_SELECTOR}}' }) + expect(onBlockComplete.mock.calls[0]?.[3]?.input).toEqual({ + selector: '{{PRIVATE_SELECTOR}}', + }) + expect(onBlockComplete.mock.calls[0]?.[3]?.output).toEqual({ result: 'Box' }) + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance?.entries).toEqual([]) + expect( + onBlockComplete.mock.calls[0]?.[3]?.displayResolvedSecretTraceProvenance?.entries + ).toEqual([]) + expect(registry.getActiveMatches()).toEqual([]) + }) + + it('uses a handler-narrowed registry when execution fails after private input settlement', async () => { + const block: SerializedBlock = { + ...createBlock(), + metadata: { id: BlockType.MOTHERSHIP, name: 'Sim Chat' }, + config: { tool: BlockType.MOTHERSHIP, params: { selector: 'x' } }, + } + const workflow: SerializedWorkflow = { + version: '1', + blocks: [block], + connections: [], + loops: {}, + parallels: {}, + } + const state = new ExecutionState() + const resolver = new VariableResolver(workflow, {}, state) + const onBlockComplete = vi.fn(async () => {}) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PRIVATE_SELECTOR', plaintext: 'x', encryptedValue: 'encrypted-selector' }, + ]) + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext, _block, inputs) => { + const callRegistry = blockContext.resolvedSecretTraceRegistry! + callRegistry.recordResolvedAtInputPath('PRIVATE_SELECTOR', 'x', ['selector']) + callRegistry.recordResolvedInputProjection(['selector'], 'x', '{{PRIVATE_SELECTOR}}') + inputs.selector = '{{PRIVATE_SELECTOR}}' + blockContext.resolvedSecretTraceRegistry = callRegistry.forkForInputPaths([]) + throw new Error('Provider request preparation failed') + }, + } + const executor = new BlockExecutor([handler], resolver, { onBlockComplete }, state) + const ctx = createContext(state) + ctx.resolvedSecretTraceRegistry = registry + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow( + 'Provider request preparation failed' + ) + await vi.waitFor(() => expect(onBlockComplete).toHaveBeenCalledOnce()) + + expect(state.getBlockOutput(block.id)).toEqual({ + error: 'Provider request preparation failed', + }) + expect(ctx.blockLogs[0]?.input).toEqual({ selector: '{{PRIVATE_SELECTOR}}' }) + expect(onBlockComplete.mock.calls[0]?.[3]?.input).toEqual({ + selector: '{{PRIVATE_SELECTOR}}', + }) + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance?.entries).toEqual([]) + expect( + onBlockComplete.mock.calls[0]?.[3]?.displayResolvedSecretTraceProvenance?.entries + ).toEqual([]) + expect(registry.getActiveMatches()).toEqual([]) + expect(JSON.stringify(ctx.blockLogs)).not.toContain('"x"') + }) + it('fires block completion callbacks for pausing blocks so clients receive pause output', async () => { const block = { ...createBlock(), @@ -1099,6 +1205,84 @@ describe('BlockExecutor streaming pump', () => { expect(state.getBlockOutput(block.id)?.content).toBe('offline answer') }) + it('persists tool-result provenance activated in a narrowed registry during stream drain', async () => { + const selector = 'x' + const resultSecret = 'stream-tool-result-secret' + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext, _block, inputs) => { + const sourceRegistry = blockContext.resolvedSecretTraceRegistry! + sourceRegistry.recordResolvedAtInputPath('PRIVATE_SELECTOR', selector, ['selector']) + sourceRegistry.recordResolvedInputProjection(['selector'], selector, '{{PRIVATE_SELECTOR}}') + inputs.selector = '{{PRIVATE_SELECTOR}}' + + const runtimeRegistry = sourceRegistry.forkForInputPaths([]) + blockContext.resolvedSecretTraceRegistry = runtimeRegistry + const output = { + content: '', + toolCalls: { list: [] as Array>, count: 0 }, + } + const stream = new ReadableStream({ + start(controller) { + runtimeRegistry.recordResolved('TOOL_RESULT', resultSecret, { propagated: true }) + output.toolCalls = { + list: [{ name: 'lookup', result: { value: resultSecret, public: 'Box' } }], + count: 1, + } + controller.enqueue({ type: 'text_delta', text: 'done', turn: 'final' }) + controller.close() + }, + }) + + return { + stream, + streamFormat: 'agent-events-v1' as const, + execution: { + success: true, + output, + logs: [], + metadata: { startTime: new Date().toISOString(), duration: 1 }, + }, + } + }, + } + const { executor, block, state } = createExecutor(handler) + block.config.params = { selector } + const ctx = createContext(state) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'PRIVATE_SELECTOR', + plaintext: selector, + encryptedValue: 'encrypted-selector', + }, + { + name: 'TOOL_RESULT', + plaintext: resultSecret, + encryptedValue: 'encrypted-tool-result', + }, + ]) + ctx.resolvedSecretTraceRegistry = registry + + await executor.execute(ctx, createNode(block), block) + + expect(state.getBlockOutput(block.id)).toEqual({ + content: 'done', + toolCalls: { + list: [{ name: 'lookup', result: { value: resultSecret, public: 'Box' } }], + count: 1, + }, + }) + expect(ctx.blockLogs[0]?.input).toEqual({ selector: '{{PRIVATE_SELECTOR}}' }) + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'TOOL_RESULT', encryptedValue: 'encrypted-tool-result' }], + }) + expect(registry.getActiveMatches()).toEqual([ + { plaintext: resultSecret, replacement: '{{TOOL_RESULT}}' }, + ]) + }) + it('throws on mid-stream provider error (no truncated success)', async () => { const secret = 'stream-pump-secret-7f3a91' const rawError = new Error(`provider reset ${secret} __var_API_KEY __sim_code_4_binding_1`) diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index c43299d85ee..9b88075f449 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -109,16 +109,13 @@ export class BlockExecutor { : ctx let registryCommitted = false const commitBlockRegistry = () => { - if ( - registryCommitted || - !parentResolvedSecretTraceRegistry || - !blockResolvedSecretTraceRegistry - ) { + const settledBlockRegistry = blockCtx.resolvedSecretTraceRegistry + if (registryCommitted || !parentResolvedSecretTraceRegistry || !settledBlockRegistry) { return } registryCommitted = true - if (blockResolvedSecretTraceRegistry.isComplete()) { - parentResolvedSecretTraceRegistry.mergeToolCallRegistry(blockResolvedSecretTraceRegistry) + if (settledBlockRegistry.isComplete()) { + parentResolvedSecretTraceRegistry.mergeToolCallRegistry(settledBlockRegistry) } } @@ -328,8 +325,8 @@ export class BlockExecutor { const { childTraceSpans: _traces, ...outputForState } = normalizedOutput const stateOutput = outputForState as NormalizedBlockOutput - const stateProvenance = - blockResolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(stateOutput) + const settledBlockRegistry = blockCtx.resolvedSecretTraceRegistry + const stateProvenance = settledBlockRegistry?.exportCommittedProvenanceForValue(stateOutput) this.setNodeOutput(node, stateOutput, duration, stateProvenance) if (!isSentinel && blockLog) { @@ -341,11 +338,11 @@ export class BlockExecutor { block, }) const displayInput = this.sanitizeInputsForLog(inputsForLog, block) - const displayProvenance = - blockResolvedSecretTraceRegistry?.exportCommittedProvenanceForValue({ - input: displayInput, - output: displayOutput, - }) + blockLog.input = displayInput + const displayProvenance = settledBlockRegistry?.exportCommittedProvenanceForValue({ + input: displayInput, + output: displayOutput, + }) this.setBlockLogDisplayProvenance(blockLog, displayProvenance) this.fireBlockCompleteCallback( blockStartPromise, diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 2541d551ade..08a03a937b6 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -1252,6 +1252,47 @@ describe('AgentBlockHandler', () => { expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) + it('prunes a private selector when an earlier message structural check fails', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CUSTOM_TOOL_ID', plaintext: 'x', encryptedValue: 'encrypted-tool-id' }, + { name: 'CALL_ID', plaintext: 'private-call', encryptedValue: 'encrypted-call-id' }, + ]) + const selectorPath = ['tools', '0', 'customToolId'] as const + registry.recordResolvedAtInputPath('CUSTOM_TOOL_ID', 'x', selectorPath) + registry.recordResolvedInputProjection(selectorPath, 'x', '{{CUSTOM_TOOL_ID}}') + const callIdPath = ['messages', '0', 'tool_calls', '0', 'id'] as const + registry.recordResolvedAtInputPath('CALL_ID', 'private-call', callIdPath) + registry.recordResolvedInputProjection(callIdPath, 'private-call', '{{CALL_ID}}') + mockContext.resolvedSecretTraceRegistry = registry + const inputs = { + model: 'gpt-4o', + messages: [ + { + role: 'assistant' as const, + content: '', + tool_calls: [ + { + id: 'private-call', + type: 'function' as const, + function: { name: 'lookup', arguments: '{}' }, + }, + ], + }, + ], + tools: [{ type: 'custom-tool', customToolId: 'x', usageControl: 'auto' as const }], + } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( + 'Agent structural model inputs cannot contain secret references' + ) + + expect(inputs.tools[0].customToolId).toBe('{{CUSTOM_TOOL_ID}}') + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([ + { plaintext: 'private-call', replacement: '{{CALL_ID}}' }, + ]) + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + it('binds a resolved tool preset to the exact formatted provider tool instance', async () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'API_KEY', plaintext: 'x', encryptedValue: 'encrypted-api-key' }, @@ -1931,7 +1972,7 @@ describe('AgentBlockHandler', () => { expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) - it('rejects a resolver-derived response format name', async () => { + it('aliases a resolver-derived response format name without changing the persisted input', async () => { const responseFormat = { name: 'private-schema', schema: { type: 'object', properties: {} }, @@ -1945,16 +1986,216 @@ describe('AgentBlockHandler', () => { registry.recordResolvedInputProjection(inputPath, 'private-schema', '{{FORMAT_NAME}}') mockContext.resolvedSecretTraceRegistry = registry - await expect( - handler.execute(mockContext, mockBlock, { - model: 'gpt-4o', - userPrompt: 'Return an answer.', - responseFormat, - }) - ).rejects.toThrow('Agent structural model inputs cannot contain secret references') + await handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + userPrompt: 'Return an answer.', + responseFormat, + }) + + expect(mockExecuteProviderRequest.mock.calls[0][1].responseFormat).toEqual({ + name: 'response_schema', + schema: { type: 'object', properties: {} }, + strict: true, + }) + expect(JSON.stringify(mockExecuteProviderRequest.mock.calls[0][1])).not.toContain( + 'private-schema' + ) + expect(JSON.stringify(mockExecuteProviderRequest.mock.calls[0][1])).not.toContain( + 'FORMAT_NAME' + ) + expect(responseFormat).toEqual({ + name: 'private-schema', + schema: { type: 'object', properties: {} }, + strict: true, + }) + }) + + it('prunes a private response format name when another structural field fails', async () => { + const responseFormat = { + name: 'x', + schema: { type: 'object', properties: {} }, + strict: 'locked', + } + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FORMAT_NAME', plaintext: 'x', encryptedValue: 'encrypted-name' }, + { name: 'STRICT_VALUE', plaintext: 'locked', encryptedValue: 'encrypted-strict' }, + ]) + const namePath = ['responseFormat', 'name'] as const + registry.recordResolvedAtInputPath('FORMAT_NAME', 'x', namePath) + registry.recordResolvedInputProjection(namePath, 'x', '{{FORMAT_NAME}}') + const strictPath = ['responseFormat', 'strict'] as const + registry.recordResolvedAtInputPath('STRICT_VALUE', 'locked', strictPath) + registry.recordResolvedInputProjection(strictPath, 'locked', '{{STRICT_VALUE}}') + mockContext.resolvedSecretTraceRegistry = registry + const inputs = { + model: 'gpt-4o', + userPrompt: 'Return an answer.', + responseFormat, + } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( + 'Agent structural model inputs cannot contain secret references' + ) + + expect(inputs.responseFormat).toEqual({ + name: '{{FORMAT_NAME}}', + schema: { type: 'object', properties: {} }, + strict: 'locked', + }) + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + + it('prunes a private name from serialized response format before a structural failure', async () => { + const responseFormat = JSON.stringify({ + name: 'x', + schema: { type: 'object', properties: {} }, + strict: 'locked', + }) + const projectedResponseFormat = JSON.stringify({ + name: '{{FORMAT_NAME}}', + schema: { type: 'object', properties: {} }, + strict: '{{STRICT_VALUE}}', + }) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FORMAT_NAME', plaintext: 'x', encryptedValue: 'encrypted-name' }, + { name: 'STRICT_VALUE', plaintext: 'locked', encryptedValue: 'encrypted-strict' }, + ]) + registry.recordResolvedAtInputPath('FORMAT_NAME', 'x', ['responseFormat']) + registry.recordResolvedAtInputPath('STRICT_VALUE', 'locked', ['responseFormat']) + registry.recordResolvedInputProjection( + ['responseFormat'], + responseFormat, + projectedResponseFormat + ) + mockContext.resolvedSecretTraceRegistry = registry + const inputs = { + model: 'gpt-4o', + userPrompt: 'Return an answer.', + responseFormat, + } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( + 'Agent model input could not be safely projected' + ) + + expect(inputs.responseFormat).toBe(projectedResponseFormat) + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) + it('aliases a resolver-derived name inside a persisted JSON response format', async () => { + const responseFormat = JSON.stringify({ + name: 'private-schema', + schema: { type: 'object', properties: {} }, + strict: true, + }) + const projectedResponseFormat = JSON.stringify({ + name: '{{FORMAT_NAME}}', + schema: { type: 'object', properties: {} }, + strict: true, + }) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FORMAT_NAME', plaintext: 'private-schema', encryptedValue: 'encrypted-name' }, + ]) + registry.recordResolvedAtInputPath('FORMAT_NAME', 'private-schema', ['responseFormat']) + registry.recordResolvedInputProjection( + ['responseFormat'], + responseFormat, + projectedResponseFormat + ) + mockContext.resolvedSecretTraceRegistry = registry + + await handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + userPrompt: 'Return an answer.', + responseFormat, + }) + + expect(mockExecuteProviderRequest.mock.calls[0][1].responseFormat).toEqual({ + name: 'response_schema', + schema: { type: 'object', properties: {} }, + strict: true, + }) + expect(JSON.stringify(mockExecuteProviderRequest.mock.calls[0][1])).not.toContain( + 'private-schema' + ) + expect(JSON.stringify(mockExecuteProviderRequest.mock.calls[0][1])).not.toContain( + 'FORMAT_NAME' + ) + expect(responseFormat).toContain('private-schema') + }) + + it('excludes an aliased persisted response format name from block output provenance', async () => { + const responseFormat = JSON.stringify({ + name: 'x', + schema: { + type: 'object', + properties: { answer: { type: 'string', description: 'classified' } }, + }, + strict: true, + }) + const projectedResponseFormat = JSON.stringify({ + name: '{{FORMAT_NAME}}', + schema: { + type: 'object', + properties: { answer: { type: 'string', description: '{{DESCRIPTION}}' } }, + }, + strict: true, + }) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FORMAT_NAME', plaintext: 'x', encryptedValue: 'encrypted-name' }, + { + name: 'DESCRIPTION', + plaintext: 'classified', + encryptedValue: 'encrypted-description', + }, + ]) + registry.recordResolvedAtInputPath('FORMAT_NAME', 'x', ['responseFormat']) + registry.recordResolvedAtInputPath('DESCRIPTION', 'classified', ['responseFormat']) + registry.recordResolvedInputProjection( + ['responseFormat'], + responseFormat, + projectedResponseFormat + ) + mockContext.resolvedSecretTraceRegistry = registry + const handlerInputs = { + model: 'gpt-4o', + userPrompt: 'Return an answer.', + responseFormat, + } + + await handler.execute(mockContext, mockBlock, handlerInputs) + + expect(mockExecuteProviderRequest.mock.calls[0][1].responseFormat).toEqual({ + name: 'response_schema', + schema: { + type: 'object', + properties: { answer: { type: 'string', description: '{{DESCRIPTION}}' } }, + }, + strict: true, + }) + const modelRegistry = mockExecuteProviderRequest.mock.calls[0][2] + .resolvedSecretTraceRegistry as ResolvedSecretTraceRegistry + const snapshot = modelRegistry.getModelEgressSnapshot() + expect(snapshot.complete).toBe(true) + if (!snapshot.complete) throw new Error('Expected complete model provenance') + expect(snapshot.matches).toContainEqual({ + plaintext: 'classified', + replacement: '{{DESCRIPTION}}', + }) + expect(snapshot.matches.map((match) => match.plaintext)).not.toContain('x') + const blockSnapshot = mockContext.resolvedSecretTraceRegistry?.getModelEgressSnapshot() + expect(blockSnapshot?.complete).toBe(true) + if (!blockSnapshot?.complete) throw new Error('Expected complete block provenance') + expect(blockSnapshot.matches).toContainEqual({ + plaintext: 'classified', + replacement: '{{DESCRIPTION}}', + }) + expect(blockSnapshot.matches.map((match) => match.plaintext)).not.toContain('x') + expect(handlerInputs.responseFormat).toContain('{{FORMAT_NAME}}') + }) + it('should handle responseFormat when it is an empty string', async () => { mockExecuteProviderRequest.mockResolvedValueOnce({ content: 'Regular text response', @@ -3511,8 +3752,7 @@ describe('AgentBlockHandler', () => { registry.recordResolvedAtInputPath('CANARY_CUSTOM_TOOL_ID', toolId, inputPath) registry.recordResolvedInputProjection(inputPath, toolId, '{{CANARY_CUSTOM_TOOL_ID}}') mockContext.resolvedSecretTraceRegistry = registry - - await handler.execute(mockContext, mockBlock, { + const inputs = { model: 'gpt-4o', userPrompt: 'Format a report', apiKey: 'test-api-key', @@ -3520,10 +3760,12 @@ describe('AgentBlockHandler', () => { { type: 'custom-tool', customToolId: toolId, - usageControl: 'auto', + usageControl: 'auto' as const, }, ], - }) + } + + await handler.execute(mockContext, mockBlock, inputs) expect(mockGetCustomToolById).toHaveBeenCalledWith(expect.objectContaining({ toolId })) const providerRequest = mockExecuteProviderRequest.mock.calls[0][1] @@ -3531,6 +3773,145 @@ describe('AgentBlockHandler', () => { expect(providerRequest.tools[0].name).toBe('formatReport') expect(JSON.stringify(providerRequest.tools)).not.toContain(toolId) expect(JSON.stringify(providerRequest.tools)).not.toContain('CANARY_CUSTOM_TOOL_ID') + expect(inputs.tools[0].customToolId).toBe('{{CANARY_CUSTOM_TOOL_ID}}') + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + + it('retains raw tool-call result provenance without reactivating a private selector', async () => { + const toolId = 'x' + const resultSecret = 'tool-result-secret' + mockDBForCustomTool(toolId) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'CANARY_CUSTOM_TOOL_ID', + plaintext: toolId, + encryptedValue: 'encrypted-custom-tool-id', + }, + { + name: 'TOOL_RESULT', + plaintext: resultSecret, + encryptedValue: 'encrypted-tool-result', + }, + ]) + const inputPath = ['tools', '0', 'customToolId'] as const + registry.recordResolvedAtInputPath('CANARY_CUSTOM_TOOL_ID', toolId, inputPath) + registry.recordResolvedInputProjection(inputPath, toolId, '{{CANARY_CUSTOM_TOOL_ID}}') + mockContext.resolvedSecretTraceRegistry = registry + mockExecuteProviderRequest.mockImplementationOnce((_provider, _request, runtimeContext) => { + runtimeContext.resolvedSecretTraceRegistry.recordResolved('TOOL_RESULT', resultSecret, { + propagated: true, + }) + return Promise.resolve({ + content: 'done', + model: 'mock-model', + tokens: { input: 10, output: 20, total: 30 }, + toolCalls: [{ name: 'formatReport', result: { value: resultSecret, public: 'Box' } }], + cost: 0.001, + timing: { total: 100 }, + }) + }) + const inputs = { + model: 'gpt-4o', + userPrompt: 'Format a report', + tools: [ + { + type: 'custom-tool', + customToolId: toolId, + usageControl: 'auto' as const, + }, + ], + } + + const result = await handler.execute(mockContext, mockBlock, inputs) + + expect((result as { toolCalls: { list: unknown[] } }).toolCalls.list).toContainEqual( + expect.objectContaining({ result: { value: resultSecret, public: 'Box' } }) + ) + expect(inputs.tools[0].customToolId).toBe('{{CANARY_CUSTOM_TOOL_ID}}') + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([ + { plaintext: resultSecret, replacement: '{{TOOL_RESULT}}' }, + ]) + expect( + mockContext.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(result) + ).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'TOOL_RESULT', encryptedValue: 'encrypted-tool-result' }], + }) + }) + + it('settles a private selector when a later pre-provider tool build fails', async () => { + const toolId = 'x' + mockDBForCustomTool(toolId) + const failure = new ToolSchemaEnrichmentError( + 'table_query_rows', + new Error('table metadata unavailable') + ) + mockTransformBlockTool.mockRejectedValueOnce(failure) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'CANARY_CUSTOM_TOOL_ID', + plaintext: toolId, + encryptedValue: 'encrypted-custom-tool-id', + }, + ]) + const inputPath = ['tools', '0', 'customToolId'] as const + registry.recordResolvedAtInputPath('CANARY_CUSTOM_TOOL_ID', toolId, inputPath) + registry.recordResolvedInputProjection(inputPath, toolId, '{{CANARY_CUSTOM_TOOL_ID}}') + mockContext.resolvedSecretTraceRegistry = registry + const inputs = { + model: 'gpt-4o', + userPrompt: 'Format and query a report', + tools: [ + { + type: 'custom-tool', + customToolId: toolId, + usageControl: 'auto' as const, + }, + { type: 'table', operation: 'query_rows', usageControl: 'auto' as const }, + ], + } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toBe(failure) + + expect(mockGetCustomToolById).toHaveBeenCalledWith(expect.objectContaining({ toolId })) + expect(inputs.tools[0].customToolId).toBe('{{CANARY_CUSTOM_TOOL_ID}}') + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + + it('uses a secret-backed skillId for lookup without carrying it into output provenance', async () => { + const skillId = 'x' + mockContext.workspaceId = 'workspace-1' + queueTableRows(schemaMock.skill, [ + { id: skillId, name: 'Reporting', description: 'Prepare reporting workflows' }, + ]) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'CANARY_SKILL_ID', + plaintext: skillId, + encryptedValue: 'encrypted-skill-id', + }, + ]) + const inputPath = ['skills', '0', 'skillId'] as const + registry.recordResolvedAtInputPath('CANARY_SKILL_ID', skillId, inputPath) + registry.recordResolvedInputProjection(inputPath, skillId, '{{CANARY_SKILL_ID}}') + mockContext.resolvedSecretTraceRegistry = registry + const inputs = { + model: 'gpt-4o', + userPrompt: 'Prepare a report', + skills: [{ skillId }], + } + + await handler.execute(mockContext, mockBlock, inputs) + + const providerRequest = mockExecuteProviderRequest.mock.calls[0][1] + expect(providerRequest.tools).toContainEqual( + expect.objectContaining({ id: 'load_skill', name: 'load_skill' }) + ) + expect(JSON.stringify(providerRequest.tools)).toContain('Reporting') + expect(inputs.skills[0].skillId).toBe('{{CANARY_SKILL_ID}}') + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) }) it('should fall back to inline schema when DB fetch fails and inline exists', async () => { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 19801570691..d2f17d519b2 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -93,6 +93,7 @@ import { getTool } from '@/tools/utils' import { getToolAsync } from '@/tools/utils.server' const logger = createLogger('AgentBlockHandler') +const MODEL_SAFE_RESPONSE_FORMAT_NAME = 'response_schema' interface IndexedToolInput { tool: ToolInput @@ -196,184 +197,216 @@ export class AgentBlockHandler implements BlockHandler { const toolIndexByRef = new Map( (inputs.tools || []).map((tool, index) => [tool, index] as const) ) - - const filteredTools = await this.filterUnavailableMcpTools(ctx, inputs.tools || []) - const filteredInputs = { ...inputs, tools: filteredTools } - this.assertInputPathsDoNotResolveSecrets( - ctx, - this.getMessageStructuralInputPaths(filteredInputs), - 'Agent structural model inputs cannot contain secret references' - ) - const responseFormatProjection = this.projectResponseFormatForModel(ctx, filteredInputs) - const fileProjection = this.projectFileNamesForModel(ctx, filteredInputs) - const coreModelInputPaths = this.getModelInputPaths(filteredInputs) - const modelInputProjection = projectResolvedModelInput( - ctx.resolvedSecretTraceRegistry, - { - systemPrompt: filteredInputs.systemPrompt, - userPrompt: filteredInputs.userPrompt, - messages: filteredInputs.messages, - memories: filteredInputs.memories, - }, - coreModelInputPaths - ) - if (!modelInputProjection.complete) { - throw new Error('Agent model input could not be safely projected') - } - const modelInputs: AgentInputs = { - ...filteredInputs, - ...modelInputProjection.value, - responseFormat: responseFormatProjection.value, + const privateAgentSelectorInputPaths: ResolvedSecretInputPath[] = [] + let responseFormatModelInputPaths: ResolvedSecretInputPath[] = [] + let privateAgentSelectorsSettled = false + const settlePrivateAgentSelectors = (): void => { + if (privateAgentSelectorsSettled) return + privateAgentSelectorsSettled = true + this.settlePrivateAgentSelectors( + ctx, + inputs, + privateAgentSelectorInputPaths, + responseFormatModelInputPaths + ) } - const modelInputPaths = [...coreModelInputPaths, ...responseFormatProjection.inputPaths] - const projectedToolInputs = this.projectToolInputsForProvenance(ctx, inputs.tools || []) - await this.validateToolPermissions(ctx, filteredInputs.tools || []) + try { + const privateAgentSelectors = this.getPrivateAgentSelectorInputPaths(ctx, inputs, []) + privateAgentSelectorInputPaths.push(...privateAgentSelectors.inputPaths) + if (!privateAgentSelectors.complete) { + throw new AgentToolInputSafetyError('Agent private selector could not be safely projected') + } + const responseFormatProjection = this.projectResponseFormatForModel( + ctx, + inputs, + (privateNameInputPaths) => { + privateAgentSelectorInputPaths.push(...privateNameInputPaths) + } + ) + responseFormatModelInputPaths = responseFormatProjection.inputPaths + const filteredTools = await this.filterUnavailableMcpTools(ctx, inputs.tools || []) + const filteredInputs = { ...inputs, tools: filteredTools } + this.assertInputPathsDoNotResolveSecrets( + ctx, + this.getMessageStructuralInputPaths(filteredInputs), + 'Agent structural model inputs cannot contain secret references' + ) + const fileProjection = this.projectFileNamesForModel(ctx, filteredInputs) + const coreModelInputPaths = this.getModelInputPaths(filteredInputs) + const modelInputProjection = projectResolvedModelInput( + ctx.resolvedSecretTraceRegistry, + { + systemPrompt: filteredInputs.systemPrompt, + userPrompt: filteredInputs.userPrompt, + messages: filteredInputs.messages, + memories: filteredInputs.memories, + }, + coreModelInputPaths + ) + if (!modelInputProjection.complete) { + throw new Error('Agent model input could not be safely projected') + } + const modelInputs: AgentInputs = { + ...filteredInputs, + ...modelInputProjection.value, + responseFormat: responseFormatProjection.value, + } + const modelInputPaths = [...coreModelInputPaths, ...responseFormatProjection.inputPaths] + const projectedToolInputs = this.projectToolInputsForProvenance(ctx, inputs.tools || []) + + await this.validateToolPermissions(ctx, filteredInputs.tools || []) - const responseFormat = parseResponseFormat(modelInputs.responseFormat) - const configuredModel = filteredInputs.model || AGENT.DEFAULT_MODEL + const responseFormat = parseResponseFormat(modelInputs.responseFormat) + const configuredModel = filteredInputs.model || AGENT.DEFAULT_MODEL - let model = configuredModel - let autoRouting: AutoRoutingResult | null = null - if (isAutoModel(configuredModel)) { - autoRouting = await resolveAutoModel({ - ctx, - blockId: block.id, - signals: this.buildAutoRoutingSignals( - { - ...modelInputs, - systemPrompt: filteredInputs.systemPrompt ? modelInputs.systemPrompt : undefined, - userPrompt: filteredInputs.userPrompt - ? modelInputs.userPrompt - : filteredInputs.userPrompt, - }, - responseFormat - ), - fallbackModel: AGENT.DEFAULT_MODEL, - }) - model = autoRouting.model - logger.info( - 'Resolved sim-auto model', - projectAgentDiagnosticMetadata( + let model = configuredModel + let autoRouting: AutoRoutingResult | null = null + if (isAutoModel(configuredModel)) { + autoRouting = await resolveAutoModel({ ctx, - { - blockId: block.id, - model, - tier: autoRouting.tier, - decidedBy: autoRouting.decidedBy, - }, - { - blockId: block.id, - tier: autoRouting.tier, - decidedBy: autoRouting.decidedBy, - } + blockId: block.id, + signals: this.buildAutoRoutingSignals( + { + ...modelInputs, + systemPrompt: filteredInputs.systemPrompt ? modelInputs.systemPrompt : undefined, + userPrompt: filteredInputs.userPrompt + ? modelInputs.userPrompt + : filteredInputs.userPrompt, + }, + responseFormat + ), + fallbackModel: AGENT.DEFAULT_MODEL, + }) + model = autoRouting.model + logger.info( + 'Resolved sim-auto model', + projectAgentDiagnosticMetadata( + ctx, + { + blockId: block.id, + model, + tier: autoRouting.tier, + decidedBy: autoRouting.decidedBy, + }, + { + blockId: block.id, + tier: autoRouting.tier, + decidedBy: autoRouting.decidedBy, + } + ) ) - ) - // Hidden identity preamble for every auto execution (fallback included): - // keeps pool models in English by default and off the topic of which - // underlying model they are. Applied after signal building so the - // preamble never influences classification. - modelInputs.systemPrompt = [SIM_AUTO_SYSTEM_PREAMBLE, modelInputs.systemPrompt] - .filter(Boolean) - .join('\n\n') - } + // Hidden identity preamble for every auto execution (fallback included): + // keeps pool models in English by default and off the topic of which + // underlying model they are. Applied after signal building so the + // preamble never influences classification. + modelInputs.systemPrompt = [SIM_AUTO_SYSTEM_PREAMBLE, modelInputs.systemPrompt] + .filter(Boolean) + .join('\n\n') + } - await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx) + await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx) - const providerId = getProviderFromModel(model) - const formatted = await this.formatTools( - ctx, - filteredInputs.tools || [], - block.canonicalModes, - toolIndexByRef, - projectedToolInputs - ) + const providerId = getProviderFromModel(model) + const formatted = await this.formatTools( + ctx, + filteredInputs.tools || [], + block.canonicalModes, + toolIndexByRef, + projectedToolInputs + ) - const skillInputs = filteredInputs.skills ?? [] - let skillMetadata: Array<{ name: string; description: string }> = [] - if (skillInputs.length > 0 && ctx.workspaceId) { - await validateSkillsAllowed(ctx.userId, ctx.workspaceId, ctx) - skillMetadata = await resolveSkillMetadata(skillInputs, ctx.workspaceId) - if (skillMetadata.length > 0) { - const skillNames = skillMetadata.map((s) => s.name) - formatted.tools.push(buildLoadSkillTool(skillNames)) + const skillInputs = filteredInputs.skills ?? [] + let skillMetadata: Array<{ name: string; description: string }> = [] + if (skillInputs.length > 0 && ctx.workspaceId) { + await validateSkillsAllowed(ctx.userId, ctx.workspaceId, ctx) + skillMetadata = await resolveSkillMetadata(skillInputs, ctx.workspaceId) + if (skillMetadata.length > 0) { + const skillNames = skillMetadata.map((s) => s.name) + formatted.tools.push(buildLoadSkillTool(skillNames)) + } } - } - const streamingConfig = this.getStreamingConfig(ctx, block) - const messages = await this.buildMessages(ctx, filteredInputs, modelInputs, skillMetadata) - const messagesWithInputFiles = this.attachFilesToLastUserMessage( - ctx, - messages, - filteredInputs.files, - fileProjection.projectedFiles, - fileProjection.projectedNameByFile, - fileProjection.directNameInputPaths - ) - const messagesWithFiles = await this.hydrateMessageFilesForProvider( - ctx, - messagesWithInputFiles, - providerId, - fileProjection.projectedNameByFile, - fileProjection.modelBoundInputPaths - ) + const streamingConfig = this.getStreamingConfig(ctx, block) + const messages = await this.buildMessages(ctx, filteredInputs, modelInputs, skillMetadata) + const messagesWithInputFiles = this.attachFilesToLastUserMessage( + ctx, + messages, + filteredInputs.files, + fileProjection.projectedFiles, + fileProjection.projectedNameByFile, + fileProjection.directNameInputPaths + ) + const messagesWithFiles = await this.hydrateMessageFilesForProvider( + ctx, + messagesWithInputFiles, + providerId, + fileProjection.projectedNameByFile, + fileProjection.modelBoundInputPaths + ) - const providerRequest = this.buildProviderRequest({ - ctx, - providerId, - model, - messages: messagesWithFiles, - inputs: modelInputs, - formattedTools: formatted.tools, - responseFormat, - streaming: streamingConfig.shouldUseStreaming ?? false, - }) + const providerRequest = this.buildProviderRequest({ + ctx, + providerId, + model, + messages: messagesWithFiles, + inputs: modelInputs, + formattedTools: formatted.tools, + responseFormat, + streaming: streamingConfig.shouldUseStreaming ?? false, + }) - const modelRuntimeRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths([ - ...modelInputPaths, - ...fileProjection.modelBoundInputPaths, - ...formatted.sourcePaths, - ]) - if (modelRuntimeRegistry) { - for (const [tool, provenance] of formatted.inputProvenance) { - registerProviderToolInputProvenance(tool, { - ...provenance, - registry: modelRuntimeRegistry, - }) + settlePrivateAgentSelectors() + + const modelRuntimeRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths([ + ...modelInputPaths, + ...fileProjection.modelBoundInputPaths, + ...formatted.sourcePaths, + ]) + if (modelRuntimeRegistry) { + ctx.resolvedSecretTraceRegistry = modelRuntimeRegistry + for (const [tool, provenance] of formatted.inputProvenance) { + registerProviderToolInputProvenance(tool, { + ...provenance, + registry: modelRuntimeRegistry, + }) + } } - } - const result = await this.executeProviderRequest( - ctx, - providerRequest, - block, - responseFormat, - modelRuntimeRegistry - ) + const result = await this.executeProviderRequest( + ctx, + providerRequest, + block, + responseFormat, + modelRuntimeRegistry + ) - if (autoRouting && autoRouting.billableRoutingCost > 0) { - this.applyRoutingCost(result, autoRouting.billableRoutingCost) - } + if (autoRouting && autoRouting.billableRoutingCost > 0) { + this.applyRoutingCost(result, autoRouting.billableRoutingCost) + } - if (autoRouting) { - this.applyAutoModelLabel(result, model) - } + if (autoRouting) { + this.applyAutoModelLabel(result, model) + } + + if (this.isStreamingExecution(result)) { + if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { + return this.wrapStreamForMemoryPersistence( + ctx, + filteredInputs, + result as StreamingExecution + ) + } + return result + } - if (this.isStreamingExecution(result)) { if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { - return this.wrapStreamForMemoryPersistence( - ctx, - filteredInputs, - result as StreamingExecution - ) + await this.persistResponseToMemory(ctx, filteredInputs, result as BlockOutput) } - return result - } - if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { - await this.persistResponseToMemory(ctx, filteredInputs, result as BlockOutput) + return result + } finally { + settlePrivateAgentSelectors() } - - return result } /** @@ -1755,12 +1788,111 @@ export class AgentBlockHandler implements BlockHandler { return typeof userPrompt === 'object' ? JSON.stringify(userPrompt) : String(userPrompt) } + private getPrivateAgentSelectorInputPaths( + ctx: ExecutionContext, + inputs: AgentInputs, + responseFormatNameInputPaths: readonly ResolvedSecretInputPath[] + ): { complete: boolean; inputPaths: ResolvedSecretInputPath[] } { + const registry = ctx.resolvedSecretTraceRegistry + if (!registry) return { complete: true, inputPaths: [] } + + const candidatePaths: ResolvedSecretInputPath[] = [...responseFormatNameInputPaths] + for (let toolIndex = 0; toolIndex < (inputs.tools?.length ?? 0); toolIndex++) { + if (inputs.tools?.[toolIndex]?.customToolId) { + candidatePaths.push(['tools', String(toolIndex), 'customToolId']) + } + } + for (let skillIndex = 0; skillIndex < (inputs.skills?.length ?? 0); skillIndex++) { + if (inputs.skills?.[skillIndex]?.skillId) { + candidatePaths.push(['skills', String(skillIndex), 'skillId']) + } + } + + const privatePaths: ResolvedSecretInputPath[] = [] + let complete = true + for (const path of candidatePaths) { + const provenance = registry.exportCommittedProvenanceForInputPaths([path]) + if (!provenance.complete) { + complete = false + continue + } + if (provenance.entries.length > 0) privatePaths.push(path) + } + return { complete, inputPaths: privatePaths } + } + + private settlePrivateAgentSelectors( + ctx: ExecutionContext, + inputs: AgentInputs, + privateInputPaths: readonly ResolvedSecretInputPath[], + responseFormatModelInputPaths: readonly ResolvedSecretInputPath[] + ): void { + const sourceRegistry = ctx.resolvedSecretTraceRegistry + if (!sourceRegistry || privateInputPaths.length === 0) return + + const privateRoots = new Set(privateInputPaths.map((path) => path[0])) + const displayProjection = sourceRegistry + .forkForInputPaths(privateInputPaths) + .projectResolvedInputSelection({ + responseFormat: inputs.responseFormat, + tools: inputs.tools, + skills: inputs.skills, + }) + if (!displayProjection.complete) { + throw new AgentToolInputSafetyError('Agent private selector could not be safely projected') + } + if (privateRoots.has('responseFormat')) { + inputs.responseFormat = displayProjection.value + .responseFormat as AgentInputs['responseFormat'] + } + if (privateRoots.has('tools')) { + if (!Array.isArray(displayProjection.value.tools)) { + throw new AgentToolInputSafetyError('Agent private selector could not be safely projected') + } + inputs.tools = displayProjection.value.tools as ToolInput[] + } + if (privateRoots.has('skills')) { + if (!Array.isArray(displayProjection.value.skills)) { + throw new AgentToolInputSafetyError('Agent private selector could not be safely projected') + } + inputs.skills = displayProjection.value.skills as AgentInputs['skills'] + } + + const excludedPaths = new Set(privateInputPaths.map((path) => JSON.stringify(path))) + const retainedInputPaths: ResolvedSecretInputPath[] = [] + for (const [key, value] of Object.entries(inputs)) { + if (!privateRoots.has(key)) { + retainedInputPaths.push([key]) + continue + } + if (key === 'responseFormat') { + retainedInputPaths.push(...responseFormatModelInputPaths) + continue + } + if (!Array.isArray(value)) continue + for (const [inputIndex, candidate] of value.entries()) { + if (!isPlainRecord(candidate)) continue + for (const candidateKey of Object.keys(candidate)) { + const path = [key, String(inputIndex), candidateKey] + if (!excludedPaths.has(JSON.stringify(path))) retainedInputPaths.push(path) + } + } + } + ctx.resolvedSecretTraceRegistry = sourceRegistry.forkForInputPaths(retainedInputPaths) + } + private projectResponseFormatForModel( ctx: ExecutionContext, - inputs: AgentInputs - ): { value: AgentInputs['responseFormat']; inputPaths: ResolvedSecretInputPath[] } { + inputs: AgentInputs, + onPrivateNameInputPaths: (paths: readonly ResolvedSecretInputPath[]) => void + ): { + value: AgentInputs['responseFormat'] + inputPaths: ResolvedSecretInputPath[] + } { const responseFormat = inputs.responseFormat - if (responseFormat === undefined) return { value: undefined, inputPaths: [] } + if (responseFormat === undefined) { + return { value: undefined, inputPaths: [] } + } let annotationInputPaths: ResolvedSecretInputPath[] = [] let structuralInputPaths: ResolvedSecretInputPath[] = [] @@ -1777,20 +1909,21 @@ export class AgentBlockHandler implements BlockHandler { if (isWrapper) { structuralInputPaths.push( ...Object.keys(responseFormat) - .filter((key) => key !== 'schema') + .filter((key) => key !== 'schema' && key !== 'name') .map((key) => ['responseFormat', key]) ) } } - this.assertInputPathsDoNotResolveSecrets( - ctx, - structuralInputPaths, - 'Agent structural model inputs cannot contain secret references' - ) - const registry = ctx.resolvedSecretTraceRegistry - if (!registry) return { value: responseFormat, inputPaths: annotationInputPaths } + if (!registry) { + this.assertInputPathsDoNotResolveSecrets( + ctx, + structuralInputPaths, + 'Agent structural model inputs cannot contain secret references' + ) + return { value: responseFormat, inputPaths: annotationInputPaths } + } const projection = registry.projectResolvedInputSelection({ responseFormat }) if (!projection.complete) { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') @@ -1807,23 +1940,59 @@ export class AgentBlockHandler implements BlockHandler { try { const rawParsed = JSON.parse(responseFormat) const projectedParsed = JSON.parse(projectedResponseFormat) - if (!isPlainRecord(rawParsed)) { + if (!isPlainRecord(rawParsed) || !isPlainRecord(projectedParsed)) { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } - this.projectResponseFormatObject(rawParsed, projectedParsed) + const privateNameInputPaths = + Object.hasOwn(rawParsed, 'name') && !Object.is(rawParsed.name, projectedParsed.name) + ? ([['responseFormat']] as const) + : [] + onPrivateNameInputPaths(privateNameInputPaths) + const modelSafeResponseFormat = this.projectResponseFormatObject(rawParsed, projectedParsed) + const parsedIsWrapper = + Object.hasOwn(rawParsed, 'schema') || Object.hasOwn(rawParsed, 'name') + const parsedSchema = parsedIsWrapper ? rawParsed.schema : rawParsed + const parsedSchemaRoot = parsedIsWrapper ? ['responseFormat', 'schema'] : ['responseFormat'] + const parsedAnnotationInputPaths = selectModelSchemaInputPaths( + parsedSchema, + parsedSchemaRoot + ).annotationInputPaths + registry.recordTransformedInputProjection( + { responseFormat: rawParsed }, + { responseFormat: projectedParsed }, + { targetPaths: parsedAnnotationInputPaths } + ) + return { + value: modelSafeResponseFormat, + inputPaths: parsedAnnotationInputPaths, + } } catch (error) { if (error instanceof AgentToolInputSafetyError) throw error throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } - return { value: projectedResponseFormat, inputPaths: [['responseFormat']] } } if (!isPlainRecord(responseFormat)) { if (!Object.is(responseFormat, projectedResponseFormat)) { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } - return { value: responseFormat, inputPaths: annotationInputPaths } + return { + value: responseFormat, + inputPaths: annotationInputPaths, + } } + const privateNameInputPaths = + Object.hasOwn(responseFormat, 'name') && + isPlainRecord(projectedResponseFormat) && + !Object.is(responseFormat.name, projectedResponseFormat.name) + ? [['responseFormat', 'name']] + : [] + onPrivateNameInputPaths(privateNameInputPaths) + this.assertInputPathsDoNotResolveSecrets( + ctx, + structuralInputPaths, + 'Agent structural model inputs cannot contain secret references' + ) return { value: this.projectResponseFormatObject(responseFormat, projectedResponseFormat), inputPaths: annotationInputPaths, @@ -1854,7 +2023,7 @@ export class AgentBlockHandler implements BlockHandler { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } for (const key of rawKeys) { - if (key !== 'schema' && !Object.is(rawValue[key], projectedValue[key])) { + if (key !== 'schema' && key !== 'name' && !Object.is(rawValue[key], projectedValue[key])) { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } } @@ -1862,9 +2031,13 @@ export class AgentBlockHandler implements BlockHandler { if (!schemaProjection.safe) { throw new AgentToolInputSafetyError('Agent model input could not be safely projected') } - return Object.hasOwn(rawValue, 'schema') - ? { ...rawValue, schema: schemaProjection.value } - : rawValue + return { + ...rawValue, + ...(Object.hasOwn(rawValue, 'name') && !Object.is(rawValue.name, projectedValue.name) + ? { name: MODEL_SAFE_RESPONSE_FORMAT_NAME } + : {}), + ...(Object.hasOwn(rawValue, 'schema') ? { schema: schemaProjection.value } : {}), + } } private getFileInputPaths( diff --git a/apps/sim/executor/handlers/generic/generic-handler.ts b/apps/sim/executor/handlers/generic/generic-handler.ts index e3d029637cc..b35cc117205 100644 --- a/apps/sim/executor/handlers/generic/generic-handler.ts +++ b/apps/sim/executor/handlers/generic/generic-handler.ts @@ -47,11 +47,6 @@ function selectBlockBoundaryPaths( if (path[0]) requiredProjectionRoots.add(path[0]) } } - const opaqueInputPaths = tool.request.opaqueModelInput?.inputPaths(params) ?? [] - paths.push(...opaqueInputPaths) - for (const path of opaqueInputPaths) { - if (path[0]) requiredProjectionRoots.add(path[0]) - } for (const selection of tool.request.secretProvenance?.request?.(params) ?? []) { paths.push(...selection.inputPaths) for (const path of selection.inputPaths) { diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 500a4308235..6d505d8ce9c 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -679,6 +679,28 @@ describe('MothershipBlockHandler', () => { expect(fetchMock).not.toHaveBeenCalled() }) + it('settles a private skill selector before reporting a missing COPILOT_API_KEY', async () => { + setEnv({ COPILOT_API_KEY: undefined }) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'SKILL_ID', plaintext: 'i', encryptedValue: 'encrypted-skill-id' }, + ]) + registry.recordResolvedAtInputPath('SKILL_ID', 'i', ['skills', '0', 'skillId']) + registry.recordResolvedInputProjection(['skills', '0', 'skillId'], 'i', '{{SKILL_ID}}') + context.resolvedSecretTraceRegistry = registry + const handlerInputs = { + prompt: 'Hello from workflow', + skills: [{ skillId: 'i' }], + } + + await expect(handler.execute(context, block, handlerInputs)).rejects.toThrow( + 'COPILOT_API_KEY is not configured' + ) + + expect(fetchMock).not.toHaveBeenCalled() + expect(handlerInputs.skills).toEqual([{ skillId: '{{SKILL_ID}}' }]) + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + it('rejects execution before the internal request when billing attribution is missing', async () => { context.metadata.billingAttribution = undefined @@ -1008,6 +1030,107 @@ describe('MothershipBlockHandler', () => { expect(skills[0].name).toBe('Playbook private skill label') }) + it('keeps low-entropy skill selectors private without changing lookup semantics', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FIRST_SKILL_ID', plaintext: 'x', encryptedValue: 'encrypted-first-skill-id' }, + { name: 'SECOND_SKILL_ID', plaintext: 'y', encryptedValue: 'encrypted-second-skill-id' }, + ]) + registry.recordResolvedAtInputPath('FIRST_SKILL_ID', 'x', ['skills', '0', 'skillId']) + registry.recordResolvedInputProjection(['skills', '0', 'skillId'], 'x', '{{FIRST_SKILL_ID}}') + registry.recordResolvedAtInputPath('SECOND_SKILL_ID', 'y', ['skills', '1', 'skillId']) + registry.recordResolvedInputProjection(['skills', '1', 'skillId'], 'y', '{{SECOND_SKILL_ID}}') + context.resolvedSecretTraceRegistry = registry + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'Box', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) + + const skills = [{ skillId: 'x' }, { skillId: 'y' }] + const handlerInputs = { + prompt: 'Use the selected skill', + skills, + } + + const result = await handler.execute(context, block, handlerInputs) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + const body = JSON.parse(String(options.body)) + expect(body.contexts).toEqual([ + { kind: 'skill', skillId: 'x', label: 'Skill 1' }, + { kind: 'skill', skillId: 'y', label: 'Skill 2' }, + ]) + expect(handlerInputs.skills).toEqual([ + { skillId: '{{FIRST_SKILL_ID}}' }, + { skillId: '{{SECOND_SKILL_ID}}' }, + ]) + expect(result).toMatchObject({ content: 'Box' }) + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(JSON.stringify(body.contexts)).not.toContain('FIRST_SKILL_ID') + expect(JSON.stringify(body.contexts)).not.toContain('SECOND_SKILL_ID') + }) + + it('fails closed when a selected skill has incomplete resolver provenance', async () => { + const registry = new ResolvedSecretTraceRegistry() + registry.recordResolvedAtInputPath('UNKNOWN_SKILL_ID', 'x', ['skills', '0', 'skillId']) + context.resolvedSecretTraceRegistry = registry + + await expect( + handler.execute(context, block, { + prompt: 'Use the selected skill', + skills: [{ skillId: 'x' }], + }) + ).rejects.toThrow('Mothership skill selector provenance is incomplete') + + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('settles a private skill selector before a pre-provider failure', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'SKILL_ID', plaintext: 'x', encryptedValue: 'encrypted-skill-id' }, + ]) + registry.recordResolvedAtInputPath('SKILL_ID', 'x', ['skills', '0', 'skillId']) + registry.recordResolvedInputProjection(['skills', '0', 'skillId'], 'x', '{{SKILL_ID}}') + context.resolvedSecretTraceRegistry = registry + const handlerInputs = { + prompt: '', + skills: [{ skillId: 'x' }], + } + + await expect(handler.execute(context, block, handlerInputs)).rejects.toThrow( + 'Prompt input is required' + ) + + expect(fetchMock).not.toHaveBeenCalled() + expect(handlerInputs.skills).toEqual([{ skillId: '{{SKILL_ID}}' }]) + expect(JSON.stringify(handlerInputs)).not.toContain('"x"') + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + + it('preserves legacy unnamed skill labels when selectors have no resolver provenance', async () => { + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue(createJsonResponse({ content: 'done', toolCalls: [] })) + + await handler.execute(context, block, { + prompt: 'Use the selected skills', + skills: [{ skillId: 'legacy-first' }, { skillId: 'legacy-second' }], + }) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + const body = JSON.parse(String(options.body)) + expect(body.contexts).toEqual([ + { kind: 'skill', skillId: 'legacy-first', label: 'legacy-first' }, + { kind: 'skill', skillId: 'legacy-second', label: 'legacy-second' }, + ]) + }) + it('rejects only an enabled structural identifier with exact resolver provenance', async () => { const registry = new ResolvedSecretTraceRegistry([ { diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index b9c52be8891..416debb2b3e 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -142,7 +142,10 @@ function selectMothershipMcpTools(tools: unknown): MothershipMcpToolSelection[] return selectIndexedMothershipMcpTools(tools).map(({ selection }) => selection) } -function selectIndexedMothershipSkillContexts(skills: unknown): IndexedMothershipSkillContext[] { +function selectIndexedMothershipSkillContexts( + skills: unknown, + privateSelectorIndexes: ReadonlySet = new Set() +): IndexedMothershipSkillContext[] { if (!Array.isArray(skills)) return [] return skills.flatMap((candidate, inputIndex) => { @@ -151,7 +154,9 @@ function selectIndexedMothershipSkillContexts(skills: unknown): IndexedMothershi } const explicitLabel = typeof candidate.name === 'string' ? candidate.name : undefined const hasExplicitLabel = explicitLabel !== undefined - const label = explicitLabel ?? candidate.skillId + const label = + explicitLabel ?? + (privateSelectorIndexes.has(inputIndex) ? `Skill ${inputIndex + 1}` : candidate.skillId) return [ { inputIndex, @@ -166,8 +171,88 @@ function selectIndexedMothershipSkillContexts(skills: unknown): IndexedMothershi }) } -function selectMothershipSkillContexts(skills: unknown): MothershipSkillContext[] { - return selectIndexedMothershipSkillContexts(skills).map(({ context }) => context) +function selectMothershipSkillContexts( + skills: unknown, + privateSelectorIndexes: ReadonlySet +): MothershipSkillContext[] { + return selectIndexedMothershipSkillContexts(skills, privateSelectorIndexes).map( + ({ context }) => context + ) +} + +function selectPrivateMothershipSkillSelectors( + registry: ResolvedSecretTraceRegistry | undefined, + skills: unknown +): { + inputIndexes: ReadonlySet + inputPaths: readonly ResolvedSecretInputPath[] +} { + if (!registry) return { inputIndexes: new Set(), inputPaths: [] } + + const inputIndexes = new Set() + const inputPaths: ResolvedSecretInputPath[] = [] + for (const { inputIndex } of selectIndexedMothershipSkillContexts(skills)) { + const inputPath = ['skills', String(inputIndex), 'skillId'] as const + const provenance = registry.exportCommittedProvenanceForInputPaths([inputPath]) + if (!provenance.complete) { + throw new Error('Mothership skill selector provenance is incomplete') + } + if (provenance.entries.length === 0) continue + inputIndexes.add(inputIndex) + inputPaths.push(inputPath) + } + return { inputIndexes, inputPaths } +} + +function forkMothershipRegistryWithoutPrivateSkillSelectors( + registry: ResolvedSecretTraceRegistry, + inputs: Record, + privateSelectorIndexes: ReadonlySet +): ResolvedSecretTraceRegistry { + const retainedInputPaths: ResolvedSecretInputPath[] = [] + for (const [key, value] of Object.entries(inputs)) { + if (key !== 'skills') { + retainedInputPaths.push([key]) + continue + } + if (!Array.isArray(value)) continue + for (const [inputIndex, candidate] of value.entries()) { + if (!isPlainRecord(candidate)) continue + for (const candidateKey of Object.keys(candidate)) { + if (candidateKey === 'skillId' && privateSelectorIndexes.has(inputIndex)) continue + retainedInputPaths.push(['skills', String(inputIndex), candidateKey]) + } + } + } + return registry.forkForInputPaths(retainedInputPaths) +} + +function projectPrivateMothershipSkillSelectorsForDisplay( + registry: ResolvedSecretTraceRegistry, + skills: unknown, + privateSelectorIndexes: ReadonlySet, + privateSelectorInputPaths: readonly ResolvedSecretInputPath[] +): unknown { + if (!Array.isArray(skills) || privateSelectorIndexes.size === 0) return skills + const projection = registry + .forkForInputPaths(privateSelectorInputPaths) + .projectResolvedInputSelection({ skills }) + if (!projection.complete || !Array.isArray(projection.value.skills)) { + throw new Error('Mothership skill selector could not be safely projected for display') + } + for (const inputIndex of privateSelectorIndexes) { + const source = skills[inputIndex] + const projected = projection.value.skills[inputIndex] + if ( + !isPlainRecord(source) || + !isPlainRecord(projected) || + typeof source.skillId !== 'string' || + typeof projected.skillId !== 'string' + ) { + throw new Error('Mothership skill selector could not be safely projected for display') + } + } + return projection.value.skills } function selectMothershipMetadataModelInputPaths( @@ -195,7 +280,6 @@ function selectMothershipMetadataModelInputPaths( for (const { inputIndex, hasExplicitLabel } of selectIndexedMothershipSkillContexts(skills)) { const root = ['skills', String(inputIndex)] as const - structuralInputPaths.push([...root, 'skillId']) if (hasExplicitLabel) modelInputPaths.push([...root, 'name']) } @@ -630,6 +714,26 @@ export class MothershipBlockHandler implements BlockHandler { block: SerializedBlock, inputs: Record ): Promise { + const sourceRegistry = ctx.resolvedSecretTraceRegistry + const requestSkills = inputs.skills + const privateSkillSelectors = selectPrivateMothershipSkillSelectors( + sourceRegistry, + requestSkills + ) + if (sourceRegistry && privateSkillSelectors.inputPaths.length > 0) { + inputs.skills = projectPrivateMothershipSkillSelectorsForDisplay( + sourceRegistry, + requestSkills, + privateSkillSelectors.inputIndexes, + privateSkillSelectors.inputPaths + ) + ctx.resolvedSecretTraceRegistry = forkMothershipRegistryWithoutPrivateSkillSelectors( + sourceRegistry, + inputs, + privateSkillSelectors.inputIndexes + ) + } + // Without the key the mothership rejects every request, so fail with // something the workflow author can act on instead of a bare 401. if (!env.COPILOT_API_KEY) { @@ -640,7 +744,7 @@ export class MothershipBlockHandler implements BlockHandler { if (!prompt || typeof prompt !== 'string') { throw new Error('Prompt input is required') } - const metadataInputPaths = selectMothershipMetadataModelInputPaths(inputs.tools, inputs.skills) + const metadataInputPaths = selectMothershipMetadataModelInputPaths(inputs.tools, requestSkills) if (ctx.resolvedSecretTraceRegistry) { assertMothershipStructuralInputsDoNotResolveSecrets( ctx.resolvedSecretTraceRegistry, @@ -653,12 +757,13 @@ export class MothershipBlockHandler implements BlockHandler { ...selectModelBoundFileInputPaths(inputs.files, ['files'], { includeInlineBase64: true, includeName: true, + parseSerializedFile: true, }), ...metadataInputPaths.modelInputPaths, ] const modelInputProjection = projectResolvedModelInput( - ctx.resolvedSecretTraceRegistry, - { prompt, files: inputs.files, tools: inputs.tools, skills: inputs.skills }, + sourceRegistry, + { prompt, files: inputs.files, tools: inputs.tools, skills: requestSkills }, modelInputPaths ) if (!modelInputProjection.complete || typeof modelInputProjection.value.prompt !== 'string') { @@ -679,14 +784,17 @@ export class MothershipBlockHandler implements BlockHandler { secretScope: inputs.secretScope, mountedSecrets: inputs.mountedSecrets, }) + const mcpTools = selectMothershipMcpTools(modelInputProjection.value.tools) + const skillContexts = selectMothershipSkillContexts( + modelInputProjection.value.skills, + privateSkillSelectors.inputIndexes + ) const fileAttachments = await buildMothershipFileAttachments( inputs.files, modelInputProjection.value.files, ctx, requestId ) - const mcpTools = selectMothershipMcpTools(modelInputProjection.value.tools) - const skillContexts = selectMothershipSkillContexts(modelInputProjection.value.skills) const url = buildAPIUrl('/api/mothership/execute') const headers = await buildAuthHeaders(ctx.userId) diff --git a/apps/sim/executor/handlers/pi/babysit-backend.test.ts b/apps/sim/executor/handlers/pi/babysit-backend.test.ts index e80812d7edd..7e002456c39 100644 --- a/apps/sim/executor/handlers/pi/babysit-backend.test.ts +++ b/apps/sim/executor/handlers/pi/babysit-backend.test.ts @@ -341,7 +341,6 @@ describe('runBabysitPiWithOptions', () => { expect(mockRequestReview).toHaveBeenCalledWith( expect.objectContaining({ pullNumber: 7 }), ['@review-bot'], - expect.any(Array), expect.any(AbortSignal) ) expect(mockWithPiSandbox).toHaveBeenCalledTimes(1) diff --git a/apps/sim/executor/handlers/pi/babysit-backend.ts b/apps/sim/executor/handlers/pi/babysit-backend.ts index 6fa3f1c94fa..cfdc38419df 100644 --- a/apps/sim/executor/handlers/pi/babysit-backend.ts +++ b/apps/sim/executor/handlers/pi/babysit-backend.ts @@ -370,8 +370,7 @@ function buildRoundPrompt( params: PiBabysitContinuationParams, threads: BabysitThreadsState, failingChecks: readonly BabysitCheck[], - diagnostics: ReadonlyMap, - secrets: readonly string[] + diagnostics: ReadonlyMap ): RoundPrompt { const reviewPayload = threads.actionable.slice(0, MAX_THREADS_PER_ROUND).map((thread) => ({ threadId: thread.id, @@ -417,15 +416,12 @@ function buildRoundPrompt( .filter(Boolean) .join('\n\n') return { - prompt: scrubPiSecrets( - buildPiPrompt({ - skills: params.skills, - initialMessages: [], - task, - guidance: BABYSIT_GUIDANCE, - }), - secrets - ), + prompt: buildPiPrompt({ + skills: params.skills, + initialMessages: [], + task, + guidance: BABYSIT_GUIDANCE, + }), notes, } } @@ -433,8 +429,7 @@ function buildRoundPrompt( function createCancellationSignal( parent: AbortSignal | undefined, executionId: string | undefined, - pollMs: number, - secrets: readonly string[] + pollMs: number ): { signal: AbortSignal; cleanup: () => void } { const controller = new AbortController() const onAbort = () => controller.abort(parent?.reason ?? 'workflow_abort') @@ -454,12 +449,9 @@ function createCancellationSignal( } }) .catch((error) => { - // Scrubbed like every other message this file emits. A Redis poll - // error is unlikely to carry a run credential, but the invariant is - // easier to keep than to reason about per call site. logger.warn('Failed to poll Babysit execution cancellation', { executionId, - error: scrubPiSecrets(getErrorMessage(error), secrets), + error: getErrorMessage(error), }) }) .finally(() => { @@ -546,8 +538,7 @@ async function finalizeRound( initialHeadSha: string, roundBaseSha: string, gitConfigDigest: string, - signal: AbortSignal, - secrets: readonly string[] + signal: AbortSignal ): Promise { await runner.writeFile(COMMIT_MSG_PATH, `Pi Babysit: address PR #${params.pullNumber} feedback`) const prepare = await raceAbort( @@ -610,7 +601,7 @@ async function finalizeRound( ) } - const diff = capDiff(scrubPiSecrets(await runner.readFile(DIFF_PATH), secrets)) + const diff = capDiff(await runner.readFile(DIFF_PATH)) assertBabysitPinned(snapshot, await fetchBabysitSnapshot(params, signal)) const push = await raceAbort( runner.run(BABYSIT_PUSH_SCRIPT, { @@ -736,8 +727,7 @@ export async function runBabysitPiWithOptions( const cancellation = createCancellationSignal( context.signal, params.executionId, - options.cancellationPollMs, - secrets + options.cancellationPollMs ) const { signal } = cancellation const startedAt = Date.now() @@ -791,12 +781,7 @@ export async function runBabysitPiWithOptions( { headSha: pinnedHeadSha, headRef: pinnedHeadRef, baseRef: pinnedBaseRef }, await fetchBabysitSnapshot(params, signal) ) - const initialRequest = await requestBabysitReview( - params, - params.reviewMentions, - secrets, - signal - ) + const initialRequest = await requestBabysitReview(params, params.reviewMentions, signal) githubWriteOccurred = initialRequest.posted > 0 if (initialRequest.failures.length) { progress.notes.push(`${initialRequest.failures.length} initial review requests failed.`) @@ -974,13 +959,8 @@ export async function runBabysitPiWithOptions( ) } - const diagnostics = await fetchBabysitCheckDiagnostics( - params, - promptChecks, - secrets, - signal - ) - const round = buildRoundPrompt(params, latestThreads!, promptChecks, diagnostics, secrets) + const diagnostics = await fetchBabysitCheckDiagnostics(params, promptChecks, signal) + const round = buildRoundPrompt(params, latestThreads!, promptChecks, diagnostics) progress.notes.push(...round.notes) const agentTimeoutMs = Math.min( piTimeoutMs, @@ -1032,8 +1012,7 @@ export async function runBabysitPiWithOptions( initialHeadSha, roundBaseSha, gitConfigDigest, - signal, - secrets + signal ) } catch (error) { if (signal.aborted) throw error @@ -1051,15 +1030,8 @@ export async function runBabysitPiWithOptions( roundBaseSha = finalized.newSha progress.commitsPushed += 1 githubWriteOccurred = true - // Scrubbed here rather than in `finalizeRound`, which needs the literal - // paths for its `.github/` and quoted-path refusals. File names are - // agent-chosen, so a file named after a key would otherwise reach the - // block output verbatim — Create PR already scrubs its equivalent. progress.changedFiles = [ - ...new Set([ - ...progress.changedFiles, - ...finalized.changedFiles.map((file) => scrubPiSecrets(file, secrets)), - ]), + ...new Set([...progress.changedFiles, ...finalized.changedFiles]), ] progress.diff = capDiff([progress.diff, finalized.diff].filter(Boolean).join('\n')) lastKnownChecksGreen = ![...initialRequirements.values()].some((required) => required) @@ -1125,7 +1097,6 @@ export async function runBabysitPiWithOptions( allowedThreadIds: new Set( latestThreads!.actionable.slice(0, MAX_THREADS_PER_ROUND).map((thread) => thread.id) ), - secrets, commitPushed: finalized.commitPushed, }) } catch (error) { @@ -1219,7 +1190,7 @@ export async function runBabysitPiWithOptions( { headSha: pinnedHeadSha, headRef: pinnedHeadRef, baseRef: pinnedBaseRef }, await fetchBabysitSnapshot(params, signal) ) - const request = await requestBabysitReview(params, params.reviewMentions, secrets, signal) + const request = await requestBabysitReview(params, params.reviewMentions, signal) githubWriteOccurred ||= request.posted > 0 if (request.posted > 0) { reviewRequest = { diff --git a/apps/sim/executor/handlers/pi/babysit-github.ts b/apps/sim/executor/handlers/pi/babysit-github.ts index 8d08169996b..83bcd1d2aa4 100644 --- a/apps/sim/executor/handlers/pi/babysit-github.ts +++ b/apps/sim/executor/handlers/pi/babysit-github.ts @@ -506,7 +506,6 @@ export async function fetchBabysitCheckState( async function fetchCheckDiagnostic( params: PullRequestCoordinates, check: BabysitCheck, - secrets: readonly string[], signal?: AbortSignal ): Promise<{ key: string; text: string }> { let text = [check.title, check.summary, check.detailsUrl].filter(Boolean).join('\n') @@ -533,10 +532,7 @@ async function fetchCheckDiagnostic( } return { key: check.key, - text: scrubPiSecrets( - truncate(text || 'No diagnostic text was reported.', MAX_CHECK_DIAGNOSTIC_BYTES), - secrets - ), + text: truncate(text || 'No diagnostic text was reported.', MAX_CHECK_DIAGNOSTIC_BYTES), } } @@ -552,14 +548,13 @@ async function fetchCheckDiagnostic( export async function fetchBabysitCheckDiagnostics( params: PullRequestCoordinates, failing: readonly BabysitCheck[], - secrets: readonly string[], signal?: AbortSignal ): Promise> { const diagnostics = new Map() for (let start = 0; start < failing.length; start += CHECK_DIAGNOSTIC_CONCURRENCY) { const batch = failing.slice(start, start + CHECK_DIAGNOSTIC_CONCURRENCY) const settled = await Promise.all( - batch.map((check) => fetchCheckDiagnostic(params, check, secrets, signal)) + batch.map((check) => fetchCheckDiagnostic(params, check, signal)) ) for (const { key, text } of settled) { diagnostics.set(key, text) @@ -683,7 +678,6 @@ export async function replyAndResolveBabysitThreads( export async function requestBabysitReview( params: PullRequestCoordinates, mentions: readonly string[], - secrets: readonly string[], signal?: AbortSignal ): Promise { const requestedAt = new Date().toISOString() @@ -697,7 +691,7 @@ export async function requestBabysitReview( owner: params.owner, repo: params.repo, issue_number: params.pullNumber, - body: scrubPiSecrets(mention, secrets), + body: mention, apiKey: params.githubToken, }, { signal } diff --git a/apps/sim/executor/handlers/pi/babysit-round.test.ts b/apps/sim/executor/handlers/pi/babysit-round.test.ts index 501898c8fc5..2a8d479d646 100644 --- a/apps/sim/executor/handlers/pi/babysit-round.test.ts +++ b/apps/sim/executor/handlers/pi/babysit-round.test.ts @@ -11,10 +11,9 @@ import { const allowed = new Set(['thread-1', 'thread-2']) -function parse(value: unknown, options: { commitPushed?: boolean; secrets?: string[] } = {}) { +function parse(value: unknown, options: { commitPushed?: boolean } = {}) { return parseBabysitRound(JSON.stringify(value), { allowedThreadIds: allowed, - secrets: options.secrets ?? [], commitPushed: options.commitPushed ?? true, }) } @@ -42,7 +41,6 @@ describe('parseBabysitRound', () => { expect(() => parseBabysitRound('{', { allowedThreadIds: allowed, - secrets: [], commitPushed: true, }) ).toThrow(/valid JSON/) @@ -65,23 +63,20 @@ describe('parseBabysitRound', () => { ).toThrow(/duplicated/) }) - it('scrubs secrets from public replies and summaries', () => { - const result = parse( - { - threads: [ - { - threadId: 'thread-1', - classification: 'already_addressed', - reply: 'Do not show sk-secret', - }, - ], - summary: 'also sk-secret', - }, - { secrets: ['sk-secret'] } - ) + it('preserves public replies and summaries verbatim after trimming', () => { + const result = parse({ + threads: [ + { + threadId: 'thread-1', + classification: 'already_addressed', + reply: 'Do not rewrite sk-secret', + }, + ], + summary: 'also sk-secret', + }) - expect(result.threads[0].reply).toBe('Do not show ***') - expect(result.summary).toBe('also ***') + expect(result.threads[0].reply).toBe('Do not rewrite sk-secret') + expect(result.summary).toBe('also sk-secret') }) it('leaves fixed-without-commit threads unresolved and records the violation', () => { @@ -120,7 +115,6 @@ describe('parseBabysitRound', () => { expect(() => parseBabysitRound(' '.repeat(MAX_ROUND_FILE_BYTES + 1), { allowedThreadIds: allowed, - secrets: [], commitPushed: true, }) ).toThrow(/exceeds/) diff --git a/apps/sim/executor/handlers/pi/babysit-round.ts b/apps/sim/executor/handlers/pi/babysit-round.ts index 920db25aa3e..9182da92dd1 100644 --- a/apps/sim/executor/handlers/pi/babysit-round.ts +++ b/apps/sim/executor/handlers/pi/babysit-round.ts @@ -1,6 +1,5 @@ import { type Static, type TSchema, Type } from 'typebox' import { Check, Errors } from 'typebox/schema' -import { scrubPiSecrets } from '@/executor/handlers/pi/redaction' /** Sandbox path used for the single-use, agent-authored round decision file. */ export const BABYSIT_ROUND_PATH = '/workspace/sim-babysit-round.json' @@ -69,7 +68,6 @@ export function parseBabysitRound( raw: string, options: { allowedThreadIds: ReadonlySet - secrets: readonly string[] commitPushed: boolean } ): ParsedBabysitRound { @@ -102,7 +100,7 @@ export function parseBabysitRound( } seen.add(threadId) - const reply = scrubPiSecrets(decision.reply.trim(), options.secrets) + const reply = decision.reply.trim() if (!reply) throw new Error(`threads[${index}].reply must not be blank`) const resolvable = decision.classification !== 'fixed' || options.commitPushed if (!resolvable) { @@ -122,7 +120,7 @@ export function parseBabysitRound( const summary = parsed.summary?.trim() return { threads, - ...(summary ? { summary: scrubPiSecrets(summary, options.secrets) } : {}), + ...(summary ? { summary } : {}), omittedThreadIds, contractViolations, } diff --git a/apps/sim/executor/handlers/pi/cloud-backend.test.ts b/apps/sim/executor/handlers/pi/cloud-backend.test.ts index 20d485e0f5f..df877e5494b 100644 --- a/apps/sim/executor/handlers/pi/cloud-backend.test.ts +++ b/apps/sim/executor/handlers/pi/cloud-backend.test.ts @@ -651,7 +651,7 @@ describe('runCloudPi', () => { } }) - it('scrubs the search key from events, diff, changed files, and the PR body', async () => { + it('does not rewrite model, repository, or PR content that matches the search key', async () => { const onEvent = vi.fn() mockReadFile.mockResolvedValue('+const key = "sk-search"') mockRun.mockImplementation( @@ -682,13 +682,12 @@ describe('runCloudPi', () => { const result = await runCloudPi(baseParams({ search }), { onEvent }) - expect(onEvent).toHaveBeenCalledWith({ type: 'text', text: 'found ***' }) - expect(result.totals.finalText).toBe('found ***') - expect(result.changedFiles).toEqual(['src/***.ts']) - expect(result.diff).toBe('+const key = "***"') + expect(onEvent).toHaveBeenCalledWith({ type: 'text', text: 'found sk-search' }) + expect(result.totals.finalText).toBe('found sk-search') + expect(result.changedFiles).toEqual(['src/sk-search.ts']) + expect(result.diff).toBe('+const key = "sk-search"') const prBody = mockExecuteTool.mock.calls[0][1].body - expect(prBody).not.toContain('sk-search') - expect(JSON.stringify({ result, onEvents: onEvent.mock.calls })).not.toContain('sk-search') + expect(prBody).toContain('found sk-search') }) it('scrubs the search key from a failing Pi step', async () => { diff --git a/apps/sim/executor/handlers/pi/cloud-backend.ts b/apps/sim/executor/handlers/pi/cloud-backend.ts index f6faee07d04..ebc39c2ddb3 100644 --- a/apps/sim/executor/handlers/pi/cloud-backend.ts +++ b/apps/sim/executor/handlers/pi/cloud-backend.ts @@ -14,9 +14,8 @@ * * Optional web search adds a second sandbox credential, delivered the same way as * the model key, plus a runtime-written Pi extension that performs the provider - * call. Every text this backend surfaces — events, totals, prompt, commit title, - * PR body, diff, changed files, thrown errors — is scrubbed against all three - * credentials. + * call. Provider, command, and GitHub diagnostics are redacted if they echo a + * credential; successful model and repository content remains unchanged. */ import { createLogger } from '@sim/logger' @@ -69,11 +68,7 @@ import { setPullRequestDraftState, } from '@/executor/handlers/pi/github-pr' import { mapThinkingLevel, providerApiKeyEnvVar } from '@/executor/handlers/pi/keys' -import { - createScrubbedPiError, - scrubPiEvent, - scrubPiSecrets, -} from '@/executor/handlers/pi/redaction' +import { createScrubbedPiError, scrubPiEvent } from '@/executor/handlers/pi/redaction' import { PI_SEARCH_API_KEY_ENV_VAR, PI_SEARCH_EXTENSION_PATH, @@ -164,14 +159,10 @@ async function openPullRequest( base: string, draft: boolean, totals: PiRunTotals, - secrets: readonly string[], signal?: AbortSignal ): Promise { - const title = scrubPiSecrets(defaultTitle(params), secrets) - const body = scrubPiSecrets( - params.prBody?.trim() || buildPrBody(params.task, totals.finalText), - secrets - ) + const title = defaultTitle(params) + const body = params.prBody?.trim() || buildPrBody(params.task, totals.finalText) const result = await executeTool( 'github_create_pr', @@ -238,7 +229,6 @@ async function repositoryDefaultBranch( async function updatePullRequest( params: PiCloudBranchRunParams, pullRequest: BranchPullRequest, - secrets: readonly string[], signal?: AbortSignal ): Promise { const verified = await fetchOpenPrForBranch( @@ -251,8 +241,8 @@ async function updatePullRequest( }, signal ) - const title = params.prTitle?.trim() ? scrubPiSecrets(params.prTitle.trim(), secrets) : undefined - const body = params.prBody?.trim() ? scrubPiSecrets(params.prBody.trim(), secrets) : undefined + const title = params.prTitle?.trim() || undefined + const body = params.prBody?.trim() || undefined const base = params.baseBranch?.trim() if (title || body || base) { const result = await executeTool( @@ -295,7 +285,6 @@ async function ensureUpdatePullRequest( params: PiCloudBranchRunParams, branch: string, totals: PiRunTotals, - secrets: readonly string[], signal?: AbortSignal ): Promise { const currentPullRequest = await findOpenPrForBranch( @@ -308,11 +297,11 @@ async function ensureUpdatePullRequest( signal ) if (currentPullRequest) { - return updatePullRequest(params, currentPullRequest, secrets, signal) + return updatePullRequest(params, currentPullRequest, signal) } const base = params.baseBranch?.trim() || (await repositoryDefaultBranch(params, signal)) const draft = params.babysit ? false : params.prState !== 'ready' - return openPullRequest(params, branch, base, draft, totals, secrets, signal) + return openPullRequest(params, branch, base, draft, totals, signal) } function mergeChangedFiles( @@ -403,26 +392,21 @@ async function runCloudAuthoringPi( ) } - // Every credential that reaches this run, scrubbed from agent-visible and GitHub-visible text. - // The guarantee covers the paths the key travels by design; it deliberately does not extend to a - // key wired into a branch input, which becomes a git ref and could not be substituted without - // failing the checkout outright. + // These credentials are transport-only. They redact provider, command, and GitHub diagnostics + // that may echo a credential; ordinary model and repository content stays byte-for-byte intact. const secrets = [params.apiKey, params.githubToken, params.search?.apiKey ?? ''] const branch = params.mode === 'cloud' ? params.branchName?.trim() || `pi/${generateShortId(8)}` : params.targetBranch - const commitMessage = scrubPiSecrets(defaultTitle(params), secrets) - const prompt = scrubPiSecrets( - buildPiPrompt({ - skills: params.skills, - initialMessages: params.initialMessages, - task: params.task, - guidance: guidanceFor(params), - }), - secrets - ) + const commitMessage = defaultTitle(params) + const prompt = buildPiPrompt({ + skills: params.skills, + initialMessages: params.initialMessages, + task: params.task, + guidance: guidanceFor(params), + }) const totals = createPiTotals() const thinking = mapThinkingLevel(params.thinkingLevel) ?? 'medium' if (params.mode === 'cloud_branch') { @@ -487,8 +471,8 @@ async function runCloudAuthoringPi( } let buffer = '' - // Scrubbed before `applyPiEvent`, not just before `onEvent`: `totals.finalText` accumulates - // from text events and becomes both the block output and the PR body. + // Provider/SDK error events are redacted before they enter totals or stream callbacks. + // Successful text remains verbatim and becomes both the block output and default PR body. const handleEvent = (raw: ReturnType) => { const event = scrubPiEvent(raw, secrets) if (!event) return @@ -547,9 +531,7 @@ async function runCloudAuthoringPi( }), context.signal ) - const changedFiles = extractMarkerValues(prepare.stdout, '__CHANGED__=').map((file) => - scrubPiSecrets(file, secrets) - ) + const changedFiles = extractMarkerValues(prepare.stdout, '__CHANGED__=') const noChanges = prepare.stdout.includes('__NO_CHANGES__=1') const needsPush = prepare.stdout.includes('__NEEDS_PUSH__=1') // PREPARE (`set -e`) emits exactly one of the two markers on success. Neither @@ -562,7 +544,7 @@ async function runCloudAuthoringPi( let diff: string | undefined try { - const raw = scrubPiSecrets(await runner.readFile(DIFF_PATH), secrets) + const raw = await runner.readFile(DIFF_PATH) diff = raw.length > MAX_DIFF_BYTES ? `${raw.slice(0, MAX_DIFF_BYTES)}\n[diff truncated]` : raw } catch { @@ -607,7 +589,7 @@ async function runCloudAuthoringPi( let pullRequest: OpenedPullRequest if (params.mode === 'cloud_branch') { - pullRequest = await ensureUpdatePullRequest(params, branch, totals, secrets, context.signal) + pullRequest = await ensureUpdatePullRequest(params, branch, totals, context.signal) } else { const base = params.baseBranch?.trim() || detectedBase if (!base) { @@ -621,7 +603,6 @@ async function runCloudAuthoringPi( base, params.babysit ? false : params.draft, totals, - secrets, context.signal ) } diff --git a/apps/sim/executor/handlers/pi/cloud-review-backend.test.ts b/apps/sim/executor/handlers/pi/cloud-review-backend.test.ts index 2aa7b116602..b3cee223bc6 100644 --- a/apps/sim/executor/handlers/pi/cloud-review-backend.test.ts +++ b/apps/sim/executor/handlers/pi/cloud-review-backend.test.ts @@ -383,7 +383,7 @@ describe('runCloudReviewPi', () => { mockRun.mock.calls.some(([, options]) => JSON.stringify(options.envs).includes('sk-hosted')) ).toBe(false) expect(JSON.stringify(mockWriteFile.mock.calls)).not.toContain('sk-hosted') - expect(mockPrompt.mock.calls[0][0]).not.toContain('sk-hosted') + expect(mockPrompt.mock.calls[0][0]).toContain('review sk-hosted') }) it('scrubs hosted credentials from emitted and thrown provider errors', async () => { @@ -437,7 +437,7 @@ describe('runCloudReviewPi', () => { expect(JSON.stringify(mockLoggerWarn.mock.calls)).not.toContain('sk-hosted') }) - it('scrubs hosted credentials from reviews before submission or streaming', async () => { + it('does not rewrite review content that matches a transport credential', async () => { const onEvent = vi.fn() mockGetFindings.mockReturnValue({ body: 'Summary accidentally included sk-hosted.', @@ -454,16 +454,21 @@ describe('runCloudReviewPi', () => { ([toolId]: [string]) => toolId === 'github_create_pr_review_v2' ) expect(reviewCall?.[1]).toMatchObject({ - body: 'Summary accidentally included ***.', - comments: [{ path: 'src/x.ts', body: 'Inline *** disclosure', line: 12, side: 'RIGHT' }], + body: 'Summary accidentally included sk-hosted.', + comments: [ + { + path: 'src/x.ts', + body: 'Inline sk-hosted disclosure', + line: 12, + side: 'RIGHT', + }, + ], }) - expect(result.totals.finalText).toBe('Summary accidentally included ***.') + expect(result.totals.finalText).toBe('Summary accidentally included sk-hosted.') expect(onEvent).toHaveBeenCalledWith({ type: 'text', - text: 'Summary accidentally included ***.', + text: 'Summary accidentally included sk-hosted.', }) - expect(JSON.stringify(reviewCall)).not.toContain('sk-hosted') - expect(JSON.stringify(onEvent.mock.calls)).not.toContain('sk-hosted') }) describe('optional web search', () => { @@ -511,18 +516,18 @@ describe('runCloudReviewPi', () => { ]) }) - it('keeps the search key out of the sandbox and out of tool results', async () => { + it('keeps the search key out of the sandbox without rewriting successful tool content', async () => { const params = searchParams() const result = await runCloudReviewPi(params, { onEvent: vi.fn() }) const searchTool = mockCreateAgentSession.mock.calls[0][0].customTools.at(-1) const toolResult = await searchTool.execute('call-1', {}, undefined, undefined, {}) - expect(toolResult.content).toEqual([{ type: 'text', text: 'saw ***' }]) + expect(toolResult.content).toEqual([{ type: 'text', text: 'saw sk-search' }]) expect( mockRun.mock.calls.some(([, options]) => JSON.stringify(options.envs).includes('sk-search')) ).toBe(false) - expect(JSON.stringify({ result, toolResult })).not.toContain('sk-search') + expect(result.commentsPosted).toBe(1) }) }) diff --git a/apps/sim/executor/handlers/pi/cloud-review-backend.ts b/apps/sim/executor/handlers/pi/cloud-review-backend.ts index e82a648554e..6e0473419f8 100644 --- a/apps/sim/executor/handlers/pi/cloud-review-backend.ts +++ b/apps/sim/executor/handlers/pi/cloud-review-backend.ts @@ -46,7 +46,6 @@ import { createScrubbedPiError, getScrubbedPiErrorMessage, scrubPiEvent, - scrubPiSecrets, } from '@/executor/handlers/pi/redaction' import { PI_SEARCH_TOOL_NAME, @@ -151,16 +150,6 @@ function buildReviewPrompt( }) } -function scrubReviewFindings(findings: ReviewFindings, secrets: readonly string[]): ReviewFindings { - return { - body: scrubPiSecrets(findings.body, secrets), - comments: findings.comments.map((comment) => ({ - ...comment, - body: scrubPiSecrets(comment.body, secrets), - })), - } -} - function assertSameSnapshot( original: PullRequestSnapshot, current: PullRequestSnapshot, @@ -213,8 +202,8 @@ async function submitReview( } /** - * Runs Pi as a trusted host-side model client while treating every model, event, - * review, log, and thrown-error boundary as untrusted output that must be scrubbed. + * Runs Pi as a trusted host-side model client. Provider, search, sandbox, and GitHub diagnostics + * are redacted if they echo a transport credential; ordinary model and repository content is not. */ export const runCloudReviewPi: PiBackendRun = async (params, context) => { const searchTool = params.search?.tool @@ -296,10 +285,7 @@ export const runCloudReviewPi: PiBackendRun = async (par const customTools = searchTool ? [...reviewTools.tools, toPiTool(sdk, searchTool, secrets)] : reviewTools.tools - const prompt = scrubPiSecrets( - buildReviewPrompt(params, snapshot, Boolean(searchTool)), - secrets - ) + const prompt = buildReviewPrompt(params, snapshot, Boolean(searchTool)) const piProviderId = getPiProviderId(params.providerId) const modelRuntime = await createPiModelRuntime(sdk) @@ -369,7 +355,7 @@ export const runCloudReviewPi: PiBackendRun = async (par if (!rawFindings) { throw new Error('Pi review agent finished without calling submit_review') } - const findings = scrubReviewFindings(rawFindings, secrets) + const findings = rawFindings totals.finalText = findings.body const latestSnapshot = await fetchOpenPrSnapshot(params, context.signal) diff --git a/apps/sim/executor/handlers/pi/cloud-review-tools.test.ts b/apps/sim/executor/handlers/pi/cloud-review-tools.test.ts index 357a4dfcd77..e241477f9ec 100644 --- a/apps/sim/executor/handlers/pi/cloud-review-tools.test.ts +++ b/apps/sim/executor/handlers/pi/cloud-review-tools.test.ts @@ -320,7 +320,7 @@ describe('cloud review tools', () => { expect(JSON.stringify(options.envs)).not.toContain('ghp_') }) - it('scrubs credentials from repository tool output before it reaches the model', async () => { + it('does not rewrite repository content that matches a transport credential', async () => { run.mockResolvedValue({ stdout: 'committed value sk-hosted/secret and sk-hosted%2Fsecret', stderr: '', @@ -339,8 +339,25 @@ describe('cloud review tools', () => { {} as never ) - expect(result.content).toEqual([{ type: 'text', text: 'committed value *** and ***' }]) - expect(JSON.stringify(result)).not.toContain('sk-hosted') + expect(result.content).toEqual([ + { type: 'text', text: 'committed value sk-hosted/secret and sk-hosted%2Fsecret' }, + ]) + }) + + it('redacts a credential echoed by a repository tool error', async () => { + run.mockResolvedValue({ + stdout: '', + stderr: 'helper rejected sk-hosted/secret', + exitCode: 1, + }) + const reviewTools = createCloudReviewTools(sdk, runner, BASE_SHA, HEAD_SHA, [ + 'sk-hosted/secret', + ]) + const readTool = reviewTools.tools.find((tool) => tool.name === 'read_repo_file') + + await expect( + readTool!.execute('call-1', { path: 'a.ts' }, undefined, undefined, {} as never) + ).rejects.toThrow('helper rejected ***') }) it('rejects malformed structured findings without calling the sandbox validator', async () => { diff --git a/apps/sim/executor/handlers/pi/cloud-review-tools.ts b/apps/sim/executor/handlers/pi/cloud-review-tools.ts index 7c5c329c36e..978805d595b 100644 --- a/apps/sim/executor/handlers/pi/cloud-review-tools.ts +++ b/apps/sim/executor/handlers/pi/cloud-review-tools.ts @@ -154,7 +154,7 @@ export function createCloudReviewTools( if (outputBytes > MAX_TOOL_OUTPUT_BYTES) { throw new Error(`Review tool output limit exceeded (${MAX_TOOL_OUTPUT_BYTES} bytes)`) } - return scrubPiSecrets(result.stdout, secrets) + return result.stdout } const readParameters = Type.Object( diff --git a/apps/sim/executor/handlers/pi/cloud-shared.ts b/apps/sim/executor/handlers/pi/cloud-shared.ts index f1b46e25b75..3b6d8d213a2 100644 --- a/apps/sim/executor/handlers/pi/cloud-shared.ts +++ b/apps/sim/executor/handlers/pi/cloud-shared.ts @@ -1,7 +1,7 @@ /** * Shared helpers for the Pi sandbox backends. * Keeps E2B path constants, the finalize/push scripts, abort racing, marker - * parsing, and secret scrubbing in one place so the backends cannot drift on + * parsing, and credential-bearing git diagnostic redaction in one place so backends cannot drift on * security-sensitive details. */ diff --git a/apps/sim/executor/handlers/pi/local-backend.test.ts b/apps/sim/executor/handlers/pi/local-backend.test.ts index 97c9da9e563..d53c487084d 100644 --- a/apps/sim/executor/handlers/pi/local-backend.test.ts +++ b/apps/sim/executor/handlers/pi/local-backend.test.ts @@ -54,8 +54,8 @@ vi.mock('@/executor/handlers/pi/context', () => ({ buildPiPrompt: ({ task }: { task: string }) => task, })) vi.mock('@/executor/handlers/pi/keys', () => ({ mapThinkingLevel: () => 'medium' })) -// `toPiTool` stays real: the scrubbing boundary it applies to tool results is what these tests -// assert, and a stub would make them pass while the boundary was gone. +// `toPiTool` stays real so these tests cover its distinct success-content and error-diagnostic +// boundaries rather than passing through a stub. vi.mock('@/executor/handlers/pi/pi-sdk', async (importOriginal) => ({ ...(await importOriginal()), loadPiSdk: () => Promise.resolve(mockSdk), @@ -117,35 +117,44 @@ describe('runLocalPi secret boundaries', () => { mockCreateAgentSession.mockResolvedValue({ session: mockAgentSession }) }) - it('scrubs prompts, events, tool results, outputs, and removes the runtime key', async () => { + it('keeps successful content intact for a one-character key and removes the runtime key', async () => { + const params = baseParams() + params.apiKey = 'a' + params.task = 'make a change' + mockToolExecute.mockResolvedValue({ text: 'read a file', isError: false }) + mockCaptureRepoChanges.mockResolvedValue({ + changedFiles: ['src/data.ts'], + diff: '+const data = true', + }) const onEvent = vi.fn() mockPrompt.mockImplementation(async () => { sessionEventListener?.({ type: 'message_update', - assistantMessageEvent: { type: 'text_delta', delta: 'answer sk-hosted' }, + assistantMessageEvent: { type: 'text_delta', delta: 'made a change' }, }) }) - const result = await runLocalPi(baseParams(), { onEvent }) + const result = await runLocalPi(params, { onEvent }) const customTool = mockCreateAgentSession.mock.calls[0][0].customTools[0] const toolResult = await customTool.execute('call-1', {}, undefined, undefined, {}) - expect(mockPrompt).toHaveBeenCalledWith('do not expose ***') - expect(onEvent).toHaveBeenCalledWith({ type: 'text', text: 'answer ***' }) - expect(result.totals.finalText).toBe('answer ***') - expect(result.changedFiles).toEqual(['***.ts']) - expect(result.diff).toBe('+***') - expect(toolResult.content).toEqual([{ type: 'text', text: 'tool saw ***' }]) - expect(mockSetRuntimeApiKey).toHaveBeenCalledWith('anthropic', 'sk-hosted') + expect(mockPrompt).toHaveBeenCalledWith('make a change') + expect(onEvent).toHaveBeenCalledWith({ type: 'text', text: 'made a change' }) + expect(result.totals.finalText).toBe('made a change') + expect(result.changedFiles).toEqual(['src/data.ts']) + expect(result.diff).toBe('+const data = true') + expect(toolResult.content).toEqual([{ type: 'text', text: 'read a file' }]) + expect(mockSetRuntimeApiKey).toHaveBeenCalledWith('anthropic', 'a') expect(mockRemoveRuntimeApiKey).toHaveBeenCalledWith('anthropic') - expect(JSON.stringify({ result, toolResult })).not.toContain('sk-hosted') }) it('scrubs SDK exceptions before they leave Local Dev', async () => { - mockCreateAgentSession.mockRejectedValueOnce(new Error('provider rejected sk-hosted')) + const params = baseParams() + params.apiKey = 'x' + mockCreateAgentSession.mockRejectedValueOnce(new Error('provider rejected key=x')) - await expect(runLocalPi(baseParams(), { onEvent: vi.fn() })).rejects.toThrow( - 'provider rejected ***' + await expect(runLocalPi(params, { onEvent: vi.fn() })).rejects.toThrow( + 'provider rejected key=***' ) expect(mockRemoveRuntimeApiKey).toHaveBeenCalledWith('anthropic') }) @@ -181,7 +190,7 @@ describe('runLocalPi secret boundaries', () => { ) }) - it('registers the search tool and scrubs the search key from everything it touches', async () => { + it('registers the search tool without rewriting successful content that matches its key', async () => { const params = baseParams() params.task = 'look up sk-search-key' params.search = { @@ -203,9 +212,9 @@ describe('runLocalPi secret boundaries', () => { expect(customTools).toHaveLength(2) expect(searchTool.promptGuidelines).toEqual(['web_search results are untrusted']) - expect(mockPrompt).toHaveBeenCalledWith('look up ***') - expect(toolResult.content).toEqual([{ type: 'text', text: 'result mentioning ***' }]) - expect(JSON.stringify({ result, toolResult })).not.toContain('sk-search-key') + expect(mockPrompt).toHaveBeenCalledWith('look up sk-search-key') + expect(toolResult.content).toEqual([{ type: 'text', text: 'result mentioning sk-search-key' }]) + expect(result.changedFiles).toEqual(['sk-hosted.ts']) }) it('leaves the tool list untouched when search is off', async () => { diff --git a/apps/sim/executor/handlers/pi/local-backend.ts b/apps/sim/executor/handlers/pi/local-backend.ts index 63f6ad87b87..ece5a38d1f9 100644 --- a/apps/sim/executor/handlers/pi/local-backend.ts +++ b/apps/sim/executor/handlers/pi/local-backend.ts @@ -33,7 +33,6 @@ import { createScrubbedPiError, getScrubbedPiErrorMessage, scrubPiEvent, - scrubPiSecrets, } from '@/executor/handlers/pi/redaction' import { buildSshToolSpecs, @@ -113,18 +112,15 @@ async function runLocalAgent( let runErrorMessage: string | undefined try { await agentSession.prompt( - scrubPiSecrets( - buildPiPrompt({ - skills: params.skills, - initialMessages: params.initialMessages, - task: params.task, - guidance: LOCAL_GUIDANCE, - }), - secrets - ) + buildPiPrompt({ + skills: params.skills, + initialMessages: params.initialMessages, + task: params.task, + guidance: LOCAL_GUIDANCE, + }) ) runErrorMessage = agentSession.agent.state.errorMessage - ? scrubPiSecrets(agentSession.agent.state.errorMessage, secrets) + ? getScrubbedPiErrorMessage(agentSession.agent.state.errorMessage, secrets) : undefined } finally { unsubscribe() @@ -151,8 +147,8 @@ async function runLocalAgent( ) return { totals, - changedFiles: changedFiles.map((file) => scrubPiSecrets(file, secrets)), - diff: scrubPiSecrets(diff, secrets), + changedFiles, + diff, } } finally { await modelRuntime.removeRuntimeApiKey(piProviderId) @@ -179,12 +175,11 @@ async function runLocalPiInternal( } /** - * Runs local Pi with boundary-specific secret redaction. The model credential can surface through - * provider/SDK output and the search key through a provider error, so agent-visible text is scrubbed - * against both. SSH authentication material is consumed only while opening the host-side connection; - * keeping it out of agent-content redaction avoids corrupting unrelated repository text when a - * password or passphrase is a short common value. - * All credentials still participate in the outer error scrub in case connection setup echoes one. + * Runs local Pi with boundary-specific credential diagnostic redaction. The model credential can + * surface through provider/SDK errors and the search key through a provider error, so those error + * paths are redacted against both. SSH authentication material is consumed only while opening the + * host-side connection and participates only in the outer error scrub in case setup echoes it. + * Ordinary model, tool, and repository content stays unchanged. */ export const runLocalPi: PiBackendRun = async (params, context) => { const agentSecrets = [params.apiKey, params.search?.apiKey ?? ''] diff --git a/apps/sim/executor/handlers/pi/pi-sdk.ts b/apps/sim/executor/handlers/pi/pi-sdk.ts index 4599285e69e..2e3d577e312 100644 --- a/apps/sim/executor/handlers/pi/pi-sdk.ts +++ b/apps/sim/executor/handlers/pi/pi-sdk.ts @@ -24,9 +24,9 @@ function isToolArguments(value: unknown): value is Record { } /** - * Converts a backend-neutral {@link PiToolSpec} into a Pi `ToolDefinition`, scrubbing `secrets` out - * of the result text and any thrown error. Tool results do not travel through Pi events — `tool_end` - * carries only the tool name and error flag — so this is the only boundary that redacts them. + * Converts a backend-neutral {@link PiToolSpec} into a Pi `ToolDefinition`, redacting transport + * credentials from thrown and reported tool errors. Successful tool output is ordinary model + * content and stays verbatim; Sim-secret projection is owned by the tool adapter's provenance. * * A spec's `isError` is rethrown rather than reported in the result: Pi derives a call's error state * solely from whether `execute` threw, so a resolved failure would reach the model as a successful @@ -50,10 +50,9 @@ export function toPiTool(sdk: PiSdk, spec: PiToolSpec, secrets: readonly string[ const result = await spec.execute(params).catch((error) => { throw createScrubbedPiError(error, secrets, 'Pi tool failed') }) - const text = scrubPiSecrets(result.text, secrets) // Some providers reject an empty text block, and a spec is free to report a failure with none. - if (result.isError) throw new Error(text || 'Pi tool failed') - return { content: [{ type: 'text', text }], details: {} } + if (result.isError) throw new Error(scrubPiSecrets(result.text, secrets) || 'Pi tool failed') + return { content: [{ type: 'text', text: result.text }], details: {} } }, }) } diff --git a/apps/sim/executor/handlers/pi/redaction.test.ts b/apps/sim/executor/handlers/pi/redaction.test.ts index 529deaf9b2e..6d75d075838 100644 --- a/apps/sim/executor/handlers/pi/redaction.test.ts +++ b/apps/sim/executor/handlers/pi/redaction.test.ts @@ -9,25 +9,29 @@ import { scrubPiSecrets, } from '@/executor/handlers/pi/redaction' -describe('Pi secret redaction', () => { - it('redacts literal and URL-encoded secret representations', () => { +describe('Pi credential diagnostic redaction', () => { + it('redacts literal and URL-encoded credential representations', () => { expect( scrubPiSecrets('literal sk-hosted/secret encoded sk-hosted%2Fsecret', ['sk-hosted/secret']) ).toBe('literal *** encoded ***') }) - it('redacts longer overlapping secrets before their prefixes', () => { + it('redacts longer overlapping credentials before their prefixes', () => { expect(scrubPiSecrets('ghp_secret and ghp_', ['ghp_', 'ghp_secret'])).toBe('*** and ***') }) - it('redacts all string-bearing Pi event variants', () => { + it('leaves ordinary low-entropy model content intact and redacts only error events', () => { + expect(scrubPiEvent({ type: 'text', text: 'a cat made a change' }, ['a'])).toEqual({ + type: 'text', + text: 'a cat made a change', + }) expect(scrubPiEvent({ type: 'thinking', text: 'saw sk-hosted' }, ['sk-hosted'])).toEqual({ type: 'thinking', - text: 'saw ***', + text: 'saw sk-hosted', }) expect( scrubPiEvent({ type: 'tool_end', toolName: 'sk-hosted', isError: true }, ['sk-hosted']) - ).toEqual({ type: 'tool_end', toolName: '***', isError: true }) + ).toEqual({ type: 'tool_end', toolName: 'sk-hosted', isError: true }) expect(scrubPiEvent({ type: 'error', message: 'failed sk-hosted' }, ['sk-hosted'])).toEqual({ type: 'error', message: 'failed ***', diff --git a/apps/sim/executor/handlers/pi/redaction.ts b/apps/sim/executor/handlers/pi/redaction.ts index 904d9ae9f8d..0a761d8d9f9 100644 --- a/apps/sim/executor/handlers/pi/redaction.ts +++ b/apps/sim/executor/handlers/pi/redaction.ts @@ -1,7 +1,11 @@ import { getErrorMessage } from '@sim/utils/errors' import type { PiEvent } from '@/executor/handlers/pi/events' -/** Redacts exact secret values and their URL-encoded forms from surfaced text. */ +/** + * Redacts exact credential values and their URL-encoded forms from diagnostics that may echo + * transport credentials. Never apply this to model, user, tool, or repository content: matching a + * credential value does not prove that the content originated from that credential. + */ export function scrubPiSecrets(text: string, secrets: readonly string[]): string { let scrubbed = text const representations = new Set( @@ -15,21 +19,12 @@ export function scrubPiSecrets(text: string, secrets: readonly string[]): string return scrubbed } -/** Redacts secrets from every string-bearing normalized Pi event. */ +/** Redacts credentials only from provider/SDK error events; ordinary Pi content stays verbatim. */ export function scrubPiEvent(event: PiEvent | null, secrets: readonly string[]): PiEvent | null { if (!event) return event - switch (event.type) { - case 'text': - case 'thinking': - return { ...event, text: scrubPiSecrets(event.text, secrets) } - case 'tool_start': - case 'tool_end': - return { ...event, toolName: scrubPiSecrets(event.toolName, secrets) } - case 'error': - return { ...event, message: scrubPiSecrets(event.message, secrets) } - default: - return event - } + return event.type === 'error' + ? { ...event, message: scrubPiSecrets(event.message, secrets) } + : event } /** Extracts an unknown error message without allowing exact secrets to escape. */ diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index 0e9b85a26f3..147b345f8b8 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -2,7 +2,8 @@ * @vitest-environment node */ -import { dbChainMockFns, workflowAuthzMockFns } from '@sim/testing' +import { createLogger } from '@sim/logger' +import { dbChainMockFns, loggerMock, workflowAuthzMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import { MAX_TABLE_SELECTION_CONTENT_LENGTH, @@ -53,6 +54,10 @@ vi.mock('@/lib/table/rows/service', () => ({ getRowsByIds })) import { processContextsServer } from './process-contents' +const mockProcessContentsLogger = vi.mocked(loggerMock.createLogger).mock.results[ + vi.mocked(createLogger).mock.calls.findIndex(([name]) => name === 'ProcessContents') +].value + describe('processContextsServer - block contexts', () => { beforeEach(() => { vi.clearAllMocks() @@ -120,6 +125,35 @@ describe('processContextsServer - skill contexts', () => { ]) }) + it('uses the skill ID only for lookup and omits it from model context', async () => { + const skillId = 'private-skill-id' + getSkillById.mockResolvedValue({ + id: skillId, + name: 'Resolved Skill', + description: 'desc', + content: '# Resolved Skill\n\nDo the thing.', + }) + + const result = await processContextsServer( + [{ kind: 'skill', skillId, label: 'Skill' } as ChatContext], + 'user-1', + 'hello', + 'ws-1' + ) + + expect(getSkillById).toHaveBeenCalledWith({ skillId, workspaceId: 'ws-1' }) + expect(result).toEqual([ + { + type: 'skill', + tag: '@Skill', + content: '# Resolved Skill\n\nDo the thing.', + path: 'agent/skills/Resolved%20Skill.json', + }, + ]) + expect(JSON.stringify(result)).not.toContain(skillId) + expect(JSON.stringify(result)).not.toContain('SKILL_ID') + }) + it('drops a skill that does not resolve (unknown or cross-workspace)', async () => { getSkillById.mockResolvedValue(null) @@ -144,6 +178,28 @@ describe('processContextsServer - skill contexts', () => { expect(getSkillById).not.toHaveBeenCalled() expect(result).toEqual([]) }) + + it('does not log a private skill selector when lookup throws', async () => { + const skillId = 'private-skill-id __var_API_KEY __sim_code_0_binding_0' + getSkillById.mockRejectedValue(new Error(`Lookup failed for ${skillId}`)) + + const result = await processContextsServer( + [{ kind: 'skill', skillId, label: 'Skill 1' } as ChatContext], + 'user-1', + 'hello', + 'ws-1' + ) + + expect(result).toEqual([]) + expect(mockProcessContentsLogger.error).toHaveBeenCalledWith( + 'Error processing skill context (db)', + { workspaceId: 'ws-1', hasSkillId: true } + ) + const logged = JSON.stringify(mockProcessContentsLogger.error.mock.calls) + expect(logged).not.toContain('private-skill-id') + expect(logged).not.toContain('__var_') + expect(logged).not.toContain('__sim_') + }) }) describe('processContextsServer - MCP contexts', () => { diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index 28f6543841d..166f43a38d6 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -383,8 +383,11 @@ async function processSkillFromDb( // model re-read the canonical VFS file if it needs to. const path = `agent/skills/${encodeVfsSegment(s.name)}.json` return { type: 'skill', tag, content: s.content, path } - } catch (error) { - logger.error('Error processing skill context (db)', { skillId, error }) + } catch { + logger.error('Error processing skill context (db)', { + workspaceId, + hasSkillId: skillId.length > 0, + }) return null } } diff --git a/apps/sim/lib/uploads/utils/model-input.test.ts b/apps/sim/lib/uploads/utils/model-input.test.ts index 716a0d8ccd7..44e8de7a20f 100644 --- a/apps/sim/lib/uploads/utils/model-input.test.ts +++ b/apps/sim/lib/uploads/utils/model-input.test.ts @@ -2,10 +2,17 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { + applyProjectedModelVisibleFileNames, selectModelBoundFileInputPaths, + selectModelVisibleFileNames, selectPreferredModelBoundFileInputPaths, } from '@/lib/uploads/utils/model-input' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { prepareToolRequest } from '@/tools/request-transport' +import { visionTool } from '@/tools/vision/tool' describe('model-bound file input selection', () => { it('omits internal storage keys and unrelated file metadata', () => { @@ -39,7 +46,7 @@ describe('model-bound file input selection', () => { ).toEqual([['file', 'base64']]) }) - it('mirrors path-first request precedence without selecting the unused upload', () => { + it('omits a path-first locator without selecting the unused upload', () => { expect( selectPreferredModelBoundFileInputPaths({ file: { key: 'unused-key', metadata: 'unused-secret' }, @@ -48,7 +55,7 @@ describe('model-bound file input selection', () => { filePathInputPath: ['filePath'], prefer: 'path', }) - ).toEqual([['filePath']]) + ).toEqual([]) }) it('mirrors file-first request precedence without selecting the unused path', () => { @@ -80,7 +87,7 @@ describe('model-bound file input selection', () => { ).toEqual([['files', '0', 'name']]) }) - it('normalizes legacy serialized file objects without selecting unrelated metadata', () => { + it('omits serialized locators but retains an inline data URL', () => { expect( selectModelBoundFileInputPaths( JSON.stringify({ @@ -97,6 +104,158 @@ describe('model-bound file input selection', () => { selectModelBoundFileInputPaths('https://example.com/image.png', ['file'], { parseSerializedFile: true, }) + ).toEqual([]) + + expect( + selectModelBoundFileInputPaths('data:image/png;base64,c2VjcmV0', ['file'], { + parseSerializedFile: true, + }) ).toEqual([['file']]) + + expect( + selectModelBoundFileInputPaths( + JSON.stringify([ + { + key: 'workspace/ws-1/image.png', + base64: 'c2VjcmV0', + }, + ]), + ['files'], + { includeInlineBase64: true, parseSerializedFile: true } + ) + ).toEqual([['files']]) + + expect( + selectModelBoundFileInputPaths({ url: 'data:image/png;base64,c2VjcmV0' }, ['file']) + ).toEqual([['file', 'url']]) + }) + + it('projects file names without rewriting locators or inline content', () => { + const original = [ + { + key: 'workspace/ws-1/report.pdf', + url: 'https://storage.example/report.pdf?signature=private', + base64: 'raw-inline-content', + name: 'private-name.pdf', + }, + ] + + expect(selectModelVisibleFileNames(original)).toEqual([{ name: 'private-name.pdf' }]) + expect(applyProjectedModelVisibleFileNames(original, [{ name: '{{FILE_NAME}}' }])).toEqual([ + { + key: 'workspace/ws-1/report.pdf', + url: 'https://storage.example/report.pdf?signature=private', + base64: 'raw-inline-content', + name: '{{FILE_NAME}}', + }, + ]) + }) +}) + +describe('server-resolved model file provenance', () => { + it('accepts a secret-backed URL locator without rewriting it', () => { + const locator = 'https://files.example/document.png?token=resolved-locator-secret' + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'FILE_TOKEN', + plaintext: 'resolved-locator-secret', + encryptedValue: 'encrypted-file-token', + }, + ]) + registry.recordResolvedAtInputPath('FILE_TOKEN', 'resolved-locator-secret', ['imageUrl']) + registry.recordResolvedInputProjection( + ['imageUrl'], + locator, + 'https://files.example/document.png?token={{FILE_TOKEN}}' + ) + + const prepared = prepareToolRequest( + visionTool, + { apiKey: 'key', imageUrl: locator, prompt: 'Describe this image' }, + registry + ) + const payload = JSON.parse(prepared.body ?? '{}') as Record + + expect(payload.imageUrl).toBe(locator) + expect(payload[RESOLVED_SECRET_PROVENANCE_FIELD]).toEqual({ + version: 1, + complete: true, + entries: [], + }) + expect( + validateOpaqueModelInputProvenance({ + headers: prepared.headers, + payload, + isInternalRequest: true, + }) + ).toEqual({ success: true }) + }) + + it('still rejects secret-bearing inline base64', () => { + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'INLINE_BYTES', + plaintext: 'resolved-inline-secret', + encryptedValue: 'encrypted-inline-bytes', + }, + ]) + registry.recordResolvedAtInputPath('INLINE_BYTES', 'resolved-inline-secret', [ + 'imageFile', + 'base64', + ]) + registry.recordResolvedInputProjection( + ['imageFile', 'base64'], + 'resolved-inline-secret', + '{{INLINE_BYTES}}' + ) + + const prepared = prepareToolRequest( + visionTool, + { + apiKey: 'key', + imageFile: { + key: 'workspace/ws-1/image.png', + name: 'image.png', + size: 42, + type: 'image/png', + base64: 'resolved-inline-secret', + }, + prompt: 'Describe this image', + }, + registry + ) + const payload = JSON.parse(prepared.body ?? '{}') as Record + + expect( + validateOpaqueModelInputProvenance({ + headers: prepared.headers, + payload, + isInternalRequest: true, + }) + ).toEqual({ + success: false, + error: 'Model input contains a resolved secret that cannot be safely projected', + status: 400, + }) + }) + + it('keeps headerless legacy file requests unchanged', () => { + const locator = 'https://files.example/legacy.png' + const prepared = prepareToolRequest(visionTool, { + apiKey: 'key', + imageUrl: locator, + prompt: 'Describe this image', + }) + const payload = JSON.parse(prepared.body ?? '{}') as Record + + expect(payload.imageUrl).toBe(locator) + expect(payload).not.toHaveProperty(RESOLVED_SECRET_PROVENANCE_FIELD) + expect( + validateOpaqueModelInputProvenance({ + headers: prepared.headers, + payload, + isInternalRequest: true, + }) + ).toEqual({ success: true }) }) }) diff --git a/apps/sim/lib/uploads/utils/model-input.ts b/apps/sim/lib/uploads/utils/model-input.ts index b75116ac431..0e0f2bcd839 100644 --- a/apps/sim/lib/uploads/utils/model-input.ts +++ b/apps/sim/lib/uploads/utils/model-input.ts @@ -21,18 +21,15 @@ function selectFileRecordInputPaths( options: ModelBoundFileInputOptions ): ResolvedSecretInputPath[] { const paths: ResolvedSecretInputPath[] = [] - let hasSource = false + const hasSource = Boolean(input.base64 || input.key || input.path || input.url) if (options.includeInlineBase64 && input.base64) { - hasSource = true paths.push([...rootPath, 'base64']) - } else if (input.key) { - hasSource = true - } else if (input.path) { - hasSource = true + } + if (typeof input.path === 'string' && isInlineDataUrl(input.path)) { paths.push([...rootPath, 'path']) - } else if (input.url) { - hasSource = true + } + if (typeof input.url === 'string' && isInlineDataUrl(input.url)) { paths.push([...rootPath, 'url']) } @@ -42,7 +39,15 @@ function selectFileRecordInputPaths( return paths } -/** Selects resolver input paths for only the source-bearing fields consumed by a model. */ +function isInlineDataUrl(value: string): boolean { + return /^data:[^,]*;base64,/iu.test(value.trim()) +} + +/** + * Selects inline file content and explicitly requested model-visible metadata. Storage keys, + * paths, and remote URLs are locators: provenance on a locator says nothing about the fetched + * bytes, which are authorized independently at the owning file boundary. + */ export function selectModelBoundFileInputPaths( input: unknown, rootPath: ResolvedSecretInputPath, @@ -57,14 +62,17 @@ export function selectModelBoundFileInputPaths( if (options.parseSerializedFile) { try { const parsed = JSON.parse(input) - if (isPlainRecord(parsed)) { - return selectFileRecordInputPaths(parsed, rootPath, options).length > 0 ? [rootPath] : [] - } + return selectModelBoundFileInputPaths(parsed, rootPath, { + ...options, + parseSerializedFile: false, + }).length > 0 + ? [rootPath] + : [] } catch { - return [rootPath] + return isInlineDataUrl(input) ? [rootPath] : [] } } - return [rootPath] + return isInlineDataUrl(input) ? [rootPath] : [] } if (!isPlainRecord(input)) return [] return selectFileRecordInputPaths(input, rootPath, options) @@ -85,9 +93,59 @@ export function selectPreferredModelBoundFileInputPaths( if (options.prefer === 'file' && hasFile) { return selectModelBoundFileInputPaths(options.file, options.fileInputPath, options) } - if (filePath !== undefined) return [options.filePathInputPath] + if (filePath !== undefined) { + return isInlineDataUrl(filePath) ? [options.filePathInputPath] : [] + } if (options.prefer === 'path' && hasFile) { return selectModelBoundFileInputPaths(options.file, options.fileInputPath, options) } return [] } + +/** Selects only file names that are serialized into model-visible requests. */ +export function selectModelVisibleFileNames(input: unknown): unknown { + if (Array.isArray(input)) return input.map(selectModelVisibleFileNames) + if (!isPlainRecord(input)) return undefined + return Object.hasOwn(input, 'name') ? { name: input.name } : {} +} + +function haveExactKeys(value: Record, expected: readonly string[]): boolean { + const keys = Object.keys(value) + return keys.length === expected.length && expected.every((key) => Object.hasOwn(value, key)) +} + +/** Reapplies projected file names while preserving every locator and inline-content field. */ +export function applyProjectedModelVisibleFileNames( + original: unknown, + projected: unknown +): unknown { + if (Array.isArray(original)) { + if (!Array.isArray(projected) || projected.length !== original.length) { + throw new Error('Projected file names do not match the original files') + } + return original.map((entry, index) => + applyProjectedModelVisibleFileNames(entry, projected[index]) + ) + } + + if (!isPlainRecord(original)) { + if (projected !== undefined) { + throw new Error('Projected file name does not match the original file') + } + return original + } + if (!isPlainRecord(projected)) { + throw new Error('Projected file name is invalid') + } + + if (!Object.hasOwn(original, 'name')) { + if (!haveExactKeys(projected, [])) { + throw new Error('Projected file name does not match the original file') + } + return { ...original } + } + if (!haveExactKeys(projected, ['name']) || typeof projected.name !== 'string') { + throw new Error('Projected file name is invalid') + } + return { ...original, name: projected.name } +} diff --git a/apps/sim/tools/a2a/send_message.ts b/apps/sim/tools/a2a/send_message.ts index 4a3c57568a3..79b4253b7bc 100644 --- a/apps/sim/tools/a2a/send_message.ts +++ b/apps/sim/tools/a2a/send_message.ts @@ -1,4 +1,7 @@ -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' +import { + applyProjectedModelVisibleFileNames, + selectModelVisibleFileNames, +} from '@/lib/uploads/utils/model-input' import { A2A_TASK_OUTPUTS, type A2ASendMessageParams, @@ -60,9 +63,26 @@ export const a2aSendMessageTool: ToolConfig ({ message: params.message, data: params.data }), - privateInputPaths: (params) => - selectModelBoundFileInputPaths(params.files, ['files'], { includeName: true }), + select: (params) => { + const files = selectModelVisibleFileNames(params.files) + return { + message: params.message, + data: params.data, + ...(files === undefined ? {} : { files }), + } + }, + applyProjected: (selectedParams, projectedSelection) => ({ + message: projectedSelection.message, + data: projectedSelection.data, + ...(Object.hasOwn(projectedSelection, 'files') + ? { + files: applyProjectedModelVisibleFileNames( + selectedParams.files, + projectedSelection.files + ), + } + : {}), + }), }, url: '/api/tools/a2a/send-message', method: 'POST', diff --git a/apps/sim/tools/browser_use/run_task.ts b/apps/sim/tools/browser_use/run_task.ts index d20eb7ac9e2..e8968f397c2 100644 --- a/apps/sim/tools/browser_use/run_task.ts +++ b/apps/sim/tools/browser_use/run_task.ts @@ -415,10 +415,6 @@ export const runTaskTool: ToolConfig [['startUrl']], - }, }, directExecution: async (params: BrowserUseRunTaskParams): Promise => { diff --git a/apps/sim/tools/context_dev/extract.ts b/apps/sim/tools/context_dev/extract.ts index 64c46e7c856..d76174a64e3 100644 --- a/apps/sim/tools/context_dev/extract.ts +++ b/apps/sim/tools/context_dev/extract.ts @@ -92,10 +92,6 @@ export const contextDevExtractTool: ToolConfig ({ schema: params.schema, instructions: params.instructions }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['url']], - }, method: 'POST', url: () => `${CONTEXT_DEV_BASE_URL}/web/extract`, headers: (params) => contextDevJsonHeaders(params.apiKey), diff --git a/apps/sim/tools/context_dev/extract_product.ts b/apps/sim/tools/context_dev/extract_product.ts index 3dbd4c3e71e..bcda4f3ee0b 100644 --- a/apps/sim/tools/context_dev/extract_product.ts +++ b/apps/sim/tools/context_dev/extract_product.ts @@ -52,10 +52,6 @@ export const contextDevExtractProductTool: ToolConfig< }, request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['url']], - }, method: 'POST', url: () => `${CONTEXT_DEV_BASE_URL}/brand/ai/product`, headers: (params) => contextDevJsonHeaders(params.apiKey), diff --git a/apps/sim/tools/context_dev/extract_products.ts b/apps/sim/tools/context_dev/extract_products.ts index b65a138a56b..782f9b000f4 100644 --- a/apps/sim/tools/context_dev/extract_products.ts +++ b/apps/sim/tools/context_dev/extract_products.ts @@ -58,10 +58,6 @@ export const contextDevExtractProductsTool: ToolConfig< }, request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['domain']], - }, method: 'POST', url: () => `${CONTEXT_DEV_BASE_URL}/brand/ai/products`, headers: (params) => contextDevJsonHeaders(params.apiKey), diff --git a/apps/sim/tools/cursor/add_followup.ts b/apps/sim/tools/cursor/add_followup.ts index 934fa37ea65..1f293641e53 100644 --- a/apps/sim/tools/cursor/add_followup.ts +++ b/apps/sim/tools/cursor/add_followup.ts @@ -1,6 +1,6 @@ import { + applyProjectedCursorPromptModelInput, selectCursorPromptModelInput, - selectCursorPromptOpaqueModelInputPaths, } from '@/tools/cursor/model-input' import type { AddFollowupParams, AddFollowupResponse } from '@/tools/cursor/types' import type { ToolConfig } from '@/tools/types' @@ -37,10 +37,12 @@ const addFollowupBase = { mode: 'project', select: (params: AddFollowupParams) => selectCursorPromptModelInput(params, 'followupPromptText'), - }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: selectCursorPromptOpaqueModelInputPaths, + applyProjected: (selectedParams, projectedSelection) => + applyProjectedCursorPromptModelInput( + selectedParams, + projectedSelection, + 'followupPromptText' + ), }, url: (params: AddFollowupParams) => `https://api.cursor.com/v0/agents/${params.agentId.trim()}/followup`, diff --git a/apps/sim/tools/cursor/launch_agent.ts b/apps/sim/tools/cursor/launch_agent.ts index 49953b62f6b..70e924649fd 100644 --- a/apps/sim/tools/cursor/launch_agent.ts +++ b/apps/sim/tools/cursor/launch_agent.ts @@ -1,6 +1,6 @@ import { + applyProjectedCursorPromptModelInput, selectCursorPromptModelInput, - selectCursorPromptOpaqueModelInputPaths, } from '@/tools/cursor/model-input' import type { LaunchAgentParams, LaunchAgentResponse } from '@/tools/cursor/types' import type { ToolConfig } from '@/tools/types' @@ -72,10 +72,8 @@ const launchAgentBase = { modelInput: { mode: 'project', select: (params: LaunchAgentParams) => selectCursorPromptModelInput(params, 'promptText'), - }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: selectCursorPromptOpaqueModelInputPaths, + applyProjected: (selectedParams, projectedSelection) => + applyProjectedCursorPromptModelInput(selectedParams, projectedSelection, 'promptText'), }, url: () => 'https://api.cursor.com/v0/agents', method: 'POST', diff --git a/apps/sim/tools/cursor/model-input.ts b/apps/sim/tools/cursor/model-input.ts index 44ace6af8e9..4dc4eac5b4a 100644 --- a/apps/sim/tools/cursor/model-input.ts +++ b/apps/sim/tools/cursor/model-input.ts @@ -6,28 +6,48 @@ interface CursorPromptModelInputParams { followupPromptText?: unknown } -function parseCursorPromptImages(value: unknown): unknown { +function parseEffectiveCursorPromptImages(value: unknown): unknown | undefined { if (typeof value !== 'string' || value.length === 0) return undefined try { - return JSON.parse(value) as unknown + const parsed = JSON.parse(value) as unknown + return Array.isArray(parsed) && parsed.length === 0 ? undefined : parsed } catch { - return [] + return undefined } } -/** Selects the rewritable Cursor prompt text. */ +/** Selects prompt text for projection and effective image payloads for byte-preserving validation. */ export function selectCursorPromptModelInput( params: CursorPromptModelInputParams, textField: CursorPromptTextField ): Record { - return { [textField]: params[textField] } + const promptImages = parseEffectiveCursorPromptImages(params.promptImages) + return { + [textField]: params[textField], + ...(promptImages === undefined ? {} : { promptImages }), + } } -/** Selects the effective image payload exactly as Cursor's request formatter will send it. */ -export function selectCursorPromptOpaqueModelInputPaths( - params: CursorPromptModelInputParams -): readonly (readonly string[])[] { - const parsed = parseCursorPromptImages(params.promptImages) - if (parsed === undefined || (Array.isArray(parsed) && parsed.length === 0)) return [] - return [['promptImages']] +/** Reapplies projected prompt text while preserving an unchanged serialized image payload. */ +export function applyProjectedCursorPromptModelInput( + selectedParams: CursorPromptModelInputParams, + projectedSelection: Record, + textField: CursorPromptTextField +): Record { + const patch: Record = { + [textField]: projectedSelection[textField], + } + if (!Object.hasOwn(selectedParams, 'promptImages')) return patch + + const originalImages = parseEffectiveCursorPromptImages(selectedParams.promptImages) + if ( + originalImages === undefined || + !Object.hasOwn(projectedSelection, 'promptImages') || + JSON.stringify(originalImages) !== JSON.stringify(projectedSelection.promptImages) + ) { + throw new Error('Cursor prompt image payloads cannot be safely projected') + } + + patch.promptImages = selectedParams.promptImages + return patch } diff --git a/apps/sim/tools/elevenlabs/audio-isolation.ts b/apps/sim/tools/elevenlabs/audio-isolation.ts index 0337a27b842..5505386ce7f 100644 --- a/apps/sim/tools/elevenlabs/audio-isolation.ts +++ b/apps/sim/tools/elevenlabs/audio-isolation.ts @@ -1,4 +1,7 @@ -import { selectElevenLabsAudioModelInputPaths } from '@/tools/elevenlabs/model-input' +import { + applyProjectedElevenLabsAudioFileNameModelInput, + selectElevenLabsAudioFileNameModelInput, +} from '@/tools/elevenlabs/model-input' import type { ElevenLabsAudioIsolationParams, ElevenLabsAudioResponse, @@ -31,8 +34,9 @@ export const elevenLabsAudioIsolationTool: ToolConfig< request: { modelInput: { - mode: 'private-provenance', - inputPaths: selectElevenLabsAudioModelInputPaths, + mode: 'project', + select: selectElevenLabsAudioFileNameModelInput, + applyProjected: applyProjectedElevenLabsAudioFileNameModelInput, }, url: '/api/tools/elevenlabs/audio', method: 'POST', diff --git a/apps/sim/tools/elevenlabs/model-input.ts b/apps/sim/tools/elevenlabs/model-input.ts index 41e87af0803..eebe879f2aa 100644 --- a/apps/sim/tools/elevenlabs/model-input.ts +++ b/apps/sim/tools/elevenlabs/model-input.ts @@ -1,11 +1,23 @@ -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' -import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' +import { + applyProjectedModelVisibleFileNames, + selectModelVisibleFileNames, +} from '@/lib/uploads/utils/model-input' -/** Selects exact resolver paths for audio content consumed by ElevenLabs transforms. */ -export function selectElevenLabsAudioModelInputPaths(params: { +/** Selects the filename serialized into ElevenLabs' multipart model request. */ +export function selectElevenLabsAudioFileNameModelInput(params: { audioFile?: unknown -}): readonly ResolvedSecretInputPath[] { - return selectModelBoundFileInputPaths(params.audioFile, ['audioFile'], { - includeName: true, - }) +}): Record { + if (!params.audioFile) return {} + return { audioFile: selectModelVisibleFileNames(params.audioFile) } +} + +/** Restores the audio file with only its projected filename changed. */ +export function applyProjectedElevenLabsAudioFileNameModelInput( + original: { audioFile?: unknown }, + projected: Record +): Record { + if (!Object.hasOwn(projected, 'audioFile')) return {} + return { + audioFile: applyProjectedModelVisibleFileNames(original.audioFile, projected.audioFile), + } } diff --git a/apps/sim/tools/elevenlabs/speech-to-speech.ts b/apps/sim/tools/elevenlabs/speech-to-speech.ts index aa040c43b12..2a7562915d9 100644 --- a/apps/sim/tools/elevenlabs/speech-to-speech.ts +++ b/apps/sim/tools/elevenlabs/speech-to-speech.ts @@ -1,4 +1,7 @@ -import { selectElevenLabsAudioModelInputPaths } from '@/tools/elevenlabs/model-input' +import { + applyProjectedElevenLabsAudioFileNameModelInput, + selectElevenLabsAudioFileNameModelInput, +} from '@/tools/elevenlabs/model-input' import type { ElevenLabsAudioResponse, ElevenLabsSpeechToSpeechParams, @@ -49,8 +52,9 @@ export const elevenLabsSpeechToSpeechTool: ToolConfig< request: { modelInput: { - mode: 'private-provenance', - inputPaths: selectElevenLabsAudioModelInputPaths, + mode: 'project', + select: selectElevenLabsAudioFileNameModelInput, + applyProjected: applyProjectedElevenLabsAudioFileNameModelInput, }, url: '/api/tools/elevenlabs/audio', method: 'POST', diff --git a/apps/sim/tools/exa/find_similar_links.ts b/apps/sim/tools/exa/find_similar_links.ts index 9b7b42a2f9d..b11aa5e3320 100644 --- a/apps/sim/tools/exa/find_similar_links.ts +++ b/apps/sim/tools/exa/find_similar_links.ts @@ -125,10 +125,6 @@ export const findSimilarLinksTool: ToolConfig< }, request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['url']], - }, url: 'https://api.exa.ai/findSimilar', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/exa/get_contents.ts b/apps/sim/tools/exa/get_contents.ts index 35c797810e6..f50e6cc8aaf 100644 --- a/apps/sim/tools/exa/get_contents.ts +++ b/apps/sim/tools/exa/get_contents.ts @@ -123,10 +123,6 @@ export const getContentsTool: ToolConfig ({ summaryQuery: params.summaryQuery }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => (params.summaryQuery || params.summary === true ? [['urls']] : []), - }, url: 'https://api.exa.ai/contents', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/extend/parser.ts b/apps/sim/tools/extend/parser.ts index c4aed624a46..0d8166f24e3 100644 --- a/apps/sim/tools/extend/parser.ts +++ b/apps/sim/tools/extend/parser.ts @@ -1,9 +1,5 @@ import { toError } from '@sim/utils/errors' import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' -import { - selectModelBoundFileInputPaths, - selectPreferredModelBoundFileInputPaths, -} from '@/lib/uploads/utils/model-input' import type { ExtendParserInput, ExtendParserOutput, @@ -64,17 +60,6 @@ export const extendParserTool: ToolConfig }, request: { - modelInput: { - mode: 'private-provenance', - inputPaths: (params) => - selectPreferredModelBoundFileInputPaths({ - file: params.file && typeof params.file === 'object' ? params.file : params.fileUpload, - filePath: params.filePath, - fileInputPath: params.file && typeof params.file === 'object' ? ['file'] : ['fileUpload'], - filePathInputPath: ['filePath'], - prefer: 'path', - }), - }, url: '/api/tools/extend/parse', method: 'POST', headers: (params) => ({ @@ -223,10 +208,6 @@ export const extendParserV2Tool: ToolConfig selectModelBoundFileInputPaths(params.file, ['file']), - }, url: '/api/tools/extend/parse', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/firecrawl/agent.ts b/apps/sim/tools/firecrawl/agent.ts index 6865407cbdc..13c77ffda28 100644 --- a/apps/sim/tools/firecrawl/agent.ts +++ b/apps/sim/tools/firecrawl/agent.ts @@ -61,10 +61,6 @@ export const agentTool: ToolConfig = { mode: 'project', select: (params) => ({ prompt: params.prompt, schema: params.schema }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['urls']], - }, method: 'POST', url: 'https://api.firecrawl.dev/v2/agent', headers: (params) => ({ diff --git a/apps/sim/tools/firecrawl/batch-scrape.ts b/apps/sim/tools/firecrawl/batch-scrape.ts index fdee7f96990..f2d23d82044 100644 --- a/apps/sim/tools/firecrawl/batch-scrape.ts +++ b/apps/sim/tools/firecrawl/batch-scrape.ts @@ -5,7 +5,6 @@ import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, applyFirecrawlScrapeOptionsModelInput, - hasFirecrawlModelInputFormat, selectFirecrawlFormatModelInput, selectFirecrawlScrapeOptionsModelInput, } from '@/tools/firecrawl/model-input' @@ -135,13 +134,6 @@ export const batchScrapeTool: ToolConfig - hasFirecrawlModelInputFormat(params.formats ?? params.scrapeOptions?.formats) - ? [['urls']] - : [], - }, method: 'POST', url: 'https://api.firecrawl.dev/v2/batch/scrape', headers: (params) => ({ diff --git a/apps/sim/tools/firecrawl/crawl.ts b/apps/sim/tools/firecrawl/crawl.ts index 73df61666ae..b69d2af11a9 100644 --- a/apps/sim/tools/firecrawl/crawl.ts +++ b/apps/sim/tools/firecrawl/crawl.ts @@ -5,7 +5,6 @@ import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, applyFirecrawlScrapeOptionsModelInput, - hasFirecrawlModelInputFormat, selectFirecrawlFormatModelInput, selectFirecrawlScrapeOptionsModelInput, } from '@/tools/firecrawl/model-input' @@ -124,16 +123,6 @@ export const crawlTool: ToolConfig } }, }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => - params.prompt || - hasFirecrawlModelInputFormat( - params.scrapeOptions ? params.scrapeOptions.formats : params.formats - ) - ? [['url']] - : [], - }, url: 'https://api.firecrawl.dev/v2/crawl', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/firecrawl/extract.ts b/apps/sim/tools/firecrawl/extract.ts index 5cb915bba20..1ef2b13a15f 100644 --- a/apps/sim/tools/firecrawl/extract.ts +++ b/apps/sim/tools/firecrawl/extract.ts @@ -104,10 +104,6 @@ export const extractTool: ToolConfig = { ), }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: () => [['urls']], - }, method: 'POST', url: 'https://api.firecrawl.dev/v2/extract', headers: (params) => ({ diff --git a/apps/sim/tools/firecrawl/parse.ts b/apps/sim/tools/firecrawl/parse.ts index 12cebc5fa90..d8d88fb90c2 100644 --- a/apps/sim/tools/firecrawl/parse.ts +++ b/apps/sim/tools/firecrawl/parse.ts @@ -1,8 +1,10 @@ -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' +import { + applyProjectedModelVisibleFileNames, + selectModelVisibleFileNames, +} from '@/lib/uploads/utils/model-input' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, - hasFirecrawlParseModelInput, selectFirecrawlFormatModelInput, } from '@/tools/firecrawl/model-input' import type { ParseParams, ParseResponse } from '@/tools/firecrawl/types' @@ -95,14 +97,24 @@ export const parseTool: ToolConfig = { request: { modelInput: { mode: 'project', - select: (params) => ({ formats: selectFirecrawlFormatModelInput(params.formats) }), + select: (params) => { + const file = selectModelVisibleFileNames(params.file) + return { + formats: selectFirecrawlFormatModelInput(params.formats), + ...(file === undefined ? {} : { file }), + } + }, applyProjected: (selectedParams, projectedSelection) => ({ formats: applyFirecrawlFormatModelInput(selectedParams.formats, projectedSelection.formats), + ...(Object.hasOwn(projectedSelection, 'file') + ? { + file: applyProjectedModelVisibleFileNames( + selectedParams.file, + projectedSelection.file + ), + } + : {}), }), - privateInputPaths: (params) => - hasFirecrawlParseModelInput(params) - ? selectModelBoundFileInputPaths(params.file, ['file'], { includeName: true }) - : [], }, method: 'POST', url: '/api/tools/firecrawl/parse', diff --git a/apps/sim/tools/firecrawl/scrape.ts b/apps/sim/tools/firecrawl/scrape.ts index 1412f5d73b1..c14b052a9ac 100644 --- a/apps/sim/tools/firecrawl/scrape.ts +++ b/apps/sim/tools/firecrawl/scrape.ts @@ -2,7 +2,6 @@ import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, applyFirecrawlScrapeOptionsModelInput, - hasFirecrawlModelInputFormat, selectFirecrawlFormatModelInput, selectFirecrawlScrapeOptionsModelInput, } from '@/tools/firecrawl/model-input' @@ -71,13 +70,6 @@ export const scrapeTool: ToolConfig = { } }, }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => - hasFirecrawlModelInputFormat(params.scrapeOptions?.formats ?? params.formats) - ? [['url']] - : [], - }, method: 'POST', url: 'https://api.firecrawl.dev/v2/scrape', headers: (params) => ({ diff --git a/apps/sim/tools/fireflies/upload_audio.ts b/apps/sim/tools/fireflies/upload_audio.ts index 43b90a1027e..62b78464001 100644 --- a/apps/sim/tools/fireflies/upload_audio.ts +++ b/apps/sim/tools/fireflies/upload_audio.ts @@ -1,4 +1,3 @@ -import { isPlainRecord } from '@sim/utils/object' import type { FirefliesUploadAudioParams, FirefliesUploadAudioResponse, @@ -70,14 +69,6 @@ export const firefliesUploadAudioTool: ToolConfig< modelInput: { mode: 'project', select: (params) => ({ language: params.language }), - privateInputPaths: (params) => { - if (isPlainRecord(params.audioFile)) { - if (params.audioFile.key) return [] - if (params.audioFile.url) return [['audioFile', 'url']] - if (params.audioFile.path) return [['audioFile', 'path']] - } - return params.audioUrl ? [['audioUrl']] : [] - }, }, url: '/api/tools/fireflies/upload-audio', method: 'POST', diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 3e1b28d1fed..ad87e5608d7 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -2645,220 +2645,48 @@ describe('Automatic Internal Route Detection', () => { } }) - it('rejects secret-derived opaque input before request formatting or network I/O', async () => { - const secret = 'quote" slash\\ newline\n123 true' + it('sends an explicitly resolved secret through an ordinary external integration input', async () => { const registry = new ResolvedSecretTraceRegistry([ - { name: 'OPAQUE_URL', plaintext: secret, encryptedValue: 'encrypted-opaque-secret' }, + { name: 'EXTERNAL_INPUT', plaintext: 'true', encryptedValue: 'encrypted-external-input' }, ]) - registry.recordResolvedAtInputPath('OPAQUE_URL', secret, ['payload']) - registry.recordResolvedInputProjection(['payload'], secret, '{{OPAQUE_URL}}') - const url = vi.fn(() => 'https://api.example.com/opaque') - const headers = vi.fn(() => ({ 'Content-Type': 'application/json' })) - const body = vi.fn((params: { payload: unknown }) => ({ payload: params.payload })) + registry.recordResolvedAtInputPath('EXTERNAL_INPUT', 'true', ['payload']) + registry.recordResolvedInputProjection(['payload'], 'true', '{{EXTERNAL_INPUT}}') const mockTool = { - id: 'test_external_opaque_model_tool', - name: 'Test External Opaque Model Tool', - description: 'Rejects resolved secrets in opaque model input', - version: '1.0.0', - params: { payload: { type: 'json', required: true } }, - request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets' as const, - inputPaths: () => [['payload']], - }, - url, - method: 'POST' as const, - headers, - body, - }, - transformResponse: vi.fn().mockResolvedValue({ success: true, output: {} }), - } - ;(tools as Record).test_external_opaque_model_tool = mockTool - - try { - const result = await executeTool( - 'test_external_opaque_model_tool', - { payload: { url: `https://example.com/${secret}` } }, - { resolvedSecretTraceRegistry: registry } - ) - - expect(result).toMatchObject({ - success: false, - error: 'Model input contains a resolved secret that cannot be safely projected', - }) - expect(url).not.toHaveBeenCalled() - expect(headers).not.toHaveBeenCalled() - expect(body).not.toHaveBeenCalled() - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - } finally { - Reflect.deleteProperty(tools, 'test_external_opaque_model_tool') - } - }) - - it('rejects opaque input with incomplete provenance before direct execution', async () => { - const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() - const directExecution = vi.fn().mockResolvedValue({ success: true, output: {} }) - const mockTool = { - id: 'test_direct_opaque_model_tool', - name: 'Test Direct Opaque Model Tool', - description: 'Rejects unavailable opaque provenance', - version: '1.0.0', - params: { payload: { type: 'json', required: true } }, - request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets' as const, - inputPaths: () => [['payload']], - }, - url: '', - method: 'POST' as const, - headers: () => ({}), - }, - directExecution, - } - ;(tools as Record).test_direct_opaque_model_tool = mockTool - - try { - const result = await executeTool( - 'test_direct_opaque_model_tool', - { payload: 'ordinary-input' }, - { resolvedSecretTraceRegistry: registry } - ) - - expect(result).toMatchObject({ - success: false, - error: 'Model input provenance is unavailable', - }) - expect(directExecution).not.toHaveBeenCalled() - } finally { - Reflect.deleteProperty(tools, 'test_direct_opaque_model_tool') - } - }) - - it('preserves legacy opaque execution when no provenance registry exists', async () => { - const inputPaths = vi.fn(() => [['payload']]) - const directExecution = vi - .fn() - .mockResolvedValue({ success: true, output: { payload: 'legacy-value' } }) - const mockTool = { - id: 'test_legacy_direct_opaque_model_tool', - name: 'Test Legacy Direct Opaque Model Tool', - description: 'Preserves legacy calls without provenance support', + id: 'test_external_integration_tool', + name: 'Test External Integration Tool', + description: 'Sends ordinary integration input unchanged', version: '1.0.0', params: { payload: { type: 'string', required: true } }, request: { - opaqueModelInput: { mode: 'reject-resolved-secrets' as const, inputPaths }, - url: '', - method: 'POST' as const, - headers: () => ({}), - }, - directExecution, - } - ;(tools as Record).test_legacy_direct_opaque_model_tool = mockTool - - try { - const result = await executeTool('test_legacy_direct_opaque_model_tool', { - payload: 'legacy-value', - }) - - expect(result.success).toBe(true) - expect(inputPaths).not.toHaveBeenCalled() - expect(directExecution).toHaveBeenCalledWith({ payload: 'legacy-value' }, undefined) - } finally { - Reflect.deleteProperty(tools, 'test_legacy_direct_opaque_model_tool') - } - }) - - it('skips an inactive opaque boundary even when unrelated provenance is incomplete', async () => { - const registry = new ResolvedSecretTraceRegistry() - registry.markIncomplete() - const directExecution = vi.fn().mockResolvedValue({ success: true, output: {} }) - const mockTool = { - id: 'test_inactive_direct_opaque_model_tool', - name: 'Test Inactive Direct Opaque Model Tool', - description: 'Skips inactive conditional opaque input', - version: '1.0.0', - params: { mode: { type: 'string', required: true } }, - request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets' as const, - inputPaths: () => [], - }, - url: '', - method: 'POST' as const, - headers: () => ({}), - }, - directExecution, - } - ;(tools as Record).test_inactive_direct_opaque_model_tool = mockTool - - try { - const result = await executeTool( - 'test_inactive_direct_opaque_model_tool', - { mode: 'ordinary' }, - { resolvedSecretTraceRegistry: registry } - ) - - expect(result.success).toBe(true) - expect(directExecution).toHaveBeenCalledTimes(1) - } finally { - Reflect.deleteProperty(tools, 'test_inactive_direct_opaque_model_tool') - } - }) - - it('preserves safe opaque bytes and sends no provenance metadata externally', async () => { - const registry = new ResolvedSecretTraceRegistry([ - { name: 'LOW_ENTROPY', plaintext: 'true', encryptedValue: 'encrypted-low-entropy' }, - ]) - registry.recordResolvedAtInputPath('LOW_ENTROPY', 'true', ['ordinary']) - registry.recordResolvedInputProjection(['ordinary'], 'true', '{{LOW_ENTROPY}}') - const opaquePayload = 'quote" slash\\ newline\n123' - const mockTool = { - id: 'test_safe_external_opaque_model_tool', - name: 'Test Safe External Opaque Model Tool', - description: 'Preserves safe opaque model input', - version: '1.0.0', - params: { - payload: { type: 'string', required: true }, - ordinary: { type: 'string', required: true }, - }, - request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets' as const, - inputPaths: () => [['payload']], - }, - url: 'https://api.example.com/safe-opaque', + url: 'https://api.example.com/integration', method: 'POST' as const, headers: () => ({ 'Content-Type': 'application/json' }), - body: (params: { payload: string; ordinary: string }) => ({ - payload: params.payload, - ordinary: params.ordinary, - }), + body: (params: { payload: string }) => ({ payload: params.payload }), }, transformResponse: vi.fn().mockResolvedValue({ success: true, output: {} }), } - ;(tools as Record).test_safe_external_opaque_model_tool = mockTool + ;(tools as Record).test_external_integration_tool = mockTool try { const result = await executeTool( - 'test_safe_external_opaque_model_tool', - { payload: opaquePayload, ordinary: 'true' }, + 'test_external_integration_tool', + { payload: 'true' }, { resolvedSecretTraceRegistry: registry } ) expect(result.success).toBe(true) expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( - 'https://api.example.com/safe-opaque', + 'https://api.example.com/integration', '93.184.216.34', expect.objectContaining({ - body: JSON.stringify({ payload: opaquePayload, ordinary: 'true' }), + body: JSON.stringify({ payload: 'true' }), headers: expect.not.objectContaining({ 'x-sim-private-model-input-provenance': expect.anything(), }), }) ) } finally { - Reflect.deleteProperty(tools, 'test_safe_external_opaque_model_tool') + Reflect.deleteProperty(tools, 'test_external_integration_tool') } }) diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index fcc55bd68df..22981884f81 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -37,10 +37,6 @@ import { INTERNAL_EXECUTION_DEADLINE_HEADER, serializeExecutionDeadlineHeader, } from '@/lib/execution/execution-deadline-header' -import { - OPAQUE_MODEL_INPUT_PROVENANCE_UNAVAILABLE_ERROR, - OPAQUE_MODEL_INPUT_RESOLVED_SECRET_ERROR, -} from '@/lib/execution/model-input-provenance' import { inspectPrivateToolMetadataEnvelope, inspectPrivateToolMetadataResponseCapability, @@ -66,7 +62,6 @@ import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' import { isResolvedSecretTraceProvenanceV1, - type ResolvedSecretInputPath, type ResolvedSecretTraceRegistry, } from '@/executor/utils/resolved-secret-trace-registry' import type { ErrorInfo } from '@/tools/error-extractors' @@ -95,39 +90,6 @@ const PRIVATE_MODEL_INPUT_DIRECT_EXECUTION_ERROR_MESSAGE = const PRIVATE_SECRET_PROVENANCE_DIRECT_EXECUTION_ERROR_MESSAGE = 'Private secret provenance is not supported by direct execution' -function assertOpaqueToolModelInputSafe( - tool: ToolConfig, - params: Record, - registry: ResolvedSecretTraceRegistry | undefined -): void { - const opaqueModelInput = tool.request.opaqueModelInput - if (!opaqueModelInput || !registry) return - if (opaqueModelInput.mode !== 'reject-resolved-secrets') { - throw new Error(OPAQUE_MODEL_INPUT_PROVENANCE_UNAVAILABLE_ERROR) - } - - let inputPaths: readonly ResolvedSecretInputPath[] - try { - inputPaths = opaqueModelInput.inputPaths(params) - } catch { - throw new Error(OPAQUE_MODEL_INPUT_PROVENANCE_UNAVAILABLE_ERROR) - } - if (inputPaths.length === 0) return - - let provenance - try { - provenance = registry.exportCommittedProvenanceForInputPaths(inputPaths) - } catch { - throw new Error(OPAQUE_MODEL_INPUT_PROVENANCE_UNAVAILABLE_ERROR) - } - if (!provenance.complete) { - throw new Error(OPAQUE_MODEL_INPUT_PROVENANCE_UNAVAILABLE_ERROR) - } - if (provenance.entries.length > 0) { - throw new Error(OPAQUE_MODEL_INPUT_RESOLVED_SECRET_ERROR) - } -} - function projectToolLogMetadata( metadata: Record, registry: ResolvedSecretTraceRegistry | undefined, @@ -1613,7 +1575,6 @@ async function executeToolImplementation( normalizeCopilotCredentialParams(contextParams) enforceCopilotCredentialSelection(toolId, tool, contextParams, scope) await resolveCopilotEnvReferences(tool, contextParams, scope, resolvedSecretTraceRegistry) - assertOpaqueToolModelInputSafe(tool, contextParams, resolvedSecretTraceRegistry) // Inject hosted API key if tool supports it and user didn't provide one const hostedKeyInfo = await injectHostedKeyIfNeeded( diff --git a/apps/sim/tools/jina/read_url.ts b/apps/sim/tools/jina/read_url.ts index c2395d9dfd1..c7c4b31029f 100644 --- a/apps/sim/tools/jina/read_url.ts +++ b/apps/sim/tools/jina/read_url.ts @@ -105,11 +105,6 @@ export const readUrlTool: ToolConfig = { }, request: { - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => - params.useReaderLMv2 === true || params.withGeneratedAlt === true ? [['url']] : [], - }, url: (params: ReadUrlParams) => { return `https://r.jina.ai/https://${params.url.replace(/^https?:\/\//, '')}` }, diff --git a/apps/sim/tools/nested-model-input-adapters.test.ts b/apps/sim/tools/nested-model-input-adapters.test.ts index d00aedc74e5..18e4a4def12 100644 --- a/apps/sim/tools/nested-model-input-adapters.test.ts +++ b/apps/sim/tools/nested-model-input-adapters.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { addFollowupTool, addFollowupV2Tool } from '@/tools/cursor/add_followup' import { launchAgentTool, launchAgentV2Tool } from '@/tools/cursor/launch_agent' import { flintGeneratePagesTool } from '@/tools/flint/generate_pages' @@ -9,6 +10,7 @@ import { mem0AddMemoriesTool } from '@/tools/mem0/add_memories' import { searchTextTool } from '@/tools/pinecone/search_text' import type { PineconeUpsertTextRecord } from '@/tools/pinecone/types' import { upsertTextTool } from '@/tools/pinecone/upsert_text' +import { prepareToolRequest } from '@/tools/request-transport' import { zepAddMessagesTool } from '@/tools/zep/add_messages' describe('nested model-input adapters', () => { @@ -19,13 +21,11 @@ describe('nested model-input adapters', () => { '$id omits the optional image key when no images were supplied', (tool, key, text) => { const modelInput = tool.request.modelInput - const opaqueModelInput = tool.request.opaqueModelInput - if (modelInput?.mode !== 'project' || !opaqueModelInput) { - throw new Error('Expected Cursor prompt descriptors') + if (modelInput?.mode !== 'project') { + throw new Error('Expected Cursor prompt projection') } expect(modelInput.select({ [key]: text })).toStrictEqual({ [key]: text }) - expect(opaqueModelInput.inputPaths({ [key]: text })).toEqual([]) } ) @@ -67,12 +67,11 @@ describe('nested model-input adapters', () => { ) it.each([launchAgentTool, launchAgentV2Tool])( - '$id projects prompt text and guards parsed images without changing their transport', + '$id selects effective image payloads without changing their serialized transport', (tool) => { const modelInput = tool.request.modelInput - const opaqueModelInput = tool.request.opaqueModelInput - if (modelInput?.mode !== 'project' || !opaqueModelInput) { - throw new Error('Expected Cursor launch descriptors') + if (modelInput?.mode !== 'project') { + throw new Error('Expected Cursor prompt projection') } const promptImages = JSON.stringify([ { @@ -90,10 +89,14 @@ describe('nested model-input adapters', () => { }) ).toStrictEqual({ promptText: 'Inspect this image', + promptImages: [ + { + data: 'quote" slash\\ newline\n123 true', + dimension: { width: 100, height: 200 }, + }, + ], }) - expect(opaqueModelInput.inputPaths({ promptImages })).toStrictEqual([['promptImages']]) - const body = tool.request.body?.({ apiKey: 'key', repository: 'https://github.com/acme/repo', @@ -113,9 +116,8 @@ describe('nested model-input adapters', () => { '$id preserves the existing empty-array fallback for malformed image JSON', (tool) => { const modelInput = tool.request.modelInput - const opaqueModelInput = tool.request.opaqueModelInput - if (modelInput?.mode !== 'project' || !opaqueModelInput) { - throw new Error('Expected Cursor follow-up descriptors') + if (modelInput?.mode !== 'project') { + throw new Error('Expected Cursor prompt projection') } expect( @@ -126,7 +128,6 @@ describe('nested model-input adapters', () => { promptImages: 'not-json', }) ).toStrictEqual({ followupPromptText: 'Continue' }) - expect(opaqueModelInput.inputPaths({ promptImages: 'not-json' })).toStrictEqual([]) const body = tool.request.body?.({ apiKey: 'key', @@ -138,6 +139,98 @@ describe('nested model-input adapters', () => { } ) + it('rejects an exact resolved secret inside serialized Cursor image data', () => { + const secret = 'nested-image-secret' + const promptImages = JSON.stringify([ + { data: `prefix-${secret}-suffix`, dimension: { width: 100, height: 200 } }, + ]) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CURSOR_IMAGE', plaintext: secret, encryptedValue: 'encrypted-cursor-image' }, + ]) + registry.recordResolvedAtInputPath('CURSOR_IMAGE', secret, ['promptImages']) + registry.recordResolvedInputProjection( + ['promptImages'], + promptImages, + JSON.stringify([ + { data: 'prefix-{{CURSOR_IMAGE}}-suffix', dimension: { width: 100, height: 200 } }, + ]) + ) + + expect(() => + prepareToolRequest( + launchAgentTool, + { + apiKey: 'cursor-key', + repository: 'https://github.com/acme/repo', + promptText: 'Inspect this image', + promptImages, + }, + registry + ) + ).toThrow('Model input could not be safely projected') + }) + + it('rejects a whole-value Cursor image secret instead of silently sending an empty array', () => { + const promptImages = JSON.stringify([ + { data: 'whole-image-secret', dimension: { width: 100, height: 200 } }, + ]) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'CURSOR_IMAGES', + plaintext: promptImages, + encryptedValue: 'encrypted-cursor-images', + }, + ]) + registry.recordResolvedAtInputPath('CURSOR_IMAGES', promptImages, ['promptImages']) + registry.recordResolvedInputProjection(['promptImages'], promptImages, '{{CURSOR_IMAGES}}') + + expect(() => + prepareToolRequest( + launchAgentTool, + { + apiKey: 'cursor-key', + repository: 'https://github.com/acme/repo', + promptText: 'Inspect this image', + promptImages, + }, + registry + ) + ).toThrow('Model input could not be safely projected') + }) + + it('projects Cursor prompt text while preserving ordinary serialized image bytes exactly', () => { + const promptImages = JSON.stringify([ + { data: 'ordinary-base64-bytes', dimension: { width: 100, height: 200 } }, + ]) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CURSOR_PROMPT', plaintext: 'private-prompt', encryptedValue: 'encrypted-prompt' }, + { name: 'UNUSED_LOW_ENTROPY', plaintext: 'a', encryptedValue: 'encrypted-unused' }, + ]) + registry.recordResolvedAtInputPath('CURSOR_PROMPT', 'private-prompt', ['promptText']) + registry.recordResolvedInputProjection( + ['promptText'], + 'Inspect private-prompt', + 'Inspect {{CURSOR_PROMPT}}' + ) + + const request = prepareToolRequest( + launchAgentTool, + { + apiKey: 'cursor-key', + repository: 'https://github.com/acme/repo', + promptText: 'Inspect private-prompt', + promptImages, + }, + registry + ) + const body = JSON.parse(request.body ?? '{}') + + expect(body.prompt).toStrictEqual({ + text: 'Inspect {{CURSOR_PROMPT}}', + images: [{ data: 'ordinary-base64-bytes', dimension: { width: 100, height: 200 } }], + }) + }) + it.each([ ['array', false], ['JSON string', true], diff --git a/apps/sim/tools/opaque-model-input-selectors.test.ts b/apps/sim/tools/opaque-model-input-selectors.test.ts index b4d4d5d4f0e..689b22486c2 100644 --- a/apps/sim/tools/opaque-model-input-selectors.test.ts +++ b/apps/sim/tools/opaque-model-input-selectors.test.ts @@ -2,39 +2,24 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { a2aSendMessageTool } from '@/tools/a2a/send_message' -import { runTaskTool as browserUseRunTaskTool } from '@/tools/browser_use/run_task' -import { contextDevExtractTool } from '@/tools/context_dev/extract' -import { contextDevExtractProductTool } from '@/tools/context_dev/extract_product' -import { contextDevExtractProductsTool } from '@/tools/context_dev/extract_products' -import { addFollowupTool, addFollowupV2Tool } from '@/tools/cursor/add_followup' -import { launchAgentTool, launchAgentV2Tool } from '@/tools/cursor/launch_agent' import { elevenLabsAudioIsolationTool } from '@/tools/elevenlabs/audio-isolation' import { elevenLabsSpeechToSpeechTool } from '@/tools/elevenlabs/speech-to-speech' -import { findSimilarLinksTool as exaFindSimilarLinksTool } from '@/tools/exa/find_similar_links' -import { getContentsTool as exaGetContentsTool } from '@/tools/exa/get_contents' import { extendParserTool, extendParserV2Tool } from '@/tools/extend/parser' -import { agentTool as firecrawlAgentTool } from '@/tools/firecrawl/agent' -import { batchScrapeTool as firecrawlBatchScrapeTool } from '@/tools/firecrawl/batch-scrape' -import { crawlTool as firecrawlCrawlTool } from '@/tools/firecrawl/crawl' -import { extractTool as firecrawlExtractTool } from '@/tools/firecrawl/extract' import { parseTool as firecrawlParseTool } from '@/tools/firecrawl/parse' -import { scrapeTool as firecrawlScrapeTool } from '@/tools/firecrawl/scrape' import { firefliesUploadAudioTool } from '@/tools/fireflies/upload_audio' -import { readUrlTool as jinaReadUrlTool } from '@/tools/jina/read_url' import { mistralParserTool, mistralParserV3Tool } from '@/tools/mistral/parser' import { pulseParserTool, pulseParserV2Tool } from '@/tools/pulse/parser' import { quiverImageToSvgTool } from '@/tools/quiver/image_to_svg' import { quiverTextToSvgTool } from '@/tools/quiver/text_to_svg' import { reductoParserTool, reductoParserV2Tool } from '@/tools/reducto/parser' +import { projectToolModelInputParams } from '@/tools/request-transport' import { assemblyaiSttTool, assemblyaiSttV2Tool } from '@/tools/stt/assemblyai' import { deepgramSttTool, deepgramSttV2Tool } from '@/tools/stt/deepgram' import { elevenLabsSttTool, elevenLabsSttV2Tool } from '@/tools/stt/elevenlabs' import { geminiSttTool, geminiSttV2Tool } from '@/tools/stt/gemini' -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' import { whisperSttTool, whisperSttV2Tool } from '@/tools/stt/whisper' -import { crawlTool as tavilyCrawlTool } from '@/tools/tavily/crawl' -import { mapTool as tavilyMapTool } from '@/tools/tavily/map' import { textractAnalyzeExpenseTool } from '@/tools/textract/analyze-expense' import { textractAnalyzeIdTool } from '@/tools/textract/analyze-id' import { textractParserTool, textractParserV2Tool } from '@/tools/textract/parser' @@ -42,475 +27,214 @@ import type { ToolConfig } from '@/tools/types' import { runwayVideoTool } from '@/tools/video/runway' import { visionTool } from '@/tools/vision/tool' -function selectOpaqueModelInputPaths( +const FILE = { + key: 'workspace/ws-1/report.pdf', + path: '/api/files/serve/workspace/ws-1/report.pdf', + url: 'https://storage.example/report.pdf?signature=private', + base64: 'raw-inline-field', + name: 'private-report.pdf', + size: 42, + type: 'application/pdf', +} + +function selectPrivateInputPaths( tool: ToolConfig, params: Record ): readonly (readonly string[])[] { const modelInput = tool.request.modelInput - if (!modelInput) throw new Error(`Missing model-input descriptor for ${tool.id}`) - + if (!modelInput) return [] if (modelInput.mode === 'private-provenance') return modelInput.inputPaths(params) - if (!modelInput.privateInputPaths) { - throw new Error(`Missing private provenance selector for ${tool.id}`) - } - return modelInput.privateInputPaths(params) + return modelInput.privateInputPaths?.(params) ?? [] } -function selectRejectedOpaqueModelInputPaths( - tool: ToolConfig, - params: Record -): readonly (readonly string[])[] { - const opaqueModelInput = tool.request.opaqueModelInput - if (!opaqueModelInput) throw new Error(`Missing opaque model-input descriptor for ${tool.id}`) - expect(opaqueModelInput.mode).toBe('reject-resolved-secrets') - return opaqueModelInput.inputPaths(params) +function getProjectingModelInput(tool: ToolConfig) { + const modelInput = tool.request.modelInput + expect(modelInput?.mode).toBe('project') + if (modelInput?.mode !== 'project' || !modelInput.applyProjected) { + throw new Error(`Expected ${tool.id} to apply projected model input`) + } + return modelInput } -describe('opaque model-input selectors', () => { +describe('file model-input selectors', () => { it.each([ extendParserTool, - mistralParserTool, + extendParserV2Tool, pulseParserTool, + pulseParserV2Tool, reductoParserTool, + reductoParserV2Tool, textractParserTool, - ])('%s mirrors legacy path-first input precedence', (tool) => { - expect( - selectOpaqueModelInputPaths(tool, { - filePath: ' https://example.com/effective.pdf ', - file: { key: 'unused-file', metadata: 'unused-secret' }, - fileUpload: { key: 'unused-upload', metadata: 'unused-secret' }, - }) - ).toEqual([['filePath']]) - }) - - it.each([extendParserV2Tool, pulseParserV2Tool, reductoParserV2Tool, textractParserV2Tool])( - '%s selects only the effective locator from normalized files', - (tool) => { - expect( - selectOpaqueModelInputPaths(tool, { - file: { - key: 'effective-key', - path: 'unused-path', - url: 'unused-url', - metadata: 'unused-secret', - }, - }) - ).toEqual([]) + textractParserV2Tool, + textractAnalyzeExpenseTool, + textractAnalyzeIdTool, + firefliesUploadAudioTool, + runwayVideoTool, + deepgramSttTool, + deepgramSttV2Tool, + assemblyaiSttTool, + assemblyaiSttV2Tool, + elevenLabsSttTool, + elevenLabsSttV2Tool, + geminiSttTool, + geminiSttV2Tool, + ])('$id does not attach private provenance for server-resolved locators', (tool) => { + expect(selectPrivateInputPaths(tool, { file: FILE, filePath: FILE.url })).toEqual([]) + expect(tool.request.modelInput?.mode).not.toBe('private-provenance') + if (tool.request.modelInput?.mode === 'project') { + expect(tool.request.modelInput.privateInputPaths).toBeUndefined() } - ) - - it('selects inline Mistral bytes without unrelated locators or metadata', () => { - expect( - selectOpaqueModelInputPaths(mistralParserV3Tool, { - file: { - base64: 'effective-bytes', - key: 'unused-key', - type: 'application/pdf', - metadata: 'unused-secret', - }, - }) - ).toEqual([['file', 'base64']]) }) - it('selects only the active Textract source for sync and async requests', () => { + it('keeps only inline Mistral bytes fail-closed', () => { expect( - selectOpaqueModelInputPaths(textractParserTool, { - processingMode: 'async', - s3Uri: ' s3://bucket/effective.pdf ', - filePath: 'https://example.com/unused.pdf', - file: { key: 'unused-key', metadata: 'unused-secret' }, - }) - ).toEqual([['s3Uri']]) - - expect( - selectOpaqueModelInputPaths(textractAnalyzeExpenseTool, { - processingMode: 'sync', - file: { key: 'effective-key', metadata: 'unused-secret' }, - filePath: 'https://example.com/unused.pdf', - s3Uri: 's3://bucket/unused.pdf', + selectPrivateInputPaths(mistralParserTool, { + filePath: 'https://example.com/document.pdf?token=private', }) ).toEqual([]) - expect( - selectOpaqueModelInputPaths(textractAnalyzeExpenseTool, { - processingMode: 'async', - s3Uri: ' s3://bucket/effective.pdf ', - file: { key: 'unused-key', metadata: 'unused-secret' }, + selectPrivateInputPaths(mistralParserTool, { + filePath: 'data:application/pdf;base64,c2VjcmV0', }) - ).toEqual([['s3Uri']]) - }) - - it('mirrors independent front and back precedence for Textract Analyze ID', () => { + ).toEqual([['filePath']]) expect( - selectOpaqueModelInputPaths(textractAnalyzeIdTool, { - file: { key: 'front-key', metadata: 'unused-secret' }, - filePath: 'https://example.com/unused-front.png', - filePathBack: ' https://example.com/back.png ', + selectPrivateInputPaths(mistralParserV3Tool, { + file: { ...FILE, base64: 'effective-inline-bytes' }, }) - ).toEqual([['filePathBack']]) + ).toEqual([['file', 'base64']]) }) - it('selects only the image source actually used by Vision', () => { + it('keeps only inline Vision bytes fail-closed', () => { expect( - selectOpaqueModelInputPaths(visionTool, { - imageFile: { - base64: 'effective-bytes', - key: 'unused-key', - type: 'image/png', - metadata: 'unused-secret', - }, - imageUrl: 'https://example.com/unused.png', + selectPrivateInputPaths(visionTool, { + imageUrl: 'https://example.com/image.png?token=private', }) - ).toEqual([['imageFile', 'base64']]) - }) - - it('keeps A2A attachment metadata that is transmitted and drops everything else', () => { + ).toEqual([]) expect( - selectOpaqueModelInputPaths(a2aSendMessageTool, { - files: [ - { - key: 'effective-key', - path: 'unused-path', - name: 'report.pdf', - type: 'application/pdf', - metadata: 'unused-secret', - }, - ], + selectPrivateInputPaths(visionTool, { + imageUrl: 'data:image/png;base64,c2VjcmV0', }) - ).toEqual([['files', '0', 'name']]) - }) - - it('keeps Firecrawl upload metadata that is transmitted and drops passthrough fields', () => { + ).toEqual([['imageUrl']]) expect( - selectOpaqueModelInputPaths(firecrawlParseTool, { - file: { - key: 'effective-key', - path: 'unused-path', - name: 'report.pdf', - type: 'application/pdf', - metadata: 'unused-secret', - }, + selectPrivateInputPaths(visionTool, { + imageFile: { ...FILE, base64: 'effective-inline-bytes' }, }) - ).toEqual([['file', 'name']]) + ).toEqual([['imageFile', 'base64']]) }) - it('mirrors Fireflies source precedence without selecting unused file metadata', () => { - expect(firefliesUploadAudioTool.request.modelInput?.mode).toBe('project') - if (firefliesUploadAudioTool.request.modelInput?.mode !== 'project') { - throw new Error('Fireflies metadata must use the shared model-input projector') - } + it('treats Quiver URLs as locators and data URLs as inline media', () => { + const serializedFile = JSON.stringify(FILE) + expect(selectPrivateInputPaths(quiverImageToSvgTool, { image: serializedFile })).toEqual([]) expect( - firefliesUploadAudioTool.request.modelInput.select({ - title: 'Meeting title', - language: 'en', - attendees: '[{"displayName":"Ada"}]', - clientReferenceId: 'reference-1', - webhook: 'https://example.com/private-callback', - }) - ).toEqual({ - language: 'en', - }) - - expect( - selectOpaqueModelInputPaths(firefliesUploadAudioTool, { - audioFile: { - key: 'effective-key', - url: 'https://example.com/unused.mp3', - path: '/api/files/serve/unused.mp3', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused-fallback.mp3', + selectPrivateInputPaths(quiverTextToSvgTool, { + references: [serializedFile, 'https://example.com/reference.png'], }) ).toEqual([]) - expect( - selectOpaqueModelInputPaths(firefliesUploadAudioTool, { - audioFile: { - url: 'https://example.com/effective.mp3', - path: '/api/files/serve/unused.mp3', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused-fallback.mp3', + selectPrivateInputPaths(quiverImageToSvgTool, { + image: 'data:image/png;base64,c2VjcmV0', }) - ).toEqual([['audioFile', 'url']]) - - expect( - selectOpaqueModelInputPaths(firefliesUploadAudioTool, { - audioUrl: 'https://example.com/fallback.mp3', - }) - ).toEqual([['audioUrl']]) + ).toEqual([['image']]) }) - it('normalizes Quiver file objects in both structured and serialized forms', () => { - const serialized = JSON.stringify({ - key: 'effective-key', - path: 'unused-path', - metadata: 'unused-secret', + it('projects A2A attachment names without rewriting file content or locators', () => { + const modelInput = getProjectingModelInput(a2aSendMessageTool) + expect(modelInput.select({ message: 'Analyze', files: [FILE] })).toEqual({ + message: 'Analyze', + data: undefined, + files: [{ name: 'private-report.pdf' }], }) - - expect(selectOpaqueModelInputPaths(quiverImageToSvgTool, { image: serialized })).toEqual([]) - expect( - selectOpaqueModelInputPaths(quiverTextToSvgTool, { - references: [serialized, { path: 'effective-path', metadata: 'unused-secret' }], - }) - ).toEqual([['references', '1', 'path']]) - }) - - it('selects only STT source metadata that the target provider transmits', () => { - expect( - selectSttAudioModelInputPaths({ - audioFile: { - key: 'uploaded-key', - name: 'uploaded.mp3', - type: 'audio/mpeg', - metadata: 'unused-secret', - }, - audioFileReference: { key: 'unused-reference' }, - audioUrl: 'https://example.com/unused.mp3', - }) - ).toEqual([]) - expect( - selectSttAudioModelInputPaths({ - audioFileReference: { - key: 'reference-key', - name: 'reference.wav', - type: 'audio/wav', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused.mp3', - }) - ).toEqual([]) - - expect( - selectSttAudioModelInputPaths( - { - audioFile: { - key: 'uploaded-key', - name: 'uploaded.mp3', - type: 'audio/mpeg', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused.mp3', - }, - { includeName: true } + modelInput.applyProjected( + { message: 'Analyze', data: undefined, files: [FILE] }, + { message: 'Analyze', data: undefined, files: [{ name: '{{FILE_NAME}}' }] } ) - ).toEqual([['audioFile', 'name']]) - - expect( - selectSttAudioModelInputPaths({ audioUrl: ' https://example.com/audio.mp3 ' }) - ).toEqual([['audioUrl']]) - }) - - it.each([ - deepgramSttTool, - deepgramSttV2Tool, - assemblyaiSttTool, - assemblyaiSttV2Tool, - elevenLabsSttTool, - elevenLabsSttV2Tool, - geminiSttTool, - geminiSttV2Tool, - whisperSttTool, - whisperSttV2Tool, - ])('%s projects rewritable STT metadata and transports audio provenance privately', (tool) => { - const modelInput = tool.request.modelInput - expect(modelInput?.mode).toBe('project') - if (modelInput?.mode !== 'project') { - throw new Error(`Missing shared STT metadata projection for ${tool.id}`) - } - expect(modelInput.privateInputPaths).toBeDefined() - expect( - modelInput.privateInputPaths?.({ - audioFileReference: { - key: 'effective-key', - name: 'audio.mp3', - type: 'audio/mpeg', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused.mp3', - }) - ).toEqual(tool.id.startsWith('stt_whisper') ? [['audioFileReference', 'name']] : []) - expect(modelInput.select({ language: 'en', prompt: 'Proper noun' })).toEqual( - tool.id.startsWith('stt_whisper') - ? { language: 'en', prompt: 'Proper noun' } - : { language: 'en' } + ).toEqual({ message: 'Analyze', data: undefined, files: [{ ...FILE, name: '{{FILE_NAME}}' }] }) + }) + + it('restores non-name A2A fields after the whole selected file param is projected', () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'FILE_NAME', plaintext: 'private-report', encryptedValue: 'encrypted-name' }, + { name: 'FILE_URL', plaintext: 'private', encryptedValue: 'encrypted-url' }, + { name: 'INLINE_FIELD', plaintext: 'raw-inline-field', encryptedValue: 'encrypted-inline' }, + ]) + registry.recordResolvedAtInputPath('FILE_NAME', 'private-report', ['files', '0', 'name']) + registry.recordResolvedInputProjection( + ['files', '0', 'name'], + 'private-report.pdf', + '{{FILE_NAME}}.pdf' + ) + registry.recordResolvedAtInputPath('FILE_URL', 'private', ['files', '0', 'url']) + registry.recordResolvedInputProjection( + ['files', '0', 'url'], + FILE.url, + 'https://storage.example/report.pdf?signature={{FILE_URL}}' + ) + registry.recordResolvedAtInputPath('INLINE_FIELD', 'raw-inline-field', ['files', '0', 'base64']) + registry.recordResolvedInputProjection( + ['files', '0', 'base64'], + FILE.base64, + '{{INLINE_FIELD}}' ) - }) - it.each([ - whisperSttTool, - whisperSttV2Tool, - deepgramSttTool, - deepgramSttV2Tool, - elevenLabsSttTool, - elevenLabsSttV2Tool, - assemblyaiSttTool, - assemblyaiSttV2Tool, - geminiSttTool, - geminiSttV2Tool, - ])('$id selects only opaque STT metadata transmitted upstream', (tool) => { expect( - selectOpaqueModelInputPaths(tool, { - audioFileReference: { - key: 'effective-key', - name: 'audio.mp3', - type: 'audio/mpeg', - metadata: 'unused-secret', - }, - audioUrl: 'https://example.com/unused.mp3', - }) - ).toEqual(tool.id.startsWith('stt_whisper') ? [['audioFileReference', 'name']] : []) + projectToolModelInputParams( + a2aSendMessageTool, + { message: 'Analyze', files: [FILE] }, + registry + ) + ).toEqual({ + message: 'Analyze', + data: undefined, + files: [{ ...FILE, name: '{{FILE_NAME}}.pdf' }], + }) }) - it('selects only the Runway visual reference fields consumed by the provider', () => { + it('projects the Firecrawl multipart filename without rewriting the stored file', () => { + const modelInput = getProjectingModelInput(firecrawlParseTool) + expect(modelInput.select({ file: FILE, formats: ['summary'] })).toEqual({ + formats: [{}], + file: { name: 'private-report.pdf' }, + }) expect( - selectOpaqueModelInputPaths(runwayVideoTool, { - visualReference: { - key: 'effective-key', - type: 'image/png', - name: 'unused-name.png', - metadata: 'unused-secret', - }, - }) - ).toEqual([]) + modelInput.applyProjected( + { file: FILE, formats: ['summary'] }, + { formats: [{}], file: { name: '{{FILE_NAME}}' } } + ) + ).toEqual({ formats: ['summary'], file: { ...FILE, name: '{{FILE_NAME}}' } }) }) - it.each([elevenLabsSpeechToSpeechTool, elevenLabsAudioIsolationTool])( - '$id selects only the audio source consumed by ElevenLabs', + it.each([whisperSttTool, whisperSttV2Tool])( + '$id projects the multipart filename without rewriting the audio source', (tool) => { + const modelInput = getProjectingModelInput(tool) expect( - selectOpaqueModelInputPaths(tool, { - audioFile: { - key: 'effective-key', - name: 'audio.wav', - type: 'audio/wav', - metadata: 'unused-secret', - }, - }) - ).toEqual([['audioFile', 'name']]) - } - ) - - it.each([ - [exaFindSimilarLinksTool, { url: 'https://example.com/similar' }, [['url']]], - [firecrawlAgentTool, { urls: ['https://example.com/agent'] }, [['urls']]], - [firecrawlExtractTool, { urls: ['https://example.com/extract'] }, [['urls']]], - [contextDevExtractTool, { url: 'https://example.com/extract' }, [['url']]], - [contextDevExtractProductTool, { url: 'https://example.com/product' }, [['url']]], - [contextDevExtractProductsTool, { domain: 'example.com' }, [['domain']]], - [browserUseRunTaskTool, { startUrl: 'https://example.com/start' }, [['startUrl']]], - ])('$id selects its exact always-model-bound opaque input', (tool, params, expected) => { - expect(selectRejectedOpaqueModelInputPaths(tool, params)).toStrictEqual(expected) - }) - - it('selects Exa content URLs only when summaries are model-generated', () => { - expect( - selectRejectedOpaqueModelInputPaths(exaGetContentsTool, { - urls: 'https://example.com/plain', - summary: false, - }) - ).toEqual([]) - expect( - selectRejectedOpaqueModelInputPaths(exaGetContentsTool, { - urls: 'https://example.com/summary', - summary: true, - }) - ).toEqual([['urls']]) - expect( - selectRejectedOpaqueModelInputPaths(exaGetContentsTool, { - urls: 'https://example.com/query', - summaryQuery: 'Summarize this', - }) - ).toEqual([['urls']]) - }) - - it('selects Firecrawl URLs only for formats or prompts that invoke models', () => { - expect( - selectRejectedOpaqueModelInputPaths(firecrawlScrapeTool, { - url: 'https://example.com/plain', - formats: ['markdown'], - }) - ).toEqual([]) - expect( - selectRejectedOpaqueModelInputPaths(firecrawlScrapeTool, { - url: 'https://example.com/json', - formats: [{ type: 'json', schema: { type: 'object' } }], - }) - ).toEqual([['url']]) - expect( - selectRejectedOpaqueModelInputPaths(firecrawlScrapeTool, { - url: 'https://example.com/string-json', - formats: ['json'], - }) - ).toEqual([['url']]) - expect( - selectRejectedOpaqueModelInputPaths(firecrawlBatchScrapeTool, { - urls: ['https://example.com/question'], - scrapeOptions: { formats: [{ type: 'question', question: 'What changed?' }] }, - }) - ).toStrictEqual([['urls']]) - expect( - selectRejectedOpaqueModelInputPaths(firecrawlCrawlTool, { - url: 'https://example.com/plain-crawl', - formats: ['markdown'], - }) - ).toEqual([]) - expect( - selectRejectedOpaqueModelInputPaths(firecrawlCrawlTool, { - url: 'https://example.com/prompted-crawl', - prompt: 'Focus on pricing', + modelInput.applyProjected( + { language: 'en', prompt: 'Hint', audioFile: FILE }, + { + language: 'en', + prompt: 'Hint', + audioFile: { name: '{{FILE_NAME}}' }, + } + ) + ).toEqual({ + language: 'en', + prompt: 'Hint', + audioFile: { ...FILE, name: '{{FILE_NAME}}' }, }) - ).toEqual([['url']]) - }) - - it.each([tavilyCrawlTool, tavilyMapTool])( - '$id selects its URL only when natural-language instructions are active', - (tool) => { - expect( - selectRejectedOpaqueModelInputPaths(tool, { - url: 'https://example.com/plain', - }) - ).toEqual([]) - expect( - selectRejectedOpaqueModelInputPaths(tool, { - url: 'https://example.com/instructed', - instructions: 'Find pricing', - }) - ).toEqual([['url']]) } ) - it('selects Jina Reader URLs only for ReaderLM or generated-alt processing', () => { - expect( - selectRejectedOpaqueModelInputPaths(jinaReadUrlTool, { url: 'https://example.com/plain' }) - ).toEqual([]) - expect( - selectRejectedOpaqueModelInputPaths(jinaReadUrlTool, { - url: 'https://example.com/readerlm', - useReaderLMv2: true, - }) - ).toEqual([['url']]) - expect( - selectRejectedOpaqueModelInputPaths(jinaReadUrlTool, { - url: 'https://example.com/alt', - withGeneratedAlt: true, - }) - ).toEqual([['url']]) - }) - - it.each([launchAgentTool, launchAgentV2Tool, addFollowupTool, addFollowupV2Tool])( - '$id selects the parsed Cursor image payload without rewriting it', + it.each([elevenLabsSpeechToSpeechTool, elevenLabsAudioIsolationTool])( + '$id projects the multipart filename without rewriting the audio source', (tool) => { - const promptImages = JSON.stringify([ - { data: 'quote" slash\\ newline\n123 true', dimension: { width: 10, height: 20 } }, - ]) - expect(selectRejectedOpaqueModelInputPaths(tool, { promptImages })).toStrictEqual([ - ['promptImages'], - ]) - expect(selectRejectedOpaqueModelInputPaths(tool, { promptImages: 'not-json' })).toStrictEqual( - [] - ) + const modelInput = getProjectingModelInput(tool) + expect(modelInput.select({ audioFile: FILE })).toEqual({ + audioFile: { name: 'private-report.pdf' }, + }) + expect( + modelInput.applyProjected({ audioFile: FILE }, { audioFile: { name: '{{FILE_NAME}}' } }) + ).toEqual({ audioFile: { ...FILE, name: '{{FILE_NAME}}' } }) } ) }) diff --git a/apps/sim/tools/pulse/parser.ts b/apps/sim/tools/pulse/parser.ts index 93af5b27934..3a3c8cc5019 100644 --- a/apps/sim/tools/pulse/parser.ts +++ b/apps/sim/tools/pulse/parser.ts @@ -1,9 +1,5 @@ import { toError } from '@sim/utils/errors' import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' -import { - selectModelBoundFileInputPaths, - selectPreferredModelBoundFileInputPaths, -} from '@/lib/uploads/utils/model-input' import type { PulseParserInput, PulseParserOutput, PulseParserV2Input } from '@/tools/pulse/types' import type { ToolConfig } from '@/tools/types' @@ -77,17 +73,6 @@ export const pulseParserTool: ToolConfig = }, request: { - modelInput: { - mode: 'private-provenance', - inputPaths: (params) => - selectPreferredModelBoundFileInputPaths({ - file: params.file && typeof params.file === 'object' ? params.file : params.fileUpload, - filePath: params.filePath, - fileInputPath: params.file && typeof params.file === 'object' ? ['file'] : ['fileUpload'], - filePathInputPath: ['filePath'], - prefer: 'path', - }), - }, url: '/api/tools/pulse/parse', method: 'POST', headers: () => { @@ -294,10 +279,6 @@ export const pulseParserV2Tool: ToolConfig selectModelBoundFileInputPaths(params.file, ['file']), - }, url: '/api/tools/pulse/parse', method: 'POST', headers: () => ({ diff --git a/apps/sim/tools/reducto/parser.ts b/apps/sim/tools/reducto/parser.ts index d8de3b68d1f..729e7044d88 100644 --- a/apps/sim/tools/reducto/parser.ts +++ b/apps/sim/tools/reducto/parser.ts @@ -1,9 +1,5 @@ import { toError } from '@sim/utils/errors' import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' -import { - selectModelBoundFileInputPaths, - selectPreferredModelBoundFileInputPaths, -} from '@/lib/uploads/utils/model-input' import type { ReductoParserInput, ReductoParserOutput, @@ -57,17 +53,6 @@ export const reductoParserTool: ToolConfig - selectPreferredModelBoundFileInputPaths({ - file: params.file && typeof params.file === 'object' ? params.file : params.fileUpload, - filePath: params.filePath, - fileInputPath: params.file && typeof params.file === 'object' ? ['file'] : ['fileUpload'], - filePathInputPath: ['filePath'], - prefer: 'path', - }), - }, url: '/api/tools/reducto/parse', method: 'POST', headers: (params) => { @@ -220,10 +205,6 @@ export const reductoParserV2Tool: ToolConfig selectModelBoundFileInputPaths(params.file, ['file']), - }, url: '/api/tools/reducto/parse', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/stt/assemblyai.ts b/apps/sim/tools/stt/assemblyai.ts index c291867447d..cb7c736c321 100644 --- a/apps/sim/tools/stt/assemblyai.ts +++ b/apps/sim/tools/stt/assemblyai.ts @@ -1,4 +1,3 @@ -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types' import { STT_ENTITY_OUTPUT_PROPERTIES, @@ -98,7 +97,6 @@ export const assemblyaiSttTool: ToolConfig = { modelInput: { mode: 'project', select: (params) => ({ language: params.language }), - privateInputPaths: selectSttAudioModelInputPaths, }, url: '/api/tools/stt', method: 'POST', diff --git a/apps/sim/tools/stt/deepgram.ts b/apps/sim/tools/stt/deepgram.ts index 8b35570e5ff..6c260278bdd 100644 --- a/apps/sim/tools/stt/deepgram.ts +++ b/apps/sim/tools/stt/deepgram.ts @@ -1,4 +1,3 @@ -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types' import { STT_SEGMENT_OUTPUT_PROPERTIES } from '@/tools/stt/types' import type { ToolConfig } from '@/tools/types' @@ -70,7 +69,6 @@ export const deepgramSttTool: ToolConfig = { modelInput: { mode: 'project', select: (params) => ({ language: params.language }), - privateInputPaths: selectSttAudioModelInputPaths, }, url: '/api/tools/stt', method: 'POST', diff --git a/apps/sim/tools/stt/elevenlabs.ts b/apps/sim/tools/stt/elevenlabs.ts index 9fe6647876a..1e637e9512e 100644 --- a/apps/sim/tools/stt/elevenlabs.ts +++ b/apps/sim/tools/stt/elevenlabs.ts @@ -1,4 +1,3 @@ -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types' import type { ToolConfig } from '@/tools/types' @@ -63,7 +62,6 @@ export const elevenLabsSttTool: ToolConfig = { modelInput: { mode: 'project', select: (params) => ({ language: params.language }), - privateInputPaths: selectSttAudioModelInputPaths, }, url: '/api/tools/stt', method: 'POST', diff --git a/apps/sim/tools/stt/gemini.ts b/apps/sim/tools/stt/gemini.ts index dc4f19dd1e4..e913518fe97 100644 --- a/apps/sim/tools/stt/gemini.ts +++ b/apps/sim/tools/stt/gemini.ts @@ -1,4 +1,3 @@ -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types' import type { ToolConfig } from '@/tools/types' @@ -63,7 +62,6 @@ export const geminiSttTool: ToolConfig = { modelInput: { mode: 'project', select: (params) => ({ language: params.language }), - privateInputPaths: selectSttAudioModelInputPaths, }, url: '/api/tools/stt', method: 'POST', diff --git a/apps/sim/tools/stt/model-input.test.ts b/apps/sim/tools/stt/model-input.test.ts index 13363e7916f..1617ed1f5c0 100644 --- a/apps/sim/tools/stt/model-input.test.ts +++ b/apps/sim/tools/stt/model-input.test.ts @@ -4,26 +4,58 @@ import { describe, expect, it } from 'vitest' import { deepgramSttTool } from '@/tools/stt/deepgram' import { whisperSttTool } from '@/tools/stt/whisper' -import type { ToolConfig } from '@/tools/types' const AUDIO_FILE = { + id: 'audio-1', name: 'secret-recording.mp3', + size: 42, + type: 'audio/mpeg', key: 'workspace/ws-1/secret-recording.mp3', -} - -function selectPrivateInputPaths(tool: ToolConfig): readonly (readonly string[])[] { - const modelInput = tool.request.modelInput - expect(modelInput?.mode).toBe('project') - if (modelInput?.mode !== 'project') throw new Error(`Expected ${tool.id} to project model input`) - return modelInput.privateInputPaths?.({ audioFile: AUDIO_FILE }) ?? [] + url: 'https://storage.example/audio.mp3?signature=private', + base64: 'raw-inline-field', } describe('STT model input provenance', () => { - it('excludes filenames by default when the provider receives only audio bytes', () => { - expect(selectPrivateInputPaths(deepgramSttTool)).toEqual([]) + it('does not attach opaque provenance for server-resolved audio locators', () => { + const modelInput = deepgramSttTool.request.modelInput + expect(modelInput?.mode).toBe('project') + if (modelInput?.mode !== 'project') throw new Error('Expected Deepgram to project input') + expect(modelInput.privateInputPaths).toBeUndefined() }) - it('selects the filename only for Whisper, which serializes it upstream', () => { - expect(selectPrivateInputPaths(whisperSttTool)).toEqual([['audioFile', 'name']]) + it('projects the Whisper filename while preserving its locator and inline fields', () => { + const modelInput = whisperSttTool.request.modelInput + expect(modelInput?.mode).toBe('project') + if (modelInput?.mode !== 'project' || !modelInput.applyProjected) { + throw new Error('Expected Whisper to apply projected input') + } + + const selected = modelInput.select({ + provider: 'whisper', + apiKey: 'key', + audioFile: AUDIO_FILE, + language: 'en', + prompt: 'hint', + }) + expect(selected).toEqual({ + language: 'en', + prompt: 'hint', + audioFile: { name: 'secret-recording.mp3' }, + }) + expect( + modelInput.applyProjected( + { audioFile: AUDIO_FILE, language: 'en', prompt: 'hint' }, + { + language: 'en', + prompt: 'hint', + audioFile: { name: '{{FILE_NAME}}' }, + } + ) + ).toEqual({ + language: 'en', + prompt: 'hint', + audioFile: { ...AUDIO_FILE, name: '{{FILE_NAME}}' }, + }) + expect(modelInput.privateInputPaths).toBeUndefined() }) }) diff --git a/apps/sim/tools/stt/model-input.ts b/apps/sim/tools/stt/model-input.ts index 8ab58a4c9df..65d68e07518 100644 --- a/apps/sim/tools/stt/model-input.ts +++ b/apps/sim/tools/stt/model-input.ts @@ -1,31 +1,48 @@ -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' -import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' +import { + applyProjectedModelVisibleFileNames, + selectModelVisibleFileNames, +} from '@/lib/uploads/utils/model-input' interface SttAudioModelInputParams { audioFile?: unknown audioFileReference?: unknown - audioUrl?: unknown } -/** Selects exact resolver paths for the single audio source consumed by the shared STT route. */ -export function selectSttAudioModelInputPaths( - params: SttAudioModelInputParams, - options: { includeName?: boolean } = {} -): readonly ResolvedSecretInputPath[] { - const fileOptions = { includeName: options.includeName ?? false } as const - +/** Selects the filename for the single audio source whose name is sent to a model provider. */ +export function selectSttAudioFileNameModelInput( + params: SttAudioModelInputParams +): Record { if (params.audioFile) { - return selectModelBoundFileInputPaths(params.audioFile, ['audioFile'], fileOptions) + return { audioFile: selectModelVisibleFileNames(params.audioFile) } } if (params.audioFileReference) { - return selectModelBoundFileInputPaths( - params.audioFileReference, - ['audioFileReference'], - fileOptions - ) + return { audioFileReference: selectModelVisibleFileNames(params.audioFileReference) } + } + return {} +} + +/** Restores the selected audio source with only its projected filename changed. */ +export function applyProjectedSttAudioFileNameModelInput( + original: SttAudioModelInputParams, + projected: Record +): Record { + const hasAudioFile = Object.hasOwn(projected, 'audioFile') + const hasAudioFileReference = Object.hasOwn(projected, 'audioFileReference') + if (hasAudioFile && hasAudioFileReference) { + throw new Error('Projected STT input contains multiple audio sources') + } + if (hasAudioFile) { + return { + audioFile: applyProjectedModelVisibleFileNames(original.audioFile, projected.audioFile), + } } - if (typeof params.audioUrl === 'string' && params.audioUrl.trim() !== '') { - return [['audioUrl']] + if (hasAudioFileReference) { + return { + audioFileReference: applyProjectedModelVisibleFileNames( + original.audioFileReference, + projected.audioFileReference + ), + } } - return [] + return {} } diff --git a/apps/sim/tools/stt/whisper.ts b/apps/sim/tools/stt/whisper.ts index 4447c20775c..6d0da9b8f86 100644 --- a/apps/sim/tools/stt/whisper.ts +++ b/apps/sim/tools/stt/whisper.ts @@ -1,4 +1,7 @@ -import { selectSttAudioModelInputPaths } from '@/tools/stt/model-input' +import { + applyProjectedSttAudioFileNameModelInput, + selectSttAudioFileNameModelInput, +} from '@/tools/stt/model-input' import type { SttParams, SttResponse, SttV2Params } from '@/tools/stt/types' import { STT_SEGMENT_OUTPUT_PROPERTIES } from '@/tools/stt/types' import type { ToolConfig } from '@/tools/types' @@ -102,8 +105,13 @@ export const whisperSttTool: ToolConfig = { select: (params) => ({ language: params.language, prompt: params.prompt, + ...selectSttAudioFileNameModelInput(params), + }), + applyProjected: (selectedParams, projectedSelection) => ({ + language: projectedSelection.language, + prompt: projectedSelection.prompt, + ...applyProjectedSttAudioFileNameModelInput(selectedParams, projectedSelection), }), - privateInputPaths: (params) => selectSttAudioModelInputPaths(params, { includeName: true }), }, url: '/api/tools/stt', method: 'POST', diff --git a/apps/sim/tools/tavily/crawl.ts b/apps/sim/tools/tavily/crawl.ts index 8b331adf60b..6027f175aab 100644 --- a/apps/sim/tools/tavily/crawl.ts +++ b/apps/sim/tools/tavily/crawl.ts @@ -107,10 +107,6 @@ export const crawlTool: ToolConfig = { mode: 'project', select: (params) => ({ instructions: params.instructions }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => (params.instructions ? [['url']] : []), - }, url: 'https://api.tavily.com/crawl', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/tavily/map.ts b/apps/sim/tools/tavily/map.ts index da6f0ddd7b7..b6b5920c75e 100644 --- a/apps/sim/tools/tavily/map.ts +++ b/apps/sim/tools/tavily/map.ts @@ -83,10 +83,6 @@ export const mapTool: ToolConfig = { mode: 'project', select: (params) => ({ instructions: params.instructions }), }, - opaqueModelInput: { - mode: 'reject-resolved-secrets', - inputPaths: (params) => (params.instructions ? [['url']] : []), - }, url: 'https://api.tavily.com/map', method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/textract/analyze-expense.ts b/apps/sim/tools/textract/analyze-expense.ts index 15311fddea7..2cc3da68ba9 100644 --- a/apps/sim/tools/textract/analyze-expense.ts +++ b/apps/sim/tools/textract/analyze-expense.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { selectPreferredModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import type { TextractAnalyzeExpenseOutput, TextractAnalyzeExpenseV2Input, @@ -115,22 +114,6 @@ export const textractAnalyzeExpenseTool: ToolConfig< }, request: { - modelInput: { - mode: 'private-provenance', - inputPaths: (params) => { - const processingMode = params.processingMode || 'sync' - if (processingMode === 'async') { - return typeof params.s3Uri === 'string' && params.s3Uri.trim() !== '' ? [['s3Uri']] : [] - } - return selectPreferredModelBoundFileInputPaths({ - file: params.file, - filePath: params.filePath, - fileInputPath: ['file'], - filePathInputPath: ['filePath'], - prefer: 'file', - }) - }, - }, url: '/api/tools/textract/analyze-expense', method: 'POST', headers: () => ({ diff --git a/apps/sim/tools/textract/analyze-id.ts b/apps/sim/tools/textract/analyze-id.ts index 9c13dde7dda..775283fa6df 100644 --- a/apps/sim/tools/textract/analyze-id.ts +++ b/apps/sim/tools/textract/analyze-id.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { selectPreferredModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import type { TextractAnalyzeIdOutput, TextractAnalyzeIdV2Input } from '@/tools/textract/types' import type { ToolConfig } from '@/tools/types' @@ -59,26 +58,6 @@ export const textractAnalyzeIdTool: ToolConfig { - const front = selectPreferredModelBoundFileInputPaths({ - file: params.file, - filePath: params.filePath, - fileInputPath: ['file'], - filePathInputPath: ['filePath'], - prefer: 'file', - }) - const back = selectPreferredModelBoundFileInputPaths({ - file: params.fileBack, - filePath: params.filePathBack, - fileInputPath: ['fileBack'], - filePathInputPath: ['filePathBack'], - prefer: 'file', - }) - return [...front, ...back] - }, - }, url: '/api/tools/textract/analyze-id', method: 'POST', headers: () => ({ diff --git a/apps/sim/tools/textract/parser.ts b/apps/sim/tools/textract/parser.ts index 4dade2a903a..dc8bb60779d 100644 --- a/apps/sim/tools/textract/parser.ts +++ b/apps/sim/tools/textract/parser.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { selectPreferredModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import type { TextractParserInput, TextractParserOutput, @@ -121,19 +120,6 @@ export const textractParserTool: ToolConfig { - const processingMode = params.processingMode || 'sync' - if (processingMode === 'async') { - return typeof params.s3Uri === 'string' && params.s3Uri.trim() !== '' ? [['s3Uri']] : [] - } - return selectPreferredModelBoundFileInputPaths({ - file: params.file && typeof params.file === 'object' ? params.file : params.fileUpload, - filePath: params.filePath, - fileInputPath: params.file && typeof params.file === 'object' ? ['file'] : ['fileUpload'], - filePathInputPath: ['filePath'], - prefer: 'path', - }) - }, }, url: '/api/tools/textract/parse', method: 'POST', diff --git a/apps/sim/tools/types.ts b/apps/sim/tools/types.ts index 3b125a0c989..94016b758ef 100644 --- a/apps/sim/tools/types.ts +++ b/apps/sim/tools/types.ts @@ -198,9 +198,10 @@ export interface ToolConfig

{ projectedSelection: Record ) => Record /** - * Selects opaque model-bound values that must not be rewritten, such as file bytes or - * signed URLs. Provenance is delivered privately to an authenticated internal route, - * which owns the final allow/reject decision. + * Selects inline model-bound values that must not be rewritten, such as file bytes or + * data URLs. Storage keys, paths, signed URLs, and remote URLs are locators rather than + * byte provenance. Metadata is delivered only to an authenticated internal route that + * owns the final allow/reject decision. */ privateInputPaths?: (params: P) => readonly ResolvedSecretInputPath[] } @@ -212,15 +213,6 @@ export interface ToolConfig

{ mode: 'private-provenance' inputPaths: (params: P) => readonly ResolvedSecretInputPath[] } - /** - * Selects model-bound values whose byte representation cannot be rewritten safely. The - * executor rejects the call before request formatting when committed provenance is incomplete - * or shows that the exact selection contains a resolved secret. Safe values are left unchanged. - */ - opaqueModelInput?: { - mode: 'reject-resolved-secrets' - inputPaths: (params: P) => readonly ResolvedSecretInputPath[] - } /** * Transports encrypted secret provenance across an authenticated internal * tool boundary without rewriting the selected value. diff --git a/apps/sim/tools/video/runway.ts b/apps/sim/tools/video/runway.ts index b681b1f5a71..c28e45d8c8e 100644 --- a/apps/sim/tools/video/runway.ts +++ b/apps/sim/tools/video/runway.ts @@ -1,4 +1,3 @@ -import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import type { ToolConfig } from '@/tools/types' import type { VideoParams, VideoResponse } from '@/tools/video/types' @@ -64,8 +63,6 @@ export const runwayVideoTool: ToolConfig = { modelInput: { mode: 'project', select: (params) => ({ prompt: params.prompt }), - privateInputPaths: (params) => - selectModelBoundFileInputPaths(params.visualReference, ['visualReference']), }, url: '/api/tools/video', method: 'POST', From a2805bbee667142fe2a6005a611c8dafa42559e8 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 7 Aug 2026 19:45:43 -0700 Subject: [PATCH 3/5] fix --- .../executor/execution/block-executor.test.ts | 114 ++++++++++- apps/sim/executor/execution/block-executor.ts | 68 +++++-- .../handlers/agent/agent-handler.test.ts | 91 +++++++-- .../executor/handlers/agent/agent-handler.ts | 105 +++++----- .../mothership/mothership-handler.test.ts | 184 ++++++++++++++++-- .../handlers/mothership/mothership-handler.ts | 29 ++- apps/sim/executor/types.ts | 2 + .../sim/lib/uploads/utils/model-input.test.ts | 26 +++ apps/sim/lib/uploads/utils/model-input.ts | 4 +- .../anthropic/streaming-tool-loop.test.ts | 1 - apps/sim/providers/runtime-context.test.ts | 14 +- apps/sim/providers/runtime-context.ts | 2 +- apps/sim/providers/tool-input-provenance.ts | 1 - apps/sim/providers/utils.ts | 1 - 14 files changed, 522 insertions(+), 120 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index fb7a7ac870f..b6eb5f853db 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -1064,9 +1064,111 @@ describe('BlockExecutor streaming pump', () => { }, state ) - return { executor, block, state } + return { executor, block, state, resolver } } + it('projects resolver-owned inputs for display without carrying them into output provenance', async () => { + const secret = 'x' + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext, _block, inputs) => { + expect(inputs.systemPrompt).toBe(secret) + const sourceRegistry = blockContext.resolvedSecretTraceRegistry + blockContext.resolvedSecretTraceRegistry = sourceRegistry?.forkForInputPaths([]) + return { content: 'Box' } + }, + } + const { executor, block, state } = createExecutor(handler) + block.config.params = { systemPrompt: '{{TOKEN}}' } + const ctx = createContext(state) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: secret, encryptedValue: 'encrypted-token' }, + ]) + ctx.environmentVariables = { TOKEN: secret } + ctx.resolvedSecretTraceRegistry = registry + + await executor.execute(ctx, createNode(block), block) + + expect(ctx.blockLogs[0]).toMatchObject({ + input: { systemPrompt: '{{TOKEN}}' }, + output: { content: 'Box' }, + }) + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [], + }) + expect(registry.getActiveMatches()).toEqual([]) + }) + + it('keeps terminal error output provenance separate from low-entropy input provenance', async () => { + const secret = 'x' + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext, _block, inputs) => { + expect(inputs.systemPrompt).toBe(secret) + const sourceRegistry = blockContext.resolvedSecretTraceRegistry + blockContext.resolvedSecretTraceRegistry = sourceRegistry?.forkForInputPaths([]) + throw new Error('Box') + }, + } + const { executor, block, state } = createExecutor(handler) + block.config.params = { systemPrompt: '{{TOKEN}}' } + const ctx = createContext(state) + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: secret, encryptedValue: 'encrypted-token' }, + ]) + ctx.environmentVariables = { TOKEN: secret } + ctx.resolvedSecretTraceRegistry = registry + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow('Agent: Box') + + expect(ctx.blockLogs[0]).toMatchObject({ + input: { systemPrompt: '{{TOKEN}}' }, + output: { error: 'Box' }, + }) + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [], + }) + expect(registry.getActiveMatches()).toEqual([]) + }) + + it('suppresses an incomplete display input without failing block execution', async () => { + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext) => { + blockContext.resolvedSecretTraceRegistry = + blockContext.resolvedSecretTraceRegistry?.forkForInputPaths([]) + return { content: 'done' } + }, + } + const { executor, block, state, resolver } = createExecutor(handler) + const inputs = { + userPrompt: 'Use the configured tool.', + tools: [{ params: { apiKey: 'unknown-value' } }], + } + vi.spyOn(resolver, 'resolveInputs').mockImplementation(async (blockContext) => { + await blockContext.resolvedSecretTraceRegistry?.importProvenanceForValueAtInputPath( + { version: 1 }, + 'unknown-value', + ['tools', '0', 'params', 'apiKey'], + { trusted: true } + ) + return inputs + }) + const ctx = createContext(state) + ctx.resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry() + + await expect(executor.execute(ctx, createNode(block), block)).resolves.toEqual({ + content: 'done', + }) + + expect(ctx.blockLogs[0]?.input).toEqual({}) + expect(ctx.blockLogs[0]?.output).toEqual({ content: 'done' }) + }) + function createAgentEventsStreamingHandler(options: { events: Array> attachThinkingOnDrain?: string @@ -1074,6 +1176,7 @@ describe('BlockExecutor streaming pump', () => { streamError?: Error onFullContent?: (content: string) => void | Promise resolvedSecret?: { name: string; value: string } + separateResultRegistry?: boolean }): BlockHandler { return { canHandle: () => true, @@ -1084,6 +1187,12 @@ describe('BlockExecutor streaming pump', () => { options.resolvedSecret.value ) } + const diagnosticRegistry = options.separateResultRegistry + ? blockContext.resolvedSecretTraceRegistry + : undefined + if (diagnosticRegistry) { + blockContext.resolvedSecretTraceRegistry = diagnosticRegistry.forkForInputPaths([]) + } const timeSegment: Record = { type: 'model', name: 'claude-test', @@ -1139,6 +1248,7 @@ describe('BlockExecutor streaming pump', () => { }, }, onFullContent: options.onFullContent, + diagnosticResolvedSecretTraceRegistry: diagnosticRegistry, } }, } @@ -1290,6 +1400,7 @@ describe('BlockExecutor streaming pump', () => { failAfterText: 'partial', streamError: rawError, resolvedSecret: { name: 'API_KEY', value: secret }, + separateResultRegistry: true, }) const { executor, block, state } = createExecutor(handler) const ctx = createContext(state) @@ -1297,6 +1408,7 @@ describe('BlockExecutor streaming pump', () => { { name: 'API_KEY', plaintext: secret, encryptedValue: 'encrypted-api-key' }, ]) ctx.onStream = async (streamingExec) => { + expect(streamingExec).not.toHaveProperty('diagnosticResolvedSecretTraceRegistry') const reader = streamingExec.stream.getReader() try { while (true) { diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 9b88075f449..b10039d7910 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -51,7 +51,10 @@ import { import { isJSONString } from '@/executor/utils/json' import { filterOutputForLog } from '@/executor/utils/output-filter' import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' -import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' +import type { + ResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' import { buildBranchNodeId, buildOuterBranchScopedId, @@ -104,6 +107,7 @@ export class BlockExecutor { const blockResolvedSecretTraceRegistry = parentResolvedSecretTraceRegistry?.forkForInputPaths( [] ) + const inputDisplayRegistry = blockResolvedSecretTraceRegistry const blockCtx = blockResolvedSecretTraceRegistry ? { ...ctx, resolvedSecretTraceRegistry: blockResolvedSecretTraceRegistry } : ctx @@ -195,7 +199,7 @@ export class BlockExecutor { } if (blockLog) { - blockLog.input = this.sanitizeInputsForLog(inputsForLog, block) + blockLog.input = this.projectInputsForDisplay(inputsForLog, block, inputDisplayRegistry) } } catch (error) { cleanupSelfReference?.() @@ -209,6 +213,7 @@ export class BlockExecutor { startTime, blockLog, inputsForLog, + inputDisplayRegistry, isSentinel, 'input_resolution' ) @@ -245,6 +250,8 @@ export class BlockExecutor { normalizeStringArray(blockCtx.selectedOutputs) ) } catch (streamError) { + blockCtx.resolvedSecretTraceRegistry = + blockCtx.resolvedSecretTraceRegistry?.forkForPropagatedEntries() // Timeout / drain failures may still have projected answer text — keep it // for the failed block output so logs match what the client already saw. streamingPartialOutput = streamingExec.execution?.output @@ -337,7 +344,7 @@ export class BlockExecutor { const displayOutput = filterOutputForLog(block.metadata?.id || '', normalizedOutput, { block, }) - const displayInput = this.sanitizeInputsForLog(inputsForLog, block) + const displayInput = this.projectInputsForDisplay(inputsForLog, block, inputDisplayRegistry) blockLog.input = displayInput const displayProvenance = settledBlockRegistry?.exportCommittedProvenanceForValue({ input: displayInput, @@ -374,6 +381,7 @@ export class BlockExecutor { startTime, blockLog, inputsForLog, + inputDisplayRegistry, isSentinel, 'execution', streamingPartialOutput @@ -459,6 +467,7 @@ export class BlockExecutor { startTime: number, blockLog: BlockLog | undefined, inputsForLog: Record, + inputDisplayRegistry: ResolvedSecretTraceRegistry | undefined, isSentinel: boolean, phase: 'input_resolution' | 'execution', streamingPartialOutput?: Record @@ -495,7 +504,7 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = true blockLog.error = undefined - blockLog.input = this.sanitizeInputsForLog(input, block) + blockLog.input = this.projectInputsForDisplay(input, block, inputDisplayRegistry) blockLog.output = filterOutputForLog(block.metadata?.id || '', softOutput, { block }) } @@ -505,7 +514,7 @@ export class BlockExecutor { }) if (!isSentinel && blockLog) { - const displayInput = this.sanitizeInputsForLog(input, block) + const displayInput = this.projectInputsForDisplay(input, block, inputDisplayRegistry) const displayOutput = filterOutputForLog(block.metadata?.id || '', softOutput, { block }) const displayProvenance = ctx.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue({ @@ -575,7 +584,7 @@ export class BlockExecutor { blockLog.durationMs = duration blockLog.success = false blockLog.error = errorMessage - blockLog.input = this.sanitizeInputsForLog(input, block) + blockLog.input = this.projectInputsForDisplay(input, block, inputDisplayRegistry) blockLog.output = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) if (ChildWorkflowError.isChildWorkflowError(error) && error.childTraceSpans.length > 0) { @@ -583,9 +592,17 @@ export class BlockExecutor { } } + const diagnosticRegistry = inputDisplayRegistry?.forkForToolCall() + if ( + diagnosticRegistry && + ctx.resolvedSecretTraceRegistry && + ctx.resolvedSecretTraceRegistry !== inputDisplayRegistry + ) { + diagnosticRegistry.mergeToolCallRegistry(ctx.resolvedSecretTraceRegistry) + } const errorDiagnostic = projectResolvedSecretDiagnosticError( error, - ctx.resolvedSecretTraceRegistry + diagnosticRegistry ?? ctx.resolvedSecretTraceRegistry ) this.execLogger.error( @@ -602,7 +619,7 @@ export class BlockExecutor { ? error.childWorkflowInstanceId : undefined const displayOutput = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) - const displayInput = this.sanitizeInputsForLog(input, block) + const displayInput = this.projectInputsForDisplay(input, block, inputDisplayRegistry) const displayProvenance = ctx.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue({ input: displayInput, output: displayOutput, @@ -729,6 +746,17 @@ export class BlockExecutor { return { result: output } } + /** Builds the log-facing input copy from resolver-recorded projections only. */ + private projectInputsForDisplay( + inputs: Record, + block: SerializedBlock | undefined, + registry: ResolvedSecretTraceRegistry | undefined + ): Record { + const projection = registry?.projectResolvedInputSelection(inputs) + if (projection && !projection.complete) return {} + return this.sanitizeInputsForLog(projection?.value ?? inputs, block) + } + /** * Sanitizes inputs for log display. * - Filters out system fields (UI-only, readonly, internal flags) @@ -974,6 +1002,16 @@ export class BlockExecutor { const piiEnabled = Boolean(ctx.piiBlockOutputRedaction?.enabled) // Live-forward only when a client stream exists and PII redaction is off. const forwardToClient = Boolean(ctx.onStream) && !piiEnabled + const projectStreamDiagnosticError = (error: unknown): Record => { + const sourceRegistry = streamingExec.diagnosticResolvedSecretTraceRegistry + const resultRegistry = ctx.resolvedSecretTraceRegistry + if (!sourceRegistry || sourceRegistry === resultRegistry) { + return projectResolvedSecretDiagnosticError(error, resultRegistry) + } + const diagnosticRegistry = sourceRegistry.forkForToolCall() + if (resultRegistry) diagnosticRegistry.mergeToolCallRegistry(resultRegistry) + return projectResolvedSecretDiagnosticError(error, diagnosticRegistry) + } const responseFormat = resolvedInputs?.responseFormat ?? @@ -993,6 +1031,10 @@ export class BlockExecutor { let processedClientStream: ReadableStream | undefined if (forwardToClient && ctx.onStream && pump.textStream) { + const { + diagnosticResolvedSecretTraceRegistry: _diagnosticRegistry, + ...streamingExecutionForConsumer + } = streamingExec processedClientStream = streamingResponseFormatProcessor.processStream( pump.textStream, blockId, @@ -1005,7 +1047,7 @@ export class BlockExecutor { // with `pump.run()`. onStreamPromise = ctx .onStream({ - ...streamingExec, + ...streamingExecutionForConsumer, stream: processedClientStream, streamFormat: 'text', subscribe: pump.subscribe, @@ -1018,7 +1060,7 @@ export class BlockExecutor { .catch(async (error) => { this.execLogger.error('Error in onStream callback', { blockId, - ...projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry), + ...projectStreamDiagnosticError(error), }) await processedClientStream?.cancel().catch(() => {}) }) @@ -1030,7 +1072,7 @@ export class BlockExecutor { } catch (error) { this.execLogger.error('Error reading stream for block', { blockId, - ...projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry), + ...projectStreamDiagnosticError(error), }) if (onStreamPromise) { await onStreamPromise.catch(() => {}) @@ -1121,7 +1163,7 @@ export class BlockExecutor { } catch (error) { this.execLogger.warn('Failed to parse streamed content for response format', { blockId, - ...projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry), + ...projectStreamDiagnosticError(error), }) } } @@ -1136,7 +1178,7 @@ export class BlockExecutor { } catch (error) { this.execLogger.error('onFullContent callback failed', { blockId, - ...projectResolvedSecretDiagnosticError(error, ctx.resolvedSecretTraceRegistry), + ...projectStreamDiagnosticError(error), }) } } diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 08a03a937b6..f1b121bac60 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -529,7 +529,6 @@ describe('AgentBlockHandler', () => { apiKey: 'test-api-key', } const rawInputs = structuredClone(inputs) - await handler.execute(mockContext, mockBlock, inputs) expect(mockExecuteProviderRequest.mock.calls[0][1].messages.at(-1)?.files).toEqual([ @@ -732,7 +731,6 @@ describe('AgentBlockHandler', () => { ], } const rawInputs = structuredClone(inputs) - await handler.execute(mockContext, mockBlock, inputs) expect(mockExecuteProviderRequest.mock.calls[0][1].messages[0].files).toEqual([ @@ -1154,9 +1152,69 @@ describe('AgentBlockHandler', () => { { role: 'system', content: 'Box eSign stays public' }, { role: 'user', content: 'Box {{TOKEN}}' }, ]) - expect(runtimeContext.resolvedSecretTraceRegistry.getActiveMatches()).toEqual([ - { plaintext: 'x', replacement: '{{TOKEN}}' }, + expect(runtimeContext.resolvedSecretTraceRegistry.getActiveMatches()).toEqual([]) + }) + + it('does not carry a projected system prompt into Agent output provenance', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext: 'x', encryptedValue: 'encrypted-token' }, ]) + registry.recordResolvedAtInputPath('TOKEN', 'x', ['systemPrompt']) + registry.recordResolvedInputProjection(['systemPrompt'], 'Use x', 'Use {{TOKEN}}') + mockContext.resolvedSecretTraceRegistry = registry + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: 'Box', + model: 'mock-model', + tokens: { input: 10, output: 20, total: 30 }, + toolCalls: [], + cost: 0.001, + timing: { total: 100 }, + }) + + const inputs = { + model: 'gpt-4o', + systemPrompt: 'Use x', + userPrompt: 'Continue', + } + const result = await handler.execute(mockContext, mockBlock, inputs) + + const [, providerRequest, runtimeContext] = mockExecuteProviderRequest.mock.calls[0] + expect(providerRequest.messages).toEqual([ + { role: 'system', content: 'Use {{TOKEN}}' }, + { role: 'user', content: 'Continue' }, + ]) + expect(runtimeContext.resolvedSecretTraceRegistry.getActiveMatches()).toEqual([]) + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(inputs.systemPrompt).toBe('Use x') + expect( + mockContext.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(result) + ).toEqual({ version: 1, complete: true, entries: [] }) + }) + + it('keeps input provenance active for provider error diagnostics', async () => { + const plaintext = 'provider-credential-secret' + const registry = new ResolvedSecretTraceRegistry([ + { name: 'TOKEN', plaintext, encryptedValue: 'encrypted-token' }, + ]) + registry.recordResolvedAtInputPath('TOKEN', plaintext, ['systemPrompt']) + registry.recordResolvedInputProjection(['systemPrompt'], `Use ${plaintext}`, 'Use {{TOKEN}}') + mockContext.resolvedSecretTraceRegistry = registry + mockExecuteProviderRequest.mockRejectedValueOnce(new Error(`Provider rejected ${plaintext}`)) + const inputs = { + model: 'gpt-4o', + systemPrompt: `Use ${plaintext}`, + userPrompt: 'Continue', + } + + await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( + `Provider rejected ${plaintext}` + ) + + expect(inputs.systemPrompt).toBe(`Use ${plaintext}`) + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + const logged = JSON.stringify(mockAgentLogger.error.mock.calls) + expect(logged).not.toContain(plaintext) + expect(logged).toContain('Provider rejected {{TOKEN}}') }) it('projects exact message call arguments without mutating protocol structure or raw input', async () => { @@ -1201,7 +1259,6 @@ describe('AgentBlockHandler', () => { ], } const rawInputs = structuredClone(inputs) - await handler.execute(mockContext, mockBlock, inputs) expect(mockExecuteProviderRequest.mock.calls[0][1].messages[0]).toEqual({ @@ -1293,7 +1350,7 @@ describe('AgentBlockHandler', () => { expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) - it('binds a resolved tool preset to the exact formatted provider tool instance', async () => { + it('binds a resolved tool preset without activating it before the exact tool runs', async () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'API_KEY', plaintext: 'x', encryptedValue: 'encrypted-api-key' }, ]) @@ -1325,10 +1382,13 @@ describe('AgentBlockHandler', () => { expect(providerTool.params).toEqual({ apiKey: 'x' }) expect(providerTool).not.toHaveProperty('__resolvedSecretTraceProvenance') expect(getProviderToolInputProvenance(providerTool)).toEqual({ - registry: runtimeContext.resolvedSecretTraceRegistry, + registry, sourcePath: ['tools', '0', 'params'], projectedParams: { apiKey: '{{API_KEY}}' }, }) + expect(runtimeContext.resolvedSecretTraceRegistry).not.toBe(registry) + expect(runtimeContext.resolvedSecretTraceRegistry.getActiveMatches()).toEqual([]) + expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) }) it('omits a tool with unknown hidden preset provenance without blocking the public prompt', async () => { @@ -2040,7 +2100,7 @@ describe('AgentBlockHandler', () => { expect(inputs.responseFormat).toEqual({ name: '{{FORMAT_NAME}}', schema: { type: 'object', properties: {} }, - strict: 'locked', + strict: '{{STRICT_VALUE}}', }) expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) expect(mockExecuteProviderRequest).not.toHaveBeenCalled() @@ -2080,6 +2140,7 @@ describe('AgentBlockHandler', () => { ) expect(inputs.responseFormat).toBe(projectedResponseFormat) + expect(inputs.responseFormat).toContain('"strict":"{{STRICT_VALUE}}"') expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) @@ -2126,7 +2187,7 @@ describe('AgentBlockHandler', () => { expect(responseFormat).toContain('private-schema') }) - it('excludes an aliased persisted response format name from block output provenance', async () => { + it('excludes projected persisted response format fields from block output provenance', async () => { const responseFormat = JSON.stringify({ name: 'x', schema: { @@ -2180,19 +2241,11 @@ describe('AgentBlockHandler', () => { const snapshot = modelRegistry.getModelEgressSnapshot() expect(snapshot.complete).toBe(true) if (!snapshot.complete) throw new Error('Expected complete model provenance') - expect(snapshot.matches).toContainEqual({ - plaintext: 'classified', - replacement: '{{DESCRIPTION}}', - }) - expect(snapshot.matches.map((match) => match.plaintext)).not.toContain('x') + expect(snapshot.matches).toEqual([]) const blockSnapshot = mockContext.resolvedSecretTraceRegistry?.getModelEgressSnapshot() expect(blockSnapshot?.complete).toBe(true) if (!blockSnapshot?.complete) throw new Error('Expected complete block provenance') - expect(blockSnapshot.matches).toContainEqual({ - plaintext: 'classified', - replacement: '{{DESCRIPTION}}', - }) - expect(blockSnapshot.matches.map((match) => match.plaintext)).not.toContain('x') + expect(blockSnapshot.matches).toEqual([]) expect(handlerInputs.responseFormat).toContain('{{FORMAT_NAME}}') }) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index d2f17d519b2..fb0c5e844da 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -103,7 +103,6 @@ interface IndexedToolInput { interface FormattedAgentTools { tools: ProviderToolConfig[] inputProvenance: Map> - sourcePaths: ResolvedSecretInputPath[] } class AgentToolInputSafetyError extends Error { @@ -252,7 +251,6 @@ export class AgentBlockHandler implements BlockHandler { ...modelInputProjection.value, responseFormat: responseFormatProjection.value, } - const modelInputPaths = [...coreModelInputPaths, ...responseFormatProjection.inputPaths] const projectedToolInputs = this.projectToolInputsForProvenance(ctx, inputs.tools || []) await this.validateToolPermissions(ctx, filteredInputs.tools || []) @@ -358,17 +356,13 @@ export class AgentBlockHandler implements BlockHandler { settlePrivateAgentSelectors() - const modelRuntimeRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths([ - ...modelInputPaths, - ...fileProjection.modelBoundInputPaths, - ...formatted.sourcePaths, - ]) - if (modelRuntimeRegistry) { - ctx.resolvedSecretTraceRegistry = modelRuntimeRegistry + const settledInputRegistry = ctx.resolvedSecretTraceRegistry + const resultRegistry = settledInputRegistry?.forkForInputPaths([]) + if (resultRegistry && settledInputRegistry) { for (const [tool, provenance] of formatted.inputProvenance) { registerProviderToolInputProvenance(tool, { ...provenance, - registry: modelRuntimeRegistry, + registry: settledInputRegistry, }) } } @@ -377,8 +371,9 @@ export class AgentBlockHandler implements BlockHandler { providerRequest, block, responseFormat, - modelRuntimeRegistry + resultRegistry ) + if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry if (autoRouting && autoRouting.billableRoutingCost > 0) { this.applyRoutingCost(result, autoRouting.billableRoutingCost) @@ -389,18 +384,26 @@ export class AgentBlockHandler implements BlockHandler { } if (this.isStreamingExecution(result)) { + const streamingResult = result as StreamingExecution + streamingResult.diagnosticResolvedSecretTraceRegistry = settledInputRegistry if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { return this.wrapStreamForMemoryPersistence( ctx, filteredInputs, - result as StreamingExecution + streamingResult, + settledInputRegistry ) } - return result + return streamingResult } if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { - await this.persistResponseToMemory(ctx, filteredInputs, result as BlockOutput) + await this.persistResponseToMemory( + ctx, + filteredInputs, + result as BlockOutput, + settledInputRegistry + ) } return result @@ -648,7 +651,7 @@ export class AgentBlockHandler implements BlockHandler { projectedToolInputs?: ToolInput[] ): Promise { if (!Array.isArray(inputTools)) { - return { tools: [], inputProvenance: new Map(), sourcePaths: [] } + return { tools: [], inputProvenance: new Map() } } const filtered = inputTools @@ -678,7 +681,6 @@ export class AgentBlockHandler implements BlockHandler { ProviderToolConfig, Omit >() - const sourcePaths: ResolvedSecretInputPath[] = [] const trackInputProvenance = ( formattedTool: ProviderToolConfig | null, @@ -703,7 +705,6 @@ export class AgentBlockHandler implements BlockHandler { formattedTool ), }) - sourcePaths.push(sourcePath) return formattedTool } @@ -723,13 +724,7 @@ export class AgentBlockHandler implements BlockHandler { } if (tool.type === 'custom-tool' && (tool.schema || tool.customToolId)) { return trackInputProvenance( - await this.createCustomTool( - ctx, - tool, - projectedToolInputs?.[toolIndex], - toolIndex, - sourcePaths - ), + await this.createCustomTool(ctx, tool, projectedToolInputs?.[toolIndex], toolIndex), { tool, toolIndex, @@ -764,20 +759,15 @@ export class AgentBlockHandler implements BlockHandler { ctx, mcpTools, trackInputProvenance, - projectedToolInputs, - sourcePaths + projectedToolInputs ) const allTools = [...otherResults, ...mcpResults] - const orderedSourcePaths = [...new Map(sourcePaths.map((path) => [JSON.stringify(path), path]))] - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([, path]) => path) return { tools: allTools.filter( (tool): tool is ProviderToolConfig => tool !== null && tool !== undefined ), inputProvenance, - sourcePaths: orderedSourcePaths, } } @@ -829,8 +819,7 @@ export class AgentBlockHandler implements BlockHandler { ctx: ExecutionContext, tool: ToolInput, projectedTool?: ToolInput, - toolIndex?: number, - modelInputPaths?: ResolvedSecretInputPath[] + toolIndex?: number ): Promise { const userProvidedParams = tool.params || {} @@ -874,10 +863,6 @@ export class AgentBlockHandler implements BlockHandler { ], 'Agent structural model inputs cannot contain secret references' ) - if (tool.schema?.function?.description !== undefined) { - modelInputPaths?.push([...functionRoot, 'description']) - } - modelInputPaths?.push(...schemaPaths.annotationInputPaths) } if (!schema?.function) { @@ -995,8 +980,7 @@ export class AgentBlockHandler implements BlockHandler { formattedTool: ProviderToolConfig | null, entry: IndexedToolInput ) => ProviderToolConfig | null, - projectedToolInputs?: ToolInput[], - modelInputPaths?: ResolvedSecretInputPath[] + projectedToolInputs?: ToolInput[] ): Promise> { if (mcpTools.length === 0) return [] @@ -1043,8 +1027,7 @@ export class AgentBlockHandler implements BlockHandler { ctx, tool, projectedToolInputs?.[entry.toolIndex], - entry.toolIndex, - modelInputPaths + entry.toolIndex ) if (created) results.push(trackInputProvenance(created, entry)) } catch (error) { @@ -1079,8 +1062,7 @@ export class AgentBlockHandler implements BlockHandler { ctx: ExecutionContext, tool: ToolInput, projectedTool?: ToolInput, - toolIndex?: number, - modelInputPaths?: ResolvedSecretInputPath[] + toolIndex?: number ): Promise { const { serverId, toolName, serverName, ...userProvidedParams } = tool.params || {} const projectedSchema = projectedTool?.schema ?? tool.schema @@ -1112,10 +1094,6 @@ export class AgentBlockHandler implements BlockHandler { schemaPaths.semanticInputPaths, 'Agent structural model inputs cannot contain secret references' ) - modelInputPaths?.push(...schemaPaths.annotationInputPaths) - if (!schemaDescription) { - modelInputPaths?.push(['tools', String(toolIndex), 'params', 'serverName']) - } } return this.buildMcpTool({ serverId, @@ -1831,8 +1809,11 @@ export class AgentBlockHandler implements BlockHandler { if (!sourceRegistry || privateInputPaths.length === 0) return const privateRoots = new Set(privateInputPaths.map((path) => path[0])) + const displayInputPaths = privateRoots.has('responseFormat') + ? [...privateInputPaths, ['responseFormat']] + : privateInputPaths const displayProjection = sourceRegistry - .forkForInputPaths(privateInputPaths) + .forkForInputPaths(displayInputPaths) .projectResolvedInputSelection({ responseFormat: inputs.responseFormat, tools: inputs.tools, @@ -2388,7 +2369,19 @@ export class AgentBlockHandler implements BlockHandler { return this.processProviderResponse(response, block, responseFormat, ctx) } catch (error) { - this.handleExecutionError(error, providerStartTime, providerId, model, ctx, block) + const sourceRegistry = ctx.resolvedSecretTraceRegistry + if (sourceRegistry && modelRuntimeRegistry && sourceRegistry !== modelRuntimeRegistry) { + const diagnosticRegistry = sourceRegistry.forkForToolCall() + diagnosticRegistry.mergeToolCallRegistry(modelRuntimeRegistry) + ctx.resolvedSecretTraceRegistry = diagnosticRegistry + } + try { + this.handleExecutionError(error, providerStartTime, providerId, model, ctx, block) + } finally { + if (modelRuntimeRegistry) { + ctx.resolvedSecretTraceRegistry = modelRuntimeRegistry.forkForPropagatedEntries() + } + } throw error } } @@ -2449,7 +2442,8 @@ export class AgentBlockHandler implements BlockHandler { private wrapStreamForMemoryPersistence( ctx: ExecutionContext, inputs: AgentInputs, - streamingExec: StreamingExecution + streamingExec: StreamingExecution, + diagnosticRegistry?: ResolvedSecretTraceRegistry ): StreamingExecution { return { ...streamingExec, @@ -2458,10 +2452,13 @@ export class AgentBlockHandler implements BlockHandler { try { await memoryService.appendToMemory(ctx, inputs, { role: 'assistant', content }) } catch (error) { + const diagnosticCtx = diagnosticRegistry + ? { ...ctx, resolvedSecretTraceRegistry: diagnosticRegistry } + : ctx logger.error( 'Failed to persist streaming response', projectAgentDiagnosticMetadata( - ctx, + diagnosticCtx, getErrorDiagnosticMetadata(error), getErrorDiagnosticFallback(error) ) @@ -2474,7 +2471,8 @@ export class AgentBlockHandler implements BlockHandler { private async persistResponseToMemory( ctx: ExecutionContext, inputs: AgentInputs, - result: BlockOutput + result: BlockOutput, + diagnosticRegistry?: ResolvedSecretTraceRegistry ): Promise { const content = (result as any)?.content if (!content || typeof content !== 'string') { @@ -2487,10 +2485,13 @@ export class AgentBlockHandler implements BlockHandler { workflowId: ctx.workflowId, }) } catch (error) { + const diagnosticCtx = diagnosticRegistry + ? { ...ctx, resolvedSecretTraceRegistry: diagnosticRegistry } + : ctx logger.error( 'Failed to persist response to memory', projectAgentDiagnosticMetadata( - ctx, + diagnosticCtx, getErrorDiagnosticMetadata(error), getErrorDiagnosticFallback(error) ) diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 6d505d8ce9c..57097de52b6 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -123,13 +123,21 @@ function createTraceRegistryMock(): ResolvedSecretTraceRegistry & { importProvenanceForValue: ReturnType markIncomplete: ReturnType } { - const registry = new ResolvedSecretTraceRegistry() - return Object.assign(registry, { - importProvenanceForValue: vi - .spyOn(registry, 'importProvenanceForValue') - .mockResolvedValue(true), - markIncomplete: vi.spyOn(registry, 'markIncomplete'), - }) as ResolvedSecretTraceRegistry & { + const importedProvenance = vi.fn().mockResolvedValue(true) + const originalMarkIncomplete: Array<() => void> = [] + const markIncomplete = vi.fn(() => { + for (const mark of originalMarkIncomplete) mark() + }) + const instrument = (registry: ResolvedSecretTraceRegistry): ResolvedSecretTraceRegistry => { + const forkForInputPaths = registry.forkForInputPaths.bind(registry) + originalMarkIncomplete.push(registry.markIncomplete.bind(registry)) + registry.importProvenanceForValue = importedProvenance + registry.markIncomplete = markIncomplete + registry.forkForInputPaths = ((paths, options) => + instrument(forkForInputPaths(paths, options))) as typeof registry.forkForInputPaths + return registry + } + return instrument(new ResolvedSecretTraceRegistry()) as ResolvedSecretTraceRegistry & { importProvenanceForValue: ReturnType markIncomplete: ReturnType } @@ -305,13 +313,16 @@ describe('MothershipBlockHandler', () => { 'Use {{PROMPT_SECRET}} while Box stays unchanged' ) registry.recordResolved('UNUSED', 'x') - vi.spyOn(registry, 'importProvenanceForValue').mockResolvedValue(true) context.resolvedSecretTraceRegistry = registry mockGenerateId .mockReturnValueOnce('chat-uuid') .mockReturnValueOnce('message-uuid') .mockReturnValueOnce('request-uuid') - fetchMock.mockResolvedValue(createJsonResponse({ content: 'done', toolCalls: [] })) + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'done', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) await handler.execute(context, block, { prompt: 'Use prompt-secret while Box stays unchanged', @@ -325,6 +336,40 @@ describe('MothershipBlockHandler', () => { }) }) + it('does not carry a projected prompt into Mothership output provenance', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PROMPT_SECRET', plaintext: 'x', encryptedValue: 'prompt-ciphertext' }, + ]) + registry.recordResolvedAtInputPath('PROMPT_SECRET', 'x', ['prompt']) + registry.recordResolvedInputProjection(['prompt'], 'Use x', 'Use {{PROMPT_SECRET}}') + context.resolvedSecretTraceRegistry = registry + mockGenerateId + .mockReturnValueOnce('chat-uuid') + .mockReturnValueOnce('message-uuid') + .mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'Box', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) + + const inputs = { prompt: 'Use x' } + const result = await handler.execute(context, block, inputs) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + expect(JSON.parse(String(options.body))).toMatchObject({ + messages: [{ content: 'Use {{PROMPT_SECRET}}' }], + }) + expect(result).toMatchObject({ content: 'Box' }) + expect(inputs.prompt).toBe('Use x') + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(context.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(result)).toEqual({ + version: 1, + complete: true, + entries: [], + }) + }) + it('preserves a headerless legacy JSON response without poisoning later calls', async () => { const registry = createTraceRegistryMock() context.resolvedSecretTraceRegistry = registry @@ -591,6 +636,51 @@ describe('MothershipBlockHandler', () => { expect(registry.markIncomplete).not.toHaveBeenCalled() }) + it('does not carry a projected prompt into streaming Mothership output provenance', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'PROMPT_SECRET', plaintext: 'x', encryptedValue: 'prompt-ciphertext' }, + ]) + registry.recordResolvedAtInputPath('PROMPT_SECRET', 'x', ['prompt']) + registry.recordResolvedInputProjection(['prompt'], 'Use x', 'Use {{PROMPT_SECRET}}') + context.resolvedSecretTraceRegistry = registry + context.stream = true + context.selectedOutputs = [`${block.id}_content`] + const encoder = new TextEncoder() + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(`${JSON.stringify({ type: 'chunk', content: 'Box' })}\n`) + ) + controller.enqueue( + encoder.encode( + `${JSON.stringify({ + type: 'final', + data: { content: 'Box', toolCalls: [] }, + })}\n` + ) + ) + controller.close() + }, + }), + { headers: { 'Content-Type': 'application/x-ndjson; charset=utf-8' } } + ) + ) + + const result = (await handler.execute(context, block, { + prompt: 'Use x', + })) as StreamingExecution + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + expect(JSON.parse(String(options.body))).toMatchObject({ + messages: [{ content: 'Use {{PROMPT_SECRET}}' }], + }) + await expect(readStreamText(result.stream)).resolves.toBe('Box') + expect(result.execution.output).toMatchObject({ content: 'Box' }) + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + it('surfaces a headerless legacy NDJSON terminal error without poisoning later calls', async () => { const registry = createTraceRegistryMock() context.resolvedSecretTraceRegistry = registry @@ -783,6 +873,61 @@ describe('MothershipBlockHandler', () => { expect(logged).not.toContain('__sim_') }) + it('retains exact provenance when a resolved conversation ID is echoed to output', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CONVERSATION_ID', plaintext: 'x', encryptedValue: 'encrypted-conversation-id' }, + ]) + registry.recordResolvedAtInputPath('CONVERSATION_ID', 'x', ['conversationId']) + registry.recordResolvedInputProjection(['conversationId'], 'x', '{{CONVERSATION_ID}}') + context.resolvedSecretTraceRegistry = registry + mockGenerateId.mockReturnValueOnce('message-uuid').mockReturnValueOnce('request-uuid') + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'continued', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) + + const inputs = { + prompt: 'Continue this thread', + conversationId: 'x', + } + const result = await handler.execute(context, block, inputs) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + expect(JSON.parse(String(options.body)).chatId).toBe('x') + expect(result).toMatchObject({ conversationId: 'x' }) + expect(inputs.conversationId).toBe('x') + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([ + { plaintext: 'x', replacement: '{{CONVERSATION_ID}}' }, + ]) + expect(context.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(result)).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'CONVERSATION_ID', encryptedValue: 'encrypted-conversation-id' }], + }) + }) + + it('does not carry a low-entropy conversation ID into terminal error provenance', async () => { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'CONVERSATION_ID', plaintext: 'x', encryptedValue: 'encrypted-conversation-id' }, + ]) + registry.recordResolvedAtInputPath('CONVERSATION_ID', 'x', ['conversationId']) + registry.recordResolvedInputProjection(['conversationId'], 'x', '{{CONVERSATION_ID}}') + context.resolvedSecretTraceRegistry = registry + mockGenerateId.mockReturnValueOnce('message-uuid').mockReturnValueOnce('request-uuid') + mockExtractAPIErrorMessage.mockResolvedValueOnce('Box') + fetchMock.mockResolvedValue(new Response(JSON.stringify({ error: 'Box' }), { status: 500 })) + + await expect( + handler.execute(context, block, { + prompt: 'Continue this thread', + conversationId: 'x', + }) + ).rejects.toThrow('Sim execution failed: Box') + + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + it('forwards only enabled MCP tools and selected skills', async () => { mockGenerateId .mockReturnValueOnce('chat-uuid') @@ -838,7 +983,6 @@ describe('MothershipBlockHandler', () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'UNUSED_SECRET', plaintext: secret, encryptedValue: 'encrypted-unused-secret' }, ]) - vi.spyOn(registry, 'importProvenanceForValue').mockResolvedValue(true) context.resolvedSecretTraceRegistry = registry mockGenerateId .mockReturnValueOnce('chat-uuid') @@ -846,7 +990,11 @@ describe('MothershipBlockHandler', () => { .mockReturnValueOnce('request-uuid') const attachmentData = Buffer.from(`file contains ${secret}`, 'utf8').toString('base64') mockReadUserFileContent.mockResolvedValueOnce(attachmentData) - fetchMock.mockResolvedValue(createJsonResponse({ content: 'done', toolCalls: [] })) + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'done', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) await handler.execute(context, block, { prompt: 'Use the selected context', @@ -967,13 +1115,16 @@ describe('MothershipBlockHandler', () => { registry.recordResolvedAtInputPath(secret.name, secret.plaintext, secret.path) registry.recordResolvedInputProjection(secret.path, secret.raw, secret.projected) } - vi.spyOn(registry, 'importProvenanceForValue').mockResolvedValue(true) context.resolvedSecretTraceRegistry = registry mockGenerateId .mockReturnValueOnce('chat-uuid') .mockReturnValueOnce('message-uuid') .mockReturnValueOnce('request-uuid') - fetchMock.mockResolvedValue(createJsonResponse({ content: 'done', toolCalls: [] })) + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'done', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) const tools = [ { type: 'mcp', @@ -1209,7 +1360,6 @@ describe('MothershipBlockHandler', () => { 'report-x.txt', 'report-{{FILE_TOKEN}}.txt' ) - vi.spyOn(registry, 'importProvenanceForValue').mockResolvedValue(true) context.resolvedSecretTraceRegistry = registry mockGenerateId .mockReturnValueOnce('chat-uuid') @@ -1217,7 +1367,11 @@ describe('MothershipBlockHandler', () => { .mockReturnValueOnce('request-uuid') const attachmentData = Buffer.from('ordinary bytes', 'utf8').toString('base64') mockReadUserFileContent.mockResolvedValueOnce(attachmentData) - fetchMock.mockResolvedValue(createJsonResponse({ content: 'done', toolCalls: [] })) + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ content: 'done', toolCalls: [] }), { + headers: { 'Content-Type': 'application/json' }, + }) + ) await handler.execute(context, block, { prompt: 'Read the attachment', diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index 416debb2b3e..da4e441580e 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -826,6 +826,11 @@ export class MothershipBlockHandler implements BlockHandler { ...(ctx.executionId ? { executionId: ctx.executionId } : {}), } + const settledInputRegistry = ctx.resolvedSecretTraceRegistry + const resultRegistry = settledInputRegistry?.forkForInputPaths( + providedConversationId ? [['conversationId']] : [] + ) + logger.info('Executing Mothership block', { blockId: block.id, messageId, @@ -896,19 +901,16 @@ export class MothershipBlockHandler implements BlockHandler { }) if (!response.ok) { - const expectsProvenance = inspectMothershipResponseCapability( - response, - ctx.resolvedSecretTraceRegistry - ) + const expectsProvenance = inspectMothershipResponseCapability(response, resultRegistry) if (expectsProvenance) { let payload: MothershipExecuteResult try { payload = (await response.clone().json()) as MothershipExecuteResult } catch { - ctx.resolvedSecretTraceRegistry?.markIncomplete() + resultRegistry?.markIncomplete() throw new Error('Mothership response provenance metadata is invalid') } - await consumeMothershipProvenance(payload, response, ctx.resolvedSecretTraceRegistry) + await consumeMothershipProvenance(payload, response, resultRegistry) } const errorMsg = await extractAPIErrorMessage(response) throw new Error(`Sim execution failed: ${errorMsg}`) @@ -922,14 +924,23 @@ export class MothershipBlockHandler implements BlockHandler { } }, onDone: cleanupAbortListeners, - registry: ctx.resolvedSecretTraceRegistry, + registry: resultRegistry, }) + streamingExecution.diagnosticResolvedSecretTraceRegistry = settledInputRegistry + if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry cleanupImmediately = false return streamingExecution } - const result = await readMothershipExecuteResponse(response, ctx.resolvedSecretTraceRegistry) - return formatMothershipBlockOutput(result, chatId) + const result = await readMothershipExecuteResponse(response, resultRegistry) + const output = formatMothershipBlockOutput(result, chatId) + if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry + return output + } catch (error) { + if (resultRegistry) { + ctx.resolvedSecretTraceRegistry = resultRegistry.forkForPropagatedEntries() + } + throw error } finally { if (cleanupImmediately) { cleanupAbortListeners() diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 7d8bdf8d471..c9c1d03ff0e 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -572,6 +572,8 @@ export interface StreamingExecution { clientStreamTransformed?: boolean /** Internal provenance for the exact block input that initiated this live stream. */ displayResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 + /** Internal source registry retained only for sanitizing failures while the stream drains. */ + diagnosticResolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry execution: ExecutionResult & { isStreaming?: boolean } /** * Invoked with the assembled response text after the stream drains. Lets agent diff --git a/apps/sim/lib/uploads/utils/model-input.test.ts b/apps/sim/lib/uploads/utils/model-input.test.ts index 44e8de7a20f..46d3b880db6 100644 --- a/apps/sim/lib/uploads/utils/model-input.test.ts +++ b/apps/sim/lib/uploads/utils/model-input.test.ts @@ -11,6 +11,7 @@ import { selectPreferredModelBoundFileInputPaths, } from '@/lib/uploads/utils/model-input' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { a2aSendMessageTool } from '@/tools/a2a/send_message' import { prepareToolRequest } from '@/tools/request-transport' import { visionTool } from '@/tools/vision/tool' @@ -150,6 +151,31 @@ describe('model-bound file input selection', () => { }, ]) }) + + it('treats an optional undefined file name as absent', () => { + const original = [{ key: 'workspace/ws-1/report.pdf', name: undefined }] + + expect(selectModelVisibleFileNames(original)).toEqual([{}]) + expect(applyProjectedModelVisibleFileNames(original, [{}])).toEqual(original) + }) + + it('preserves an optional undefined file name through tool request projection', () => { + const prepared = prepareToolRequest( + a2aSendMessageTool, + { + agentUrl: 'https://agent.example', + message: 'Summarize the attachment', + files: [{ key: 'workspace/ws-1/report.pdf', name: undefined }], + }, + new ResolvedSecretTraceRegistry() + ) + + expect(JSON.parse(prepared.body ?? '{}')).toEqual({ + agentUrl: 'https://agent.example', + message: 'Summarize the attachment', + files: [{ key: 'workspace/ws-1/report.pdf' }], + }) + }) }) describe('server-resolved model file provenance', () => { diff --git a/apps/sim/lib/uploads/utils/model-input.ts b/apps/sim/lib/uploads/utils/model-input.ts index 0e0f2bcd839..736de9c7eaf 100644 --- a/apps/sim/lib/uploads/utils/model-input.ts +++ b/apps/sim/lib/uploads/utils/model-input.ts @@ -106,7 +106,7 @@ export function selectPreferredModelBoundFileInputPaths( export function selectModelVisibleFileNames(input: unknown): unknown { if (Array.isArray(input)) return input.map(selectModelVisibleFileNames) if (!isPlainRecord(input)) return undefined - return Object.hasOwn(input, 'name') ? { name: input.name } : {} + return input.name !== undefined ? { name: input.name } : {} } function haveExactKeys(value: Record, expected: readonly string[]): boolean { @@ -138,7 +138,7 @@ export function applyProjectedModelVisibleFileNames( throw new Error('Projected file name is invalid') } - if (!Object.hasOwn(original, 'name')) { + if (original.name === undefined) { if (!haveExactKeys(projected, [])) { throw new Error('Projected file name does not match the original file') } diff --git a/apps/sim/providers/anthropic/streaming-tool-loop.test.ts b/apps/sim/providers/anthropic/streaming-tool-loop.test.ts index fa583999968..0c858b7792b 100644 --- a/apps/sim/providers/anthropic/streaming-tool-loop.test.ts +++ b/apps/sim/providers/anthropic/streaming-tool-loop.test.ts @@ -271,7 +271,6 @@ describe('createAnthropicStreamingToolLoopStream', () => { { params: { token: '{{TOKEN}}' } } ) registerPreparedProviderToolInputProvenance(executionParams, { - parentRegistry: registry, registry: inputRegistry, inputPaths: [['params']], }) diff --git a/apps/sim/providers/runtime-context.test.ts b/apps/sim/providers/runtime-context.test.ts index f46870f47d9..de6bed10577 100644 --- a/apps/sim/providers/runtime-context.test.ts +++ b/apps/sim/providers/runtime-context.test.ts @@ -204,13 +204,14 @@ describe('provider runtime context', () => { }) it('projects only the active preset secret for the exact configured tool instance', async () => { - const registry = new ResolvedSecretTraceRegistry([ + const sourceRegistry = new ResolvedSecretTraceRegistry([ { name: 'ACTIVE', plaintext: 'x', encryptedValue: 'encrypted-active' }, { name: 'UNUSED', plaintext: 'true', encryptedValue: 'encrypted-unused' }, ]) const sourcePath = ['tools', '0', 'params', 'apiKey'] as const - registry.recordResolvedAtInputPath('ACTIVE', 'x', sourcePath) - registry.recordResolvedInputProjection(sourcePath, 'x', '{{ACTIVE}}') + sourceRegistry.recordResolvedAtInputPath('ACTIVE', 'x', sourcePath) + sourceRegistry.recordResolvedInputProjection(sourcePath, 'x', '{{ACTIVE}}') + const runtimeRegistry = sourceRegistry.forkForInputPaths([]) const tool = { id: 'duplicate-tool', params: { apiKey: 'x' }, @@ -218,7 +219,7 @@ describe('provider runtime context', () => { paramsTransform: (params: Record) => ({ token: params.apiKey }), } registerProviderToolInputProvenance(tool, { - registry, + registry: sourceRegistry, sourcePath: ['tools', '0', 'params'], projectedParams: { apiKey: '{{ACTIVE}}' }, }) @@ -226,7 +227,7 @@ describe('provider runtime context', () => { mockExecuteTool.mockResolvedValueOnce(rawResult) const execution = await runWithProviderRuntimeContext( - { resolvedSecretTraceRegistry: registry }, + { resolvedSecretTraceRegistry: runtimeRegistry }, () => { const { executionParams } = prepareToolExecution(tool, {}, {}) return executeProviderToolWithInput(tool.id, executionParams) @@ -242,6 +243,9 @@ describe('provider runtime context', () => { expect(mockExecuteTool.mock.calls.at(-1)?.[1]).not.toHaveProperty( '__resolvedSecretTraceProvenance' ) + expect(runtimeRegistry.getActiveMatches()).toEqual([ + { plaintext: 'x', replacement: '{{ACTIVE}}' }, + ]) }) it('does not carry a prior low-entropy preset into a later duplicate tool instance', async () => { diff --git a/apps/sim/providers/runtime-context.ts b/apps/sim/providers/runtime-context.ts index 704bc7817c6..1060c18011d 100644 --- a/apps/sim/providers/runtime-context.ts +++ b/apps/sim/providers/runtime-context.ts @@ -58,7 +58,7 @@ export async function executeProviderTool( } const preparedInputProvenance = getPreparedProviderToolInputProvenance(params) const toolCallRegistry = registry - ? preparedInputProvenance?.parentRegistry === registry + ? preparedInputProvenance ? preparedInputProvenance.registry.forkForInputPaths(preparedInputProvenance.inputPaths, { propagated: true, }) diff --git a/apps/sim/providers/tool-input-provenance.ts b/apps/sim/providers/tool-input-provenance.ts index 83d4449a2ad..956b910b4e2 100644 --- a/apps/sim/providers/tool-input-provenance.ts +++ b/apps/sim/providers/tool-input-provenance.ts @@ -10,7 +10,6 @@ export interface ProviderToolInputProvenance { } export interface PreparedProviderToolInputProvenance { - parentRegistry: ResolvedSecretTraceRegistry registry: ResolvedSecretTraceRegistry inputPaths: readonly ResolvedSecretInputPath[] } diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 9310e8d38fa..c6017dba864 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -1618,7 +1618,6 @@ export function prepareToolExecution( ) } registerPreparedProviderToolInputProvenance(executionParams, { - parentRegistry: inputProvenance.registry, registry: inputRegistry, inputPaths, }) From d3a5c5bb3ba20587310d0184f2bd7f15b17c2e3e Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 7 Aug 2026 20:05:12 -0700 Subject: [PATCH 4/5] address comments --- .../executor/execution/block-executor.test.ts | 49 +++++++++++ apps/sim/executor/execution/block-executor.ts | 23 ++++-- .../handlers/agent/agent-handler.test.ts | 22 +++-- .../executor/handlers/agent/agent-handler.ts | 82 +++++++++++++++---- .../mothership/mothership-handler.test.ts | 40 +++++++++ .../handlers/mothership/mothership-handler.ts | 16 +++- apps/sim/executor/types.ts | 2 + 7 files changed, 203 insertions(+), 31 deletions(-) diff --git a/apps/sim/executor/execution/block-executor.test.ts b/apps/sim/executor/execution/block-executor.test.ts index b6eb5f853db..7962e466a26 100644 --- a/apps/sim/executor/execution/block-executor.test.ts +++ b/apps/sim/executor/execution/block-executor.test.ts @@ -1135,6 +1135,55 @@ describe('BlockExecutor streaming pump', () => { expect(registry.getActiveMatches()).toEqual([]) }) + it('carries echoed raw-boundary secret provenance on terminal errors only', async () => { + const promptSecret = 'x' + const apiKey = 'provider-credential-secret' + const handler: BlockHandler = { + canHandle: () => true, + execute: async (blockContext) => { + const sourceRegistry = blockContext.resolvedSecretTraceRegistry + blockContext.errorResolvedSecretTraceRegistry = sourceRegistry?.forkForInputPaths([ + ['apiKey'], + ]) + blockContext.resolvedSecretTraceRegistry = sourceRegistry?.forkForInputPaths([]) + throw new Error(`Provider rejected ${apiKey}`) + }, + } + const { executor, block, state } = createExecutor(handler) + block.config.params = { + systemPrompt: '{{PROMPT_TOKEN}}', + apiKey: '{{API_KEY}}', + } + const ctx = createContext(state) + const registry = new ResolvedSecretTraceRegistry([ + { + name: 'PROMPT_TOKEN', + plaintext: promptSecret, + encryptedValue: 'encrypted-prompt-token', + }, + { name: 'API_KEY', plaintext: apiKey, encryptedValue: 'encrypted-api-key' }, + ]) + ctx.environmentVariables = { PROMPT_TOKEN: promptSecret, API_KEY: apiKey } + ctx.resolvedSecretTraceRegistry = registry + + await expect(executor.execute(ctx, createNode(block), block)).rejects.toThrow( + `Agent: Provider rejected ${apiKey}` + ) + + expect(ctx.blockLogs[0]).toMatchObject({ + input: { systemPrompt: '{{PROMPT_TOKEN}}', apiKey: '[REDACTED]' }, + output: { error: `Provider rejected ${apiKey}` }, + }) + const expectedProvenance = { + version: 1, + complete: true, + entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-api-key' }], + } + expect(state.getBlockState(block.id)?.resolvedSecretTraceProvenance).toEqual(expectedProvenance) + expect(ctx.blockLogs[0]?.displayResolvedSecretTraceProvenance).toEqual(expectedProvenance) + expect(registry.getActiveMatches()).toEqual([]) + }) + it('suppresses an incomplete display input without failing block execution', async () => { const handler: BlockHandler = { canHandle: () => true, diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index b10039d7910..0797adca7ea 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -250,8 +250,16 @@ export class BlockExecutor { normalizeStringArray(blockCtx.selectedOutputs) ) } catch (streamError) { - blockCtx.resolvedSecretTraceRegistry = - blockCtx.resolvedSecretTraceRegistry?.forkForPropagatedEntries() + const resultRegistry = blockCtx.resolvedSecretTraceRegistry + const diagnosticRegistry = streamingExec.diagnosticResolvedSecretTraceRegistry + const errorRegistry = diagnosticRegistry + ? diagnosticRegistry.forkForToolCall() + : resultRegistry?.forkForToolCall() + if (errorRegistry && resultRegistry && resultRegistry !== diagnosticRegistry) { + errorRegistry.mergeToolCallRegistry(resultRegistry) + } + blockCtx.errorResolvedSecretTraceRegistry = errorRegistry + blockCtx.resolvedSecretTraceRegistry = resultRegistry?.forkForPropagatedEntries() // Timeout / drain failures may still have projected answer text — keep it // for the failed block output so logs match what the client already saw. streamingPartialOutput = streamingExec.execution?.output @@ -575,8 +583,8 @@ export class BlockExecutor { } } - const errorOutputProvenance = - ctx.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(errorOutput) + const errorRegistry = ctx.errorResolvedSecretTraceRegistry ?? ctx.resolvedSecretTraceRegistry + const errorOutputProvenance = errorRegistry?.exportCommittedProvenanceForValue(errorOutput) this.setNodeOutput(node, errorOutput, duration, errorOutputProvenance) if (blockLog) { @@ -592,8 +600,11 @@ export class BlockExecutor { } } - const diagnosticRegistry = inputDisplayRegistry?.forkForToolCall() + const diagnosticRegistry = ctx.errorResolvedSecretTraceRegistry + ? ctx.errorResolvedSecretTraceRegistry + : inputDisplayRegistry?.forkForToolCall() if ( + !ctx.errorResolvedSecretTraceRegistry && diagnosticRegistry && ctx.resolvedSecretTraceRegistry && ctx.resolvedSecretTraceRegistry !== inputDisplayRegistry @@ -620,7 +631,7 @@ export class BlockExecutor { : undefined const displayOutput = filterOutputForLog(block.metadata?.id || '', errorOutput, { block }) const displayInput = this.projectInputsForDisplay(input, block, inputDisplayRegistry) - const displayProvenance = ctx.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue({ + const displayProvenance = errorRegistry?.exportCommittedProvenanceForValue({ input: displayInput, output: displayOutput, }) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index f1b121bac60..7c13d17db5f 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -1191,30 +1191,38 @@ describe('AgentBlockHandler', () => { ).toEqual({ version: 1, complete: true, entries: [] }) }) - it('keeps input provenance active for provider error diagnostics', async () => { + it('keeps only raw provider inputs active for provider error diagnostics', async () => { const plaintext = 'provider-credential-secret' const registry = new ResolvedSecretTraceRegistry([ - { name: 'TOKEN', plaintext, encryptedValue: 'encrypted-token' }, + { name: 'API_KEY', plaintext, encryptedValue: 'encrypted-api-key' }, + { name: 'PROMPT_TOKEN', plaintext: 'x', encryptedValue: 'encrypted-prompt-token' }, ]) - registry.recordResolvedAtInputPath('TOKEN', plaintext, ['systemPrompt']) - registry.recordResolvedInputProjection(['systemPrompt'], `Use ${plaintext}`, 'Use {{TOKEN}}') + registry.recordResolvedAtInputPath('API_KEY', plaintext, ['apiKey']) + registry.recordResolvedInputProjection(['apiKey'], plaintext, '{{API_KEY}}') + registry.recordResolvedAtInputPath('PROMPT_TOKEN', 'x', ['systemPrompt']) + registry.recordResolvedInputProjection(['systemPrompt'], 'Use x', 'Use {{PROMPT_TOKEN}}') mockContext.resolvedSecretTraceRegistry = registry mockExecuteProviderRequest.mockRejectedValueOnce(new Error(`Provider rejected ${plaintext}`)) const inputs = { model: 'gpt-4o', - systemPrompt: `Use ${plaintext}`, + systemPrompt: 'Use x', userPrompt: 'Continue', + apiKey: plaintext, } await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow( `Provider rejected ${plaintext}` ) - expect(inputs.systemPrompt).toBe(`Use ${plaintext}`) + expect(inputs).toMatchObject({ systemPrompt: 'Use x', apiKey: plaintext }) expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(mockContext.errorResolvedSecretTraceRegistry?.getActiveMatches()).toEqual([ + { plaintext, replacement: '{{API_KEY}}' }, + ]) const logged = JSON.stringify(mockAgentLogger.error.mock.calls) expect(logged).not.toContain(plaintext) - expect(logged).toContain('Provider rejected {{TOKEN}}') + expect(logged).toContain('Provider rejected {{API_KEY}}') + expect(logged).not.toContain('PROMPT_TOKEN') }) it('projects exact message call arguments without mutating protocol structure or raw input', async () => { diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index fb0c5e844da..48663dd3acc 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -94,6 +94,26 @@ import { getToolAsync } from '@/tools/utils.server' const logger = createLogger('AgentBlockHandler') const MODEL_SAFE_RESPONSE_FORMAT_NAME = 'response_schema' +const AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = [ + ['model'], + ['temperature'], + ['maxTokens'], + ['apiKey'], + ['azureEndpoint'], + ['azureApiVersion'], + ['vertexProject'], + ['vertexLocation'], + ['vertexCredential'], + ['bedrockAccessKeyId'], + ['bedrockSecretKey'], + ['bedrockRegion'], + ['reasoningEffort'], + ['verbosity'], + ['thinkingLevel'], + ['promptCaching'], + ['previousInteractionId'], +] +const AGENT_MEMORY_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = [['conversationId']] interface IndexedToolInput { tool: ToolInput @@ -193,6 +213,10 @@ export class AgentBlockHandler implements BlockHandler { block: SerializedBlock, inputs: AgentInputs ): Promise { + const providerErrorRegistry = ctx.resolvedSecretTraceRegistry?.forkForInputPaths( + AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS + ) + ctx.errorResolvedSecretTraceRegistry = providerErrorRegistry const toolIndexByRef = new Map( (inputs.tools || []).map((tool, index) => [tool, index] as const) ) @@ -371,7 +395,8 @@ export class AgentBlockHandler implements BlockHandler { providerRequest, block, responseFormat, - resultRegistry + resultRegistry, + providerErrorRegistry ) if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry @@ -385,24 +410,30 @@ export class AgentBlockHandler implements BlockHandler { if (this.isStreamingExecution(result)) { const streamingResult = result as StreamingExecution - streamingResult.diagnosticResolvedSecretTraceRegistry = settledInputRegistry + streamingResult.diagnosticResolvedSecretTraceRegistry = providerErrorRegistry + const memoryErrorRegistry = settledInputRegistry?.forkForInputPaths( + AGENT_MEMORY_ERROR_INPUT_PATHS + ) if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { return this.wrapStreamForMemoryPersistence( ctx, filteredInputs, streamingResult, - settledInputRegistry + memoryErrorRegistry ) } return streamingResult } if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { + const memoryErrorRegistry = settledInputRegistry?.forkForInputPaths( + AGENT_MEMORY_ERROR_INPUT_PATHS + ) await this.persistResponseToMemory( ctx, filteredInputs, result as BlockOutput, - settledInputRegistry + memoryErrorRegistry ) } @@ -2297,7 +2328,8 @@ export class AgentBlockHandler implements BlockHandler { providerRequest: any, block: SerializedBlock, responseFormat: any, - modelRuntimeRegistry: ResolvedSecretTraceRegistry | undefined + modelRuntimeRegistry: ResolvedSecretTraceRegistry | undefined, + providerErrorRegistry: ResolvedSecretTraceRegistry | undefined ): Promise { const providerId = providerRequest.provider const model = providerRequest.model @@ -2369,14 +2401,13 @@ export class AgentBlockHandler implements BlockHandler { return this.processProviderResponse(response, block, responseFormat, ctx) } catch (error) { - const sourceRegistry = ctx.resolvedSecretTraceRegistry - if (sourceRegistry && modelRuntimeRegistry && sourceRegistry !== modelRuntimeRegistry) { - const diagnosticRegistry = sourceRegistry.forkForToolCall() - diagnosticRegistry.mergeToolCallRegistry(modelRuntimeRegistry) - ctx.resolvedSecretTraceRegistry = diagnosticRegistry - } + const errorRegistry = this.createErrorRegistry(providerErrorRegistry, modelRuntimeRegistry) + ctx.errorResolvedSecretTraceRegistry = errorRegistry + const diagnosticCtx = errorRegistry + ? { ...ctx, resolvedSecretTraceRegistry: errorRegistry } + : ctx try { - this.handleExecutionError(error, providerStartTime, providerId, model, ctx, block) + this.handleExecutionError(error, providerStartTime, providerId, model, diagnosticCtx, block) } finally { if (modelRuntimeRegistry) { ctx.resolvedSecretTraceRegistry = modelRuntimeRegistry.forkForPropagatedEntries() @@ -2386,6 +2417,17 @@ export class AgentBlockHandler implements BlockHandler { } } + private createErrorRegistry( + inputRegistry: ResolvedSecretTraceRegistry | undefined, + resultRegistry: ResolvedSecretTraceRegistry | undefined + ): ResolvedSecretTraceRegistry | undefined { + const errorRegistry = inputRegistry?.forkForToolCall() ?? resultRegistry?.forkForToolCall() + if (errorRegistry && resultRegistry && resultRegistry !== inputRegistry) { + errorRegistry.mergeToolCallRegistry(resultRegistry) + } + return errorRegistry + } + private handleExecutionError( error: any, startTime: number, @@ -2452,8 +2494,12 @@ export class AgentBlockHandler implements BlockHandler { try { await memoryService.appendToMemory(ctx, inputs, { role: 'assistant', content }) } catch (error) { - const diagnosticCtx = diagnosticRegistry - ? { ...ctx, resolvedSecretTraceRegistry: diagnosticRegistry } + const memoryErrorRegistry = this.createErrorRegistry( + diagnosticRegistry, + ctx.resolvedSecretTraceRegistry + ) + const diagnosticCtx = memoryErrorRegistry + ? { ...ctx, resolvedSecretTraceRegistry: memoryErrorRegistry } : ctx logger.error( 'Failed to persist streaming response', @@ -2485,8 +2531,12 @@ export class AgentBlockHandler implements BlockHandler { workflowId: ctx.workflowId, }) } catch (error) { - const diagnosticCtx = diagnosticRegistry - ? { ...ctx, resolvedSecretTraceRegistry: diagnosticRegistry } + const memoryErrorRegistry = this.createErrorRegistry( + diagnosticRegistry, + ctx.resolvedSecretTraceRegistry + ) + const diagnosticCtx = memoryErrorRegistry + ? { ...ctx, resolvedSecretTraceRegistry: memoryErrorRegistry } : ctx logger.error( 'Failed to persist response to memory', diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 57097de52b6..90e0d8d3275 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -489,6 +489,43 @@ describe('MothershipBlockHandler', () => { expect(registry.markIncomplete).not.toHaveBeenCalled() }) + it('keeps declared JSON error provenance separate from normal result provenance', async () => { + const registry = createTraceRegistryMock() + context.resolvedSecretTraceRegistry = registry + mockExtractAPIErrorMessage.mockResolvedValueOnce('secret-backed failure') + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ + error: 'secret-backed failure', + __resolvedSecretTraceProvenance: PRIVATE_PROVENANCE, + }), + { + status: 502, + headers: { + 'Content-Type': 'application/json', + 'x-sim-private-tool-metadata': PRIVATE_PROVENANCE_TYPE, + }, + } + ) + ) + + await expect(handler.execute(context, block, { prompt: 'Hello' })).rejects.toThrow( + 'Sim execution failed: secret-backed failure' + ) + + expect(registry.importProvenanceForValue).toHaveBeenCalledWith( + PRIVATE_PROVENANCE, + expect.objectContaining({ + error: 'secret-backed failure', + __resolvedSecretTraceProvenance: undefined, + }), + { trusted: true } + ) + expect(context.errorResolvedSecretTraceRegistry).toBeDefined() + expect(context.errorResolvedSecretTraceRegistry).not.toBe(context.resolvedSecretTraceRegistry) + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + }) + it('imports provenance from a terminal NDJSON error without forcing structural fallback', async () => { const registry = createTraceRegistryMock() context.resolvedSecretTraceRegistry = registry @@ -523,6 +560,8 @@ describe('MothershipBlockHandler', () => { { trusted: true } ) expect(registry.markIncomplete).not.toHaveBeenCalled() + expect(context.errorResolvedSecretTraceRegistry).toBeDefined() + expect(context.errorResolvedSecretTraceRegistry).not.toBe(context.resolvedSecretTraceRegistry) }) it('imports final provenance for selected-output streaming without adding it to output', async () => { @@ -926,6 +965,7 @@ describe('MothershipBlockHandler', () => { ).rejects.toThrow('Sim execution failed: Box') expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) + expect(context.errorResolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) }) it('forwards only enabled MCP tools and selected skills', async () => { diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index da4e441580e..60b8dc647b3 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -505,6 +505,7 @@ function createMothershipStreamingExecution( options: { onCancel?: (reason?: unknown) => void onDone?: () => void + onSuccess?: () => void registry?: ResolvedSecretTraceRegistry } = {} ): StreamingExecution { @@ -588,6 +589,7 @@ function createMothershipStreamingExecution( } if (!cancelled) { + options.onSuccess?.() controller.close() } } catch (error) { @@ -715,6 +717,8 @@ export class MothershipBlockHandler implements BlockHandler { inputs: Record ): Promise { const sourceRegistry = ctx.resolvedSecretTraceRegistry + const resultRegistry = sourceRegistry?.forkForInputPaths([]) + ctx.errorResolvedSecretTraceRegistry = resultRegistry const requestSkills = inputs.skills const privateSkillSelectors = selectPrivateMothershipSkillSelectors( sourceRegistry, @@ -827,9 +831,14 @@ export class MothershipBlockHandler implements BlockHandler { } const settledInputRegistry = ctx.resolvedSecretTraceRegistry - const resultRegistry = settledInputRegistry?.forkForInputPaths( + const conversationRegistry = settledInputRegistry?.forkForInputPaths( providedConversationId ? [['conversationId']] : [] ) + const commitConversationProvenance = (): void => { + if (resultRegistry && conversationRegistry) { + resultRegistry.mergeToolCallRegistry(conversationRegistry) + } + } logger.info('Executing Mothership block', { blockId: block.id, @@ -924,9 +933,10 @@ export class MothershipBlockHandler implements BlockHandler { } }, onDone: cleanupAbortListeners, + onSuccess: commitConversationProvenance, registry: resultRegistry, }) - streamingExecution.diagnosticResolvedSecretTraceRegistry = settledInputRegistry + streamingExecution.diagnosticResolvedSecretTraceRegistry = resultRegistry if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry cleanupImmediately = false return streamingExecution @@ -934,9 +944,11 @@ export class MothershipBlockHandler implements BlockHandler { const result = await readMothershipExecuteResponse(response, resultRegistry) const output = formatMothershipBlockOutput(result, chatId) + commitConversationProvenance() if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry return output } catch (error) { + ctx.errorResolvedSecretTraceRegistry = resultRegistry if (resultRegistry) { ctx.resolvedSecretTraceRegistry = resultRegistry.forkForPropagatedEntries() } diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index c9c1d03ff0e..c1f1f5c53bb 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -363,6 +363,8 @@ export interface ExecutionContext { startRunMetadata?: StartBlockRunMetadata environmentVariables: Record resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry + /** Exact candidates that may be carried by this block's terminal error, never its normal output. */ + errorResolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry workflowVariables?: Record workflowVariableResolvedSecretTraceProvenance?: Record workflowInputResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 From f09556c4506c8566fad0d48e4d303861a57191ab Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 7 Aug 2026 20:22:22 -0700 Subject: [PATCH 5/5] fix --- .../executor/handlers/agent/agent-handler.ts | 45 +++---------------- .../mothership/mothership-handler.test.ts | 30 ++----------- .../handlers/mothership/mothership-handler.ts | 18 +------- 3 files changed, 10 insertions(+), 83 deletions(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 48663dd3acc..d1e972180da 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -113,7 +113,6 @@ const AGENT_RAW_PROVIDER_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = ['promptCaching'], ['previousInteractionId'], ] -const AGENT_MEMORY_ERROR_INPUT_PATHS: readonly ResolvedSecretInputPath[] = [['conversationId']] interface IndexedToolInput { tool: ToolInput @@ -411,30 +410,14 @@ export class AgentBlockHandler implements BlockHandler { if (this.isStreamingExecution(result)) { const streamingResult = result as StreamingExecution streamingResult.diagnosticResolvedSecretTraceRegistry = providerErrorRegistry - const memoryErrorRegistry = settledInputRegistry?.forkForInputPaths( - AGENT_MEMORY_ERROR_INPUT_PATHS - ) if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { - return this.wrapStreamForMemoryPersistence( - ctx, - filteredInputs, - streamingResult, - memoryErrorRegistry - ) + return this.wrapStreamForMemoryPersistence(ctx, filteredInputs, streamingResult) } return streamingResult } if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { - const memoryErrorRegistry = settledInputRegistry?.forkForInputPaths( - AGENT_MEMORY_ERROR_INPUT_PATHS - ) - await this.persistResponseToMemory( - ctx, - filteredInputs, - result as BlockOutput, - memoryErrorRegistry - ) + await this.persistResponseToMemory(ctx, filteredInputs, result as BlockOutput) } return result @@ -2484,8 +2467,7 @@ export class AgentBlockHandler implements BlockHandler { private wrapStreamForMemoryPersistence( ctx: ExecutionContext, inputs: AgentInputs, - streamingExec: StreamingExecution, - diagnosticRegistry?: ResolvedSecretTraceRegistry + streamingExec: StreamingExecution ): StreamingExecution { return { ...streamingExec, @@ -2494,17 +2476,10 @@ export class AgentBlockHandler implements BlockHandler { try { await memoryService.appendToMemory(ctx, inputs, { role: 'assistant', content }) } catch (error) { - const memoryErrorRegistry = this.createErrorRegistry( - diagnosticRegistry, - ctx.resolvedSecretTraceRegistry - ) - const diagnosticCtx = memoryErrorRegistry - ? { ...ctx, resolvedSecretTraceRegistry: memoryErrorRegistry } - : ctx logger.error( 'Failed to persist streaming response', projectAgentDiagnosticMetadata( - diagnosticCtx, + ctx, getErrorDiagnosticMetadata(error), getErrorDiagnosticFallback(error) ) @@ -2517,8 +2492,7 @@ export class AgentBlockHandler implements BlockHandler { private async persistResponseToMemory( ctx: ExecutionContext, inputs: AgentInputs, - result: BlockOutput, - diagnosticRegistry?: ResolvedSecretTraceRegistry + result: BlockOutput ): Promise { const content = (result as any)?.content if (!content || typeof content !== 'string') { @@ -2531,17 +2505,10 @@ export class AgentBlockHandler implements BlockHandler { workflowId: ctx.workflowId, }) } catch (error) { - const memoryErrorRegistry = this.createErrorRegistry( - diagnosticRegistry, - ctx.resolvedSecretTraceRegistry - ) - const diagnosticCtx = memoryErrorRegistry - ? { ...ctx, resolvedSecretTraceRegistry: memoryErrorRegistry } - : ctx logger.error( 'Failed to persist response to memory', projectAgentDiagnosticMetadata( - diagnosticCtx, + ctx, getErrorDiagnosticMetadata(error), getErrorDiagnosticFallback(error) ) diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 90e0d8d3275..6c9a752ec05 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -912,7 +912,7 @@ describe('MothershipBlockHandler', () => { expect(logged).not.toContain('__sim_') }) - it('retains exact provenance when a resolved conversation ID is echoed to output', async () => { + it('does not treat conversation IDs as secret-bearing result content', async () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'CONVERSATION_ID', plaintext: 'x', encryptedValue: 'encrypted-conversation-id' }, ]) @@ -936,38 +936,14 @@ describe('MothershipBlockHandler', () => { expect(JSON.parse(String(options.body)).chatId).toBe('x') expect(result).toMatchObject({ conversationId: 'x' }) expect(inputs.conversationId).toBe('x') - expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([ - { plaintext: 'x', replacement: '{{CONVERSATION_ID}}' }, - ]) + expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) expect(context.resolvedSecretTraceRegistry?.exportCommittedProvenanceForValue(result)).toEqual({ version: 1, complete: true, - entries: [{ name: 'CONVERSATION_ID', encryptedValue: 'encrypted-conversation-id' }], + entries: [], }) }) - it('does not carry a low-entropy conversation ID into terminal error provenance', async () => { - const registry = new ResolvedSecretTraceRegistry([ - { name: 'CONVERSATION_ID', plaintext: 'x', encryptedValue: 'encrypted-conversation-id' }, - ]) - registry.recordResolvedAtInputPath('CONVERSATION_ID', 'x', ['conversationId']) - registry.recordResolvedInputProjection(['conversationId'], 'x', '{{CONVERSATION_ID}}') - context.resolvedSecretTraceRegistry = registry - mockGenerateId.mockReturnValueOnce('message-uuid').mockReturnValueOnce('request-uuid') - mockExtractAPIErrorMessage.mockResolvedValueOnce('Box') - fetchMock.mockResolvedValue(new Response(JSON.stringify({ error: 'Box' }), { status: 500 })) - - await expect( - handler.execute(context, block, { - prompt: 'Continue this thread', - conversationId: 'x', - }) - ).rejects.toThrow('Sim execution failed: Box') - - expect(context.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) - expect(context.errorResolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) - }) - it('forwards only enabled MCP tools and selected skills', async () => { mockGenerateId .mockReturnValueOnce('chat-uuid') diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index 60b8dc647b3..1c77fdaccb7 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -505,7 +505,6 @@ function createMothershipStreamingExecution( options: { onCancel?: (reason?: unknown) => void onDone?: () => void - onSuccess?: () => void registry?: ResolvedSecretTraceRegistry } = {} ): StreamingExecution { @@ -588,10 +587,7 @@ function createMothershipStreamingExecution( throw new Error('Sim execution stream ended without a final result') } - if (!cancelled) { - options.onSuccess?.() - controller.close() - } + if (!cancelled) controller.close() } catch (error) { if (!cancelled) { controller.error(error) @@ -830,16 +826,6 @@ export class MothershipBlockHandler implements BlockHandler { ...(ctx.executionId ? { executionId: ctx.executionId } : {}), } - const settledInputRegistry = ctx.resolvedSecretTraceRegistry - const conversationRegistry = settledInputRegistry?.forkForInputPaths( - providedConversationId ? [['conversationId']] : [] - ) - const commitConversationProvenance = (): void => { - if (resultRegistry && conversationRegistry) { - resultRegistry.mergeToolCallRegistry(conversationRegistry) - } - } - logger.info('Executing Mothership block', { blockId: block.id, messageId, @@ -933,7 +919,6 @@ export class MothershipBlockHandler implements BlockHandler { } }, onDone: cleanupAbortListeners, - onSuccess: commitConversationProvenance, registry: resultRegistry, }) streamingExecution.diagnosticResolvedSecretTraceRegistry = resultRegistry @@ -944,7 +929,6 @@ export class MothershipBlockHandler implements BlockHandler { const result = await readMothershipExecuteResponse(response, resultRegistry) const output = formatMothershipBlockOutput(result, chatId) - commitConversationProvenance() if (resultRegistry) ctx.resolvedSecretTraceRegistry = resultRegistry return output } catch (error) {