From fe8585d8a9c8891df2469d0fff36741e13f84b18 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 03:16:24 -0600 Subject: [PATCH 1/4] fix(codex): capture custom tools and subagents --- src/adapters/codex.ts | 94 +++++++++++++++++++++++-- src/pipelines.ts | 18 ++++- tests/adapters.test.ts | 147 ++++++++++++++++++++++++++++++++++++++++ tests/pipelines.test.ts | 15 ++++ 4 files changed, 267 insertions(+), 7 deletions(-) diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index ed3f785..8b7213d 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -4,7 +4,8 @@ * Line types: `session_meta` (id + cwd), `turn_context`, `event_msg` * (carries `token_count` with per-turn `last_token_usage`), and * `response_item` (the OpenAI Responses items: `message`, `reasoning`, - * `function_call`, `function_call_output`). + * function/custom tool calls, and their outputs). Current Codex builds also + * emit `sub_agent_activity` events for delegated agents. * * Token trajectory comes from the `token_count` deltas (Codex puts usage * on events, not on the message). Tools come from `function_call`, with @@ -37,8 +38,14 @@ interface CodexLine { name?: string content?: unknown arguments?: string + input?: string call_id?: string output?: unknown + event_id?: string + occurred_at_ms?: number + agent_thread_id?: string + agent_path?: string + kind?: string info?: { last_token_usage?: { input_tokens?: number; output_tokens?: number }; model_context_window?: number } } } @@ -78,6 +85,28 @@ function outputIsError(output: unknown): { error: boolean; message: string } { return { error, message: error ? s.slice(0, 500) : '' } } +/** A custom `exec` call is a small JavaScript program around one or more real tools. */ +function singleNestedToolName(input: string | undefined): string | null { + if (!input) return null + const names = [...input.matchAll(/\btools\.([A-Za-z][A-Za-z0-9_]*)\s*\(/g)].map((match) => match[1]!) + const unique = [...new Set(names)] + return unique.length === 1 ? unique[0]! : null +} + +const verificationCommand = + /\b(?:pnpm|npm|yarn|bun)\s+(?:run\s+)?(?:test|typecheck|lint|build|check)(?::[A-Za-z0-9:_-]+)?\b|\b(?:vitest|jest|pytest|tsc|biome|eslint|sha256sum|pdfinfo|pdftotext)\b|\bgo\s+test\b|\bcargo\s+(?:test|check|clippy|build)\b|\bgit\s+(?:status|diff|show|merge-tree)\b|\bgh-drew\s+pr\s+(?:view|checks)\b/i + +function hasReadOnlyCurl(input: string): boolean { + if (!/\bcurl\b/i.test(input)) return false + return !/(?:^|\s)(?:-X|--request)(?:=|\s*)(?:POST|PUT|PATCH|DELETE)\b|(?:^|\s)(?:-d|--data(?:-ascii|-binary|-raw|-urlencode)?)(?:=|\s)/i.test(input) +} + +function classifyNestedTool(name: string, input: string | undefined): string { + return name === 'exec_command' && input && (verificationCommand.test(input) || hasReadOnlyCurl(input)) + ? 'exec_command.verify' + : name +} + async function* walkRollouts(root: string): AsyncGenerator { let years: string[] try { @@ -184,6 +213,7 @@ export class CodexAdapter implements HarnessTraceAdapter { ] const toolByCallId = new Map() + const subagentByThreadId = new Map() let step = 0 let lastLlm = rootId let sawUserTurn = false @@ -213,9 +243,15 @@ export class CodexAdapter implements HarnessTraceAdapter { lastLlm = id step += 1 } - } else if (l.type === 'response_item' && l.payload?.type === 'function_call') { - const name = l.payload.name ?? 'tool' + } else if ( + l.type === 'response_item' && + (l.payload?.type === 'function_call' || l.payload?.type === 'custom_tool_call') + ) { + const outerName = l.payload.name ?? 'tool' const callId = l.payload.call_id ?? `${step}` + const input = l.payload.type === 'custom_tool_call' ? l.payload.input : l.payload.arguments + const nestedName = l.payload.type === 'custom_tool_call' ? singleNestedToolName(input) : null + const name = classifyNestedTool(nestedName ?? outerName, input) const toolSpan = span({ traceId, spanId: `tool:${callId}`, @@ -227,18 +263,66 @@ export class CodexAdapter implements HarnessTraceAdapter { agent: SERVICE, tool: name, step, - content: l.payload.arguments ?? null, + content: input ?? null, + extra: { + 'traces.codex.call_type': l.payload.type, + ...(name !== outerName ? { 'traces.codex.outer_tool_name': outerName } : {}), + ...(nestedName ? { 'traces.codex.nested_tool_name': nestedName } : {}), + }, }) spans.push(toolSpan) toolByCallId.set(callId, toolSpan) step += 1 - } else if (l.type === 'response_item' && l.payload?.type === 'function_call_output') { + } else if ( + l.type === 'response_item' && + (l.payload?.type === 'function_call_output' || l.payload?.type === 'custom_tool_call_output') + ) { const t = toolByCallId.get(l.payload.call_id ?? '') if (t) { const { error, message } = outputIsError(l.payload.output) t.end_time = ts t.status = error ? { code: 'ERROR', message } : { code: 'OK' } } + } else if (l.type === 'event_msg' && l.payload?.type === 'sub_agent_activity') { + const threadId = l.payload.agent_thread_id + const occurredAtMs = l.payload.occurred_at_ms + const eventTime = typeof occurredAtMs === 'number' && Number.isFinite(occurredAtMs) + ? new Date(occurredAtMs).toISOString() + : ts + if (l.payload.kind === 'started' && threadId && !subagentByThreadId.has(threadId)) { + const agentPath = l.payload.agent_path ?? 'subagent' + const subagentType = agentPath.split('/').filter(Boolean).at(-1) ?? 'subagent' + const toolSpan = span({ + traceId, + spanId: `subagent:${threadId}`, + parentSpanId: lastLlm, + name: 'tool.Agent', + kind: 'TOOL', + startTime: eventTime, + service: SERVICE, + agent: SERVICE, + tool: 'Agent', + step, + content: JSON.stringify({ + subagent_type: subagentType, + agent_path: agentPath, + agent_thread_id: threadId, + }), + extra: { + 'traces.codex.subagent_path': agentPath, + 'traces.codex.subagent_thread_id': threadId, + }, + }) + spans.push(toolSpan) + subagentByThreadId.set(threadId, toolSpan) + step += 1 + } else if (l.payload.kind === 'interrupted' && threadId) { + const toolSpan = subagentByThreadId.get(threadId) + if (toolSpan) { + toolSpan.end_time = eventTime + toolSpan.status = { code: 'ERROR', message: 'subagent interrupted' } + } + } } else if (l.type === 'response_item' && l.payload?.type === 'message' && l.payload.role === 'user') { // The human's prompt text. Codex drops the user turn from token events, // so capture it here as its own CHAIN span (no text → no span). diff --git a/src/pipelines.ts b/src/pipelines.ts index 2a5a73e..d557801 100644 --- a/src/pipelines.ts +++ b/src/pipelines.ts @@ -13,7 +13,8 @@ * error rates above, so adding it would only emit a misleading waste %. * * Both are cheap ($0, deterministic), so they're safe to run continuously in - * `watch` mode, over the OTLP spans traces already produces. + * `watch` mode, over the OTLP spans traces already produces. Blocking wait and + * stdin-poll calls stay in usage totals but are excluded from stuck-loop findings. */ import { computeToolUseMetrics } from '@tangle-network/agent-eval' @@ -33,9 +34,22 @@ export interface PipelineOptions { minLoopOccurrences?: number } +const expectedBlockingTools = new Set(['wait', 'write_stdin']) + +function toolNameOf(span: OtlpSpan): string | null { + const value = span.attributes['tool.name'] + if (typeof value === 'string') return value + return span.name.startsWith('tool.') ? span.name.slice('tool.'.length) : null +} + export async function runPipelines(spans: readonly OtlpSpan[], opts: PipelineOptions = {}): Promise { const { store, runIds } = await toRuntimeStore(spans) - const stuckLoops = await stuckLoopView(store, { minOccurrences: opts.minLoopOccurrences ?? 3 }) + const loopEligible = spans.filter((item) => { + const name = toolNameOf(item) + return name === null || !expectedBlockingTools.has(name) + }) + const { store: loopStore } = await toRuntimeStore(loopEligible) + const stuckLoops = await stuckLoopView(loopStore, { minOccurrences: opts.minLoopOccurrences ?? 3 }) const toolUse = await Promise.all(runIds.map((runId) => computeToolUseMetrics(store, runId))) return { stuckLoops, toolUse } } diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 88b3064..aa0fc69 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -206,6 +206,153 @@ describe('conversation capture — JSONL adapters', () => { } }) +describe('codex current tool and subagent events', () => { + it('captures custom tool calls, joins their outputs, and counts one subagent start', async () => { + const path = join(dir, 'rollout-codex-current.jsonl') + const startedAt = Date.parse('2026-07-11T09:00:05.000Z') + writeFileSync( + path, + [ + { type: 'session_meta', timestamp: '2026-07-11T09:00:00.000Z', payload: { id: 'codex-current', cwd: '/x' } }, + { type: 'turn_context', timestamp: '2026-07-11T09:00:01.000Z', payload: { model: 'gpt-5' } }, + { type: 'event_msg', timestamp: '2026-07-11T09:00:02.000Z', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 20, output_tokens: 4 } } } }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:03.000Z', + payload: { + type: 'custom_tool_call', + call_id: 'custom-1', + name: 'exec', + input: "const r = await tools.exec_command({ cmd: 'pnpm test' })", + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.000Z', + payload: { + type: 'custom_tool_call_output', + call_id: 'custom-1', + output: [{ type: 'input_text', text: 'Command failed with exit code 1' }], + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.100Z', + payload: { + type: 'custom_tool_call', + call_id: 'custom-2', + name: 'exec', + input: "const r = await tools.exec_command({ cmd: 'rm -rf build' })", + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.200Z', + payload: { type: 'custom_tool_call_output', call_id: 'custom-2', output: 'Script completed' }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.300Z', + payload: { + type: 'custom_tool_call', + call_id: 'custom-3', + name: 'exec', + input: "const r = await tools.exec_command({ cmd: 'curl -X POST https://example.test/release' })", + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.400Z', + payload: { type: 'custom_tool_call_output', call_id: 'custom-3', output: 'Script completed' }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.500Z', + payload: { + type: 'custom_tool_call', + call_id: 'custom-4', + name: 'exec', + input: "const r = await tools.exec_command({ cmd: 'curl https://example.test/health' })", + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.600Z', + payload: { type: 'custom_tool_call_output', call_id: 'custom-4', output: 'Script completed' }, + }, + { + type: 'event_msg', + timestamp: '2026-07-11T09:00:05.500Z', + payload: { + type: 'sub_agent_activity', + event_id: 'spawn-1', + occurred_at_ms: startedAt, + agent_thread_id: 'thread-1', + agent_path: '/root/paper_audit', + kind: 'started', + }, + }, + { + type: 'event_msg', + timestamp: '2026-07-11T09:00:06.000Z', + payload: { + type: 'sub_agent_activity', + event_id: 'message-1', + occurred_at_ms: startedAt + 1_000, + agent_thread_id: 'thread-1', + agent_path: '/root/paper_audit', + kind: 'interacted', + }, + }, + { + type: 'event_msg', + timestamp: '2026-07-11T09:00:07.000Z', + payload: { + type: 'sub_agent_activity', + event_id: 'interrupt-1', + occurred_at_ms: startedAt + 2_000, + agent_thread_id: 'thread-1', + agent_path: '/root/paper_audit', + kind: 'interrupted', + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n'), + ) + + const spans = await new CodexAdapter().parse(refFor(path, 'codex')) + const tools = spans.filter((item) => item.attributes['openinference.span.kind'] === 'TOOL') + expect(tools).toHaveLength(5) + const verifications = tools.filter((item) => item.attributes['tool.name'] === 'exec_command.verify') + expect(verifications).toHaveLength(2) + const failedVerification = verifications.find((item) => item.status.code === 'ERROR') + expect(failedVerification?.attributes.content).toContain('tools.exec_command') + expect(failedVerification?.attributes['traces.codex.call_type']).toBe('custom_tool_call') + expect(failedVerification?.attributes['traces.codex.outer_tool_name']).toBe('exec') + expect(failedVerification?.attributes['traces.codex.nested_tool_name']).toBe('exec_command') + expect(failedVerification?.status.message).toContain('Command failed') + expect(verifications.find((item) => item.status.code === 'OK')?.attributes.content).toContain('/health') + const mutations = tools.filter((item) => item.attributes['tool.name'] === 'exec_command') + expect(mutations).toHaveLength(2) + expect(mutations.map((item) => item.attributes.content)).toEqual([ + expect.stringContaining('rm -rf build'), + expect.stringContaining('curl -X POST'), + ]) + expect(mutations.every((item) => item.status.code === 'OK')).toBe(true) + + const agent = tools.find((item) => item.attributes['tool.name'] === 'Agent') + expect(JSON.parse(String(agent?.attributes.content))).toEqual({ + subagent_type: 'paper_audit', + agent_path: '/root/paper_audit', + agent_thread_id: 'thread-1', + }) + expect(agent?.start_time).toBe('2026-07-11T09:00:05.000Z') + expect(agent?.end_time).toBe('2026-07-11T09:00:07.000Z') + expect(agent?.status).toEqual({ code: 'ERROR', message: 'subagent interrupted' }) + }) +}) + describe('conversation capture — single-JSON adapters', () => { it('gemini captures user.prompt + assistant text + a tool call', async () => { const path = join(dir, 'gem.json') diff --git a/tests/pipelines.test.ts b/tests/pipelines.test.ts index 6971661..f112a56 100644 --- a/tests/pipelines.test.ts +++ b/tests/pipelines.test.ts @@ -47,4 +47,19 @@ describe('runPipelines (reuses agent-eval stuckLoopView + computeToolUseMetrics) const r = await runPipelines(spans) expect(r.stuckLoops.findings.length).toBe(0) }) + + it('does not call repeated blocking waits a stuck loop', async () => { + const spans = [ + span({ traceId: 'sess', spanId: 'root', name: 'session', kind: 'AGENT', startTime: new Date(0).toISOString(), service: 'codex' }), + toolCall(1, 'write_stdin', { session_id: 7, chars: '' }), + toolCall(2, 'write_stdin', { session_id: 7, chars: '' }), + toolCall(3, 'write_stdin', { session_id: 7, chars: '' }), + toolCall(4, 'wait', { cell_id: 'a' }), + toolCall(5, 'wait', { cell_id: 'a' }), + toolCall(6, 'wait', { cell_id: 'a' }), + ] + const r = await runPipelines(spans) + expect(r.stuckLoops.findings).toHaveLength(0) + expect(r.toolUse[0]!.totalCalls).toBe(6) + }) }) From 46289473016ab70553e1d03b00718eb146593409 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 03:22:12 -0600 Subject: [PATCH 2/4] fix(pipelines): scope blocking wait suppression --- src/adapters/codex.ts | 2 ++ src/pipelines.ts | 18 ++++-------------- tests/adapters.test.ts | 21 ++++++++++++++++++++- tests/pipelines.test.ts | 33 ++++++++++++++++++++++++++------- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 8b7213d..6de075d 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -24,6 +24,7 @@ import { capText, userPromptSpan } from './conversation.js' import type { HarnessTraceAdapter, LocateOptions, SessionRef } from '../types.js' const SERVICE = 'codex' +const EXPECTED_BLOCKING_TOOLS = new Set(['wait', 'write_stdin']) interface CodexLine { timestamp?: string @@ -268,6 +269,7 @@ export class CodexAdapter implements HarnessTraceAdapter { 'traces.codex.call_type': l.payload.type, ...(name !== outerName ? { 'traces.codex.outer_tool_name': outerName } : {}), ...(nestedName ? { 'traces.codex.nested_tool_name': nestedName } : {}), + ...(EXPECTED_BLOCKING_TOOLS.has(name) ? { 'traces.expected_blocking': true } : {}), }, }) spans.push(toolSpan) diff --git a/src/pipelines.ts b/src/pipelines.ts index d557801..e0afcba 100644 --- a/src/pipelines.ts +++ b/src/pipelines.ts @@ -13,8 +13,9 @@ * error rates above, so adding it would only emit a misleading waste %. * * Both are cheap ($0, deterministic), so they're safe to run continuously in - * `watch` mode, over the OTLP spans traces already produces. Blocking wait and - * stdin-poll calls stay in usage totals but are excluded from stuck-loop findings. + * `watch` mode, over the OTLP spans traces already produces. Calls explicitly + * marked as expected blocking stay in usage totals but are excluded from + * stuck-loop findings. */ import { computeToolUseMetrics } from '@tangle-network/agent-eval' @@ -34,20 +35,9 @@ export interface PipelineOptions { minLoopOccurrences?: number } -const expectedBlockingTools = new Set(['wait', 'write_stdin']) - -function toolNameOf(span: OtlpSpan): string | null { - const value = span.attributes['tool.name'] - if (typeof value === 'string') return value - return span.name.startsWith('tool.') ? span.name.slice('tool.'.length) : null -} - export async function runPipelines(spans: readonly OtlpSpan[], opts: PipelineOptions = {}): Promise { const { store, runIds } = await toRuntimeStore(spans) - const loopEligible = spans.filter((item) => { - const name = toolNameOf(item) - return name === null || !expectedBlockingTools.has(name) - }) + const loopEligible = spans.filter((item) => item.attributes['traces.expected_blocking'] !== true) const { store: loopStore } = await toRuntimeStore(loopEligible) const stuckLoops = await stuckLoopView(loopStore, { minOccurrences: opts.minLoopOccurrences ?? 3 }) const toolUse = await Promise.all(runIds.map((runId) => computeToolUseMetrics(store, runId))) diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index aa0fc69..02c6fb5 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -280,6 +280,21 @@ describe('codex current tool and subagent events', () => { timestamp: '2026-07-11T09:00:04.600Z', payload: { type: 'custom_tool_call_output', call_id: 'custom-4', output: 'Script completed' }, }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.700Z', + payload: { + type: 'function_call', + call_id: 'blocking-1', + name: 'wait', + arguments: '{"cell_id":"running-1"}', + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.800Z', + payload: { type: 'function_call_output', call_id: 'blocking-1', output: 'Completed' }, + }, { type: 'event_msg', timestamp: '2026-07-11T09:00:05.500Z', @@ -323,7 +338,7 @@ describe('codex current tool and subagent events', () => { const spans = await new CodexAdapter().parse(refFor(path, 'codex')) const tools = spans.filter((item) => item.attributes['openinference.span.kind'] === 'TOOL') - expect(tools).toHaveLength(5) + expect(tools).toHaveLength(6) const verifications = tools.filter((item) => item.attributes['tool.name'] === 'exec_command.verify') expect(verifications).toHaveLength(2) const failedVerification = verifications.find((item) => item.status.code === 'ERROR') @@ -341,6 +356,10 @@ describe('codex current tool and subagent events', () => { ]) expect(mutations.every((item) => item.status.code === 'OK')).toBe(true) + const blocking = tools.find((item) => item.attributes['tool.name'] === 'wait') + expect(blocking?.attributes['traces.expected_blocking']).toBe(true) + expect(blocking?.status.code).toBe('OK') + const agent = tools.find((item) => item.attributes['tool.name'] === 'Agent') expect(JSON.parse(String(agent?.attributes.content))).toEqual({ subagent_type: 'paper_audit', diff --git a/tests/pipelines.test.ts b/tests/pipelines.test.ts index f112a56..c317ec4 100644 --- a/tests/pipelines.test.ts +++ b/tests/pipelines.test.ts @@ -3,7 +3,13 @@ import { span } from '../src/otlp.js' import { runPipelines } from '../src/pipelines.js' /** Build a tool call span with given name + identical input (→ same argHash). */ -function toolCall(i: number, name: string, input: unknown, status: 'OK' | 'ERROR' = 'OK') { +function toolCall( + i: number, + name: string, + input: unknown, + status: 'OK' | 'ERROR' = 'OK', + extra?: Record, +) { return span({ traceId: 'sess', spanId: `t${i}`, @@ -16,6 +22,7 @@ function toolCall(i: number, name: string, input: unknown, status: 'OK' | 'ERROR tool: name, content: JSON.stringify(input), step: i, + extra, }) } @@ -51,15 +58,27 @@ describe('runPipelines (reuses agent-eval stuckLoopView + computeToolUseMetrics) it('does not call repeated blocking waits a stuck loop', async () => { const spans = [ span({ traceId: 'sess', spanId: 'root', name: 'session', kind: 'AGENT', startTime: new Date(0).toISOString(), service: 'codex' }), - toolCall(1, 'write_stdin', { session_id: 7, chars: '' }), - toolCall(2, 'write_stdin', { session_id: 7, chars: '' }), - toolCall(3, 'write_stdin', { session_id: 7, chars: '' }), - toolCall(4, 'wait', { cell_id: 'a' }), - toolCall(5, 'wait', { cell_id: 'a' }), - toolCall(6, 'wait', { cell_id: 'a' }), + toolCall(1, 'write_stdin', { session_id: 7, chars: '' }, 'OK', { 'traces.expected_blocking': true }), + toolCall(2, 'write_stdin', { session_id: 7, chars: '' }, 'OK', { 'traces.expected_blocking': true }), + toolCall(3, 'write_stdin', { session_id: 7, chars: '' }, 'OK', { 'traces.expected_blocking': true }), + toolCall(4, 'wait', { cell_id: 'a' }, 'OK', { 'traces.expected_blocking': true }), + toolCall(5, 'wait', { cell_id: 'a' }, 'OK', { 'traces.expected_blocking': true }), + toolCall(6, 'wait', { cell_id: 'a' }, 'OK', { 'traces.expected_blocking': true }), ] const r = await runPipelines(spans) expect(r.stuckLoops.findings).toHaveLength(0) expect(r.toolUse[0]!.totalCalls).toBe(6) }) + + it('still flags repeated domain waits that are not marked as expected blocking', async () => { + const spans = [ + span({ traceId: 'sess', spanId: 'root', name: 'session', kind: 'AGENT', startTime: new Date(0).toISOString(), service: 'other' }), + toolCall(1, 'wait', { job_id: 7 }), + toolCall(2, 'wait', { job_id: 7 }), + toolCall(3, 'wait', { job_id: 7 }), + ] + const r = await runPipelines(spans) + expect(r.stuckLoops.findings).toHaveLength(1) + expect(r.stuckLoops.findings[0]!.toolName).toBe('wait') + }) }) From 9c558899f73403b1d2550cb05c44ca7db4617534 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 03:24:19 -0600 Subject: [PATCH 3/4] fix(codex): identify process polls by arguments --- src/adapters/codex.ts | 10 ++++++++-- tests/adapters.test.ts | 25 +++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 6de075d..de59812 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -24,7 +24,6 @@ import { capText, userPromptSpan } from './conversation.js' import type { HarnessTraceAdapter, LocateOptions, SessionRef } from '../types.js' const SERVICE = 'codex' -const EXPECTED_BLOCKING_TOOLS = new Set(['wait', 'write_stdin']) interface CodexLine { timestamp?: string @@ -108,6 +107,13 @@ function classifyNestedTool(name: string, input: string | undefined): string { : name } +function isExpectedBlockingTool(name: string, input: string | undefined): boolean { + if (!input) return false + if (name === 'wait') return /\bcell_id\b["']?\s*:/.test(input) + if (name === 'write_stdin') return /\bsession_id\b["']?\s*:/.test(input) + return false +} + async function* walkRollouts(root: string): AsyncGenerator { let years: string[] try { @@ -269,7 +275,7 @@ export class CodexAdapter implements HarnessTraceAdapter { 'traces.codex.call_type': l.payload.type, ...(name !== outerName ? { 'traces.codex.outer_tool_name': outerName } : {}), ...(nestedName ? { 'traces.codex.nested_tool_name': nestedName } : {}), - ...(EXPECTED_BLOCKING_TOOLS.has(name) ? { 'traces.expected_blocking': true } : {}), + ...(isExpectedBlockingTool(name, input) ? { 'traces.expected_blocking': true } : {}), }, }) spans.push(toolSpan) diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 02c6fb5..3899715 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -295,6 +295,21 @@ describe('codex current tool and subagent events', () => { timestamp: '2026-07-11T09:00:04.800Z', payload: { type: 'function_call_output', call_id: 'blocking-1', output: 'Completed' }, }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:04.900Z', + payload: { + type: 'function_call', + call_id: 'domain-wait-1', + name: 'wait', + arguments: '{"job_id":"domain-1"}', + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:05.000Z', + payload: { type: 'function_call_output', call_id: 'domain-wait-1', output: 'Completed' }, + }, { type: 'event_msg', timestamp: '2026-07-11T09:00:05.500Z', @@ -338,7 +353,7 @@ describe('codex current tool and subagent events', () => { const spans = await new CodexAdapter().parse(refFor(path, 'codex')) const tools = spans.filter((item) => item.attributes['openinference.span.kind'] === 'TOOL') - expect(tools).toHaveLength(6) + expect(tools).toHaveLength(7) const verifications = tools.filter((item) => item.attributes['tool.name'] === 'exec_command.verify') expect(verifications).toHaveLength(2) const failedVerification = verifications.find((item) => item.status.code === 'ERROR') @@ -356,9 +371,11 @@ describe('codex current tool and subagent events', () => { ]) expect(mutations.every((item) => item.status.code === 'OK')).toBe(true) - const blocking = tools.find((item) => item.attributes['tool.name'] === 'wait') - expect(blocking?.attributes['traces.expected_blocking']).toBe(true) - expect(blocking?.status.code).toBe('OK') + const waits = tools.filter((item) => item.attributes['tool.name'] === 'wait') + expect(waits).toHaveLength(2) + expect(waits.find((item) => String(item.attributes.content).includes('cell_id'))?.attributes['traces.expected_blocking']).toBe(true) + expect(waits.find((item) => String(item.attributes.content).includes('job_id'))?.attributes['traces.expected_blocking']).toBeUndefined() + expect(waits.every((item) => item.status.code === 'OK')).toBe(true) const agent = tools.find((item) => item.attributes['tool.name'] === 'Agent') expect(JSON.parse(String(agent?.attributes.content))).toEqual({ From 7dd9dce62d2acc8f0a025be7612ad6a87d8fd850 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 09:38:34 -0600 Subject: [PATCH 4/4] fix(codex): harden current event parsing --- src/adapters/codex.ts | 34 +++++++++--- tests/adapters.test.ts | 113 ++++++++++++++++++++++++++++++++++++++-- tests/pipelines.test.ts | 1 + 3 files changed, 139 insertions(+), 9 deletions(-) diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index de59812..d790da8 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -37,8 +37,8 @@ interface CodexLine { role?: string name?: string content?: unknown - arguments?: string - input?: string + arguments?: unknown + input?: unknown call_id?: string output?: unknown event_id?: string @@ -93,12 +93,22 @@ function singleNestedToolName(input: string | undefined): string | null { return unique.length === 1 ? unique[0]! : null } +function toolInputToString(input: unknown): string | undefined { + if (typeof input === 'string') return input + if (input == null) return undefined + return JSON.stringify(input) +} + const verificationCommand = /\b(?:pnpm|npm|yarn|bun)\s+(?:run\s+)?(?:test|typecheck|lint|build|check)(?::[A-Za-z0-9:_-]+)?\b|\b(?:vitest|jest|pytest|tsc|biome|eslint|sha256sum|pdfinfo|pdftotext)\b|\bgo\s+test\b|\bcargo\s+(?:test|check|clippy|build)\b|\bgit\s+(?:status|diff|show|merge-tree)\b|\bgh-drew\s+pr\s+(?:view|checks)\b/i function hasReadOnlyCurl(input: string): boolean { if (!/\bcurl\b/i.test(input)) return false - return !/(?:^|\s)(?:-X|--request)(?:=|\s*)(?:POST|PUT|PATCH|DELETE)\b|(?:^|\s)(?:-d|--data(?:-ascii|-binary|-raw|-urlencode)?)(?:=|\s)/i.test(input) + const method = + input.match(/(?:^|\s)-X(?:=|\s*)([A-Za-z]+)\b/)?.[1] ?? + input.match(/(?:^|\s)--request(?:=|\s+)([A-Za-z]+)\b/i)?.[1] + if (method && !/^(?:GET|HEAD)$/i.test(method)) return false + return !/(?:^|\s)-(?:d|F|T)(?:\S*|\s+\S+)|(?:^|\s)--(?:data(?:-ascii|-binary|-raw|-urlencode)?|form(?:-string)?|json|upload-file)(?:=|\s)/i.test(input) } function classifyNestedTool(name: string, input: string | undefined): string { @@ -256,7 +266,9 @@ export class CodexAdapter implements HarnessTraceAdapter { ) { const outerName = l.payload.name ?? 'tool' const callId = l.payload.call_id ?? `${step}` - const input = l.payload.type === 'custom_tool_call' ? l.payload.input : l.payload.arguments + const input = toolInputToString( + l.payload.type === 'custom_tool_call' ? l.payload.input : l.payload.arguments, + ) const nestedName = l.payload.type === 'custom_tool_call' ? singleNestedToolName(input) : null const name = classifyNestedTool(nestedName ?? outerName, input) const toolSpan = span({ @@ -324,13 +336,23 @@ export class CodexAdapter implements HarnessTraceAdapter { spans.push(toolSpan) subagentByThreadId.set(threadId, toolSpan) step += 1 - } else if (l.payload.kind === 'interrupted' && threadId) { + } else if (l.payload.kind === 'completed' && threadId) { + const toolSpan = subagentByThreadId.get(threadId) + if (toolSpan) { + toolSpan.end_time = eventTime + toolSpan.status = { code: 'OK' } + } + } else if ( + ['interrupted', 'failed', 'timed_out'].includes(l.payload.kind ?? '') && + threadId + ) { const toolSpan = subagentByThreadId.get(threadId) if (toolSpan) { toolSpan.end_time = eventTime - toolSpan.status = { code: 'ERROR', message: 'subagent interrupted' } + toolSpan.status = { code: 'ERROR', message: `subagent ${l.payload.kind}` } } } + // `interacted` is a progress event, not a terminal state. } else if (l.type === 'response_item' && l.payload?.type === 'message' && l.payload.role === 'user') { // The human's prompt text. Codex drops the user turn from token events, // so capture it here as its own CHAIN span (no text → no span). diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 3899715..c3eb515 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -207,7 +207,7 @@ describe('conversation capture — JSONL adapters', () => { }) describe('codex current tool and subagent events', () => { - it('captures custom tool calls, joins their outputs, and counts one subagent start', async () => { + it('captures custom tool calls, joins their outputs, and tracks subagent lifecycles', async () => { const path = join(dir, 'rollout-codex-current.jsonl') const startedAt = Date.parse('2026-07-11T09:00:05.000Z') writeFileSync( @@ -310,6 +310,36 @@ describe('codex current tool and subagent events', () => { timestamp: '2026-07-11T09:00:05.000Z', payload: { type: 'function_call_output', call_id: 'domain-wait-1', output: 'Completed' }, }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:05.100Z', + payload: { + type: 'custom_tool_call', + call_id: 'malformed-input-1', + name: 'exec', + input: { cmd: 'ls' }, + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:05.200Z', + payload: { type: 'custom_tool_call_output', call_id: 'malformed-input-1', output: 'Completed' }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:05.300Z', + payload: { + type: 'custom_tool_call', + call_id: 'write-stdin-1', + name: 'exec', + input: "const r = await tools.write_stdin({ session_id: 7, chars: '' })", + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:05.400Z', + payload: { type: 'custom_tool_call_output', call_id: 'write-stdin-1', output: 'Completed' }, + }, { type: 'event_msg', timestamp: '2026-07-11T09:00:05.500Z', @@ -346,6 +376,30 @@ describe('codex current tool and subagent events', () => { kind: 'interrupted', }, }, + { + type: 'event_msg', + timestamp: '2026-07-11T09:00:08.000Z', + payload: { + type: 'sub_agent_activity', + event_id: 'spawn-2', + occurred_at_ms: startedAt + 3_000, + agent_thread_id: 'thread-2', + agent_path: '/root/runtime_audit', + kind: 'started', + }, + }, + { + type: 'event_msg', + timestamp: '2026-07-11T09:00:09.000Z', + payload: { + type: 'sub_agent_activity', + event_id: 'complete-2', + occurred_at_ms: startedAt + 4_000, + agent_thread_id: 'thread-2', + agent_path: '/root/runtime_audit', + kind: 'completed', + }, + }, ] .map((event) => JSON.stringify(event)) .join('\n'), @@ -353,7 +407,7 @@ describe('codex current tool and subagent events', () => { const spans = await new CodexAdapter().parse(refFor(path, 'codex')) const tools = spans.filter((item) => item.attributes['openinference.span.kind'] === 'TOOL') - expect(tools).toHaveLength(7) + expect(tools).toHaveLength(10) const verifications = tools.filter((item) => item.attributes['tool.name'] === 'exec_command.verify') expect(verifications).toHaveLength(2) const failedVerification = verifications.find((item) => item.status.code === 'ERROR') @@ -377,7 +431,17 @@ describe('codex current tool and subagent events', () => { expect(waits.find((item) => String(item.attributes.content).includes('job_id'))?.attributes['traces.expected_blocking']).toBeUndefined() expect(waits.every((item) => item.status.code === 'OK')).toBe(true) - const agent = tools.find((item) => item.attributes['tool.name'] === 'Agent') + const malformed = tools.find((item) => item.attributes['tool.name'] === 'exec') + expect(malformed?.attributes.content).toBe('{"cmd":"ls"}') + expect(malformed?.status.code).toBe('OK') + + const writeStdin = tools.find((item) => item.attributes['tool.name'] === 'write_stdin') + expect(writeStdin?.attributes['traces.expected_blocking']).toBe(true) + expect(writeStdin?.status.code).toBe('OK') + + const agents = tools.filter((item) => item.attributes['tool.name'] === 'Agent') + expect(agents).toHaveLength(2) + const agent = agents.find((item) => String(item.attributes.content).includes('paper_audit')) expect(JSON.parse(String(agent?.attributes.content))).toEqual({ subagent_type: 'paper_audit', agent_path: '/root/paper_audit', @@ -386,6 +450,49 @@ describe('codex current tool and subagent events', () => { expect(agent?.start_time).toBe('2026-07-11T09:00:05.000Z') expect(agent?.end_time).toBe('2026-07-11T09:00:07.000Z') expect(agent?.status).toEqual({ code: 'ERROR', message: 'subagent interrupted' }) + + const completed = agents.find((item) => String(item.attributes.content).includes('runtime_audit')) + expect(completed?.start_time).toBe('2026-07-11T09:00:08.000Z') + expect(completed?.end_time).toBe('2026-07-11T09:00:09.000Z') + expect(completed?.status).toEqual({ code: 'OK' }) + }) + + it.each([ + ['curl https://example.test/health', 'exec_command.verify'], + ['curl -X HEAD https://example.test/health', 'exec_command.verify'], + ['curl -F file=@x https://example.test/upload', 'exec_command'], + ["curl --json '{\"ok\":true}' https://example.test", 'exec_command'], + ['curl -T ./x https://example.test/upload', 'exec_command'], + ['curl -d@payload.json https://example.test', 'exec_command'], + ['curl -X PURGE https://example.test/cache', 'exec_command'], + ])('classifies curl command %s as %s', async (command, expectedTool) => { + const path = join(dir, `rollout-codex-curl-${expectedTool}-${command.length}.jsonl`) + writeFileSync( + path, + [ + { type: 'session_meta', timestamp: '2026-07-11T09:00:00.000Z', payload: { id: `curl-${command}`, cwd: '/x' } }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:01.000Z', + payload: { + type: 'custom_tool_call', + call_id: 'curl-1', + name: 'exec', + input: `const r = await tools.exec_command({ cmd: ${JSON.stringify(command)} })`, + }, + }, + { + type: 'response_item', + timestamp: '2026-07-11T09:00:02.000Z', + payload: { type: 'custom_tool_call_output', call_id: 'curl-1', output: 'Completed' }, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n'), + ) + const spans = await new CodexAdapter().parse(refFor(path, 'codex')) + const tool = spans.find((item) => item.attributes['openinference.span.kind'] === 'TOOL') + expect(tool?.attributes['tool.name']).toBe(expectedTool) }) }) diff --git a/tests/pipelines.test.ts b/tests/pipelines.test.ts index c317ec4..75935a6 100644 --- a/tests/pipelines.test.ts +++ b/tests/pipelines.test.ts @@ -80,5 +80,6 @@ describe('runPipelines (reuses agent-eval stuckLoopView + computeToolUseMetrics) const r = await runPipelines(spans) expect(r.stuckLoops.findings).toHaveLength(1) expect(r.stuckLoops.findings[0]!.toolName).toBe('wait') + expect(r.stuckLoops.findings[0]!.occurrences).toBe(3) }) })