diff --git a/src/agents/exceptions.py b/src/agents/exceptions.py index 8c086b2c08..349004c97d 100644 --- a/src/agents/exceptions.py +++ b/src/agents/exceptions.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -11,7 +11,9 @@ from .tool_guardrails import ( ToolGuardrailFunctionOutput, ToolInputGuardrail, + ToolInputGuardrailResult, ToolOutputGuardrail, + ToolOutputGuardrailResult, ) from .util._pretty_print import pretty_print_run_error_details @@ -38,6 +40,11 @@ class RunErrorDetails: context_wrapper: RunContextWrapper[Any] input_guardrail_results: list[InputGuardrailResult] output_guardrail_results: list[OutputGuardrailResult] + tool_input_guardrail_results: list[ToolInputGuardrailResult] = field(default_factory=list) + """Tool input guardrail results accumulated from completed turns before the run failed.""" + + tool_output_guardrail_results: list[ToolOutputGuardrailResult] = field(default_factory=list) + """Tool output guardrail results accumulated from completed turns before the run failed.""" def __str__(self) -> str: return pretty_print_run_error_details(self) diff --git a/src/agents/result.py b/src/agents/result.py index daf8516927..6482cd2813 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -947,6 +947,8 @@ def _create_error_details(self) -> RunErrorDetails | None: context_wrapper=self.context_wrapper, input_guardrail_results=self.input_guardrail_results, output_guardrail_results=self.output_guardrail_results, + tool_input_guardrail_results=self.tool_input_guardrail_results, + tool_output_guardrail_results=self.tool_output_guardrail_results, ) def _check_errors(self): diff --git a/src/agents/run.py b/src/agents/run.py index ee8125ddab..286a0e9fa5 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -1601,6 +1601,8 @@ def _finalize_result(result: RunResult) -> RunResult: context_wrapper=context_wrapper, input_guardrail_results=input_guardrail_results, output_guardrail_results=output_guardrail_results, + tool_input_guardrail_results=tool_input_guardrail_results, + tool_output_guardrail_results=tool_output_guardrail_results, ) raise finally: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 65d4e794e8..a1389670f5 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -1410,6 +1410,8 @@ async def _save_stream_items_without_count( context_wrapper=context_wrapper, input_guardrail_results=streamed_result.input_guardrail_results, output_guardrail_results=streamed_result.output_guardrail_results, + tool_input_guardrail_results=streamed_result.tool_input_guardrail_results, + tool_output_guardrail_results=streamed_result.tool_output_guardrail_results, ) raise except Exception as e: diff --git a/src/agents/util/_pretty_print.py b/src/agents/util/_pretty_print.py index 9af5a3a1de..4c869f3a84 100644 --- a/src/agents/util/_pretty_print.py +++ b/src/agents/util/_pretty_print.py @@ -46,6 +46,8 @@ def pretty_print_run_error_details(result: "RunErrorDetails") -> str: output += f"\n- {len(result.raw_responses)} raw response(s)" output += f"\n- {len(result.input_guardrail_results)} input guardrail result(s)" output += f"\n- {len(result.output_guardrail_results)} output guardrail result(s)" + output += f"\n- {len(result.tool_input_guardrail_results)} tool input guardrail result(s)" + output += f"\n- {len(result.tool_output_guardrail_results)} tool output guardrail result(s)" output += "\n(See `RunErrorDetails` for more details)" return output diff --git a/tests/test_pretty_print.py b/tests/test_pretty_print.py index 5d76e0cc0a..1bb6814bd5 100644 --- a/tests/test_pretty_print.py +++ b/tests/test_pretty_print.py @@ -84,6 +84,8 @@ def test_pretty_run_error_details(): - 0 raw response(s) - 0 input guardrail result(s) - 0 output guardrail result(s) +- 0 tool input guardrail result(s) +- 0 tool output guardrail result(s) (See `RunErrorDetails` for more details)\ """) diff --git a/tests/test_source_compat_constructors.py b/tests/test_source_compat_constructors.py index 9e9aae6e34..7274cfda1d 100644 --- a/tests/test_source_compat_constructors.py +++ b/tests/test_source_compat_constructors.py @@ -16,6 +16,7 @@ MultiProvider, RunConfig, RunContextWrapper, + RunErrorDetails, RunResult, RunResultStreaming, SessionSettings, @@ -40,6 +41,33 @@ async def keep_handoff_input(data: HandoffInputData) -> HandoffInputData: assert config.session_settings is None +def test_run_error_details_positional_prefix_and_defaults_are_preserved() -> None: + first = RunErrorDetails( + "input", + [], + [], + Agent(name="agent"), + RunContextWrapper(context=None), + [], + [], + ) + second = RunErrorDetails( + "input", + [], + [], + Agent(name="agent"), + RunContextWrapper(context=None), + [], + [], + ) + + first.tool_input_guardrail_results.append(cast(Any, object())) + first.tool_output_guardrail_results.append(cast(Any, object())) + + assert second.tool_input_guardrail_results == [] + assert second.tool_output_guardrail_results == [] + + def test_run_config_session_settings_positional_binding_is_preserved() -> None: session_settings = SessionSettings(limit=123) config = RunConfig( diff --git a/tests/test_tool_guardrails.py b/tests/test_tool_guardrails.py index 30e862f1fa..9402edf247 100644 --- a/tests/test_tool_guardrails.py +++ b/tests/test_tool_guardrails.py @@ -7,6 +7,8 @@ from agents import ( Agent, + MaxTurnsExceeded, + Runner, ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData, @@ -15,10 +17,14 @@ ToolOutputGuardrailData, ToolOutputGuardrailTripwireTriggered, UserError, + function_tool, ) from agents.tool_context import ToolContext from agents.tool_guardrails import tool_input_guardrail, tool_output_guardrail +from .fake_model import FakeModel +from .test_responses import get_function_tool_call + def get_mock_tool_context(tool_arguments: str = '{"param": "value"}') -> ToolContext: """Helper to create a mock tool context for testing.""" @@ -520,6 +526,119 @@ def mixed_guardrail(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutpu assert result.output_info["status"] == "clean" +def _agent_with_repeated_guarded_tool_calls( + *, + input_guardrails: list[ToolInputGuardrail[Any]] | None = None, + output_guardrails: list[ToolOutputGuardrail[Any]] | None = None, +) -> Agent[Any]: + @function_tool + def guarded(query: str) -> str: + return "tool output" + + guarded.tool_input_guardrails = input_guardrails or [] + guarded.tool_output_guardrails = output_guardrails or [] + + model = FakeModel() + tool_call = [get_function_tool_call("guarded", '{"query": "secret"}')] + model.add_multiple_turn_outputs([tool_call, tool_call]) + return Agent(name="guarded_tool_agent", model=model, tools=[guarded]) + + +async def _run_until_max_turns(agent: Agent[Any], *, streaming: bool) -> MaxTurnsExceeded: + with pytest.raises(MaxTurnsExceeded) as exc_info: + if streaming: + result = Runner.run_streamed(agent, "go", max_turns=2) + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "go", max_turns=2) + return exc_info.value + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [False, True]) +async def test_tool_input_guardrail_results_reported_on_max_turns(streaming: bool): + async def reject(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + return ToolGuardrailFunctionOutput.reject_content( + message="blocked by policy", output_info="input_rejected" + ) + + guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail( + guardrail_function=reject, + name="input_rejects", + ) + exc = await _run_until_max_turns( + _agent_with_repeated_guarded_tool_calls(input_guardrails=[guardrail]), + streaming=streaming, + ) + + assert exc.run_data is not None + assert [ + result.guardrail.get_name() for result in exc.run_data.tool_input_guardrail_results + ] == ["input_rejects", "input_rejects"] + assert exc.run_data.tool_output_guardrail_results == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [False, True]) +async def test_tool_output_guardrail_results_reported_on_max_turns(streaming: bool): + async def reject(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + return ToolGuardrailFunctionOutput.reject_content( + message="blocked by policy", output_info="output_rejected" + ) + + guardrail: ToolOutputGuardrail[Any] = ToolOutputGuardrail( + guardrail_function=reject, + name="output_rejects", + ) + exc = await _run_until_max_turns( + _agent_with_repeated_guarded_tool_calls(output_guardrails=[guardrail]), + streaming=streaming, + ) + + assert exc.run_data is not None + assert [ + result.guardrail.get_name() for result in exc.run_data.tool_output_guardrail_results + ] == ["output_rejects", "output_rejects"] + assert exc.run_data.tool_input_guardrail_results == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [False, True]) +async def test_tool_tripwire_preserves_completed_turn_results(streaming: bool): + guardrail_runs = 0 + + async def allow_then_raise(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + nonlocal guardrail_runs + guardrail_runs += 1 + if guardrail_runs == 2: + return ToolGuardrailFunctionOutput.raise_exception(output_info="second_turn") + return ToolGuardrailFunctionOutput.allow(output_info="first_turn") + + guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail( + guardrail_function=allow_then_raise, + name="allow_then_raise", + ) + agent = _agent_with_repeated_guarded_tool_calls(input_guardrails=[guardrail]) + + with pytest.raises(ToolInputGuardrailTripwireTriggered) as exc_info: + if streaming: + result = Runner.run_streamed(agent, "go") + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, "go") + + exc = exc_info.value + assert exc.guardrail is guardrail + assert exc.output.output_info == "second_turn" + assert exc.run_data is not None + assert [result.output.output_info for result in exc.run_data.tool_input_guardrail_results] == [ + "first_turn" + ] + assert exc.run_data.tool_output_guardrail_results == [] + + if __name__ == "__main__": # Run a simple test to verify functionality async def main():