Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/agents/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
Expand All @@ -11,7 +11,9 @@
from .tool_guardrails import (
ToolGuardrailFunctionOutput,
ToolInputGuardrail,
ToolInputGuardrailResult,
ToolOutputGuardrail,
ToolOutputGuardrailResult,
)

from .util._pretty_print import pretty_print_run_error_details
Expand All @@ -38,6 +40,11 @@ class RunErrorDetails:
context_wrapper: RunContextWrapper[Any]
input_guardrail_results: list[InputGuardrailResult]
output_guardrail_results: list[OutputGuardrailResult]
tool_input_guardrail_results: list[ToolInputGuardrailResult] = field(default_factory=list)
"""Tool input guardrail results accumulated from completed turns before the run failed."""

tool_output_guardrail_results: list[ToolOutputGuardrailResult] = field(default_factory=list)
"""Tool output guardrail results accumulated from completed turns before the run failed."""

def __str__(self) -> str:
return pretty_print_run_error_details(self)
Expand Down
2 changes: 2 additions & 0 deletions src/agents/result.py
Original file line number Diff line number Diff line change
Expand Up @@ -947,6 +947,8 @@ def _create_error_details(self) -> RunErrorDetails | None:
context_wrapper=self.context_wrapper,
input_guardrail_results=self.input_guardrail_results,
output_guardrail_results=self.output_guardrail_results,
tool_input_guardrail_results=self.tool_input_guardrail_results,
tool_output_guardrail_results=self.tool_output_guardrail_results,
)

def _check_errors(self):
Expand Down
2 changes: 2 additions & 0 deletions src/agents/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1601,6 +1601,8 @@ def _finalize_result(result: RunResult) -> RunResult:
context_wrapper=context_wrapper,
input_guardrail_results=input_guardrail_results,
output_guardrail_results=output_guardrail_results,
tool_input_guardrail_results=tool_input_guardrail_results,
tool_output_guardrail_results=tool_output_guardrail_results,
)
raise
finally:
Expand Down
2 changes: 2 additions & 0 deletions src/agents/run_internal/run_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -1410,6 +1410,8 @@ async def _save_stream_items_without_count(
context_wrapper=context_wrapper,
input_guardrail_results=streamed_result.input_guardrail_results,
output_guardrail_results=streamed_result.output_guardrail_results,
tool_input_guardrail_results=streamed_result.tool_input_guardrail_results,
tool_output_guardrail_results=streamed_result.tool_output_guardrail_results,
)
raise
except Exception as e:
Expand Down
2 changes: 2 additions & 0 deletions src/agents/util/_pretty_print.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ def pretty_print_run_error_details(result: "RunErrorDetails") -> str:
output += f"\n- {len(result.raw_responses)} raw response(s)"
output += f"\n- {len(result.input_guardrail_results)} input guardrail result(s)"
output += f"\n- {len(result.output_guardrail_results)} output guardrail result(s)"
output += f"\n- {len(result.tool_input_guardrail_results)} tool input guardrail result(s)"
output += f"\n- {len(result.tool_output_guardrail_results)} tool output guardrail result(s)"
output += "\n(See `RunErrorDetails` for more details)"

return output
Expand Down
2 changes: 2 additions & 0 deletions tests/test_pretty_print.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ def test_pretty_run_error_details():
- 0 raw response(s)
- 0 input guardrail result(s)
- 0 output guardrail result(s)
- 0 tool input guardrail result(s)
- 0 tool output guardrail result(s)
(See `RunErrorDetails` for more details)\
""")

Expand Down
28 changes: 28 additions & 0 deletions tests/test_source_compat_constructors.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
MultiProvider,
RunConfig,
RunContextWrapper,
RunErrorDetails,
RunResult,
RunResultStreaming,
SessionSettings,
Expand All @@ -40,6 +41,33 @@ async def keep_handoff_input(data: HandoffInputData) -> HandoffInputData:
assert config.session_settings is None


def test_run_error_details_positional_prefix_and_defaults_are_preserved() -> None:
first = RunErrorDetails(
"input",
[],
[],
Agent(name="agent"),
RunContextWrapper(context=None),
[],
[],
)
second = RunErrorDetails(
"input",
[],
[],
Agent(name="agent"),
RunContextWrapper(context=None),
[],
[],
)

first.tool_input_guardrail_results.append(cast(Any, object()))
first.tool_output_guardrail_results.append(cast(Any, object()))

assert second.tool_input_guardrail_results == []
assert second.tool_output_guardrail_results == []


def test_run_config_session_settings_positional_binding_is_preserved() -> None:
session_settings = SessionSettings(limit=123)
config = RunConfig(
Expand Down
119 changes: 119 additions & 0 deletions tests/test_tool_guardrails.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

from agents import (
Agent,
MaxTurnsExceeded,
Runner,
ToolGuardrailFunctionOutput,
ToolInputGuardrail,
ToolInputGuardrailData,
Expand All @@ -15,10 +17,14 @@
ToolOutputGuardrailData,
ToolOutputGuardrailTripwireTriggered,
UserError,
function_tool,
)
from agents.tool_context import ToolContext
from agents.tool_guardrails import tool_input_guardrail, tool_output_guardrail

from .fake_model import FakeModel
from .test_responses import get_function_tool_call


def get_mock_tool_context(tool_arguments: str = '{"param": "value"}') -> ToolContext:
"""Helper to create a mock tool context for testing."""
Expand Down Expand Up @@ -520,6 +526,119 @@ def mixed_guardrail(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutpu
assert result.output_info["status"] == "clean"


def _agent_with_repeated_guarded_tool_calls(
*,
input_guardrails: list[ToolInputGuardrail[Any]] | None = None,
output_guardrails: list[ToolOutputGuardrail[Any]] | None = None,
) -> Agent[Any]:
@function_tool
def guarded(query: str) -> str:
return "tool output"

guarded.tool_input_guardrails = input_guardrails or []
guarded.tool_output_guardrails = output_guardrails or []

model = FakeModel()
tool_call = [get_function_tool_call("guarded", '{"query": "secret"}')]
model.add_multiple_turn_outputs([tool_call, tool_call])
return Agent(name="guarded_tool_agent", model=model, tools=[guarded])


async def _run_until_max_turns(agent: Agent[Any], *, streaming: bool) -> MaxTurnsExceeded:
with pytest.raises(MaxTurnsExceeded) as exc_info:
if streaming:
result = Runner.run_streamed(agent, "go", max_turns=2)
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, "go", max_turns=2)
return exc_info.value


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_tool_input_guardrail_results_reported_on_max_turns(streaming: bool):
async def reject(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.reject_content(
message="blocked by policy", output_info="input_rejected"
)

guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail(
guardrail_function=reject,
name="input_rejects",
)
exc = await _run_until_max_turns(
_agent_with_repeated_guarded_tool_calls(input_guardrails=[guardrail]),
streaming=streaming,
)

assert exc.run_data is not None
assert [
result.guardrail.get_name() for result in exc.run_data.tool_input_guardrail_results
] == ["input_rejects", "input_rejects"]
assert exc.run_data.tool_output_guardrail_results == []


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_tool_output_guardrail_results_reported_on_max_turns(streaming: bool):
async def reject(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.reject_content(
message="blocked by policy", output_info="output_rejected"
)

guardrail: ToolOutputGuardrail[Any] = ToolOutputGuardrail(
guardrail_function=reject,
name="output_rejects",
)
exc = await _run_until_max_turns(
_agent_with_repeated_guarded_tool_calls(output_guardrails=[guardrail]),
streaming=streaming,
)

assert exc.run_data is not None
assert [
result.guardrail.get_name() for result in exc.run_data.tool_output_guardrail_results
] == ["output_rejects", "output_rejects"]
assert exc.run_data.tool_input_guardrail_results == []


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_tool_tripwire_preserves_completed_turn_results(streaming: bool):
guardrail_runs = 0

async def allow_then_raise(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
nonlocal guardrail_runs
guardrail_runs += 1
if guardrail_runs == 2:
return ToolGuardrailFunctionOutput.raise_exception(output_info="second_turn")
return ToolGuardrailFunctionOutput.allow(output_info="first_turn")

guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail(
guardrail_function=allow_then_raise,
name="allow_then_raise",
)
agent = _agent_with_repeated_guarded_tool_calls(input_guardrails=[guardrail])

with pytest.raises(ToolInputGuardrailTripwireTriggered) as exc_info:
if streaming:
result = Runner.run_streamed(agent, "go")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, "go")

exc = exc_info.value
assert exc.guardrail is guardrail
assert exc.output.output_info == "second_turn"
assert exc.run_data is not None
assert [result.output.output_info for result in exc.run_data.tool_input_guardrail_results] == [
"first_turn"
]
assert exc.run_data.tool_output_guardrail_results == []


if __name__ == "__main__":
# Run a simple test to verify functionality
async def main():
Expand Down