fix(models): record model-call failures on the provider's own span

src/agents/extensions/models/any_llm_model.py

@@ -53,6 +53,7 @@
 from ...tracing.span_data import GenerationSpanData
 from ...tracing.spans import Span
 from ...usage import Usage
+from ...util._error_tracing import model_span_errors, record_model_error_on_span
 from ...util._json import _to_dump_compatible
 
 try:
@@ -361,7 +362,14 @@ async def _get_response_via_responses(
         conversation_id: str | None,
         prompt: ResponsePromptParam | None,
     ) -> ModelResponse:
-        with response_span(disabled=tracing.is_disabled()) as span_response:
+        with (
+            response_span(disabled=tracing.is_disabled()) as span_response,
+            model_span_errors(
+                span_response,
+                message="Error getting response",
+                trace_include_sensitive_data=tracing.include_data(),
+            ),
+        ):
             response = await self._fetch_responses_response(
                 system_instructions=system_instructions,
                 input=input,
@@ -425,7 +433,14 @@ async def _stream_response_via_responses(
         conversation_id: str | None,
         prompt: ResponsePromptParam | None,
     ) -> AsyncGenerator[ResponseStreamEvent, None]:
-        with response_span(disabled=tracing.is_disabled()) as span_response:
+        with (
+            response_span(disabled=tracing.is_disabled()) as span_response,
+            model_span_errors(
+                span_response,
+                message="Error streaming response",
+                trace_include_sensitive_data=tracing.include_data(),
+            ),
+        ):
             stream = await self._fetch_responses_response(
                 system_instructions=system_instructions,
                 input=input,
@@ -472,6 +487,17 @@ async def _stream_response_via_responses(
                         if tracing.include_data() and final_response:
                             span_response.span_data.response = final_response
                             span_response.span_data.input = input
+                        if terminal_failure_error is not None:
+                            # The failure is already known here. A consumer that stops at
+                            # this event closes the generator, which raises GeneratorExit
+                            # at the yield below and skips the raise after the loop, so
+                            # recording later would miss it entirely.
+                            record_model_error_on_span(
+                                span_response,
+                                message="Error streaming response",
+                                error=terminal_failure_error,
+                                trace_include_sensitive_data=tracing.include_data(),
+                            )
                     yield chunk
             except asyncio.CancelledError:
                 close_stream_in_background = True
@@ -506,15 +532,22 @@ async def _get_response_via_chat(
         tracing: ModelTracing,
         prompt: ResponsePromptParam | None,
     ) -> ModelResponse:
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(
-                model_settings,
-                base_url=self.base_url or "",
-                extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(
+                    model_settings,
+                    base_url=self.base_url or "",
+                    extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
+                ),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error getting response",
+                trace_include_sensitive_data=tracing.include_data(),
             ),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        ):
             response = await self._fetch_chat_response(
                 system_instructions=system_instructions,
                 input=input,
@@ -608,15 +641,22 @@ async def _stream_response_via_chat(
         tracing: ModelTracing,
         prompt: ResponsePromptParam | None,
     ) -> AsyncGenerator[TResponseStreamEvent, None]:
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(
-                model_settings,
-                base_url=self.base_url or "",
-                extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(
+                    model_settings,
+                    base_url=self.base_url or "",
+                    extra_config={"provider": self._provider_name, "model_impl": "any-llm"},
+                ),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error streaming response",
+                trace_include_sensitive_data=tracing.include_data(),
             ),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        ):
             response, stream = await self._fetch_chat_response(
                 system_instructions=system_instructions,
                 input=input,

src/agents/extensions/models/litellm_model.py

@@ -59,6 +59,7 @@
 from ...tracing.span_data import GenerationSpanData
 from ...tracing.spans import Span
 from ...usage import Usage, _cache_write_tokens, _make_input_tokens_details
+from ...util._error_tracing import model_span_errors
 from ...util._json import _to_dump_compatible
 
 
@@ -214,15 +215,22 @@ async def get_response(
         conversation_id: str | None = None,  # unused
         prompt: Any | None = None,
     ) -> ModelResponse:
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(
-                model_settings,
-                base_url=self.base_url or "",
-                extra_config={"model_impl": "litellm"},
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(
+                    model_settings,
+                    base_url=self.base_url or "",
+                    extra_config={"model_impl": "litellm"},
+                ),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error getting response",
+                trace_include_sensitive_data=tracing.include_data(),
             ),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        ):
             response = await self._fetch_response(
                 system_instructions,
                 input,
@@ -377,15 +385,22 @@ async def stream_response(
         conversation_id: str | None = None,  # unused
         prompt: Any | None = None,
     ) -> AsyncIterator[TResponseStreamEvent]:
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(
-                model_settings,
-                base_url=self.base_url or "",
-                extra_config={"model_impl": "litellm"},
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(
+                    model_settings,
+                    base_url=self.base_url or "",
+                    extra_config={"model_impl": "litellm"},
+                ),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error streaming response",
+                trace_include_sensitive_data=tracing.include_data(),
             ),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        ):
             response, stream = await self._fetch_response(
                 system_instructions,
                 input,

src/agents/models/openai_chatcompletions.py

@@ -32,6 +32,7 @@
 from ..tracing.span_data import GenerationSpanData
 from ..tracing.spans import Span
 from ..usage import Usage
+from ..util._error_tracing import model_span_errors
 from ..util._json import _to_dump_compatible
 from ._openai_retry import get_openai_retry_advice
 from ._retry_runtime import should_disable_provider_managed_retries
@@ -206,11 +207,18 @@ async def get_response(
         )
         self._handle_unsupported_prompt(prompt)
 
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error getting response",
+                trace_include_sensitive_data=tracing.include_data(),
+            ),
+        ):
             response = await self._fetch_response(
                 system_instructions,
                 input,
@@ -340,11 +348,18 @@ async def stream_response(
         )
         self._handle_unsupported_prompt(prompt)
 
-        with generation_span(
-            model=str(self.model),
-            model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
-            disabled=tracing.is_disabled(),
-        ) as span_generation:
+        with (
+            generation_span(
+                model=str(self.model),
+                model_config=model_config_for_trace(model_settings, base_url=self._client.base_url),
+                disabled=tracing.is_disabled(),
+            ) as span_generation,
+            model_span_errors(
+                span_generation,
+                message="Error streaming response",
+                trace_include_sensitive_data=tracing.include_data(),
+            ),
+        ):
             response, stream = await self._fetch_response(
                 system_instructions,
                 input,

src/agents/models/openai_responses.py

@@ -75,6 +75,7 @@
 )
 from ..tracing import SpanError, response_span
 from ..usage import Usage, _response_usage_to_usage, model_usage_to_span_usage
+from ..util._error_tracing import record_model_error_on_span
 from ..util._json import _to_dump_compatible
 from ..version import __version__
 from ._openai_retry import get_openai_retry_advice
@@ -593,6 +594,17 @@ async def stream_response(
                             "response.error",
                         }:
                             yielded_terminal_event = True
+                            if terminal_failure_error is not None:
+                                # A consumer that stops at this event closes the
+                                # generator, which raises GeneratorExit at the yield
+                                # below and skips the raise after the loop, so the
+                                # span has to be annotated here or not at all.
+                                record_model_error_on_span(
+                                    span_response,
+                                    message="Error streaming response",
+                                    error=terminal_failure_error,
+                                    trace_include_sensitive_data=tracing.include_data(),
+                                )
                         yield chunk
                 except asyncio.CancelledError:
                     close_stream_in_background = True

src/agents/util/_error_tracing.py

@@ -1,3 +1,5 @@
+import contextlib
+from collections.abc import Iterator
 from typing import Any
 
 from .. import _debug
@@ -29,3 +31,82 @@ def attach_error_to_current_span(error: SpanError) -> None:
         logger.warning("No active span; trace error was not attached")
     else:
         logger.warning("No span to add error %s to", error)
+
+
+def _model_error_text(error: Exception, *, trace_include_sensitive_data: bool) -> str:
+    """Render the span text for a failed model call.
+
+    The exception is only stringified when its text will actually be exported, so a
+    provider exception with a side-effecting `__str__` is not invoked just to have
+    its output thrown away by redaction.
+    """
+    if not trace_include_sensitive_data:
+        return REDACTED_TRACE_ERROR_MESSAGE
+    try:
+        return str(error)
+    except Exception:
+        logger.warning(
+            "Could not stringify %s for the model span; recording the type only",
+            type(error).__name__,
+        )
+        return f"Unrenderable {type(error).__name__}"
+
+
+def record_model_error_on_span(
+    span: Span[Any],
+    *,
+    message: str,
+    error: Exception,
+    trace_include_sensitive_data: bool,
+) -> None:
+    """Record an already-known model failure on its span.
+
+    Streaming providers learn about a terminal failure before they raise it, and a
+    consumer that stops at that terminal event closes the generator, so the raise
+    never happens. Recording at the point of knowledge keeps the span accurate in
+    that case. Best-effort: never raises, so annotating a span cannot change what
+    the caller sees.
+    """
+    try:
+        attach_error_to_span(
+            span,
+            SpanError(
+                message=message,
+                data={
+                    "error": _model_error_text(
+                        error,
+                        trace_include_sensitive_data=trace_include_sensitive_data,
+                    )
+                },
+            ),
+        )
+    except Exception:
+        logger.warning("Could not record the model error on the span", exc_info=True)
+
+
+@contextlib.contextmanager
+def model_span_errors(
+    span: Span[Any],
+    *,
+    message: str,
+    trace_include_sensitive_data: bool,
+) -> Iterator[None]:
+    """Record a failing model call on the span it happened in, then re-raise.
+
+    `Span.__exit__` finishes a span without attaching an exception, so a provider
+    that does not annotate its own span exports a failed model call that is
+    indistinguishable from a successful one.
+
+    Recording is best-effort on purpose: the exception the caller sees is always the
+    one the provider raised, never one produced while annotating the span.
+    """
+    try:
+        yield
+    except Exception as error:
+        record_model_error_on_span(
+            span,
+            message=message,
+            error=error,
+            trace_include_sensitive_data=trace_include_sensitive_data,
+        )
+        raise