session: classify litellm exceptions raised inside user callables

The rollout isolation in the previous commit converts target-side
LLMInputError into a recorded transcript event with
stop_reason="target_input_refused". It only fires when the exception
reaches the rollout helpers as an already-classified LLM error class.

For callable: targets (LangGraph agents, framework wrappers, raw
litellm callers, etc.) the user's own code makes the provider calls
and bypasses generate()/_with_retries entirely, so provider errors
bubble up as raw litellm exceptions (BadRequestError,
ContentPolicyViolationError, RateLimitError, ...). The rollout's
isinstance(_, LLMInputError) checks miss them and the stage aborts on
the first content-filter rejection \u2014 the same failure pattern the
isolation was supposed to fix.

Fix CallableSession.run_turn so any exception escaping invoke_callable
is passed through _classify_llm_error before propagating. Classified
errors are re-raised with __cause__ preserved; unclassified errors
(user agent crashes, ValueError from misconfigured tools, etc.) pass
through unchanged so they don't get smuggled into one of the four LLM
error classes.

Tests:

- test_run_turn_reclassifies_litellm_bad_request_as_input_error: a
  fake litellm BadRequestError raised from inside the user callable
  emerges as LLMInputError with cause preserved.
- test_run_turn_passes_through_unclassified_exceptions: a custom
  RuntimeError from inside the user callable propagates as itself,
  not as any LLM*Error class.

All 50 rollout + exception-handling tests pass. Local smoke against
azure/gpt-5.4-mini with the LangGraph travel-planner callable
(30 prompts + 10 scenarios, C=4) completed end-to-end in 176.9s.

Author: jakepresent

p2m/core/session.py

@@ -15,6 +15,7 @@
     Message,
     ModelResponse,
     ToolCall,
+    _classify_llm_error,
     build_llm_call_trace,
     generate,
     generate_with_tools,
@@ -512,15 +513,35 @@ async def run_turn(self, messages: list[Message]) -> TurnResult:
                 for msg in messages
                 if msg.role in ("user", "assistant")
             ]
-            raw_result = await invoke_callable(
-                self._callable, user_text, history=history,
-                timeout_s=self._message_timeout_s,
-            )
+            try:
+                raw_result = await invoke_callable(
+                    self._callable, user_text, history=history,
+                    timeout_s=self._message_timeout_s,
+                )
+            except Exception as exc:
+                # The user callable typically makes its own LLM calls
+                # (LangGraph, agent frameworks, raw litellm) which bypass
+                # our generate()/_with_retries wrapper, so provider errors
+                # bubble up unclassified. Re-raise them as the right p2m
+                # error class (LLMInputError for content-filter / 400s,
+                # LLMRateLimitError for 429s, LLMProviderError for 5xx,
+                # LLMAuthError for 401/403) so the rollout stage's per-seed
+                # isolation paths can do their job.
+                classified = _classify_llm_error(exc)
+                if classified is exc:
+                    raise
+                raise classified from exc
         else:
-            raw_result = await invoke_callable(
-                self._callable, user_text,
-                timeout_s=self._message_timeout_s,
-            )
+            try:
+                raw_result = await invoke_callable(
+                    self._callable, user_text,
+                    timeout_s=self._message_timeout_s,
+                )
+            except Exception as exc:
+                classified = _classify_llm_error(exc)
+                if classified is exc:
+                    raise
+                raise classified from exc
 
         result = self._normalize_callable_result(raw_result)
 

tests/test_exception_handling.py

@@ -123,6 +123,75 @@ async def test_valid_callable_opens_successfully(self) -> None:
         await session.open()
         await session.close()
 
+    async def test_run_turn_reclassifies_litellm_bad_request_as_input_error(self) -> None:
+        """User callables (LangGraph, agent frameworks, raw litellm) bypass
+        ``generate()``/``_with_retries`` and so emit unclassified provider
+        errors. ``CallableSession.run_turn`` must re-classify them so the
+        rollout stage's per-seed isolation paths can route content-filter
+        rejections to a recorded transcript event instead of aborting the
+        whole batch.
+        """
+        from p2m.core.session import CallableSession
+        from p2m.core.model_client import LLMInputError, Message
+
+        # Lightweight stand-in for ``litellm.BadRequestError`` that avoids
+        # importing litellm in the unit test. ``_classify_llm_error`` reads
+        # exception types from the live litellm module at call time, so we
+        # patch it directly to make the substitution explicit.
+        class FakeLitellmBadRequest(Exception):
+            pass
+
+        def _classified(exc: Exception) -> Exception:
+            if isinstance(exc, FakeLitellmBadRequest):
+                err = LLMInputError(f"Bad request: {exc}")
+                err.__cause__ = exc
+                return err
+            return exc
+
+        async def fake_invoke_callable(fn, *args, **kwargs):
+            raise FakeLitellmBadRequest(
+                "Invalid prompt: your prompt was flagged as potentially "
+                "violating our usage policy"
+            )
+
+        session = CallableSession(callable_ref="json:dumps")
+        await session.open()
+        try:
+            with (
+                patch("p2m.core.session.invoke_callable", new=fake_invoke_callable),
+                patch("p2m.core.session._classify_llm_error", new=_classified),
+            ):
+                with self.assertRaises(LLMInputError) as ctx:
+                    await session.run_turn([Message(role="user", content="hi")])
+            self.assertIn("flagged as potentially violating", str(ctx.exception))
+            self.assertIsInstance(ctx.exception.__cause__, FakeLitellmBadRequest)
+        finally:
+            await session.close()
+
+    async def test_run_turn_passes_through_unclassified_exceptions(self) -> None:
+        """Errors that the classifier doesn't recognise (e.g. user agent
+        crashes, ValueError from misconfigured tools) must propagate as-is
+        rather than being smuggled into one of the four LLM error classes.
+        """
+        from p2m.core.session import CallableSession
+        from p2m.core.model_client import Message
+
+        class CustomAgentError(RuntimeError):
+            pass
+
+        async def fake_invoke_callable(fn, *args, **kwargs):
+            raise CustomAgentError("user agent blew up")
+
+        session = CallableSession(callable_ref="json:dumps")
+        await session.open()
+        try:
+            with patch("p2m.core.session.invoke_callable", new=fake_invoke_callable):
+                with self.assertRaises(CustomAgentError) as ctx:
+                    await session.run_turn([Message(role="user", content="hi")])
+            self.assertIn("user agent blew up", str(ctx.exception))
+        finally:
+            await session.close()
+
 
 class OTelTracedSessionErrorTest(unittest.IsolatedAsyncioTestCase):
     async def test_missing_module_raises_value_error(self) -> None: