From 1e4e7e059dcc7f7c43f93f0b8755c7a9f1f74346 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 17 Apr 2026 08:58:32 +0100 Subject: [PATCH 1/3] feat(prompts): add capture_errors option to report fetch failures to error tracking When enabled (opt-in, default off), prompt fetch failures are reported to PostHog error tracking via `capture_exception()` before the existing fallback logic runs (stale cache, code fallback, or re-raise). Guarded with `hasattr` check so clients without `capture_exception` don't blow up. Capture failures are silently logged and never affect fallback resolution. --- .sampo/changesets/prompts-capture-errors.md | 5 + posthog/ai/prompts.py | 21 ++++ posthog/test/ai/test_prompts.py | 115 ++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .sampo/changesets/prompts-capture-errors.md diff --git a/.sampo/changesets/prompts-capture-errors.md b/.sampo/changesets/prompts-capture-errors.md new file mode 100644 index 00000000..fd84a95e --- /dev/null +++ b/.sampo/changesets/prompts-capture-errors.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Add `capture_errors` option to `Prompts` that reports prompt fetch failures to PostHog error tracking via `capture_exception()` when enabled. diff --git a/posthog/ai/prompts.py b/posthog/ai/prompts.py index 4e116cac..93c51836 100644 --- a/posthog/ai/prompts.py +++ b/posthog/ai/prompts.py @@ -94,6 +94,9 @@ class Prompts: host='https://us.posthog.com', ) + # With error tracking: prompt fetch failures are reported to PostHog + prompts = Prompts(posthog, capture_errors=True) + # Fetch with caching and fallback template = prompts.get('support-system-prompt', fallback='You are a helpful assistant.') @@ -116,6 +119,7 @@ def __init__( project_api_key: Optional[str] = None, host: Optional[str] = None, default_cache_ttl_seconds: Optional[int] = None, + capture_errors: bool = False, ): """ Initialize Prompts. @@ -126,12 +130,16 @@ def __init__( project_api_key: Direct project API key (optional if posthog provided) host: PostHog host (defaults to app endpoint) default_cache_ttl_seconds: Default cache TTL (defaults to 300) + capture_errors: If True and a PostHog client is provided, prompt fetch + failures are reported to PostHog error tracking via capture_exception(). """ self._default_cache_ttl_seconds = ( default_cache_ttl_seconds or DEFAULT_CACHE_TTL_SECONDS ) self._cache: Dict[PromptCacheKey, CachedPrompt] = {} self._has_warned_deprecation = False + self._client = posthog + self._capture_errors = capture_errors if posthog is not None: self._personal_api_key = getattr(posthog, "personal_api_key", None) or "" @@ -296,6 +304,8 @@ def _get_internal( ) except Exception as error: + self._maybe_capture_error(error) + prompt_reference = _prompt_reference(name, version) # Return stale cache (with warning) if cached is not None: @@ -363,6 +373,17 @@ def clear_cache( for key in keys_to_clear: self._cache.pop(key, None) + def _maybe_capture_error(self, error: Exception) -> None: + """Report a prompt fetch error to PostHog error tracking if enabled.""" + if not self._capture_errors or self._client is None: + return + if not hasattr(self._client, "capture_exception"): + return + try: + self._client.capture_exception(error) + except Exception: + log.debug("[PostHog Prompts] Failed to capture exception to error tracking") + def _fetch_prompt_from_api( self, name: str, version: Optional[int] = None ) -> Dict[str, Any]: diff --git a/posthog/test/ai/test_prompts.py b/posthog/test/ai/test_prompts.py index a77cdd5f..5167e412 100644 --- a/posthog/test/ai/test_prompts.py +++ b/posthog/test/ai/test_prompts.py @@ -848,6 +848,121 @@ def test_handle_variables_with_dots(self): self.assertEqual(result, "Company: Acme") +class TestPromptsCaptureErrors(TestPrompts): + """Tests for the capture_errors option.""" + + @patch("posthog.ai.prompts._get_session") + def test_capture_exception_called_on_fetch_failure_with_fallback( + self, mock_get_session + ): + """Should call capture_exception on fetch failure when capture_errors=True.""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = Exception("Network error") + + posthog = self.create_mock_posthog() + prompts = Prompts(posthog, capture_errors=True) + + result = prompts.get("test-prompt", fallback="fallback prompt") + + self.assertEqual(result, "fallback prompt") + posthog.capture_exception.assert_called_once() + captured_exc = posthog.capture_exception.call_args[0][0] + self.assertIn("Network error", str(captured_exc)) + + @patch("posthog.ai.prompts._get_session") + @patch("posthog.ai.prompts.time.time") + def test_capture_exception_called_on_fetch_failure_with_stale_cache( + self, mock_time, mock_get_session + ): + """Should call capture_exception when falling back to stale cache.""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = [ + MockResponse(json_data=self.mock_prompt_response), + Exception("Network error"), + ] + mock_time.return_value = 1000.0 + + posthog = self.create_mock_posthog() + prompts = Prompts(posthog, capture_errors=True) + + # First call populates cache + prompts.get("test-prompt", cache_ttl_seconds=60) + + # Expire cache + mock_time.return_value = 1061.0 + + # Second call falls back to stale cache + result = prompts.get("test-prompt", cache_ttl_seconds=60) + self.assertEqual(result, self.mock_prompt_response["prompt"]) + posthog.capture_exception.assert_called_once() + + @patch("posthog.ai.prompts._get_session") + def test_capture_exception_called_when_error_is_raised(self, mock_get_session): + """Should call capture_exception even when the error is re-raised (no fallback, no cache).""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = Exception("Network error") + + posthog = self.create_mock_posthog() + prompts = Prompts(posthog, capture_errors=True) + + with self.assertRaises(Exception): + prompts.get("test-prompt") + + posthog.capture_exception.assert_called_once() + + @patch("posthog.ai.prompts._get_session") + def test_no_capture_exception_when_capture_errors_is_false(self, mock_get_session): + """Should NOT call capture_exception when capture_errors=False (default).""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = Exception("Network error") + + posthog = self.create_mock_posthog() + prompts = Prompts(posthog) + + prompts.get("test-prompt", fallback="fallback prompt") + + posthog.capture_exception.assert_not_called() + + @patch("posthog.ai.prompts._get_session") + def test_no_capture_exception_without_client(self, mock_get_session): + """Should not error when capture_errors=True but no client provided.""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = Exception("Network error") + + prompts = Prompts(personal_api_key="phx_test_key", capture_errors=True) + + result = prompts.get("test-prompt", fallback="fallback prompt") + + self.assertEqual(result, "fallback prompt") + + @patch("posthog.ai.prompts._get_session") + def test_no_capture_exception_on_successful_fetch(self, mock_get_session): + """Should NOT call capture_exception on successful fetch.""" + mock_get = mock_get_session.return_value.get + mock_get.return_value = MockResponse(json_data=self.mock_prompt_response) + + posthog = self.create_mock_posthog() + prompts = Prompts(posthog, capture_errors=True) + + prompts.get("test-prompt") + + posthog.capture_exception.assert_not_called() + + @patch("posthog.ai.prompts._get_session") + def test_capture_exception_failure_does_not_affect_fallback(self, mock_get_session): + """If capture_exception itself throws, the fallback should still be returned.""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = Exception("Network error") + + posthog = self.create_mock_posthog() + posthog.capture_exception.side_effect = Exception("capture failed") + prompts = Prompts(posthog, capture_errors=True) + + result = prompts.get("test-prompt", fallback="fallback prompt") + + self.assertEqual(result, "fallback prompt") + + class TestPromptsClearCache(TestPrompts): """Tests for the Prompts.clear_cache() method.""" From c76225b215a9c585e00014195c2c825f3517c97c Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 17 Apr 2026 11:03:12 +0100 Subject: [PATCH 2/3] feat(prompts): attach prompt metadata as properties on captured exceptions Include prompt_name, prompt_version, posthog_host, and $lib_feature so users can build suppression/grouping rules in error tracking without string-matching on exception messages. --- posthog/ai/prompts.py | 16 +++++++++++++--- posthog/test/ai/test_prompts.py | 8 +++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/posthog/ai/prompts.py b/posthog/ai/prompts.py index 93c51836..1d257ec0 100644 --- a/posthog/ai/prompts.py +++ b/posthog/ai/prompts.py @@ -304,7 +304,7 @@ def _get_internal( ) except Exception as error: - self._maybe_capture_error(error) + self._maybe_capture_error(error, name=name, version=version) prompt_reference = _prompt_reference(name, version) # Return stale cache (with warning) @@ -373,14 +373,24 @@ def clear_cache( for key in keys_to_clear: self._cache.pop(key, None) - def _maybe_capture_error(self, error: Exception) -> None: + def _maybe_capture_error( + self, error: Exception, *, name: str, version: Optional[int] + ) -> None: """Report a prompt fetch error to PostHog error tracking if enabled.""" if not self._capture_errors or self._client is None: return if not hasattr(self._client, "capture_exception"): return try: - self._client.capture_exception(error) + self._client.capture_exception( + error, + properties={ + "$lib_feature": "ai.prompts", + "prompt_name": name, + "prompt_version": version, + "posthog_host": self._host, + }, + ) except Exception: log.debug("[PostHog Prompts] Failed to capture exception to error tracking") diff --git a/posthog/test/ai/test_prompts.py b/posthog/test/ai/test_prompts.py index 5167e412..deac4840 100644 --- a/posthog/test/ai/test_prompts.py +++ b/posthog/test/ai/test_prompts.py @@ -862,13 +862,19 @@ def test_capture_exception_called_on_fetch_failure_with_fallback( posthog = self.create_mock_posthog() prompts = Prompts(posthog, capture_errors=True) - result = prompts.get("test-prompt", fallback="fallback prompt") + result = prompts.get("test-prompt", fallback="fallback prompt", version=3) self.assertEqual(result, "fallback prompt") posthog.capture_exception.assert_called_once() captured_exc = posthog.capture_exception.call_args[0][0] self.assertIn("Network error", str(captured_exc)) + properties = posthog.capture_exception.call_args.kwargs["properties"] + self.assertEqual(properties["$lib_feature"], "ai.prompts") + self.assertEqual(properties["prompt_name"], "test-prompt") + self.assertEqual(properties["prompt_version"], 3) + self.assertEqual(properties["posthog_host"], "https://us.posthog.com") + @patch("posthog.ai.prompts._get_session") @patch("posthog.ai.prompts.time.time") def test_capture_exception_called_on_fetch_failure_with_stale_cache( From bde0b7597d499dac36a03a2374555df531831f8c Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 17 Apr 2026 11:06:09 +0100 Subject: [PATCH 3/3] test(prompts): fix dead mock in test_no_capture_exception_without_client Test was missing project_api_key so _fetch_prompt_from_api raised 'project_api_key is required' before the mocked HTTP error was reached. Adding both keys so the intended 'capture_errors=True with no client' path is actually exercised. --- posthog/test/ai/test_prompts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/posthog/test/ai/test_prompts.py b/posthog/test/ai/test_prompts.py index deac4840..47105fbd 100644 --- a/posthog/test/ai/test_prompts.py +++ b/posthog/test/ai/test_prompts.py @@ -935,7 +935,11 @@ def test_no_capture_exception_without_client(self, mock_get_session): mock_get = mock_get_session.return_value.get mock_get.side_effect = Exception("Network error") - prompts = Prompts(personal_api_key="phx_test_key", capture_errors=True) + prompts = Prompts( + personal_api_key="phx_test_key", + project_api_key="phc_test_key", + capture_errors=True, + ) result = prompts.get("test-prompt", fallback="fallback prompt")