From 182c4c3d4f8a40f3a525aa1fd1ac49c71f45fcc5 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Wed, 5 Aug 2026 15:48:35 -0400 Subject: [PATCH] feat(anthropic): Gate prompt collection on data_collection option Replace include_prompts and send_default_pii checks with the new data_collection configuration for controlling whether messages and system instructions are captured. Maintain backwards compatibility: when data_collection is not configured, fall back to the legacy pii/include_prompts behavior. Tools are always collected regardless of the message collection setting. Refs PY-2588 --- sentry_sdk/integrations/anthropic.py | 219 +++++--- .../integrations/anthropic/test_anthropic.py | 474 ++++++++++++++++++ 2 files changed, 623 insertions(+), 70 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index dfa4aef34c..5402c9aabb 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -26,6 +26,7 @@ from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, + has_data_collection_enabled, package_version, reraise, safe_serialize, @@ -390,74 +391,6 @@ def _set_common_input_data( ) set_on_span(SPANDATA.GEN_AI_SYSTEM, "anthropic") set_on_span(SPANDATA.GEN_AI_OPERATION_NAME, "chat") - if ( - messages is not None - and len(messages) > 0 # type: ignore - and should_send_default_pii() - and integration.include_prompts - ): - if isinstance(system, str) or isinstance(system, Iterable): - set_on_span( - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - json.dumps(_transform_system_instructions(system)), - ) - - normalized_messages = [] - for message in messages: - if ( - message.get("role") == GEN_AI_ALLOWED_MESSAGE_ROLES.USER - and "content" in message - and isinstance(message["content"], (list, tuple)) - ): - transformed_content = [] - for item in message["content"]: - # Skip tool_result items - they can contain images/documents - # with nested structures that are difficult to redact properly - if isinstance(item, dict) and item.get("type") == "tool_result": - continue - - # Transform content blocks (images, documents, etc.) - transformed_content.append( - _transform_anthropic_content_block(item) - if isinstance(item, dict) - else item - ) - - # If there are non-tool-result items, add them as a message - if transformed_content: - normalized_messages.append( - { - "role": message.get("role"), - "content": transformed_content, - } - ) - else: - # Transform content for non-list messages or assistant messages - transformed_message = message.copy() - if "content" in transformed_message: - content = transformed_message["content"] - if isinstance(content, (list, tuple)): - transformed_message["content"] = [ - _transform_anthropic_content_block(item) - if isinstance(item, dict) - else item - for item in content - ] - normalized_messages.append(transformed_message) - - role_normalized_messages = normalize_message_roles(normalized_messages) - - client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(role_normalized_messages, span, scope) - if should_truncate_gen_ai_input(client.options) - else role_normalized_messages - ) - if messages_data is not None: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False - ) if max_tokens is not None and _is_given(max_tokens): set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) @@ -470,8 +403,154 @@ def _set_common_input_data( if top_p is not None and _is_given(top_p): set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) - if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore - set_on_span(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools)) + client = sentry_sdk.get_client() + + if has_data_collection_enabled(client.options): + if client.options["data_collection"]["gen_ai"]["inputs"]: + if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore + set_on_span( + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) + ) + else: + # Tools were unconditionally added pre-data collection configuration. + # This can be removed once data collection is fully rolled out + if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore + set_on_span(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools)) + + if messages is not None and len(messages) > 0: # type: ignore + if has_data_collection_enabled(client.options): + if client.options["data_collection"]["gen_ai"]["inputs"]: + if isinstance(system, str) or isinstance(system, Iterable): + set_on_span( + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + json.dumps(_transform_system_instructions(system)), + ) + + normalized_messages = [] + for message in messages: + if ( + message.get("role") == GEN_AI_ALLOWED_MESSAGE_ROLES.USER + and "content" in message + and isinstance(message["content"], (list, tuple)) + ): + transformed_content = [] + for item in message["content"]: + # Skip tool_result items - they can contain images/documents + # with nested structures that are difficult to redact properly + if ( + isinstance(item, dict) + and item.get("type") == "tool_result" + ): + continue + + # Transform content blocks (images, documents, etc.) + transformed_content.append( + _transform_anthropic_content_block(item) + if isinstance(item, dict) + else item + ) + + # If there are non-tool-result items, add them as a message + if transformed_content: + normalized_messages.append( + { + "role": message.get("role"), + "content": transformed_content, + } + ) + else: + # Transform content for non-list messages or assistant messages + transformed_message = message.copy() + if "content" in transformed_message: + content = transformed_message["content"] + if isinstance(content, (list, tuple)): + transformed_message["content"] = [ + _transform_anthropic_content_block(item) + if isinstance(item, dict) + else item + for item in content + ] + normalized_messages.append(transformed_message) + + role_normalized_messages = normalize_message_roles(normalized_messages) + + scope = sentry_sdk.get_current_scope() + messages_data = ( + truncate_and_annotate_messages( + role_normalized_messages, span, scope + ) + if should_truncate_gen_ai_input(client.options) + else role_normalized_messages + ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, + ) + elif should_send_default_pii() and integration.include_prompts: + if isinstance(system, str) or isinstance(system, Iterable): + set_on_span( + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + json.dumps(_transform_system_instructions(system)), + ) + + normalized_messages = [] + for message in messages: + if ( + message.get("role") == GEN_AI_ALLOWED_MESSAGE_ROLES.USER + and "content" in message + and isinstance(message["content"], (list, tuple)) + ): + transformed_content = [] + for item in message["content"]: + # Skip tool_result items - they can contain images/documents + # with nested structures that are difficult to redact properly + if isinstance(item, dict) and item.get("type") == "tool_result": + continue + + # Transform content blocks (images, documents, etc.) + transformed_content.append( + _transform_anthropic_content_block(item) + if isinstance(item, dict) + else item + ) + + # If there are non-tool-result items, add them as a message + if transformed_content: + normalized_messages.append( + { + "role": message.get("role"), + "content": transformed_content, + } + ) + else: + # Transform content for non-list messages or assistant messages + transformed_message = message.copy() + if "content" in transformed_message: + content = transformed_message["content"] + if isinstance(content, (list, tuple)): + transformed_message["content"] = [ + _transform_anthropic_content_block(item) + if isinstance(item, dict) + else item + for item in content + ] + normalized_messages.append(transformed_message) + + role_normalized_messages = normalize_message_roles(normalized_messages) + + scope = sentry_sdk.get_current_scope() + messages_data = ( + truncate_and_annotate_messages(role_normalized_messages, span, scope) + if should_truncate_gen_ai_input(client.options) + else role_normalized_messages + ) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) def _set_create_input_data( diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 741c5b7d15..6298fbc720 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -85,6 +85,29 @@ async def __call__(self, *args, **kwargs): ) +DATA_COLLECTION_EXAMPLE_TOOLS = [ + { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + } +] + +DATA_COLLECTION_EXPECTED_INPUT_DATA = { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: [ + {"type": "text", "content": "You are a helpful assistant."} + ], + SPANDATA.GEN_AI_REQUEST_MESSAGES: [{"role": "user", "content": "Hello, Claude"}], + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS: DATA_COLLECTION_EXAMPLE_TOOLS, +} + +DATA_COLLECTION_INPUT_DATA_KEYS = list(DATA_COLLECTION_EXPECTED_INPUT_DATA.keys()) + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -290,6 +313,317 @@ def test_nonstreaming_create_message( assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + False, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="gen-ai-inputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + {}, + DATA_COLLECTION_INPUT_DATA_KEYS, + id="gen-ai-inputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {}}, + True, + True, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="gen-ai-inputs-omitted-defaults-to-enabled", + ), + pytest.param( + None, + True, + True, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + {SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS: DATA_COLLECTION_EXAMPLE_TOOLS}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + ], + id="legacy-pii-disabled-tools-still-collected", + ), + ], +) +def test_nonstreaming_create_message_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expected_present, + expected_absent, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = Anthropic(api_key="z") + client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) + + create_kwargs = dict( + max_tokens=1024, + model="model", + system="You are a helpful assistant.", + messages=[{"role": "user", "content": "Hello, Claude"}], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + (event,) = events + (span,) = event["spans"] + span_data = span["data"] + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024 + + for key, expected_value in expected_present.items(): + assert json.loads(span_data[key]) == expected_value + + for key in expected_absent: + assert key not in span_data + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,expected_present,expected_absent", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + {SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS: DATA_COLLECTION_EXAMPLE_TOOLS}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + ], + id="gen-ai-inputs-enabled-tools-collected-without-messages", + ), + pytest.param( + None, + {SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS: DATA_COLLECTION_EXAMPLE_TOOLS}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + ], + id="legacy-tools-collected-without-messages", + ), + ], +) +def test_nonstreaming_create_message_data_collection_tools_without_messages( + sentry_init, + capture_events, + capture_items, + data_collection, + expected_present, + expected_absent, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=False)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=False, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = Anthropic(api_key="z") + client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) + + create_kwargs = dict( + max_tokens=1024, + model="model", + messages=[], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) + + (event,) = events + (span,) = event["spans"] + span_data = span["data"] + + for key, expected_value in expected_present.items(): + assert json.loads(span_data[key]) == expected_value + + for key in expected_absent: + assert key not in span_data + + +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + False, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="gen-ai-inputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + {}, + DATA_COLLECTION_INPUT_DATA_KEYS, + id="gen-ai-inputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + None, + True, + True, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="legacy-pii-and-include-prompts-enabled", + ), + pytest.param( + None, + False, + True, + {SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS: DATA_COLLECTION_EXAMPLE_TOOLS}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + ], + id="legacy-pii-disabled-tools-still-collected", + ), + ], +) +async def test_nonstreaming_create_message_data_collection_async( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expected_present, + expected_absent, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = AsyncAnthropic(api_key="z") + client.messages._post = AsyncMock(return_value=EXAMPLE_MESSAGE) + + create_kwargs = dict( + max_tokens=1024, + model="model", + system="You are a helpful assistant.", + messages=[{"role": "user", "content": "Hello, Claude"}], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) + + (event,) = events + (span,) = event["spans"] + span_data = span["data"] + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024 + + for key, expected_value in expected_present.items(): + assert json.loads(span_data[key]) == expected_value + + for key in expected_absent: + assert key not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -739,6 +1073,146 @@ def test_streaming_create_message( assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", + [ + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + False, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="gen-ai-inputs-enabled-overrides-pii-and-include-prompts", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + {}, + DATA_COLLECTION_INPUT_DATA_KEYS, + id="gen-ai-inputs-disabled-overrides-pii-and-include-prompts", + ), + pytest.param( + None, + True, + True, + DATA_COLLECTION_EXPECTED_INPUT_DATA, + [], + id="legacy-pii-and-include-prompts-enabled", + ), + ], +) +def test_streaming_create_message_data_collection( + sentry_init, + capture_events, + capture_items, + data_collection, + send_default_pii, + include_prompts, + expected_present, + expected_absent, + get_model_response, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, +): + sentry_init_kwargs = dict( + integrations=[AnthropicIntegration(include_prompts=include_prompts)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + if data_collection is not None: + sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} + sentry_init(**sentry_init_kwargs) + + client = Anthropic(api_key="z") + + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ) + + create_kwargs = dict( + max_tokens=1024, + model="model", + system="You are a helpful assistant.", + messages=[{"role": "user", "content": "Hello, Claude"}], + tools=DATA_COLLECTION_EXAMPLE_TOOLS, + stream=True, + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + (event,) = events + span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) + span_data = span["data"] + + assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span_data[SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024 + assert span_data[SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + + for key, expected_value in expected_present.items(): + assert json.loads(span_data[key]) == expected_value + + for key in expected_absent: + assert key not in span_data + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_streaming_create_message_close(