From 0321d6c996cfd8716d6720d326f85735e9e39bac Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 10:02:17 +0200 Subject: [PATCH 1/5] chore(anthropic): Remove transaction-based tracing --- sentry_sdk/integrations/anthropic.py | 184 +- .../integrations/anthropic/test_anthropic.py | 5494 +++++------------ 2 files changed, 1581 insertions(+), 4097 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 37c9120630..d873b5ff09 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -8,20 +8,15 @@ from sentry_sdk.ai.monitoring import record_token_usage from sentry_sdk.ai.utils import ( GEN_AI_ALLOWED_MESSAGE_ROLES, - get_start_span_function, normalize_message_roles, set_data_normalized, transform_anthropic_content_part, - truncate_and_annotate_messages, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span -from sentry_sdk.tracing_utils import ( - has_span_streaming_enabled, -) from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, @@ -89,7 +84,7 @@ from sentry_sdk._types import TextPart class _PatchedRawMessageStream(Stream[RawMessageStreamEvent]): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -99,7 +94,7 @@ class _PatchedRawMessageStream(Stream[RawMessageStreamEvent]): _finish_reason: Optional[str] class _PatchedMessageStream(MessageStream): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -109,7 +104,7 @@ class _PatchedMessageStream(MessageStream): _finish_reason: Optional[str] class _PatchedRawAsyncMessageStream(AsyncStream[RawMessageStreamEvent]): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -119,7 +114,7 @@ class _PatchedRawAsyncMessageStream(AsyncStream[RawMessageStreamEvent]): _finish_reason: Optional[str] class _PatchedAsyncMessageStream(AsyncMessageStream): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -129,7 +124,7 @@ class _PatchedAsyncMessageStream(AsyncMessageStream): _finish_reason: Optional[str] class _PatchedMessageStreamManager(MessageStreamManager): - _span: Union[Span, StreamedSpan] + _span: StreamedSpan _integration: "AnthropicIntegration" _max_tokens: int @@ -142,7 +137,7 @@ class _PatchedMessageStreamManager(MessageStreamManager): _tools: Optional[Iterable[ToolUnionParam]] class _PatchedAsyncMessageStreamManager(AsyncMessageStreamManager[Any]): - _span: Union[Span, StreamedSpan] + _span: StreamedSpan _integration: "AnthropicIntegration" _max_tokens: int @@ -447,7 +442,7 @@ def _transform_system_instructions( def _set_common_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "AnthropicIntegration", max_tokens: "int", messages: "Iterable[MessageParam]", @@ -461,36 +456,35 @@ def _set_common_input_data( """ Set input data for the span based on the provided keyword arguments for the anthropic message creation. """ - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_on_span(SPANDATA.GEN_AI_SYSTEM, "anthropic") - set_on_span(SPANDATA.GEN_AI_OPERATION_NAME, "chat") + span.set_attribute(SPANDATA.GEN_AI_SYSTEM, "anthropic") + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "chat") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) if model is not None and _is_given(model): - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) if top_k is not None and _is_given(top_k): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_K, top_k) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_K, top_k) if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): if client.options["data_collection"]["gen_ai"]["inputs"]: if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) ) else: # Tools were unconditionally added pre-data collection configuration. # This can be removed once data collection is fully rolled out if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore - set_on_span(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools)) + span.set_attribute( + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) + ) if messages is None or len(messages) == 0: # type: ignore return @@ -504,7 +498,7 @@ def _set_common_input_data( if record_inputs: if isinstance(system, str) or isinstance(system, Iterable): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(_transform_system_instructions(system)), ) @@ -554,35 +548,24 @@ def _set_common_input_data( role_normalized_messages = normalize_message_roles(normalized_messages) - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(role_normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else role_normalized_messages - ) - if messages_data is not None: + if role_normalized_messages is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, + role_normalized_messages, unpack=False, ) def _set_create_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "AnthropicIntegration", ) -> None: """ Set input data for the span based on the provided keyword arguments for the anthropic message creation. """ - if isinstance(span, StreamedSpan): - span.set_attribute( - SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False) - ) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False)) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False)) _set_common_input_data( span=span, @@ -659,7 +642,7 @@ async def _wrap_asynchronous_message_iterator( def _set_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "AnthropicIntegration", model: "str | None", input_tokens: "int | None", @@ -672,15 +655,12 @@ def _set_output_data( ) -> None: """ Set output data for the span based on the AI response.""" - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) if model is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, model) if response_id is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_ID, response_id) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_ID, response_id) if finish_reason is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) client = sentry_sdk.get_client() record_outputs = False @@ -741,22 +721,13 @@ def _sentry_patched_create_sync(f: "Any", *args: "Any", **kwargs: "Any") -> "Any model = kwargs.get("model", "") - span_streaming = has_span_streaming_enabled(sentry_sdk.get_client().options) - if span_streaming: - span = sentry_sdk.traces.start_span( - name=f"chat {model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name=f"chat {model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() + span = sentry_sdk.traces.start_span( + name=f"chat {model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + }, + ) _set_create_input_data(span, kwargs, integration) @@ -839,22 +810,13 @@ async def _sentry_patched_create_async( model = kwargs.get("model", "") - span_streaming = has_span_streaming_enabled(sentry_sdk.get_client().options) - if span_streaming: - span = sentry_sdk.traces.start_span( - name=f"chat {model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name=f"chat {model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() + span = sentry_sdk.traces.start_span( + name=f"chat {model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + }, + ) _set_create_input_data(span, kwargs, integration) @@ -1088,28 +1050,16 @@ def _sentry_patched_enter(self: "MessageStreamManager") -> "MessageStream": except TypeError: return f(self) - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - SPANDATA.GEN_AI_RESPONSE_STREAMING: True, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + span = sentry_sdk.traces.start_span( + name="chat" + if patched_self._model is None + else f"chat {patched_self._model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + SPANDATA.GEN_AI_RESPONSE_STREAMING: True, + }, + ) _set_common_input_data( span=span, @@ -1201,28 +1151,16 @@ async def _sentry_patched_aenter( except TypeError: return await f(self) - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - SPANDATA.GEN_AI_RESPONSE_STREAMING: True, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + span = sentry_sdk.traces.start_span( + name="chat" + if patched_self._model is None + else f"chat {patched_self._model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + SPANDATA.GEN_AI_RESPONSE_STREAMING: True, + }, + ) _set_common_input_data( span=span, diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 5723d52740..06a6605ac4 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -57,8 +57,7 @@ async def __call__(self, *args, **kwargs): except ImportError: from anthropic.types.content_block import ContentBlock as TextBlock -from sentry_sdk import start_span, start_transaction -from sentry_sdk._types import BLOB_DATA_SUBSTITUTE +from sentry_sdk import start_transaction from sentry_sdk.ai.utils import transform_content_part, transform_message_content from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.anthropic import ( @@ -138,7 +137,6 @@ def data_collection_tool_use_message(): ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -150,19 +148,16 @@ def data_collection_tool_use_message(): ) def test_nonstreaming_create_message( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -178,113 +173,59 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) + with sentry_sdk.traces.start_span(name="anthropic"): + response = client.messages.create( + max_tokens=1024, messages=messages, model="model" + ) - assert response == EXAMPLE_MESSAGE - usage = response.usage + assert response == EXAMPLE_MESSAGE + usage = response.usage - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -324,22 +265,19 @@ def test_nonstreaming_create_message( ) def test_nonstreaming_create_message_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -354,26 +292,15 @@ def test_nonstreaming_create_message_data_collection( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -391,7 +318,6 @@ def test_nonstreaming_create_message_data_collection( ANTHROPIC_VERSION < (0, 27), reason="Tools are not supported in this version of the anthropic package", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,tools_collected", [ @@ -414,19 +340,16 @@ def test_nonstreaming_create_message_data_collection( ) def test_nonstreaming_create_message_data_collection_tools( sentry_init, - capture_events, capture_items, data_collection, tools_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -441,26 +364,15 @@ def test_nonstreaming_create_message_data_collection_tools( messages=[], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] if tools_collected: assert ( @@ -471,7 +383,6 @@ def test_nonstreaming_create_message_data_collection_tools( assert SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", @@ -504,22 +415,19 @@ def test_nonstreaming_create_message_data_collection_tools( ) async def test_nonstreaming_create_message_data_collection_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -534,26 +442,15 @@ async def test_nonstreaming_create_message_data_collection_async( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -571,7 +468,6 @@ async def test_nonstreaming_create_message_data_collection_async( ANTHROPIC_VERSION < (0, 27), reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -621,21 +517,18 @@ async def test_nonstreaming_create_message_data_collection_async( ) def test_nonstreaming_create_message_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, outputs_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -650,26 +543,15 @@ def test_nonstreaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + with start_transaction(name="anthropic"): + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -696,7 +578,6 @@ def test_nonstreaming_create_message_data_collection_outputs( ANTHROPIC_VERSION < (0, 27), reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -747,21 +628,18 @@ def test_nonstreaming_create_message_data_collection_outputs( ) async def test_nonstreaming_create_message_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, outputs_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -776,26 +654,15 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + with start_transaction(name="anthropic"): + await client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -818,7 +685,6 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -831,19 +697,16 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( ) async def test_nonstreaming_create_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -859,109 +722,58 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + with sentry_sdk.traces.start_span(name="anthropic"): + response = await client.messages.create( + max_tokens=1024, messages=messages, model="model" ) - else: - events = capture_events() - with start_transaction(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + assert response == EXAMPLE_MESSAGE + usage = response.usage - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - assert len(events) == 1 - (event,) = events + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -973,13 +785,11 @@ async def test_nonstreaming_create_message_async( ) def test_streaming_create_message( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1025,8 +835,7 @@ def test_streaming_create_message( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1039,116 +848,61 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + for _ in message: + pass - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -1180,7 +934,6 @@ def test_streaming_create_message( ) def test_streaming_create_message_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1189,15 +942,13 @@ def test_streaming_create_message_data_collection( expected_absent, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -1239,38 +990,21 @@ def test_streaming_create_message_data_collection( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -1285,7 +1019,6 @@ def test_streaming_create_message_data_collection( assert key not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -1328,7 +1061,6 @@ def test_streaming_create_message_data_collection( ) def test_streaming_create_message_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1336,15 +1068,13 @@ def test_streaming_create_message_data_collection_outputs( outputs_collected, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -1390,38 +1120,21 @@ def test_streaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -1436,14 +1149,11 @@ def test_streaming_create_message_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_create_message_close( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1489,8 +1199,7 @@ def test_streaming_create_message_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1499,109 +1208,60 @@ def test_streaming_create_message_close( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - messages = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - next(messages) - - messages.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + messages = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - messages = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - next(messages) - - messages.close() - assert len(events) == 1 - (event,) = events + for _ in range(4): + next(messages) - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + messages.close() - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", ) def test_streaming_create_message_api_error( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1642,8 +1302,7 @@ def test_streaming_create_message_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1652,101 +1311,52 @@ def test_streaming_create_message_api_error( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("transaction", "span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + for _ in message: + pass - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + assert spans[1]["status"] == SpanStatus.ERROR + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1758,13 +1368,11 @@ def test_streaming_create_message_api_error( ) def test_stream_messages( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1810,8 +1418,7 @@ def test_stream_messages( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1824,115 +1431,61 @@ def test_stream_messages( "content": "Hello, Claude", }, ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -1975,7 +1528,6 @@ def test_stream_messages( ) def test_stream_messages_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1983,15 +1535,13 @@ def test_stream_messages_data_collection_outputs( outputs_collected, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -2036,40 +1586,22 @@ def test_stream_messages_data_collection_outputs( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"), client.messages.stream( - **stream_kwargs - ) as stream: - for _ in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"), client.messages.stream( - **stream_kwargs - ) as stream: - for _ in stream: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"), client.messages.stream( + **stream_kwargs + ) as stream: + for _ in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -2084,14 +1616,11 @@ def test_stream_messages_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_close( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -2137,8 +1666,7 @@ def test_stream_messages_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2147,117 +1675,64 @@ def test_stream_messages_close( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - next(stream) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in range(4): + next(stream) - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance(next(stream), TextEvent): - next(stream) + # New versions add TextEvent, so consume one more event. + if TextEvent is not None and isinstance(next(stream), TextEvent): + next(stream) - stream.close() + stream.close() - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - next(stream) - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance(next(stream), TextEvent): - next(stream) - - stream.close() - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", ) def test_stream_messages_api_error( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -2298,8 +1773,7 @@ def test_stream_messages_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2308,101 +1782,52 @@ def test_stream_messages_api_error( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + assert spans[1]["status"] == SpanStatus.ERROR + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -2415,14 +1840,12 @@ def test_stream_messages_api_error( ) async def test_streaming_create_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2471,8 +1894,7 @@ async def test_streaming_create_message_async( traces_sample_rate=1.0, default_integrations=False, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2485,117 +1907,61 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" - ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + async for _ in message: + pass - assert len(event["spans"]) == 1 - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert spans[1]["name"] == "anthropic" + (span, _) = spans - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -2639,7 +2005,6 @@ async def test_streaming_create_message_async( ) async def test_streaming_create_message_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -2648,15 +2013,13 @@ async def test_streaming_create_message_data_collection_outputs_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -2704,38 +2067,21 @@ async def test_streaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = await client.messages.create(**create_kwargs) - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = await client.messages.create(**create_kwargs) - async for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + message = await client.messages.create(**create_kwargs) + async for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -2750,16 +2096,13 @@ async def test_streaming_create_message_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_streaming_create_message_async_close( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2807,8 +2150,7 @@ async def test_streaming_create_message_async_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2817,96 +2159,50 @@ async def test_streaming_create_message_async_close( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - messages = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - await messages.__anext__() - await messages.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + messages = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - messages = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - await messages.__anext__() - await messages.close() - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + for _ in range(4): + await messages.__anext__() + await messages.close() - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", @@ -2914,12 +2210,10 @@ async def test_streaming_create_message_async_close( @pytest.mark.asyncio async def test_streaming_create_message_async_api_error( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2962,8 +2256,7 @@ async def test_streaming_create_message_async_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2972,101 +2265,52 @@ async def test_streaming_create_message_async_api_error( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("transaction", "span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + async for _ in message: + pass - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + assert spans[1]["status"] == SpanStatus.ERROR + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -3079,14 +2323,12 @@ async def test_streaming_create_message_async_api_error( ) async def test_stream_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3134,8 +2376,7 @@ async def test_stream_message_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3148,114 +2389,61 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -3299,7 +2487,6 @@ async def test_stream_message_async( ) async def test_stream_messages_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -3308,15 +2495,13 @@ async def test_stream_messages_data_collection_outputs_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -3363,38 +2548,21 @@ async def test_stream_messages_data_collection_outputs_async( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ), start_transaction(name="anthropic"): + async with client.messages.stream(**stream_kwargs) as stream: + async for _ in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - async with client.messages.stream(**stream_kwargs) as stream: - async for _ in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - async with client.messages.stream(**stream_kwargs) as stream: - async for _ in stream: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -3409,7 +2577,6 @@ async def test_stream_messages_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", @@ -3417,12 +2584,10 @@ async def test_stream_messages_data_collection_outputs_async( @pytest.mark.asyncio async def test_stream_messages_async_api_error( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3465,8 +2630,7 @@ async def test_stream_messages_async_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3475,111 +2639,59 @@ async def test_stream_messages_async_api_error( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_stream_messages_async_close( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3627,8 +2739,7 @@ async def test_stream_messages_async_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3637,112 +2748,58 @@ async def test_stream_messages_async_close( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in range(4): + await stream.__anext__() - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - await stream.__anext__() - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance( - await stream.__anext__(), TextEvent - ): - await stream.__anext__() - - await stream.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - await stream.__anext__() - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance( - await stream.__anext__(), TextEvent - ): - await stream.__anext__() - - await stream.close() - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + # New versions add TextEvent, so consume one more event. + if TextEvent is not None and isinstance( + await stream.__anext__(), TextEvent + ): + await stream.__anext__() - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) + await stream.close() - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["name"] == "anthropic" + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta, which was introduced in >=0.27.0 along with a new message delta type for tool calling.", @@ -3758,13 +2815,11 @@ async def test_stream_messages_async_close( ) def test_streaming_create_message_with_input_json_delta( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -3840,8 +2895,7 @@ def test_streaming_create_message_with_input_json_delta( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3850,103 +2904,52 @@ def test_streaming_create_message_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) + for _ in message: + pass - for _ in message: - pass + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert spans[1]["name"] == "anthropic" - (span, _) = spans + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta, which was introduced in >=0.27.0 along with a new message delta type for tool calling.", @@ -3962,13 +2965,11 @@ def test_streaming_create_message_with_input_json_delta( ) def test_stream_messages_with_input_json_delta( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -4044,8 +3045,7 @@ def test_stream_messages_with_input_json_delta( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4054,102 +3054,52 @@ def test_stream_messages_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), @@ -4166,14 +3116,12 @@ def test_stream_messages_with_input_json_delta( ) async def test_streaming_create_message_with_input_json_delta_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") response = get_model_response( @@ -4254,8 +3202,7 @@ async def test_streaming_create_message_with_input_json_delta_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4264,104 +3211,53 @@ async def test_streaming_create_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + async for _ in message: + pass - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert spans[1]["name"] == "anthropic" + (span, _) = spans - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), @@ -4378,14 +3274,12 @@ async def test_streaming_create_message_with_input_json_delta_async( ) async def test_stream_message_with_input_json_delta_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") response = get_model_response( @@ -4466,8 +3360,7 @@ async def test_stream_message_with_input_json_delta_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4476,342 +3369,188 @@ async def test_stream_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert spans[1]["name"] == "anthropic" + (span, _) = spans - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) def test_exception_message_create( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") client.messages._post = mock.Mock( side_effect=AnthropicError("API rate limit reached") ) + items = capture_items("event") - if span_streaming: - items = capture_items("event") - - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) - - (event,) = (item.payload for item in items) - assert event["level"] == "error" - else: - events = capture_events() - - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) + with pytest.raises(AnthropicError): + client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items) + assert event["level"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_status_error( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - client = Anthropic(api_key="z") - client.messages._post = mock.Mock( - side_effect=AnthropicError("API rate limit reached") + with sentry_sdk.traces.start_span(name="anthropic"): + client = Anthropic(api_key="z") + client.messages._post = mock.Mock( + side_effect=AnthropicError("API rate limit reached") + ) + with pytest.raises(AnthropicError): + client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, ) - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - with start_transaction(name="anthropic"): - client = Anthropic(api_key="z") - client.messages._post = mock.Mock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - -@pytest.mark.parametrize("span_streaming", [True, False]) + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + + @pytest.mark.asyncio async def test_span_status_error_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - if span_streaming: - items = capture_items("event", "span") + items = capture_items("event", "span") - with sentry_sdk.traces.start_span(name="anthropic"): - client = AsyncAnthropic(api_key="z") - client.messages._post = AsyncMock( - side_effect=AnthropicError("API rate limit reached") + with sentry_sdk.traces.start_span(name="anthropic"): + client = AsyncAnthropic(api_key="z") + client.messages._post = AsyncMock( + side_effect=AnthropicError("API rate limit reached") + ) + with pytest.raises(AnthropicError): + await client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, ) - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - with start_transaction(name="anthropic"): - client = AsyncAnthropic(api_key="z") - client.messages._post = AsyncMock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - -@pytest.mark.parametrize("span_streaming", [True, False]) + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + + @pytest.mark.asyncio async def test_exception_message_create_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") client.messages._post = AsyncMock( side_effect=AnthropicError("API rate limit reached") ) + items = capture_items("event") - if span_streaming: - items = capture_items("event") - - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) - - (event,) = (item.payload for item in items) - assert event["level"] == "error" - else: - events = capture_events() - - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) + with pytest.raises(AnthropicError): + await client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items) + assert event["level"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -4823,46 +3562,29 @@ def test_span_origin( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with sentry_sdk.traces.start_span(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - (event,) = events - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -4874,43 +3596,24 @@ async def test_span_origin_async( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - (event,) = events + with sentry_sdk.traces.start_span(name="anthropic"): + await client.messages.create(max_tokens=1024, messages=messages, model="model") - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_collect_ai_data_with_input_json_delta(span_streaming): +def test_collect_ai_data_with_input_json_delta(): event = ContentBlockDeltaEvent( delta=InputJSONDelta(partial_json="test", type="input_json_delta"), index=0, @@ -4945,11 +3648,11 @@ def test_set_output_data_with_input_json_delta(sentry_init): disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) with start_transaction(name="test"): - span = start_span() + span = sentry_sdk.traces.start_span(name="test") integration = AnthropicIntegration() json_deltas = ["{'test': 'data',", "'more': 'json'}"] _set_output_data( @@ -4964,16 +3667,15 @@ def test_set_output_data_with_input_json_delta(sentry_init): ) assert ( - span._data.get(SPANDATA.GEN_AI_RESPONSE_TEXT) + span._attributes.get(SPANDATA.GEN_AI_RESPONSE_TEXT) == "{'test': 'data','more': 'json'}" ) - assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 - assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 - assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 # Test messages with mixed roles including "ai" that should be mapped to "assistant" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "test_message,expected_role", [ @@ -4991,11 +3693,9 @@ def test_set_output_data_with_input_json_delta(sentry_init): ) def test_anthropic_message_role_mapping( sentry_init, - capture_events, capture_items, test_message, expected_role, - span_streaming, ): """Test that Anthropic integration properly maps message roles like 'ai' to 'assistant'""" sentry_init( @@ -5003,8 +3703,7 @@ def test_anthropic_message_role_mapping( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -5024,159 +3723,28 @@ def mock_messages_create(*args, **kwargs): client.messages._post = mock.Mock(return_value=mock_messages_create()) test_messages = [test_message] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic tx"): - client.messages.create( - model="claude-3-opus", max_tokens=10, messages=test_messages - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - # Verify that the span was created correctly - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - - # Parse the stored messages - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + with start_transaction(name="anthropic tx"): + client.messages.create( + model="claude-3-opus", max_tokens=10, messages=test_messages ) - else: - events = capture_events() - - with start_transaction(name="anthropic tx"): - client.messages.create( - model="claude-3-opus", max_tokens=10, messages=test_messages - ) - (event,) = events - span = event["spans"][0] + sentry_sdk.flush() + span = next(item.payload for item in items) - # Verify that the span was created correctly - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + # Verify that the span was created correctly + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - # Parse the stored messages - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + # Parse the stored messages + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert stored_messages[0]["role"] == expected_role -def test_anthropic_message_truncation(sentry_init, capture_events): - """Test that large messages are truncated properly in Anthropic integration.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - messages = [ - {"role": "user", "content": "small message 1"}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": "small message 4"}, - {"role": "user", "content": "small message 5"}, - ] - - with start_transaction(): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - -@pytest.mark.asyncio -async def test_anthropic_message_truncation_async(sentry_init, capture_events): - """Test that large messages are truncated properly in Anthropic integration.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - - client = AsyncAnthropic(api_key="z") - client.messages._post = mock.AsyncMock(return_value=EXAMPLE_MESSAGE) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - messages = [ - {"role": "user", "content": "small message 1"}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": "small message 4"}, - {"role": "user", "content": "small message 5"}, - ] - - with start_transaction(): - await client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5188,11 +3756,9 @@ async def test_anthropic_message_truncation_async(sentry_init, capture_events): ) def test_nonstreaming_create_message_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that system prompts are properly captured in GEN_AI_REQUEST_MESSAGES.""" sentry_init( @@ -5200,8 +3766,7 @@ def test_nonstreaming_create_message_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -5213,128 +3778,64 @@ def test_nonstreaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + with sentry_sdk.traces.start_span(name="anthropic"): + response = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert response == EXAMPLE_MESSAGE + usage = response.usage - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert spans[1]["name"] == "anthropic" + (span, _) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} ] - else: - events = capture_events() - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -5347,11 +3848,9 @@ def test_nonstreaming_create_message_with_system_prompt( ) async def test_nonstreaming_create_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that system prompts are properly captured in GEN_AI_REQUEST_MESSAGES (async).""" sentry_init( @@ -5359,8 +3858,7 @@ async def test_nonstreaming_create_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -5372,128 +3870,64 @@ async def test_nonstreaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + with sentry_sdk.traces.start_span(name="anthropic"): + response = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert response == EXAMPLE_MESSAGE + usage = response.usage - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert spans[1]["name"] == "anthropic" + (span, _) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} ] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5505,13 +3939,11 @@ async def test_nonstreaming_create_message_with_system_prompt_async( ) def test_streaming_create_message_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") @@ -5558,8 +3990,7 @@ def test_streaming_create_message_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5568,131 +3999,66 @@ def test_streaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans + for _ in message: + pass - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5704,13 +4070,11 @@ def test_streaming_create_message_with_system_prompt( ) def test_stream_messages_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") @@ -5757,8 +4121,7 @@ def test_stream_messages_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5767,120 +4130,61 @@ def test_stream_messages_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) as stream: + for event in stream: + pass - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -5893,14 +4197,12 @@ def test_stream_messages_with_system_prompt( ) async def test_stream_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -5949,8 +4251,7 @@ async def test_stream_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5959,125 +4260,63 @@ async def test_stream_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 2 - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -6090,14 +4329,12 @@ async def test_stream_message_with_system_prompt_async( ) async def test_streaming_create_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -6146,8 +4383,7 @@ async def test_streaming_create_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -6156,135 +4392,70 @@ async def test_streaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, sentry_sdk.traces.start_span(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + async for _ in message: + pass - assert spans[1]["name"] == "anthropic" - assert len(spans) == 2 - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert spans[1]["name"] == "anthropic" + assert len(spans) == 2 + (span, _) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) def test_system_prompt_with_complex_structure( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that complex system prompt structures (list of text blocks) are properly captured.""" sentry_init( @@ -6292,8 +4463,7 @@ def test_system_prompt_with_complex_structure( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6311,76 +4481,39 @@ def test_system_prompt_with_complex_structure( "content": "Hello", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model", system=system_prompt - ) - - assert response == EXAMPLE_MESSAGE - - sentry_sdk.flush() - spans = [item.payload for item in items] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - - # System content should be a list of text blocks - assert isinstance(system_instructions, list) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + with sentry_sdk.traces.start_span(name="anthropic"): + response = client.messages.create( + max_tokens=1024, messages=messages, model="model", system=system_prompt ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model", system=system_prompt - ) - assert response == EXAMPLE_MESSAGE + assert response == EXAMPLE_MESSAGE - assert len(events) == 1 - (event,) = events + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 2 - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert spans[1]["name"] == "anthropic" + (span, _) = spans - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) - # System content should be a list of text blocks - assert isinstance(system_instructions, list) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] + # System content should be a list of text blocks + assert isinstance(system_instructions, list) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" @@ -6582,16 +4715,19 @@ def test_transform_message_content_list_anthropic(): # Integration tests for binary data in messages -def test_message_with_base64_image(sentry_init, capture_events): - """Test that messages with base64 images are properly captured.""" +def test_message_with_url_image( + sentry_init, + capture_items, +): + """Test that messages with URL-referenced images are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - events = capture_events() + client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -6599,57 +4735,48 @@ def test_message_with_base64_image(sentry_init, capture_events): { "role": "user", "content": [ - {"type": "text", "text": "What's in this image?"}, + {"type": "text", "text": "Describe this image."}, { "type": "image", "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": "base64encodeddatahere...", + "type": "url", + "url": "https://example.com/photo.png", }, }, ], } ] + items = capture_items("span") with start_transaction(name="anthropic"): client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" content = stored_messages[0]["content"] - assert len(content) == 2 - assert content[0] == {"type": "text", "text": "What's in this image?"} assert content[1] == { - "type": "blob", + "type": "uri", "modality": "image", - "mime_type": "image/jpeg", - "content": BLOB_DATA_SUBSTITUTE, + "mime_type": "", + "uri": "https://example.com/photo.png", } -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_message_with_url_image( +def test_message_with_file_image( sentry_init, - capture_events, capture_items, - span_streaming, ): - """Test that messages with URL-referenced images are properly captured.""" + """Test that messages with file_id-referenced images are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6659,67 +4786,49 @@ def test_message_with_url_image( { "role": "user", "content": [ - {"type": "text", "text": "Describe this image."}, + {"type": "text", "text": "What do you see?"}, { "type": "image", "source": { - "type": "url", - "url": "https://example.com/photo.png", + "type": "file", + "file_id": "file_img_12345", + "media_type": "image/webp", }, }, ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { - "type": "uri", + "type": "file", "modality": "image", - "mime_type": "", - "uri": "https://example.com/photo.png", + "mime_type": "image/webp", + "file_id": "file_img_12345", } -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_message_with_file_image( +def test_message_with_url_pdf( sentry_init, - capture_events, capture_items, - span_streaming, ): - """Test that messages with file_id-referenced images are properly captured.""" + """Test that messages with URL-referenced PDF documents are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6729,127 +4838,9 @@ def test_message_with_file_image( { "role": "user", "content": [ - {"type": "text", "text": "What do you see?"}, + {"type": "text", "text": "What is in this PDF?"}, { - "type": "image", - "source": { - "type": "file", - "file_id": "file_img_12345", - "media_type": "image/webp", - }, - }, - ], - } - ] - - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - - content = stored_messages[0]["content"] - assert content[1] == { - "type": "file", - "modality": "image", - "mime_type": "image/webp", - "file_id": "file_img_12345", - } - - -def test_message_with_base64_pdf(sentry_init, capture_events): - """Test that messages with base64-encoded PDF documents are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Summarize this document."}, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0xLjQKJeLj...base64pdfdata", - }, - }, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - assert content[1] == { - "type": "blob", - "modality": "document", - "mime_type": "application/pdf", - "content": BLOB_DATA_SUBSTITUTE, - } - - -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_message_with_url_pdf( - sentry_init, - capture_events, - capture_items, - span_streaming, -): - """Test that messages with URL-referenced PDF documents are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this PDF?"}, - { - "type": "document", + "type": "document", "source": { "type": "url", "url": "https://example.com/report.pdf", @@ -6858,31 +4849,16 @@ def test_message_with_url_pdf( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6893,12 +4869,9 @@ def test_message_with_url_pdf( } -@pytest.mark.parametrize("span_streaming", [True, False]) def test_message_with_file_document( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that messages with file_id-referenced documents are properly captured.""" sentry_init( @@ -6906,8 +4879,7 @@ def test_message_with_file_document( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6929,31 +4901,16 @@ def test_message_with_file_document( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6964,174 +4921,9 @@ def test_message_with_file_document( } -def test_message_with_mixed_content(sentry_init, capture_events): - """Test that messages with mixed content (text, images, documents) are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Compare this image with the document."}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "iVBORw0KGgo...base64imagedata", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/comparison.jpg", - }, - }, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0xLjQK...base64pdfdata", - }, - }, - {"type": "text", "text": "Please provide a detailed analysis."}, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - - assert len(content) == 5 - assert content[0] == { - "type": "text", - "text": "Compare this image with the document.", - } - assert content[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/png", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[2] == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/comparison.jpg", - } - assert content[3] == { - "type": "blob", - "modality": "document", - "mime_type": "application/pdf", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[4] == { - "type": "text", - "text": "Please provide a detailed analysis.", - } - - -def test_message_with_multiple_images_different_formats(sentry_init, capture_events): - """Test that messages with multiple images of different source types are handled.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": "base64data1...", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/img2.gif", - }, - }, - { - "type": "image", - "source": { - "type": "file", - "file_id": "file_img_789", - "media_type": "image/webp", - }, - }, - {"type": "text", "text": "Compare these three images."}, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - - assert len(content) == 4 - assert content[0] == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[1] == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/img2.gif", - } - assert content[2] == { - "type": "file", - "modality": "image", - "mime_type": "image/webp", - "file_id": "file_img_789", - } - assert content[3] == {"type": "text", "text": "Compare these three images."} - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_binary_content_not_stored_when_pii_disabled( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that binary content is not stored when send_default_pii is False.""" sentry_init( @@ -7139,8 +4931,7 @@ def test_binary_content_not_stored_when_pii_disabled( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7162,39 +4953,22 @@ def test_binary_content_not_stored_when_pii_disabled( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + # Messages should not be stored + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] -@pytest.mark.parametrize("span_streaming", [True, False]) def test_binary_content_not_stored_when_prompts_disabled( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that binary content is not stored when include_prompts is False.""" sentry_init( @@ -7202,8 +4976,7 @@ def test_binary_content_not_stored_when_prompts_disabled( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7225,47 +4998,29 @@ def test_binary_content_not_stored_when_prompts_disabled( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + # Messages should not be stored + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] -@pytest.mark.parametrize("span_streaming", [True, False]) def test_cache_tokens_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test cache read/write tokens are tracked for non-streaming responses.""" sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7285,50 +5040,28 @@ def test_cache_tokens_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) + with start_transaction(name="anthropic"): + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + ) - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_write_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_write tokens (non-streaming). @@ -7345,8 +5078,7 @@ def test_input_tokens_include_cache_write_nonstreaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7366,54 +5098,28 @@ def test_input_tokens_include_cache_write_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 3+3?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 + with start_transaction(name="anthropic"): + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 3+3?"}], + model="claude-sonnet-4-20250514", ) - else: - events = capture_events() - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 3+3?"}], - model="claude-sonnet-4-20250514", - ) - - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 + # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_read_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (non-streaming). @@ -7430,8 +5136,7 @@ def test_input_tokens_include_cache_read_nonstreaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7451,54 +5156,30 @@ def test_input_tokens_include_cache_read_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = [item.payload for item in items] - - # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) + with start_transaction(name="anthropic"): + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + ) - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = [item.payload for item in items] - # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_read_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -7539,69 +5220,38 @@ def test_input_tokens_include_cache_read_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + stream=True, + ): + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - stream=True, - ): - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - stream=True, - ): - pass - - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_input_tokens_include_cache_read_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -7641,65 +5291,35 @@ def test_stream_messages_input_tokens_include_cache_read_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 + 2846 = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + ) as stream: + for event in stream: + pass - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 + 2846 = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 + 2846 = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_unchanged_without_caching( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that input_tokens is unchanged when there are no cached tokens. @@ -7711,8 +5331,7 @@ def test_input_tokens_unchanged_without_caching( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7730,46 +5349,27 @@ def test_input_tokens_unchanged_without_caching( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 2+2?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 2+2?"}], - model="claude-sonnet-4-20250514", - ) + with start_transaction(name="anthropic"): + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="claude-sonnet-4-20250514", + ) - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_cache_tokens_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") @@ -7806,67 +5406,38 @@ def test_cache_tokens_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + stream=True, + ): + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - stream=True, - ): - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - stream=True, - ): - pass - - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_cache_tokens( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") @@ -7903,52 +5474,27 @@ def test_stream_messages_cache_tokens( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) as stream: - for event in stream: - pass - - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 From d748e9ec16895ee4fe4361b2297e8b21f6e6ac1e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 10:07:29 +0200 Subject: [PATCH 2/5] fix mypy --- sentry_sdk/integrations/anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index d873b5ff09..4b4ebf1924 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -928,7 +928,7 @@ def _accumulate_event_data( def _set_streaming_output_data( - span: "Span", + span: "StreamedSpan", integration: "AnthropicIntegration", model: "Optional[str]", usage: "_RecordedUsage", From 868826f0ab71ab9be48b5835dd15d49e535d9e99 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 10:21:04 +0200 Subject: [PATCH 3/5] remove start_transaction --- .../integrations/anthropic/test_anthropic.py | 144 ++++++++---------- 1 file changed, 61 insertions(+), 83 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 06a6605ac4..255c8e6594 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -57,7 +57,6 @@ async def __call__(self, *args, **kwargs): except ImportError: from anthropic.types.content_block import ContentBlock as TextBlock -from sentry_sdk import start_transaction from sentry_sdk.ai.utils import transform_content_part, transform_message_content from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.anthropic import ( @@ -294,8 +293,7 @@ def test_nonstreaming_create_message_data_collection( ) items = capture_items("transaction", "span") - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] @@ -366,8 +364,7 @@ def test_nonstreaming_create_message_data_collection_tools( ) items = capture_items("transaction", "span") - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] @@ -444,8 +441,7 @@ async def test_nonstreaming_create_message_data_collection_async( ) items = capture_items("transaction", "span") - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + await client.messages.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] @@ -545,8 +541,7 @@ def test_nonstreaming_create_message_data_collection_outputs( ) items = capture_items("transaction", "span") - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] @@ -656,8 +651,7 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( ) items = capture_items("transaction", "span") - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + await client.messages.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] @@ -996,7 +990,7 @@ def test_streaming_create_message_data_collection( client._client, "send", return_value=response, - ), start_transaction(name="anthropic"): + ): message = client.messages.create(**create_kwargs) for _ in message: pass @@ -1126,7 +1120,7 @@ def test_streaming_create_message_data_collection_outputs( client._client, "send", return_value=response, - ), start_transaction(name="anthropic"): + ): message = client.messages.create(**create_kwargs) for _ in message: pass @@ -1592,9 +1586,7 @@ def test_stream_messages_data_collection_outputs( client._client, "send", return_value=response, - ), start_transaction(name="anthropic"), client.messages.stream( - **stream_kwargs - ) as stream: + ), client.messages.stream(**stream_kwargs) as stream: for _ in stream: pass @@ -2073,7 +2065,7 @@ async def test_streaming_create_message_data_collection_outputs_async( client._client, "send", return_value=response, - ), start_transaction(name="anthropic"): + ): message = await client.messages.create(**create_kwargs) async for _ in message: pass @@ -2554,7 +2546,7 @@ async def test_stream_messages_data_collection_outputs_async( client._client, "send", return_value=response, - ), start_transaction(name="anthropic"): + ): async with client.messages.stream(**stream_kwargs) as stream: async for _ in stream: pass @@ -3651,28 +3643,27 @@ def test_set_output_data_with_input_json_delta(sentry_init): trace_lifecycle="stream", ) - with start_transaction(name="test"): - span = sentry_sdk.traces.start_span(name="test") - integration = AnthropicIntegration() - json_deltas = ["{'test': 'data',", "'more': 'json'}"] - _set_output_data( - span, - integration, - model="", - input_tokens=10, - output_tokens=20, - cache_read_input_tokens=0, - cache_write_input_tokens=0, - content_blocks=[{"text": "".join(json_deltas), "type": "text"}], - ) + span = sentry_sdk.traces.start_span(name="test") + integration = AnthropicIntegration() + json_deltas = ["{'test': 'data',", "'more': 'json'}"] + _set_output_data( + span, + integration, + model="", + input_tokens=10, + output_tokens=20, + cache_read_input_tokens=0, + cache_write_input_tokens=0, + content_blocks=[{"text": "".join(json_deltas), "type": "text"}], + ) - assert ( - span._attributes.get(SPANDATA.GEN_AI_RESPONSE_TEXT) - == "{'test': 'data','more': 'json'}" - ) - assert span._attributes.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 - assert span._attributes.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 - assert span._attributes.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 + assert ( + span._attributes.get(SPANDATA.GEN_AI_RESPONSE_TEXT) + == "{'test': 'data','more': 'json'}" + ) + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 # Test messages with mixed roles including "ai" that should be mapped to "assistant" @@ -3725,10 +3716,7 @@ def mock_messages_create(*args, **kwargs): test_messages = [test_message] items = capture_items("span") - with start_transaction(name="anthropic tx"): - client.messages.create( - model="claude-3-opus", max_tokens=10, messages=test_messages - ) + client.messages.create(model="claude-3-opus", max_tokens=10, messages=test_messages) sentry_sdk.flush() span = next(item.payload for item in items) @@ -4748,8 +4736,7 @@ def test_message_with_url_image( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -4800,8 +4787,7 @@ def test_message_with_file_image( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -4851,8 +4837,7 @@ def test_message_with_url_pdf( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -4903,8 +4888,7 @@ def test_message_with_file_document( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -4955,8 +4939,7 @@ def test_binary_content_not_stored_when_pii_disabled( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -5000,8 +4983,7 @@ def test_binary_content_not_stored_when_prompts_disabled( ] items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() spans = [item.payload for item in items] @@ -5042,12 +5024,11 @@ def test_cache_tokens_nonstreaming( ) items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -5100,12 +5081,11 @@ def test_input_tokens_include_cache_write_nonstreaming( ) items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 3+3?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 3+3?"}], + model="claude-sonnet-4-20250514", + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -5158,12 +5138,11 @@ def test_input_tokens_include_cache_read_nonstreaming( ) items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + ) sentry_sdk.flush() (span,) = [item.payload for item in items] @@ -5228,7 +5207,7 @@ def test_input_tokens_include_cache_read_streaming( client._client, "send", return_value=response, - ) as _, start_transaction(name="anthropic"): + ): for _ in client.messages.create( max_tokens=1024, messages=[{"role": "user", "content": "What is 5+5?"}], @@ -5299,7 +5278,7 @@ def test_stream_messages_input_tokens_include_cache_read_streaming( client._client, "send", return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=[{"role": "user", "content": "What is 5+5?"}], model="claude-sonnet-4-20250514", @@ -5351,12 +5330,11 @@ def test_input_tokens_unchanged_without_caching( ) items = capture_items("span") - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 2+2?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="claude-sonnet-4-20250514", + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -5414,7 +5392,7 @@ def test_cache_tokens_streaming( client._client, "send", return_value=response, - ) as _, start_transaction(name="anthropic"): + ): for _ in client.messages.create( max_tokens=1024, messages=[{"role": "user", "content": "Hello"}], @@ -5482,7 +5460,7 @@ def test_stream_messages_cache_tokens( client._client, "send", return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=[{"role": "user", "content": "Hello"}], model="claude-3-5-sonnet-20241022", From 0f03f9534e2dbac62e0dc44fc20df72e0bead66b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 10:35:54 +0200 Subject: [PATCH 4/5] stop capturing transactions --- .../integrations/anthropic/test_anthropic.py | 273 +++++++++--------- 1 file changed, 131 insertions(+), 142 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 255c8e6594..6e6a87f870 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -172,12 +172,9 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) + response = client.messages.create(max_tokens=1024, messages=messages, model="model") assert response == EXAMPLE_MESSAGE usage = response.usage @@ -186,7 +183,7 @@ def test_nonstreaming_create_message( assert usage.output_tokens == 20 sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -291,12 +288,12 @@ def test_nonstreaming_create_message_data_collection( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) - items = capture_items("transaction", "span") + items = capture_items("span") client.messages.create(**create_kwargs) sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -362,12 +359,12 @@ def test_nonstreaming_create_message_data_collection_tools( messages=[], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) - items = capture_items("transaction", "span") + items = capture_items("span") client.messages.create(**create_kwargs) sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -439,12 +436,12 @@ async def test_nonstreaming_create_message_data_collection_async( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) - items = capture_items("transaction", "span") + items = capture_items("span") await client.messages.create(**create_kwargs) sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -539,12 +536,12 @@ def test_nonstreaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) - items = capture_items("transaction", "span") + items = capture_items("span") client.messages.create(**create_kwargs) sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -649,12 +646,12 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) - items = capture_items("transaction", "span") + items = capture_items("span") await client.messages.create(**create_kwargs) sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -716,12 +713,11 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) + response = await client.messages.create( + max_tokens=1024, messages=messages, model="model" + ) assert response == EXAMPLE_MESSAGE usage = response.usage @@ -730,7 +726,7 @@ async def test_nonstreaming_create_message_async( assert usage.output_tokens == 20 sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -842,13 +838,13 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -857,7 +853,7 @@ def test_streaming_create_message( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -984,7 +980,7 @@ def test_streaming_create_message_data_collection( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, @@ -996,7 +992,7 @@ def test_streaming_create_message_data_collection( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -1114,7 +1110,7 @@ def test_streaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, @@ -1126,7 +1122,7 @@ def test_streaming_create_message_data_collection_outputs( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -1202,13 +1198,13 @@ def test_streaming_create_message_close( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): messages = client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -1219,7 +1215,7 @@ def test_streaming_create_message_close( messages.close() sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -1305,13 +1301,13 @@ def test_streaming_create_message_api_error( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with pytest.raises(APIStatusError), mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -1320,7 +1316,7 @@ def test_streaming_create_message_api_error( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" assert spans[1]["status"] == SpanStatus.ERROR span = next( @@ -1425,13 +1421,13 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=messages, model="model", @@ -1440,7 +1436,7 @@ def test_stream_messages( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -1580,7 +1576,7 @@ def test_stream_messages_data_collection_outputs( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, @@ -1591,7 +1587,7 @@ def test_stream_messages_data_collection_outputs( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -1667,13 +1663,13 @@ def test_stream_messages_close( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=messages, model="model", @@ -1688,7 +1684,7 @@ def test_stream_messages_close( stream.close() sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -1774,13 +1770,13 @@ def test_stream_messages_api_error( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with pytest.raises(APIStatusError), mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=messages, model="model", @@ -1789,7 +1785,7 @@ def test_stream_messages_api_error( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" assert spans[1]["status"] == SpanStatus.ERROR span = next( @@ -1899,13 +1895,13 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = await client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -1914,7 +1910,7 @@ async def test_streaming_create_message_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -2059,7 +2055,7 @@ async def test_streaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, @@ -2071,7 +2067,7 @@ async def test_streaming_create_message_data_collection_outputs_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -2151,13 +2147,13 @@ async def test_streaming_create_message_async_close( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): messages = await client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -2167,7 +2163,7 @@ async def test_streaming_create_message_async_close( await messages.close() sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -2257,13 +2253,13 @@ async def test_streaming_create_message_async_api_error( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with pytest.raises(APIStatusError), mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = await client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -2272,7 +2268,7 @@ async def test_streaming_create_message_async_api_error( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" assert spans[1]["status"] == SpanStatus.ERROR span = next( @@ -2381,13 +2377,13 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): async with client.messages.stream( max_tokens=1024, messages=messages, @@ -2397,7 +2393,7 @@ async def test_stream_message_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -2540,7 +2536,7 @@ async def test_stream_messages_data_collection_outputs_async( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, @@ -2552,7 +2548,7 @@ async def test_stream_messages_data_collection_outputs_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] span_data = span["attributes"] @@ -2631,13 +2627,13 @@ async def test_stream_messages_async_api_error( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with pytest.raises(APIStatusError), mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): async with client.messages.stream( max_tokens=1024, messages=messages, @@ -2647,7 +2643,7 @@ async def test_stream_messages_async_api_error( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -2740,13 +2736,13 @@ async def test_stream_messages_async_close( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): async with client.messages.stream( max_tokens=1024, messages=messages, @@ -2764,7 +2760,7 @@ async def test_stream_messages_async_close( await stream.close() sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT @@ -2896,13 +2892,13 @@ def test_streaming_create_message_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -2911,7 +2907,7 @@ def test_streaming_create_message_with_input_json_delta( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3046,13 +3042,13 @@ def test_stream_messages_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=messages, model="model", @@ -3061,7 +3057,7 @@ def test_stream_messages_with_input_json_delta( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3203,13 +3199,13 @@ async def test_streaming_create_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = await client.messages.create( max_tokens=1024, messages=messages, model="model", stream=True ) @@ -3218,7 +3214,7 @@ async def test_streaming_create_message_with_input_json_delta_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3361,13 +3357,13 @@ async def test_stream_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): async with client.messages.stream( max_tokens=1024, messages=messages, @@ -3377,7 +3373,7 @@ async def test_stream_message_with_input_json_delta_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3448,17 +3444,16 @@ def test_span_status_error( ) items = capture_items("event", "span") - with sentry_sdk.traces.start_span(name="anthropic"): - client = Anthropic(api_key="z") - client.messages._post = mock.Mock( - side_effect=AnthropicError("API rate limit reached") + client = Anthropic(api_key="z") + client.messages._post = mock.Mock( + side_effect=AnthropicError("API rate limit reached") + ) + with pytest.raises(AnthropicError): + client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, ) - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) (error,) = (item.payload for item in items if item.type == "event") assert error["level"] == "error" @@ -3483,17 +3478,16 @@ async def test_span_status_error_async( ) items = capture_items("event", "span") - with sentry_sdk.traces.start_span(name="anthropic"): - client = AsyncAnthropic(api_key="z") - client.messages._post = AsyncMock( - side_effect=AnthropicError("API rate limit reached") + client = AsyncAnthropic(api_key="z") + client.messages._post = AsyncMock( + side_effect=AnthropicError("API rate limit reached") + ) + with pytest.raises(AnthropicError): + await client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, ) - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) (error,) = (item.payload for item in items if item.type == "event") assert error["level"] == "error" @@ -3554,13 +3548,12 @@ def test_span_origin( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" @@ -3588,13 +3581,12 @@ async def test_span_origin_async( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - await client.messages.create(max_tokens=1024, messages=messages, model="model") + await client.messages.create(max_tokens=1024, messages=messages, model="model") sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" @@ -3766,15 +3758,14 @@ def test_nonstreaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) + response = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) assert response == EXAMPLE_MESSAGE usage = response.usage @@ -3783,7 +3774,7 @@ def test_nonstreaming_create_message_with_system_prompt( assert usage.output_tokens == 20 sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3858,15 +3849,14 @@ async def test_nonstreaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) + response = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) assert response == EXAMPLE_MESSAGE usage = response.usage @@ -3875,7 +3865,7 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert usage.output_tokens == 20 sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -3987,13 +3977,13 @@ def test_streaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = client.messages.create( max_tokens=1024, messages=messages, @@ -4006,7 +3996,7 @@ def test_streaming_create_message_with_system_prompt( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -4118,13 +4108,13 @@ def test_stream_messages_with_system_prompt( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( + ), client.messages.stream( max_tokens=1024, messages=messages, model="model", @@ -4134,7 +4124,7 @@ def test_stream_messages_with_system_prompt( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -4248,13 +4238,13 @@ async def test_stream_message_with_system_prompt_async( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): async with client.messages.stream( max_tokens=1024, messages=messages, @@ -4265,7 +4255,7 @@ async def test_stream_message_with_system_prompt_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert len(spans) == 2 assert spans[1]["name"] == "anthropic" @@ -4380,13 +4370,13 @@ async def test_streaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] - items = capture_items("transaction", "span") + items = capture_items("span") with mock.patch.object( client._client, "send", return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): + ): message = await client.messages.create( max_tokens=1024, messages=messages, @@ -4399,7 +4389,7 @@ async def test_streaming_create_message_with_system_prompt_async( pass sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + spans = [item.payload for item in items] assert spans[1]["name"] == "anthropic" assert len(spans) == 2 @@ -4471,10 +4461,9 @@ def test_system_prompt_with_complex_structure( ] items = capture_items("span") - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model", system=system_prompt - ) + response = client.messages.create( + max_tokens=1024, messages=messages, model="model", system=system_prompt + ) assert response == EXAMPLE_MESSAGE From 185717a8d98afc77b173f437064244feeb6573ce Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 10:45:26 +0200 Subject: [PATCH 5/5] fix tests --- .../integrations/anthropic/test_anthropic.py | 90 ++++++------------- 1 file changed, 29 insertions(+), 61 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 6e6a87f870..5d4c640c8c 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -67,7 +67,6 @@ async def __call__(self, *args, **kwargs): _transform_anthropic_content_block, ) from sentry_sdk.integrations.stdlib import StdlibIntegration -from sentry_sdk.traces import SpanStatus from sentry_sdk.utils import package_version ANTHROPIC_VERSION = package_version("anthropic") @@ -184,10 +183,9 @@ def test_nonstreaming_create_message( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -727,10 +725,9 @@ async def test_nonstreaming_create_message_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -854,7 +851,6 @@ def test_streaming_create_message( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1216,7 +1212,6 @@ def test_streaming_create_message_close( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1317,8 +1312,6 @@ def test_streaming_create_message_api_error( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1437,7 +1430,6 @@ def test_stream_messages( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1685,7 +1677,6 @@ def test_stream_messages_close( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1786,8 +1777,6 @@ def test_stream_messages_api_error( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -1911,10 +1900,9 @@ async def test_streaming_create_message_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -2164,7 +2152,6 @@ async def test_streaming_create_message_async_close( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -2269,8 +2256,6 @@ async def test_streaming_create_message_async_api_error( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -2394,10 +2379,9 @@ async def test_stream_message_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -2644,7 +2628,6 @@ async def test_stream_messages_async_api_error( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -2761,7 +2744,6 @@ async def test_stream_messages_async_close( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" span = next( span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT ) @@ -2908,10 +2890,9 @@ def test_streaming_create_message_with_input_json_delta( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3058,10 +3039,9 @@ def test_stream_messages_with_input_json_delta( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3215,10 +3195,9 @@ async def test_streaming_create_message_with_input_json_delta_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3374,10 +3353,9 @@ async def test_stream_message_with_input_json_delta_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3554,7 +3532,6 @@ def test_span_origin( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -3587,7 +3564,6 @@ async def test_span_origin_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -3775,10 +3751,9 @@ def test_nonstreaming_create_message_with_system_prompt( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3866,10 +3841,9 @@ async def test_nonstreaming_create_message_with_system_prompt_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -3997,10 +3971,9 @@ def test_streaming_create_message_with_system_prompt( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -4125,10 +4098,9 @@ def test_stream_messages_with_system_prompt( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -4256,10 +4228,9 @@ async def test_stream_message_with_system_prompt_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -4391,9 +4362,8 @@ async def test_streaming_create_message_with_system_prompt_async( sentry_sdk.flush() spans = [item.payload for item in items] - assert spans[1]["name"] == "anthropic" - assert len(spans) == 2 - (span, _) = spans + assert len(spans) == 1 + (span,) = spans assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT assert span["name"] == "chat model" @@ -4469,10 +4439,8 @@ def test_system_prompt_with_complex_structure( sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"