diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 37c9120630..4b4ebf1924 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -8,20 +8,15 @@ from sentry_sdk.ai.monitoring import record_token_usage from sentry_sdk.ai.utils import ( GEN_AI_ALLOWED_MESSAGE_ROLES, - get_start_span_function, normalize_message_roles, set_data_normalized, transform_anthropic_content_part, - truncate_and_annotate_messages, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span -from sentry_sdk.tracing_utils import ( - has_span_streaming_enabled, -) from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, @@ -89,7 +84,7 @@ from sentry_sdk._types import TextPart class _PatchedRawMessageStream(Stream[RawMessageStreamEvent]): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -99,7 +94,7 @@ class _PatchedRawMessageStream(Stream[RawMessageStreamEvent]): _finish_reason: Optional[str] class _PatchedMessageStream(MessageStream): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -109,7 +104,7 @@ class _PatchedMessageStream(MessageStream): _finish_reason: Optional[str] class _PatchedRawAsyncMessageStream(AsyncStream[RawMessageStreamEvent]): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -119,7 +114,7 @@ class _PatchedRawAsyncMessageStream(AsyncStream[RawMessageStreamEvent]): _finish_reason: Optional[str] class _PatchedAsyncMessageStream(AsyncMessageStream): - _span: Span + _span: StreamedSpan _integration: "AnthropicIntegration" _model: Optional[ModelParam] @@ -129,7 +124,7 @@ class _PatchedAsyncMessageStream(AsyncMessageStream): _finish_reason: Optional[str] class _PatchedMessageStreamManager(MessageStreamManager): - _span: Union[Span, StreamedSpan] + _span: StreamedSpan _integration: "AnthropicIntegration" _max_tokens: int @@ -142,7 +137,7 @@ class _PatchedMessageStreamManager(MessageStreamManager): _tools: Optional[Iterable[ToolUnionParam]] class _PatchedAsyncMessageStreamManager(AsyncMessageStreamManager[Any]): - _span: Union[Span, StreamedSpan] + _span: StreamedSpan _integration: "AnthropicIntegration" _max_tokens: int @@ -447,7 +442,7 @@ def _transform_system_instructions( def _set_common_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "AnthropicIntegration", max_tokens: "int", messages: "Iterable[MessageParam]", @@ -461,36 +456,35 @@ def _set_common_input_data( """ Set input data for the span based on the provided keyword arguments for the anthropic message creation. """ - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_on_span(SPANDATA.GEN_AI_SYSTEM, "anthropic") - set_on_span(SPANDATA.GEN_AI_OPERATION_NAME, "chat") + span.set_attribute(SPANDATA.GEN_AI_SYSTEM, "anthropic") + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "chat") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) if model is not None and _is_given(model): - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) if top_k is not None and _is_given(top_k): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_K, top_k) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_K, top_k) if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): if client.options["data_collection"]["gen_ai"]["inputs"]: if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) ) else: # Tools were unconditionally added pre-data collection configuration. # This can be removed once data collection is fully rolled out if tools is not None and _is_given(tools) and len(tools) > 0: # type: ignore - set_on_span(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools)) + span.set_attribute( + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) + ) if messages is None or len(messages) == 0: # type: ignore return @@ -504,7 +498,7 @@ def _set_common_input_data( if record_inputs: if isinstance(system, str) or isinstance(system, Iterable): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(_transform_system_instructions(system)), ) @@ -554,35 +548,24 @@ def _set_common_input_data( role_normalized_messages = normalize_message_roles(normalized_messages) - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(role_normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else role_normalized_messages - ) - if messages_data is not None: + if role_normalized_messages is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, + role_normalized_messages, unpack=False, ) def _set_create_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "AnthropicIntegration", ) -> None: """ Set input data for the span based on the provided keyword arguments for the anthropic message creation. """ - if isinstance(span, StreamedSpan): - span.set_attribute( - SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False) - ) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False)) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_STREAMING, kwargs.get("stream", False)) _set_common_input_data( span=span, @@ -659,7 +642,7 @@ async def _wrap_asynchronous_message_iterator( def _set_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "AnthropicIntegration", model: "str | None", input_tokens: "int | None", @@ -672,15 +655,12 @@ def _set_output_data( ) -> None: """ Set output data for the span based on the AI response.""" - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) if model is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, model) if response_id is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_ID, response_id) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_ID, response_id) if finish_reason is not None: - set_on_span(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) client = sentry_sdk.get_client() record_outputs = False @@ -741,22 +721,13 @@ def _sentry_patched_create_sync(f: "Any", *args: "Any", **kwargs: "Any") -> "Any model = kwargs.get("model", "") - span_streaming = has_span_streaming_enabled(sentry_sdk.get_client().options) - if span_streaming: - span = sentry_sdk.traces.start_span( - name=f"chat {model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name=f"chat {model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() + span = sentry_sdk.traces.start_span( + name=f"chat {model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + }, + ) _set_create_input_data(span, kwargs, integration) @@ -839,22 +810,13 @@ async def _sentry_patched_create_async( model = kwargs.get("model", "") - span_streaming = has_span_streaming_enabled(sentry_sdk.get_client().options) - if span_streaming: - span = sentry_sdk.traces.start_span( - name=f"chat {model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name=f"chat {model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() + span = sentry_sdk.traces.start_span( + name=f"chat {model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + }, + ) _set_create_input_data(span, kwargs, integration) @@ -966,7 +928,7 @@ def _accumulate_event_data( def _set_streaming_output_data( - span: "Span", + span: "StreamedSpan", integration: "AnthropicIntegration", model: "Optional[str]", usage: "_RecordedUsage", @@ -1088,28 +1050,16 @@ def _sentry_patched_enter(self: "MessageStreamManager") -> "MessageStream": except TypeError: return f(self) - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - SPANDATA.GEN_AI_RESPONSE_STREAMING: True, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + span = sentry_sdk.traces.start_span( + name="chat" + if patched_self._model is None + else f"chat {patched_self._model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + SPANDATA.GEN_AI_RESPONSE_STREAMING: True, + }, + ) _set_common_input_data( span=span, @@ -1201,28 +1151,16 @@ async def _sentry_patched_aenter( except TypeError: return await f(self) - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - attributes={ - "sentry.op": OP.GEN_AI_CHAT, - "sentry.origin": AnthropicIntegration.origin, - SPANDATA.GEN_AI_RESPONSE_STREAMING: True, - }, - ) - else: - span = get_start_span_function()( - op=OP.GEN_AI_CHAT, - name="chat" - if patched_self._model is None - else f"chat {patched_self._model}".strip(), - origin=AnthropicIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + span = sentry_sdk.traces.start_span( + name="chat" + if patched_self._model is None + else f"chat {patched_self._model}".strip(), + attributes={ + "sentry.op": OP.GEN_AI_CHAT, + "sentry.origin": AnthropicIntegration.origin, + SPANDATA.GEN_AI_RESPONSE_STREAMING: True, + }, + ) _set_common_input_data( span=span, diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 5723d52740..5d4c640c8c 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -57,8 +57,6 @@ async def __call__(self, *args, **kwargs): except ImportError: from anthropic.types.content_block import ContentBlock as TextBlock -from sentry_sdk import start_span, start_transaction -from sentry_sdk._types import BLOB_DATA_SUBSTITUTE from sentry_sdk.ai.utils import transform_content_part, transform_message_content from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.anthropic import ( @@ -69,7 +67,6 @@ async def __call__(self, *args, **kwargs): _transform_anthropic_content_block, ) from sentry_sdk.integrations.stdlib import StdlibIntegration -from sentry_sdk.traces import SpanStatus from sentry_sdk.utils import package_version ANTHROPIC_VERSION = package_version("anthropic") @@ -138,7 +135,6 @@ def data_collection_tool_use_message(): ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -150,19 +146,16 @@ def data_collection_tool_use_message(): ) def test_nonstreaming_create_message( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -178,113 +171,55 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") + response = client.messages.create(max_tokens=1024, messages=messages, model="model") - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + assert response == EXAMPLE_MESSAGE + usage = response.usage - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -324,22 +259,19 @@ def test_nonstreaming_create_message( ) def test_nonstreaming_create_message_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -354,26 +286,14 @@ def test_nonstreaming_create_message_data_collection( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -391,7 +311,6 @@ def test_nonstreaming_create_message_data_collection( ANTHROPIC_VERSION < (0, 27), reason="Tools are not supported in this version of the anthropic package", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,tools_collected", [ @@ -414,19 +333,16 @@ def test_nonstreaming_create_message_data_collection( ) def test_nonstreaming_create_message_data_collection_tools( sentry_init, - capture_events, capture_items, data_collection, tools_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -441,26 +357,14 @@ def test_nonstreaming_create_message_data_collection_tools( messages=[], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] if tools_collected: assert ( @@ -471,7 +375,6 @@ def test_nonstreaming_create_message_data_collection_tools( assert SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", @@ -504,22 +407,19 @@ def test_nonstreaming_create_message_data_collection_tools( ) async def test_nonstreaming_create_message_data_collection_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -534,26 +434,14 @@ async def test_nonstreaming_create_message_data_collection_async( system="You are a helpful assistant.", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + await client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -571,7 +459,6 @@ async def test_nonstreaming_create_message_data_collection_async( ANTHROPIC_VERSION < (0, 27), reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -621,21 +508,18 @@ async def test_nonstreaming_create_message_data_collection_async( ) def test_nonstreaming_create_message_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, outputs_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -650,26 +534,14 @@ def test_nonstreaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(**create_kwargs) + client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -696,7 +568,6 @@ def test_nonstreaming_create_message_data_collection_outputs( ANTHROPIC_VERSION < (0, 27), reason="anthropic.types.ToolUseBlock was added in 0.27.0. Before that, tool use was only available under the beta namespace and could not appear in a standard Message.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -747,21 +618,18 @@ def test_nonstreaming_create_message_data_collection_outputs( ) async def test_nonstreaming_create_message_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, include_prompts, outputs_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -776,26 +644,14 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], tools=DATA_COLLECTION_EXAMPLE_TOOLS, ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create(**create_kwargs) + await client.messages.create(**create_kwargs) - (event,) = events - (span,) = event["spans"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -818,7 +674,6 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -831,19 +686,16 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( ) async def test_nonstreaming_create_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -859,109 +711,56 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) + response = await client.messages.create( + max_tokens=1024, messages=messages, model="model" + ) - assert response == EXAMPLE_MESSAGE - usage = response.usage + assert response == EXAMPLE_MESSAGE + usage = response.usage - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -973,13 +772,11 @@ async def test_nonstreaming_create_message_async( ) def test_streaming_create_message( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1025,8 +822,7 @@ def test_streaming_create_message( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1039,116 +835,60 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + for _ in message: + pass - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -1180,7 +920,6 @@ def test_streaming_create_message( ) def test_streaming_create_message_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1189,15 +928,13 @@ def test_streaming_create_message_data_collection( expected_absent, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -1239,38 +976,21 @@ def test_streaming_create_message_data_collection( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "anthropic" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @@ -1285,7 +1005,6 @@ def test_streaming_create_message_data_collection( assert key not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -1328,7 +1047,6 @@ def test_streaming_create_message_data_collection( ) def test_streaming_create_message_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1336,15 +1054,13 @@ def test_streaming_create_message_data_collection_outputs( outputs_collected, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -1390,38 +1106,21 @@ def test_streaming_create_message_data_collection_outputs( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = client.messages.create(**create_kwargs) - for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create(**create_kwargs) + for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -1436,14 +1135,11 @@ def test_streaming_create_message_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_create_message_close( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1489,8 +1185,7 @@ def test_streaming_create_message_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1499,109 +1194,59 @@ def test_streaming_create_message_close( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - messages = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - next(messages) - - messages.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + messages = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - messages = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - next(messages) - - messages.close() - assert len(events) == 1 - (event,) = events + for _ in range(4): + next(messages) - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + messages.close() - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", ) def test_streaming_create_message_api_error( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1642,8 +1287,7 @@ def test_streaming_create_message_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1652,101 +1296,50 @@ def test_streaming_create_message_api_error( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + for _ in message: + pass - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1758,13 +1351,11 @@ def test_streaming_create_message_api_error( ) def test_stream_messages( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -1810,8 +1401,7 @@ def test_stream_messages( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -1824,115 +1414,60 @@ def test_stream_messages( "content": "Hello, Claude", }, ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", [ @@ -1975,7 +1510,6 @@ def test_stream_messages( ) def test_stream_messages_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -1983,15 +1517,13 @@ def test_stream_messages_data_collection_outputs( outputs_collected, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -2036,40 +1568,20 @@ def test_stream_messages_data_collection_outputs( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"), client.messages.stream( - **stream_kwargs - ) as stream: - for _ in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"), client.messages.stream( - **stream_kwargs - ) as stream: - for _ in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream(**stream_kwargs) as stream: + for _ in stream: + pass - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -2084,14 +1596,11 @@ def test_stream_messages_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_close( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -2137,8 +1646,7 @@ def test_stream_messages_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2147,119 +1655,65 @@ def test_stream_messages_close( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in range(4): + next(stream) - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - next(stream) + # New versions add TextEvent, so consume one more event. + if TextEvent is not None and isinstance(next(stream), TextEvent): + next(stream) - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance(next(stream), TextEvent): - next(stream) + stream.close() - stream.close() + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - next(stream) - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance(next(stream), TextEvent): - next(stream) - - stream.close() - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - -@pytest.mark.parametrize("span_streaming", [True, False]) -@pytest.mark.skipif( - ANTHROPIC_VERSION < (0, 41), - reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", -) -def test_stream_messages_api_error( - sentry_init, - capture_events, - capture_items, - get_model_response, - server_side_event_chunks, - span_streaming, -): - client = Anthropic(api_key="z") +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 41), + reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", +) +def test_stream_messages_api_error( + sentry_init, + capture_items, + get_model_response, + server_side_event_chunks, +): + client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( @@ -2298,8 +1752,7 @@ def test_stream_messages_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2308,101 +1761,50 @@ def test_stream_messages_api_error( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -2415,14 +1817,12 @@ def test_stream_messages_api_error( ) async def test_streaming_create_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2471,8 +1871,7 @@ async def test_streaming_create_message_async( traces_sample_rate=1.0, default_integrations=False, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2485,117 +1884,60 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "max_tokens" - ] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + async for _ in message: + pass - assert len(event["spans"]) == 1 - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + (span,) = spans - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["max_tokens"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -2639,7 +1981,6 @@ async def test_streaming_create_message_async( ) async def test_streaming_create_message_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -2648,15 +1989,13 @@ async def test_streaming_create_message_data_collection_outputs_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -2704,38 +2043,21 @@ async def test_streaming_create_message_data_collection_outputs_async( messages=[{"role": "user", "content": "Hello, Claude"}], stream=True, ) - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = await client.messages.create(**create_kwargs) - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - message = await client.messages.create(**create_kwargs) - async for _ in message: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = await client.messages.create(**create_kwargs) + async for _ in message: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -2750,16 +2072,13 @@ async def test_streaming_create_message_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_streaming_create_message_async_close( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2807,8 +2126,7 @@ async def test_streaming_create_message_async_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2817,96 +2135,49 @@ async def test_streaming_create_message_async_close( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - messages = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - await messages.__anext__() - await messages.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + messages = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - messages = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in range(4): - await messages.__anext__() - await messages.close() - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + for _ in range(4): + await messages.__anext__() + await messages.close() - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", @@ -2914,12 +2185,10 @@ async def test_streaming_create_message_async_close( @pytest.mark.asyncio async def test_streaming_create_message_async_api_error( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -2962,8 +2231,7 @@ async def test_streaming_create_message_async_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -2972,101 +2240,50 @@ async def test_streaming_create_message_async_api_error( "content": "Hello, Claude", } ] - - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - assert spans[1]["status"] == SpanStatus.ERROR - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" + items = capture_items("span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True ) - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + async for _ in message: + pass - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -3079,14 +2296,12 @@ async def test_streaming_create_message_async_api_error( ) async def test_stream_message_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3134,8 +2349,7 @@ async def test_stream_message_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3148,114 +2362,60 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "Hello, Claude", - }, - ] - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + (span,) = spans - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,outputs_collected", @@ -3299,7 +2459,6 @@ async def test_stream_message_async( ) async def test_stream_messages_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, @@ -3308,15 +2467,13 @@ async def test_stream_messages_data_collection_outputs_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -3363,38 +2520,21 @@ async def test_stream_messages_data_collection_outputs_async( model="model", messages=[{"role": "user", "content": "Hello, Claude"}], ) + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream(**stream_kwargs) as stream: + async for _ in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - async with client.messages.stream(**stream_kwargs) as stream: - async for _ in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ), start_transaction(name="anthropic"): - async with client.messages.stream(**stream_kwargs) as stream: - async for _ in stream: - pass - - (event,) = events - span = next(s for s in event["spans"] if s["op"] == OP.GEN_AI_CHAT) - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = [s for s in spans if s["attributes"]["sentry.op"] == OP.GEN_AI_CHAT] + span_data = span["attributes"] # Output data that is not gated on data collection assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "model" @@ -3409,7 +2549,6 @@ async def test_stream_messages_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 41), reason="Error classes moved in https://github.com/anthropics/anthropic-sdk-python/commit/4e0b15e22fe40e9aa513459564f641bf97c90954.", @@ -3417,12 +2556,10 @@ async def test_stream_messages_data_collection_outputs_async( @pytest.mark.asyncio async def test_stream_messages_async_api_error( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3465,8 +2602,7 @@ async def test_stream_messages_async_api_error( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3475,111 +2611,58 @@ async def test_stream_messages_async_api_error( "content": "Hello, Claude", } ] + items = capture_items("span") + + with pytest.raises(APIStatusError), mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(APIStatusError), mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) - assert span["status"] == "internal_error" - assert span["tags"]["status"] == "internal_error" - assert event["contexts"]["trace"]["status"] == "internal_error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_stream_messages_async_close( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") @@ -3627,8 +2710,7 @@ async def test_stream_messages_async_close( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3637,112 +2719,57 @@ async def test_stream_messages_async_close( "content": "Hello, Claude", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in range(4): + await stream.__anext__() - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - await stream.__anext__() - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance( - await stream.__anext__(), TextEvent - ): - await stream.__anext__() - - await stream.close() - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["name"] == "anthropic" - span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + # New versions add TextEvent, so consume one more event. + if TextEvent is not None and isinstance( + await stream.__anext__(), TextEvent + ): + await stream.__anext__() - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + await stream.close() - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] - == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for _ in range(4): - await stream.__anext__() - - # New versions add TextEvent, so consume one more event. - if TextEvent is not None and isinstance( - await stream.__anext__(), TextEvent - ): - await stream.__anext__() - - await stream.close() - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + span = next( + span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + ) - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_ID] == "msg_01XFDUDYJgAACzvnptvVoYEL" - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "Hello, Claude"}]' + ) + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi!" + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_ID] + == "msg_01XFDUDYJgAACzvnptvVoYEL" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta, which was introduced in >=0.27.0 along with a new message delta type for tool calling.", @@ -3758,13 +2785,11 @@ async def test_stream_messages_async_close( ) def test_streaming_create_message_with_input_json_delta( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -3840,8 +2865,7 @@ def test_streaming_create_message_with_input_json_delta( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -3850,103 +2874,51 @@ def test_streaming_create_message_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans + for _ in message: + pass - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + (span,) = spans - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta, which was introduced in >=0.27.0 along with a new message delta type for tool calling.", @@ -3962,13 +2934,11 @@ def test_streaming_create_message_with_input_json_delta( ) def test_stream_messages_with_input_json_delta( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): client = Anthropic(api_key="z") @@ -4044,8 +3014,7 @@ def test_stream_messages_with_input_json_delta( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4054,102 +3023,51 @@ def test_stream_messages_with_input_json_delta( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for event in stream: + pass - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), @@ -4166,14 +3084,12 @@ def test_stream_messages_with_input_json_delta( ) async def test_streaming_create_message_with_input_json_delta_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") response = get_model_response( @@ -4254,8 +3170,7 @@ async def test_streaming_create_message_with_input_json_delta_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4264,104 +3179,52 @@ async def test_streaming_create_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + async for _ in message: + pass - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + (span,) = spans - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), @@ -4378,14 +3241,12 @@ async def test_streaming_create_message_with_input_json_delta_async( ) async def test_stream_message_with_input_json_delta_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): client = AsyncAnthropic(api_key="z") response = get_model_response( @@ -4466,8 +3327,7 @@ async def test_stream_message_with_input_json_delta_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -4476,342 +3336,185 @@ async def test_stream_message_with_input_json_delta_async( "content": "What is the weather like in San Francisco?", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' - ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 + + (span,) = spans - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + + if send_default_pii and include_prompts: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' + ) + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == '{"location": "San Francisco, CA"}' + ) + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 407 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) def test_exception_message_create( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") client.messages._post = mock.Mock( side_effect=AnthropicError("API rate limit reached") ) + items = capture_items("event") - if span_streaming: - items = capture_items("event") - - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) - - (event,) = (item.payload for item in items) - assert event["level"] == "error" - else: - events = capture_events() - - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) + with pytest.raises(AnthropicError): + client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items) + assert event["level"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_status_error( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", + ) + items = capture_items("event", "span") + + client = Anthropic(api_key="z") + client.messages._post = mock.Mock( + side_effect=AnthropicError("API rate limit reached") ) + with pytest.raises(AnthropicError): + client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) - if span_streaming: - items = capture_items("event", "span") + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - with sentry_sdk.traces.start_span(name="anthropic"): - client = Anthropic(api_key="z") - client.messages._post = mock.Mock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - with start_transaction(name="anthropic"): - client = Anthropic(api_key="z") - client.messages._post = mock.Mock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_status_error_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - if span_streaming: - items = capture_items("event", "span") + items = capture_items("event", "span") + + client = AsyncAnthropic(api_key="z") + client.messages._post = AsyncMock( + side_effect=AnthropicError("API rate limit reached") + ) + with pytest.raises(AnthropicError): + await client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) + + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - with sentry_sdk.traces.start_span(name="anthropic"): - client = AsyncAnthropic(api_key="z") - client.messages._post = AsyncMock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - with start_transaction(name="anthropic"): - client = AsyncAnthropic(api_key="z") - client.messages._post = AsyncMock( - side_effect=AnthropicError("API rate limit reached") - ) - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[ - {"role": "system", "content": "I'm throwing an exception"} - ], - max_tokens=1024, - ) - - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert transaction["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_exception_message_create_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") client.messages._post = AsyncMock( side_effect=AnthropicError("API rate limit reached") ) + items = capture_items("event") - if span_streaming: - items = capture_items("event") - - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) - - (event,) = (item.payload for item in items) - assert event["level"] == "error" - else: - events = capture_events() - - with pytest.raises(AnthropicError): - await client.messages.create( - model="some-model", - messages=[{"role": "system", "content": "I'm throwing an exception"}], - max_tokens=1024, - ) + with pytest.raises(AnthropicError): + await client.messages.create( + model="some-model", + messages=[{"role": "system", "content": "I'm throwing an exception"}], + max_tokens=1024, + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items) + assert event["level"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -4823,46 +3526,27 @@ def test_span_origin( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") - (event,) = events - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + sentry_sdk.flush() + spans = [item.payload for item in items] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -4874,43 +3558,22 @@ async def test_span_origin_async( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - else: - events = capture_events() - - with start_transaction(name="anthropic"): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - (event,) = events + await client.messages.create(max_tokens=1024, messages=messages, model="model") - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert event["spans"][0]["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + sentry_sdk.flush() + spans = [item.payload for item in items] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert spans[0]["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_collect_ai_data_with_input_json_delta(span_streaming): +def test_collect_ai_data_with_input_json_delta(): event = ContentBlockDeltaEvent( delta=InputJSONDelta(partial_json="test", type="input_json_delta"), index=0, @@ -4945,35 +3608,33 @@ def test_set_output_data_with_input_json_delta(sentry_init): disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=False, - ) - - with start_transaction(name="test"): - span = start_span() - integration = AnthropicIntegration() - json_deltas = ["{'test': 'data',", "'more': 'json'}"] - _set_output_data( - span, - integration, - model="", - input_tokens=10, - output_tokens=20, - cache_read_input_tokens=0, - cache_write_input_tokens=0, - content_blocks=[{"text": "".join(json_deltas), "type": "text"}], - ) + trace_lifecycle="stream", + ) - assert ( - span._data.get(SPANDATA.GEN_AI_RESPONSE_TEXT) - == "{'test': 'data','more': 'json'}" - ) - assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 - assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 - assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 + span = sentry_sdk.traces.start_span(name="test") + integration = AnthropicIntegration() + json_deltas = ["{'test': 'data',", "'more': 'json'}"] + _set_output_data( + span, + integration, + model="", + input_tokens=10, + output_tokens=20, + cache_read_input_tokens=0, + cache_write_input_tokens=0, + content_blocks=[{"text": "".join(json_deltas), "type": "text"}], + ) + + assert ( + span._attributes.get(SPANDATA.GEN_AI_RESPONSE_TEXT) + == "{'test': 'data','more': 'json'}" + ) + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 + assert span._attributes.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 # Test messages with mixed roles including "ai" that should be mapped to "assistant" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "test_message,expected_role", [ @@ -4991,11 +3652,9 @@ def test_set_output_data_with_input_json_delta(sentry_init): ) def test_anthropic_message_role_mapping( sentry_init, - capture_events, capture_items, test_message, expected_role, - span_streaming, ): """Test that Anthropic integration properly maps message roles like 'ai' to 'assistant'""" sentry_init( @@ -5003,8 +3662,7 @@ def test_anthropic_message_role_mapping( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -5024,159 +3682,25 @@ def mock_messages_create(*args, **kwargs): client.messages._post = mock.Mock(return_value=mock_messages_create()) test_messages = [test_message] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic tx"): - client.messages.create( - model="claude-3-opus", max_tokens=10, messages=test_messages - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - # Verify that the span was created correctly - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - - # Parse the stored messages - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic tx"): - client.messages.create( - model="claude-3-opus", max_tokens=10, messages=test_messages - ) + client.messages.create(model="claude-3-opus", max_tokens=10, messages=test_messages) - (event,) = events - span = event["spans"][0] + sentry_sdk.flush() + span = next(item.payload for item in items) - # Verify that the span was created correctly - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + # Verify that the span was created correctly + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - # Parse the stored messages - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + # Parse the stored messages + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert stored_messages[0]["role"] == expected_role -def test_anthropic_message_truncation(sentry_init, capture_events): - """Test that large messages are truncated properly in Anthropic integration.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - messages = [ - {"role": "user", "content": "small message 1"}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": "small message 4"}, - {"role": "user", "content": "small message 5"}, - ] - - with start_transaction(): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - -@pytest.mark.asyncio -async def test_anthropic_message_truncation_async(sentry_init, capture_events): - """Test that large messages are truncated properly in Anthropic integration.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - - client = AsyncAnthropic(api_key="z") - client.messages._post = mock.AsyncMock(return_value=EXAMPLE_MESSAGE) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - messages = [ - {"role": "user", "content": "small message 1"}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": "small message 4"}, - {"role": "user", "content": "small message 5"}, - ] - - with start_transaction(): - await client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5188,11 +3712,9 @@ async def test_anthropic_message_truncation_async(sentry_init, capture_events): ) def test_nonstreaming_create_message_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that system prompts are properly captured in GEN_AI_REQUEST_MESSAGES.""" sentry_init( @@ -5200,8 +3722,7 @@ def test_nonstreaming_create_message_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -5213,128 +3734,62 @@ def test_nonstreaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + response = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert response == EXAMPLE_MESSAGE + usage = response.usage - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + (span,) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} ] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -5347,11 +3802,9 @@ def test_nonstreaming_create_message_with_system_prompt( ) async def test_nonstreaming_create_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that system prompts are properly captured in GEN_AI_REQUEST_MESSAGES (async).""" sentry_init( @@ -5359,8 +3812,7 @@ async def test_nonstreaming_create_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncAnthropic(api_key="z") @@ -5372,128 +3824,62 @@ async def test_nonstreaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage + response = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 + assert response == EXAMPLE_MESSAGE + usage = response.usage - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + assert usage.input_tokens == 10 + assert usage.output_tokens == 20 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + (span,) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "end_turn" + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} ] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) - - assert response == EXAMPLE_MESSAGE - usage = response.usage - - assert usage.input_tokens == 10 - assert usage.output_tokens == 20 - - assert len(events) == 1 - (event,) = events - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["end_turn"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5505,13 +3891,11 @@ async def test_nonstreaming_create_message_with_system_prompt_async( ) def test_streaming_create_message_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") @@ -5558,8 +3942,7 @@ def test_streaming_create_message_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5568,131 +3951,65 @@ def test_streaming_create_message_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans + for _ in message: + pass - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + (span,) = spans - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -5704,13 +4021,11 @@ def test_streaming_create_message_with_system_prompt( ) def test_stream_messages_with_system_prompt( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") @@ -5757,8 +4072,7 @@ def test_stream_messages_with_system_prompt( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5767,120 +4081,60 @@ def test_stream_messages_with_system_prompt( "content": "Hello, Claude", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) as stream: + for event in stream: + pass - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert spans[1]["name"] == "anthropic" - (span, _) = spans + (span,) = spans - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert len(event["spans"]) == 1 - (span,) = event["spans"] + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] - - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -5893,14 +4147,12 @@ def test_stream_messages_with_system_prompt( ) async def test_stream_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -5949,8 +4201,7 @@ async def test_stream_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -5959,125 +4210,62 @@ async def test_stream_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + async with client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + system="You are a helpful assistant.", + ) as stream: + async for event in stream: + pass - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - async for event in stream: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + sentry_sdk.flush() + spans = [item.payload for item in items] + assert len(spans) == 1 - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - async with client.messages.stream( - max_tokens=1024, - messages=messages, - model="model", - system="You are a helpful assistant.", - ) as stream: - async for event in stream: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + (span,) = spans - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -6090,14 +4278,12 @@ async def test_stream_message_with_system_prompt_async( ) async def test_streaming_create_message_with_system_prompt_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -6146,8 +4332,7 @@ async def test_streaming_create_message_with_system_prompt_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) messages = [ @@ -6156,135 +4341,69 @@ async def test_streaming_create_message_with_system_prompt_async( "content": "Hello, Claude", } ] + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + message = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - if span_streaming: - items = capture_items("transaction", "span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, sentry_sdk.traces.start_span(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - async for _ in message: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] + async for _ in message: + pass - assert spans[1]["name"] == "anthropic" - assert len(spans) == 2 - (span, _) = spans + sentry_sdk.flush() + spans = [item.payload for item in items] - assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - assert span["name"] == "chat model" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" + assert len(spans) == 1 + (span,) = spans - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] + assert span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT + assert span["name"] == "chat model" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads( + span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert len(stored_messages) == 1 + assert stored_messages[0]["role"] == "user" + assert stored_messages[0]["content"] == "Hello, Claude" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) - - async for _ in message: - pass - - assert len(events) == 1 - (event,) = events - - assert event["type"] == "transaction" - assert event["transaction"] == "anthropic" - - assert len(event["spans"]) == 1 - (span,) = event["spans"] - - assert span["op"] == OP.GEN_AI_CHAT - assert span["description"] == "chat model" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" - - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."} - ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" - - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True -@pytest.mark.parametrize("span_streaming", [True, False]) def test_system_prompt_with_complex_structure( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that complex system prompt structures (list of text blocks) are properly captured.""" sentry_init( @@ -6292,8 +4411,7 @@ def test_system_prompt_with_complex_structure( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6311,76 +4429,36 @@ def test_system_prompt_with_complex_structure( "content": "Hello", } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model", system=system_prompt - ) - - assert response == EXAMPLE_MESSAGE - - sentry_sdk.flush() - spans = [item.payload for item in items] - assert len(spans) == 2 - - assert spans[1]["name"] == "anthropic" - (span, _) = spans - - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] - system_instructions = json.loads( - span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - - # System content should be a list of text blocks - assert isinstance(system_instructions, list) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - response = client.messages.create( - max_tokens=1024, messages=messages, model="model", system=system_prompt - ) + response = client.messages.create( + max_tokens=1024, messages=messages, model="model", system=system_prompt + ) - assert response == EXAMPLE_MESSAGE + assert response == EXAMPLE_MESSAGE - assert len(events) == 1 - (event,) = events + sentry_sdk.flush() + spans = [item.payload for item in items] - assert len(event["spans"]) == 1 - (span,) = event["spans"] + (span,) = spans - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"] - system_instructions = json.loads( - span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["attributes"] + system_instructions = json.loads( + span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) - # System content should be a list of text blocks - assert isinstance(system_instructions, list) - assert system_instructions == [ - {"type": "text", "content": "You are a helpful assistant."}, - {"type": "text", "content": "Be concise and clear."}, - ] + # System content should be a list of text blocks + assert isinstance(system_instructions, list) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."}, + {"type": "text", "content": "Be concise and clear."}, + ] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" @@ -6582,65 +4660,9 @@ def test_transform_message_content_list_anthropic(): # Integration tests for binary data in messages -def test_message_with_base64_image(sentry_init, capture_events): - """Test that messages with base64 images are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What's in this image?"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": "base64encodeddatahere...", - }, - }, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - - assert len(stored_messages) == 1 - assert stored_messages[0]["role"] == "user" - content = stored_messages[0]["content"] - assert len(content) == 2 - assert content[0] == {"type": "text", "text": "What's in this image?"} - assert content[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": BLOB_DATA_SUBSTITUTE, - } - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_message_with_url_image( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that messages with URL-referenced images are properly captured.""" sentry_init( @@ -6648,8 +4670,7 @@ def test_message_with_url_image( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6670,31 +4691,15 @@ def test_message_with_url_image( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6705,12 +4710,9 @@ def test_message_with_url_image( } -@pytest.mark.parametrize("span_streaming", [True, False]) def test_message_with_file_image( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that messages with file_id-referenced images are properly captured.""" sentry_init( @@ -6718,8 +4720,7 @@ def test_message_with_file_image( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6741,31 +4742,15 @@ def test_message_with_file_image( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6776,59 +4761,9 @@ def test_message_with_file_image( } -def test_message_with_base64_pdf(sentry_init, capture_events): - """Test that messages with base64-encoded PDF documents are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Summarize this document."}, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0xLjQKJeLj...base64pdfdata", - }, - }, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - assert content[1] == { - "type": "blob", - "modality": "document", - "mime_type": "application/pdf", - "content": BLOB_DATA_SUBSTITUTE, - } - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_message_with_url_pdf( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that messages with URL-referenced PDF documents are properly captured.""" sentry_init( @@ -6836,8 +4771,7 @@ def test_message_with_url_pdf( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6858,31 +4792,15 @@ def test_message_with_url_pdf( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans + client.messages.create(max_tokens=1024, messages=messages, model="model") - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6893,12 +4811,9 @@ def test_message_with_url_pdf( } -@pytest.mark.parametrize("span_streaming", [True, False]) def test_message_with_file_document( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that messages with file_id-referenced documents are properly captured.""" sentry_init( @@ -6906,8 +4821,7 @@ def test_message_with_file_document( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -6929,31 +4843,15 @@ def test_message_with_file_document( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { @@ -6964,174 +4862,9 @@ def test_message_with_file_document( } -def test_message_with_mixed_content(sentry_init, capture_events): - """Test that messages with mixed content (text, images, documents) are properly captured.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Compare this image with the document."}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "iVBORw0KGgo...base64imagedata", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/comparison.jpg", - }, - }, - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0xLjQK...base64pdfdata", - }, - }, - {"type": "text", "text": "Please provide a detailed analysis."}, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - - assert len(content) == 5 - assert content[0] == { - "type": "text", - "text": "Compare this image with the document.", - } - assert content[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/png", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[2] == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/comparison.jpg", - } - assert content[3] == { - "type": "blob", - "modality": "document", - "mime_type": "application/pdf", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[4] == { - "type": "text", - "text": "Please provide a detailed analysis.", - } - - -def test_message_with_multiple_images_different_formats(sentry_init, capture_events): - """Test that messages with multiple images of different source types are handled.""" - sentry_init( - integrations=[AnthropicIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - client = Anthropic(api_key="z") - client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) - - messages = [ - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": "base64data1...", - }, - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/img2.gif", - }, - }, - { - "type": "image", - "source": { - "type": "file", - "file_id": "file_img_789", - "media_type": "image/webp", - }, - }, - {"type": "text", "text": "Compare these three images."}, - ], - } - ] - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) - content = stored_messages[0]["content"] - - assert len(content) == 4 - assert content[0] == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": BLOB_DATA_SUBSTITUTE, - } - assert content[1] == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/img2.gif", - } - assert content[2] == { - "type": "file", - "modality": "image", - "mime_type": "image/webp", - "file_id": "file_img_789", - } - assert content[3] == {"type": "text", "text": "Compare these three images."} - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_binary_content_not_stored_when_pii_disabled( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that binary content is not stored when send_default_pii is False.""" sentry_init( @@ -7139,8 +4872,7 @@ def test_binary_content_not_stored_when_pii_disabled( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7162,39 +4894,21 @@ def test_binary_content_not_stored_when_pii_disabled( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans + # Messages should not be stored + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_binary_content_not_stored_when_prompts_disabled( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that binary content is not stored when include_prompts is False.""" sentry_init( @@ -7202,8 +4916,7 @@ def test_binary_content_not_stored_when_prompts_disabled( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7225,47 +4938,28 @@ def test_binary_content_not_stored_when_prompts_disabled( ], } ] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + client.messages.create(max_tokens=1024, messages=messages, model="model") - sentry_sdk.flush() - spans = [item.payload for item in items] - (span,) = spans + sentry_sdk.flush() + spans = [item.payload for item in items] + (span,) = spans - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - else: - events = capture_events() + # Messages should not be stored + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - # Messages should not be stored - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_cache_tokens_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test cache read/write tokens are tracked for non-streaming responses.""" sentry_init( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7285,50 +4979,27 @@ def test_cache_tokens_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + ) - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 50 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 250 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_write_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_write tokens (non-streaming). @@ -7345,8 +5016,7 @@ def test_input_tokens_include_cache_write_nonstreaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7366,54 +5036,27 @@ def test_input_tokens_include_cache_write_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 3+3?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 3+3?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 3+3?"}], + model="claude-sonnet-4-20250514", + ) - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 + # input_tokens should be total: 19 (non-cached) + 2846 (cache_write) = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 0 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 2846 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_read_nonstreaming( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (non-streaming). @@ -7430,8 +5073,7 @@ def test_input_tokens_include_cache_read_nonstreaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7451,54 +5093,29 @@ def test_input_tokens_include_cache_read_nonstreaming( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = [item.payload for item in items] - - # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + ) - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = [item.payload for item in items] - # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 (non-cached) + 2846 (cache_read) = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_include_cache_read_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -7539,69 +5156,38 @@ def test_input_tokens_include_cache_read_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - stream=True, - ): - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - stream=True, - ): - pass + trace_lifecycle="stream", + ) + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + stream=True, + ): + pass - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 + 2846 = test_stream_messages_input_tokens_include_cache_read_streaming + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_input_tokens_include_cache_read_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -7641,65 +5227,35 @@ def test_stream_messages_input_tokens_include_cache_read_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - # input_tokens should be total: 19 + 2846 = 2865 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert ( - span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 - ) # 2865 + 14 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - ) as stream: - for event in stream: - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + ) as stream: + for event in stream: + pass - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - # input_tokens should be total: 19 + 2846 = 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 + # input_tokens should be total: 19 + 2846 = 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 2865 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 2879 # 2865 + 14 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 2846 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_input_tokens_unchanged_without_caching( sentry_init, - capture_events, capture_items, - span_streaming, ): """ Test that input_tokens is unchanged when there are no cached tokens. @@ -7711,8 +5267,7 @@ def test_input_tokens_unchanged_without_caching( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = Anthropic(api_key="z") @@ -7730,46 +5285,26 @@ def test_input_tokens_unchanged_without_caching( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 2+2?"}], - model="claude-sonnet-4-20250514", - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 2+2?"}], - model="claude-sonnet-4-20250514", - ) + client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="claude-sonnet-4-20250514", + ) - (span,) = events[0]["spans"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 20 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_cache_tokens_streaming( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") @@ -7806,67 +5341,38 @@ def test_cache_tokens_streaming( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - stream=True, - ): - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - stream=True, - ): - pass + trace_lifecycle="stream", + ) + items = capture_items("span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + stream=True, + ): + pass - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_stream_messages_cache_tokens( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") @@ -7903,52 +5409,27 @@ def test_stream_messages_cache_tokens( integrations=[AnthropicIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) as stream: - for event in stream: - pass - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 - else: - events = capture_events() - - with mock.patch.object( - client._client, - "send", - return_value=response, - ) as _, start_transaction(name="anthropic"), client.messages.stream( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - ) as stream: - for event in stream: - pass - - (span,) = events[0]["spans"] - # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 - assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 - assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 - assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20 + with mock.patch.object( + client._client, + "send", + return_value=response, + ), client.messages.stream( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + ) as stream: + for event in stream: + pass + + sentry_sdk.flush() + (span,) = (item.payload for item in items) + # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 200 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 210 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80 + assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20