diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index e91fed097c..d3c70ddfa4 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -472,6 +472,12 @@ class TextPart(TypedDict): type: Literal["text"] content: str + class ToolDefinition(TypedDict): + type: str + name: NotRequired[str] + description: NotRequired[str] + parameters: NotRequired[dict[str, object]] + IgnoreSpansName = Union[str, Pattern[str]] IgnoreSpansContext = TypedDict( "IgnoreSpansContext", diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index b5eb8c55ef..a61a3e7814 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -8,9 +8,10 @@ ChatCompletionContentPartParam, ChatCompletionMessageParam, ChatCompletionSystemMessageParam, + ChatCompletionToolUnionParam, ) - from sentry_sdk._types import TextPart + from sentry_sdk._types import TextPart, ToolDefinition def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: @@ -64,3 +65,55 @@ def _transform_system_instructions( instruction_text_parts += text_parts return instruction_text_parts + + +def _transform_tool_definitions( + tools: "Iterable[ChatCompletionToolUnionParam]", +) -> "list[ToolDefinition]": + """ + Transform tool definitions to the schema used by the "gen_ai.tool.definitions" attribute. + """ + if not isinstance(tools, Iterable): + return [] + + tool_definitions = [] + for tool in tools: + if not isinstance(tool, dict) or "type" not in tool: + continue + + if tool["type"] == "function": + tool_definition: ToolDefinition = { + "type": "function", + } + + if "function" not in tool: + tool_definitions.append(tool_definition) + continue + + if "name" in tool["function"]: + tool_definition["name"] = tool["function"]["name"] + + if "description" in tool["function"]: + tool_definition["description"] = tool["function"]["description"] + + if "parameters" in tool["function"]: + tool_definition["parameters"] = tool["function"]["parameters"] + + tool_definitions.append(tool_definition) + continue + + if tool["type"] == "custom": + tool_definition = { + "type": "custom", + } + + if "name" in tool["custom"]: + tool_definition["name"] = tool["custom"]["name"] + + if "description" in tool["custom"]: + tool_definition["description"] = tool["custom"]["description"] + + tool_definitions.append(tool_definition) + continue + + return tool_definitions diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 8a77668329..1b25b6b962 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -17,6 +17,9 @@ from sentry_sdk.ai._openai_completions_api import ( _is_system_instruction as _is_system_instruction_completions, ) +from sentry_sdk.ai._openai_completions_api import ( + _transform_tool_definitions as _transform_tool_definitions_completions, +) from sentry_sdk.ai._openai_responses_api import ( _get_system_instructions as _get_system_instructions_responses, ) @@ -463,15 +466,16 @@ def _set_completions_api_input_data( "messages" ) - tools = kwargs.get("tools") - if tools is not None and _is_given(tools) and len(tools) > 0: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) - ) - set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data ) + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_completions(tools)), + ) + model = kwargs.get("model") if model is not None: set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 7d243657f7..53f993b5c0 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -23,6 +23,16 @@ from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage +try: + from openai.types.chat import ( + ChatCompletionCustomToolParam, + ChatCompletionFunctionToolParam, + ) + from openai.types.chat.chat_completion_custom_tool_param import Custom + from openai.types.shared_params import FunctionDefinition +except ImportError: + pass + SKIP_RESPONSES_TESTS = False try: @@ -107,6 +117,169 @@ async def __call__(self, *args, **kwargs): ) +@pytest.mark.skipif( + OPENAI_VERSION <= (2, 10, 0), + reason="ChatCompletionCustomToolParam is unavailable before.", +) +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +def test_chat_completion_tool_definitions( + sentry_init, + capture_events, + capture_items, + nonstreaming_chat_completions_model_response, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response( + response_id="chat-id", + response_model="gpt-3.5-turbo", + message_content="the model response", + created=10000000, + usage=CompletionUsage( + prompt_tokens=20, + completion_tokens=10, + total_tokens=30, + ), + ) + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + strict=True, + ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", + ), + ), + ], + ) + + sentry_sdk.flush() + span = next(item.payload for item in items) + + assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] + else: + events = capture_events() + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + strict=True, + ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", + ), + ), + ], + ) + + tx = events[0] + assert tx["type"] == "transaction" + + assert json.loads(tx["spans"][0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, + }, + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -5617,78 +5790,6 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -@pytest.mark.skipif( - OPENAI_VERSION <= (1, 1, 0), - reason="OpenAI versions <=1.1.0 do not support the tools parameter.", -) -@pytest.mark.parametrize( - "tools", - [[], None, NOT_GIVEN, omit], -) -def test_empty_tools_in_chat_completion( - sentry_init, - capture_events, - capture_items, - tools, - nonstreaming_chat_completions_model_response, - stream_gen_ai_spans, - span_streaming, -): - sentry_init( - integrations=[OpenAIIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - stream_gen_ai_spans=stream_gen_ai_spans, - trace_lifecycle="stream" if span_streaming else "static", - ) - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - return_value=nonstreaming_chat_completions_model_response( - response_id="chat-id", - response_model="gpt-3.5-turbo", - message_content="the model response", - created=10000000, - usage=CompletionUsage( - prompt_tokens=20, - completion_tokens=10, - total_tokens=30, - ), - ) - ) - - if span_streaming or stream_gen_ai_spans: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - assert "gen_ai.request.available_tools" not in span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - (event,) = events - span = event["spans"][0] - - assert "gen_ai.request.available_tools" not in span["data"] - - @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) # Feature added in https://github.com/openai/openai-python/pull/1952