feat(openai): Set gen_ai.tool.definitions attribute for the Completions API

sentry_sdk/_types.py

@@ -472,6 +472,12 @@ class TextPart(TypedDict):
         type: Literal["text"]
         content: str
 
+    class ToolDefinition(TypedDict):
+        type: str
+        name: NotRequired[str]
+        description: NotRequired[str]
+        parameters: NotRequired[dict[str, object]]
+
     IgnoreSpansName = Union[str, Pattern[str]]
     IgnoreSpansContext = TypedDict(
         "IgnoreSpansContext",

sentry_sdk/ai/_openai_completions_api.py

@@ -8,9 +8,10 @@
         ChatCompletionContentPartParam,
         ChatCompletionMessageParam,
         ChatCompletionSystemMessageParam,
+        ChatCompletionToolUnionParam,
     )
 
-    from sentry_sdk._types import TextPart
+    from sentry_sdk._types import TextPart, ToolDefinition
 
 
 def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool:
@@ -64,3 +65,55 @@
         instruction_text_parts += text_parts
 
     return instruction_text_parts
+
+
+def _transform_tool_definitions(
+    tools: "Iterable[ChatCompletionToolUnionParam]",
+) -> "list[ToolDefinition]":
+    """
+    Transform tool definitions to the schema used by the "gen_ai.tool.definitions" attribute.
+    """
+    if not isinstance(tools, Iterable):
+        return []
+
+    tool_definitions = []
+    for tool in tools:
+        if not isinstance(tool, dict) or "type" not in tool:
+            continue
+
+        if tool["type"] == "function":
+            tool_definition: ToolDefinition = {
+                "type": "function",
+            }
+
+            if "function" not in tool:
+                tool_definitions.append(tool_definition)
+                continue
+
+            if "name" in tool["function"]:
+                tool_definition["name"] = tool["function"]["name"]
+
+            if "description" in tool["function"]:
+                tool_definition["description"] = tool["function"]["description"]
+
+            if "parameters" in tool["function"]:
+                tool_definition["parameters"] = tool["function"]["parameters"]
+
+            tool_definitions.append(tool_definition)
+            continue
+
+        if tool["type"] == "custom":
+            tool_definition = {
+                "type": "custom",
+            }
+
+            if "name" in tool["custom"]:
+                tool_definition["name"] = tool["custom"]["name"]
+
+            if "description" in tool["custom"]:
+                tool_definition["description"] = tool["custom"]["description"]
+
+            tool_definitions.append(tool_definition)
+            continue
+
+    return tool_definitions

sentry_sdk/integrations/openai.py

@@ -17,6 +17,9 @@
 from sentry_sdk.ai._openai_completions_api import (
     _is_system_instruction as _is_system_instruction_completions,
 )
+from sentry_sdk.ai._openai_completions_api import (
+    _transform_tool_definitions as _transform_tool_definitions_completions,
+)
 from sentry_sdk.ai._openai_responses_api import (
     _get_system_instructions as _get_system_instructions_responses,
 )
@@ -463,15 +466,16 @@
         "messages"
     )
 
-    tools = kwargs.get("tools")
-    if tools is not None and _is_given(tools) and len(tools) > 0:
-        set_data_normalized(
-            span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools)
-        )
-
     set_on_span = (
         span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
     )
+    tools = kwargs.get("tools")
+    if tools is not None and _is_given(tools):
+        set_on_span(
+            SPANDATA.GEN_AI_TOOL_DEFINITIONS,
+            json.dumps(_transform_tool_definitions_completions(tools)),
+        )
+
     model = kwargs.get("model")
     if model is not None:
         set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model)

tests/integrations/openai/test_openai.py

@@ -23,6 +23,16 @@
 from openai.types.chat.chat_completion_chunk import ChoiceDelta
 from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage
 
+try:
+    from openai.types.chat import (
+        ChatCompletionCustomToolParam,
+        ChatCompletionFunctionToolParam,
+    )
+    from openai.types.chat.chat_completion_custom_tool_param import Custom
+    from openai.types.shared_params import FunctionDefinition
+except ImportError:
+    pass
+
 SKIP_RESPONSES_TESTS = False
 
 try:
@@ -107,6 +117,169 @@ async def __call__(self, *args, **kwargs):
     )
 
 
+@pytest.mark.skipif(
+    OPENAI_VERSION <= (2, 10, 0),
+    reason="ChatCompletionCustomToolParam is unavailable before.",
+)
+@pytest.mark.parametrize("span_streaming", [True, False])
+@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
+def test_chat_completion_tool_definitions(
+    sentry_init,
+    capture_events,
+    capture_items,
+    nonstreaming_chat_completions_model_response,
+    stream_gen_ai_spans,
+    span_streaming,
+):
+    sentry_init(
+        integrations=[OpenAIIntegration()],
+        disabled_integrations=[StdlibIntegration],
+        traces_sample_rate=1.0,
+        stream_gen_ai_spans=stream_gen_ai_spans,
+        trace_lifecycle="stream" if span_streaming else "static",
+    )
+
+    client = OpenAI(api_key="z")
+    client.chat.completions._post = mock.Mock(
+        return_value=nonstreaming_chat_completions_model_response(
+            response_id="chat-id",
+            response_model="gpt-3.5-turbo",
+            message_content="the model response",
+            created=10000000,
+            usage=CompletionUsage(
+                prompt_tokens=20,
+                completion_tokens=10,
+                total_tokens=30,
+            ),
+        )
+    )
+
+    if span_streaming or stream_gen_ai_spans:
+        items = capture_items("span")
+
+        with start_transaction(name="openai tx"):
+            client.chat.completions.create(
+                model="some-model",
+                messages=[
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": "hello"},
+                ],
+                tools=[
+                    ChatCompletionFunctionToolParam(
+                        type="function",
+                        function=FunctionDefinition(
+                            name="name",
+                            description="description",
+                            parameters={
+                                "type": "object",
+                                "properties": {
+                                    "city": {"type": "string"},
+                                    "state": {"type": "string"},
+                                },
+                                "required": ["city", "state"],
+                                "additionalProperties": False,
+                            },
+                            strict=True,
+                        ),
+                    ),
+                    ChatCompletionCustomToolParam(
+                        type="custom",
+                        custom=Custom(
+                            name="name",
+                            description="description",
+                        ),
+                    ),
+                ],
+            )
+
+        sentry_sdk.flush()
+        span = next(item.payload for item in items)
+
+        assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [
+            {
+                "type": "function",
+                "name": "name",
+                "description": "description",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "city": {"type": "string"},
+                        "state": {"type": "string"},
+                    },
+                    "required": ["city", "state"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "type": "custom",
+                "name": "name",
+                "description": "description",
+            },
+        ]
+    else:
+        events = capture_events()
+
+        with start_transaction(name="openai tx"):
+            client.chat.completions.create(
+                model="some-model",
+                messages=[
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": "hello"},
+                ],
+                tools=[
+                    ChatCompletionFunctionToolParam(
+                        type="function",
+                        function=FunctionDefinition(
+                            name="name",
+                            description="description",
+                            parameters={
+                                "type": "object",
+                                "properties": {
+                                    "city": {"type": "string"},
+                                    "state": {"type": "string"},
+                                },
+                                "required": ["city", "state"],
+                                "additionalProperties": False,
+                            },
+                            strict=True,
+                        ),
+                    ),
+                    ChatCompletionCustomToolParam(
+                        type="custom",
+                        custom=Custom(
+                            name="name",
+                            description="description",
+                        ),
+                    ),
+                ],
+            )
+
+        tx = events[0]
+        assert tx["type"] == "transaction"
+
+        assert json.loads(tx["spans"][0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [
+            {
+                "type": "function",
+                "name": "name",
+                "description": "description",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "city": {"type": "string"},
+                        "state": {"type": "string"},
+                    },
+                    "required": ["city", "state"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "type": "custom",
+                "name": "name",
+                "description": "description",
+            },
+        ]
+
+
 @pytest.mark.parametrize("span_streaming", [True, False])
 @pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
 @pytest.mark.parametrize(
@@ -5617,78 +5790,6 @@ async def test_streaming_responses_api_async(
         assert span["data"]["gen_ai.usage.total_tokens"] == 30
 
 
-@pytest.mark.parametrize("span_streaming", [True, False])
-@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
-@pytest.mark.skipif(
-    OPENAI_VERSION <= (1, 1, 0),
-    reason="OpenAI versions <=1.1.0 do not support the tools parameter.",
-)
-@pytest.mark.parametrize(
-    "tools",
-    [[], None, NOT_GIVEN, omit],
-)
-def test_empty_tools_in_chat_completion(
-    sentry_init,
-    capture_events,
-    capture_items,
-    tools,
-    nonstreaming_chat_completions_model_response,
-    stream_gen_ai_spans,
-    span_streaming,
-):
-    sentry_init(
-        integrations=[OpenAIIntegration()],
-        disabled_integrations=[StdlibIntegration],
-        traces_sample_rate=1.0,
-        stream_gen_ai_spans=stream_gen_ai_spans,
-        trace_lifecycle="stream" if span_streaming else "static",
-    )
-
-    client = OpenAI(api_key="z")
-    client.chat.completions._post = mock.Mock(
-        return_value=nonstreaming_chat_completions_model_response(
-            response_id="chat-id",
-            response_model="gpt-3.5-turbo",
-            message_content="the model response",
-            created=10000000,
-            usage=CompletionUsage(
-                prompt_tokens=20,
-                completion_tokens=10,
-                total_tokens=30,
-            ),
-        )
-    )
-
-    if span_streaming or stream_gen_ai_spans:
-        items = capture_items("span")
-
-        with start_transaction(name="openai tx"):
-            client.chat.completions.create(
-                model="some-model",
-                messages=[{"role": "system", "content": "hello"}],
-                tools=tools,
-            )
-
-        sentry_sdk.flush()
-        span = next(item.payload for item in items)
-
-        assert "gen_ai.request.available_tools" not in span["attributes"]
-    else:
-        events = capture_events()
-
-        with start_transaction(name="openai tx"):
-            client.chat.completions.create(
-                model="some-model",
-                messages=[{"role": "system", "content": "hello"}],
-                tools=tools,
-            )
-
-        (event,) = events
-        span = event["spans"][0]
-
-        assert "gen_ai.request.available_tools" not in span["data"]
-
-
 @pytest.mark.parametrize("span_streaming", [True, False])
 @pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
 # Feature added in https://github.com/openai/openai-python/pull/1952