diff --git a/packages/ai-semantic-conventions/src/SemanticAttributes.ts b/packages/ai-semantic-conventions/src/SemanticAttributes.ts index 45b3f7de..884f9fce 100644 --- a/packages/ai-semantic-conventions/src/SemanticAttributes.ts +++ b/packages/ai-semantic-conventions/src/SemanticAttributes.ts @@ -22,6 +22,8 @@ export const SpanAttributes = { LLM_REQUEST_TOP_P: "gen_ai.request.top_p", LLM_PROMPTS: "gen_ai.prompt", LLM_COMPLETIONS: "gen_ai.completion", + LLM_INPUT_MESSAGES: "gen_ai.input.messages", + LLM_OUTPUT_MESSAGES: "gen_ai.output.messages", LLM_RESPONSE_MODEL: "gen_ai.response.model", LLM_USAGE_PROMPT_TOKENS: "gen_ai.usage.prompt_tokens", LLM_USAGE_COMPLETION_TOKENS: "gen_ai.usage.completion_tokens", diff --git a/packages/traceloop-sdk/recordings/Test-AI-SDK-Integration-with-Recording_156038438/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har b/packages/traceloop-sdk/recordings/Test-AI-SDK-Integration-with-Recording_156038438/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har new file mode 100644 index 00000000..f824af40 --- /dev/null +++ b/packages/traceloop-sdk/recordings/Test-AI-SDK-Integration-with-Recording_156038438/should-set-LLM_INPUT_MESSAGES-and-LLM_OUTPUT_MESSAGES-attributes-for-chat-completions_99541399/recording.har @@ -0,0 +1,172 @@ +{ + "log": { + "_recordingName": "Test AI SDK Integration with Recording/should set LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES attributes for chat completions", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "9e0d86c7cc2553331485a7095b7c79c7", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 128, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 273, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"model\":\"gpt-3.5-turbo\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is 2+2? Give a brief answer.\"}]}]}" + }, + "queryString": [], + "url": "https://api.openai.com/v1/responses" + }, + "response": { + "bodySize": 1368, + "content": { + "mimeType": "application/json", + "size": 1368, + "text": "{\n \"id\": \"resp_68ab8bb3a6a48196b327616ac8519242061ee2084aba21f9\",\n \"object\": \"response\",\n \"created_at\": 1756072883,\n \"status\": \"completed\",\n \"background\": false,\n \"error\": null,\n \"incomplete_details\": null,\n \"instructions\": null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": \"gpt-3.5-turbo-0125\",\n \"output\": [\n {\n \"id\": \"msg_68ab8bb44de48196bc6fd27015c46560061ee2084aba21f9\",\n \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": [\n {\n \"type\": \"output_text\",\n \"annotations\": [],\n \"logprobs\": [],\n \"text\": \"2+2 equals 4.\"\n }\n ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"reasoning\": {\n \"effort\": null,\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": \"auto\",\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 19,\n \"input_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"output_tokens\": 8,\n \"output_tokens_details\": {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 27\n },\n \"user\": null,\n \"metadata\": {}\n}" + }, + "cookies": [ + { + "domain": ".api.openai.com", + "httpOnly": true, + "name": "_cfuvid", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "YvH22Dd0_.ZSsD0IiE4C_SZLye9RnIosHw8rvV.a5xI-1756072884584-0.0.1.1-604800000" + } + ], + "headers": [ + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "cf-ray", + "value": "974620c03ebe7da4-TLV" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "br" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Sun, 24 Aug 2025 22:01:24 GMT" + }, + { + "name": "openai-organization", + "value": "traceloop" + }, + { + "name": "openai-processing-ms", + "value": "860" + }, + { + "name": "openai-project", + "value": "proj_tzz1TbPPOXaf6j9tEkVUBIAa" + }, + { + "name": "openai-version", + "value": "2020-10-01" + }, + { + "name": "server", + "value": "cloudflare" + }, + { + "name": "set-cookie", + "value": "_cfuvid=YvH22Dd0_.ZSsD0IiE4C_SZLye9RnIosHw8rvV.a5xI-1756072884584-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000; includeSubDomains; preload" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "866" + }, + { + "name": "x-ratelimit-limit-requests", + "value": "10000" + }, + { + "name": "x-ratelimit-limit-tokens", + "value": "50000000" + }, + { + "name": "x-ratelimit-remaining-requests", + "value": "9999" + }, + { + "name": "x-ratelimit-remaining-tokens", + "value": "49999976" + }, + { + "name": "x-ratelimit-reset-requests", + "value": "6ms" + }, + { + "name": "x-ratelimit-reset-tokens", + "value": "0s" + }, + { + "name": "x-request-id", + "value": "req_ca7b8860830628b84350a7485599e844" + } + ], + "headersSize": 953, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-08-24T22:01:23.016Z", + "time": 1409, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1409 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts b/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts index 7929a3fa..8263eca8 100644 --- a/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts +++ b/packages/traceloop-sdk/src/lib/tracing/ai-sdk-transformations.ts @@ -19,6 +19,10 @@ const AI_USAGE_PROMPT_TOKENS = "ai.usage.promptTokens"; const AI_USAGE_COMPLETION_TOKENS = "ai.usage.completionTokens"; const AI_MODEL_PROVIDER = "ai.model.provider"; const AI_PROMPT_TOOLS = "ai.prompt.tools"; +const TYPE_TEXT = "text"; +const TYPE_TOOL_CALL = "tool_call"; +const ROLE_ASSISTANT = "assistant"; +const ROLE_USER = "user"; // Vendor mapping from AI SDK provider prefixes to standardized LLM_SYSTEM values // Uses prefixes to match AI SDK patterns like "openai.chat", "anthropic.messages", etc. @@ -55,7 +59,21 @@ const transformResponseText = (attributes: Record): void => { if (AI_RESPONSE_TEXT in attributes) { attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.content`] = attributes[AI_RESPONSE_TEXT]; - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + + const outputMessage = { + role: ROLE_ASSISTANT, + parts: [ + { + type: TYPE_TEXT, + content: attributes[AI_RESPONSE_TEXT], + }, + ], + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + delete attributes[AI_RESPONSE_TEXT]; } }; @@ -64,7 +82,21 @@ const transformResponseObject = (attributes: Record): void => { if (AI_RESPONSE_OBJECT in attributes) { attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.content`] = attributes[AI_RESPONSE_OBJECT]; - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + + const outputMessage = { + role: ROLE_ASSISTANT, + parts: [ + { + type: TYPE_TEXT, + content: attributes[AI_RESPONSE_OBJECT], + }, + ], + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + delete attributes[AI_RESPONSE_OBJECT]; } }; @@ -76,8 +108,9 @@ const transformResponseToolCalls = (attributes: Record): void => { attributes[AI_RESPONSE_TOOL_CALLS] as string, ); - attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = "assistant"; + attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.role`] = ROLE_ASSISTANT; + const toolCallParts: any[] = []; toolCalls.forEach((toolCall: any, index: number) => { if (toolCall.toolCallType === "function") { attributes[ @@ -86,9 +119,27 @@ const transformResponseToolCalls = (attributes: Record): void => { attributes[ `${SpanAttributes.LLM_COMPLETIONS}.0.tool_calls.${index}.arguments` ] = toolCall.args; + + toolCallParts.push({ + type: TYPE_TOOL_CALL, + tool_call: { + name: toolCall.toolName, + arguments: toolCall.args, + }, + }); } }); + if (toolCallParts.length > 0) { + const outputMessage = { + role: ROLE_ASSISTANT, + parts: toolCallParts, + }; + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] = JSON.stringify([ + outputMessage, + ]); + } + delete attributes[AI_RESPONSE_TOOL_CALLS]; } catch { // Ignore parsing errors @@ -100,7 +151,10 @@ const processMessageContent = (content: any): string => { if (Array.isArray(content)) { const textItems = content.filter( (item: any) => - item && typeof item === "object" && item.type === "text" && item.text, + item && + typeof item === "object" && + item.type === TYPE_TEXT && + item.text, ); if (textItems.length > 0) { @@ -112,7 +166,7 @@ const processMessageContent = (content: any): string => { } if (content && typeof content === "object") { - if (content.type === "text" && content.text) { + if (content.type === TYPE_TEXT && content.text) { return content.text; } return JSON.stringify(content); @@ -126,7 +180,7 @@ const processMessageContent = (content: any): string => { (item: any) => item && typeof item === "object" && - item.type === "text" && + item.type === TYPE_TEXT && item.text, ); @@ -205,12 +259,32 @@ const transformPrompts = (attributes: Record): void => { } const messages = JSON.parse(jsonString); + const inputMessages: any[] = []; + messages.forEach((msg: { role: string; content: any }, index: number) => { const processedContent = processMessageContent(msg.content); const contentKey = `${SpanAttributes.LLM_PROMPTS}.${index}.content`; attributes[contentKey] = processedContent; attributes[`${SpanAttributes.LLM_PROMPTS}.${index}.role`] = msg.role; + + // Add to OpenTelemetry standard gen_ai.input.messages format + inputMessages.push({ + role: msg.role, + parts: [ + { + type: TYPE_TEXT, + content: processedContent, + }, + ], + }); }); + + // Set the OpenTelemetry standard input messages attribute + if (inputMessages.length > 0) { + attributes[SpanAttributes.LLM_INPUT_MESSAGES] = + JSON.stringify(inputMessages); + } + delete attributes[AI_PROMPT_MESSAGES]; } catch { // Ignore parsing errors @@ -223,7 +297,21 @@ const transformPrompts = (attributes: Record): void => { if (promptData.prompt && typeof promptData.prompt === "string") { attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`] = promptData.prompt; - attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`] = "user"; + attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`] = ROLE_USER; + + const inputMessage = { + role: ROLE_USER, + parts: [ + { + type: TYPE_TEXT, + content: promptData.prompt, + }, + ], + }; + attributes[SpanAttributes.LLM_INPUT_MESSAGES] = JSON.stringify([ + inputMessage, + ]); + delete attributes[AI_PROMPT]; } } catch { diff --git a/packages/traceloop-sdk/test/ai-sdk-integration.test.ts b/packages/traceloop-sdk/test/ai-sdk-integration.test.ts index e50c007b..52cb29bb 100644 --- a/packages/traceloop-sdk/test/ai-sdk-integration.test.ts +++ b/packages/traceloop-sdk/test/ai-sdk-integration.test.ts @@ -19,6 +19,7 @@ import * as assert from "assert"; import { openai as vercel_openai } from "@ai-sdk/openai"; import { google as vercel_google } from "@ai-sdk/google"; import { generateText } from "ai"; +import { SpanAttributes } from "@traceloop/ai-semantic-conventions"; import * as traceloop from "../src"; @@ -216,4 +217,61 @@ describe("Test AI SDK Integration with Recording", function () { assert.ok(generateTextSpan.attributes["gen_ai.usage.completion_tokens"]); assert.ok(generateTextSpan.attributes["llm.usage.total_tokens"]); }); + + it("should set LLM_INPUT_MESSAGES and LLM_OUTPUT_MESSAGES attributes for chat completions", async () => { + const result = await traceloop.withWorkflow( + { name: "test_transformations_workflow" }, + async () => { + return await generateText({ + messages: [ + { role: "user", content: "What is 2+2? Give a brief answer." }, + ], + model: vercel_openai("gpt-3.5-turbo"), + experimental_telemetry: { isEnabled: true }, + }); + }, + ); + + assert.ok(result); + assert.ok(result.text); + + const spans = memoryExporter.getFinishedSpans(); + const aiSdkSpan = spans.find((span) => + span.name.startsWith("ai.generateText"), + ); + + assert.ok(aiSdkSpan); + + // Verify LLM_INPUT_MESSAGES attribute exists and is valid JSON + assert.ok(aiSdkSpan.attributes[SpanAttributes.LLM_INPUT_MESSAGES]); + const inputMessages = JSON.parse( + aiSdkSpan.attributes[SpanAttributes.LLM_INPUT_MESSAGES] as string, + ); + assert.ok(Array.isArray(inputMessages)); + assert.strictEqual(inputMessages.length, 1); + + // Check user message structure + assert.strictEqual(inputMessages[0].role, "user"); + assert.ok(Array.isArray(inputMessages[0].parts)); + assert.strictEqual(inputMessages[0].parts[0].type, "text"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "What is 2+2? Give a brief answer.", + ); + + // Verify LLM_OUTPUT_MESSAGES attribute exists and is valid JSON + assert.ok(aiSdkSpan.attributes[SpanAttributes.LLM_OUTPUT_MESSAGES]); + const outputMessages = JSON.parse( + aiSdkSpan.attributes[SpanAttributes.LLM_OUTPUT_MESSAGES] as string, + ); + assert.ok(Array.isArray(outputMessages)); + assert.strictEqual(outputMessages.length, 1); + + // Check assistant response structure + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.ok(Array.isArray(outputMessages[0].parts)); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.ok(outputMessages[0].parts[0].content); + assert.ok(typeof outputMessages[0].parts[0].content === "string"); + }); }); diff --git a/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts b/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts index f50b1cbf..8ec61e68 100644 --- a/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts +++ b/packages/traceloop-sdk/test/ai-sdk-transformations.test.ts @@ -1180,6 +1180,361 @@ describe("AI SDK Transformations", () => { }); }); + describe("transformAiSdkAttributes - gen_ai input/output messages", () => { + it("should create gen_ai.input.messages for conversation with text", () => { + const messages = [ + { role: "system", content: "You are a helpful assistant" }, + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing well, thank you!" }, + { role: "user", content: "Can you help me with something?" }, + ]; + const attributes = { + "ai.prompt.messages": JSON.stringify(messages), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.input.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_INPUT_MESSAGES], + "string", + ); + + const inputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(inputMessages.length, 4); + + // Check system message + assert.strictEqual(inputMessages[0].role, "system"); + assert.strictEqual(inputMessages[0].parts.length, 1); + assert.strictEqual(inputMessages[0].parts[0].type, "text"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "You are a helpful assistant", + ); + + // Check user messages + assert.strictEqual(inputMessages[1].role, "user"); + assert.strictEqual( + inputMessages[1].parts[0].content, + "Hello, how are you?", + ); + + assert.strictEqual(inputMessages[2].role, "assistant"); + assert.strictEqual( + inputMessages[2].parts[0].content, + "I'm doing well, thank you!", + ); + + assert.strictEqual(inputMessages[3].role, "user"); + assert.strictEqual( + inputMessages[3].parts[0].content, + "Can you help me with something?", + ); + }); + + it("should create gen_ai.output.messages for text response", () => { + const attributes = { + "ai.response.text": "I'd be happy to help you with that!", + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 1); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.strictEqual( + outputMessages[0].parts[0].content, + "I'd be happy to help you with that!", + ); + }); + + it("should create gen_ai.output.messages for tool calls", () => { + const toolCallsData = [ + { + toolCallType: "function", + toolCallId: "call_weather_123", + toolName: "getWeather", + args: '{"location": "San Francisco", "unit": "celsius"}', + }, + { + toolCallType: "function", + toolCallId: "call_restaurant_456", + toolName: "findRestaurants", + args: '{"location": "San Francisco", "cuisine": "italian"}', + }, + ]; + + const attributes = { + "ai.response.toolCalls": JSON.stringify(toolCallsData), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 2); + + // Check first tool call + assert.strictEqual(outputMessages[0].parts[0].type, "tool_call"); + assert.strictEqual( + outputMessages[0].parts[0].tool_call.name, + "getWeather", + ); + assert.strictEqual( + outputMessages[0].parts[0].tool_call.arguments, + '{"location": "San Francisco", "unit": "celsius"}', + ); + + // Check second tool call + assert.strictEqual(outputMessages[0].parts[1].type, "tool_call"); + assert.strictEqual( + outputMessages[0].parts[1].tool_call.name, + "findRestaurants", + ); + assert.strictEqual( + outputMessages[0].parts[1].tool_call.arguments, + '{"location": "San Francisco", "cuisine": "italian"}', + ); + }); + + it("should create both gen_ai.input.messages and gen_ai.output.messages for complete conversation with tools", () => { + const inputMessages = [ + { + role: "system", + content: + "You are a helpful travel assistant. Use the available tools to help users plan their trips.", + }, + { + role: "user", + content: + "I'm planning a trip to San Francisco. Can you tell me about the weather and recommend some good Italian restaurants?", + }, + ]; + + const toolCallsData = [ + { + toolCallType: "function", + toolCallId: "call_weather_789", + toolName: "getWeather", + args: '{"location": "San Francisco", "forecast_days": 3}', + }, + { + toolCallType: "function", + toolCallId: "call_restaurants_101", + toolName: "searchRestaurants", + args: '{"location": "San Francisco", "cuisine": "italian", "rating_min": 4.0}', + }, + ]; + + const attributes = { + "ai.prompt.messages": JSON.stringify(inputMessages), + "ai.response.toolCalls": JSON.stringify(toolCallsData), + "ai.prompt.tools": [ + { + name: "getWeather", + description: "Get weather forecast for a location", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + forecast_days: { type: "number" }, + }, + required: ["location"], + }, + }, + { + name: "searchRestaurants", + description: "Search for restaurants in a location", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + cuisine: { type: "string" }, + rating_min: { type: "number" }, + }, + required: ["location"], + }, + }, + ], + }; + + transformAiSdkAttributes(attributes); + + // Check input messages + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_INPUT_MESSAGES], + "string", + ); + const parsedInputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(parsedInputMessages.length, 2); + assert.strictEqual(parsedInputMessages[0].role, "system"); + assert.strictEqual( + parsedInputMessages[0].parts[0].content, + "You are a helpful travel assistant. Use the available tools to help users plan their trips.", + ); + assert.strictEqual(parsedInputMessages[1].role, "user"); + assert.strictEqual( + parsedInputMessages[1].parts[0].content, + "I'm planning a trip to San Francisco. Can you tell me about the weather and recommend some good Italian restaurants?", + ); + + // Check output messages (tool calls) + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + const parsedOutputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(parsedOutputMessages.length, 1); + assert.strictEqual(parsedOutputMessages[0].role, "assistant"); + assert.strictEqual(parsedOutputMessages[0].parts.length, 2); + + // Verify tool calls in output + assert.strictEqual(parsedOutputMessages[0].parts[0].type, "tool_call"); + assert.strictEqual( + parsedOutputMessages[0].parts[0].tool_call.name, + "getWeather", + ); + assert.strictEqual(parsedOutputMessages[0].parts[1].type, "tool_call"); + assert.strictEqual( + parsedOutputMessages[0].parts[1].tool_call.name, + "searchRestaurants", + ); + + // Check that tools are also properly transformed + assert.strictEqual( + attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name`], + "getWeather", + ); + assert.strictEqual( + attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.name`], + "searchRestaurants", + ); + }); + + it("should create gen_ai.output.messages for object response", () => { + const objectResponse = { + destination: "San Francisco", + weather: "sunny, 22°C", + recommendations: ["Visit Golden Gate Bridge", "Try local sourdough"], + confidence: 0.95, + }; + + const attributes = { + "ai.response.object": JSON.stringify(objectResponse), + }; + + transformAiSdkAttributes(attributes); + + // Check that gen_ai.output.messages is properly set + assert.strictEqual( + typeof attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + "string", + ); + + const outputMessages = JSON.parse( + attributes[SpanAttributes.LLM_OUTPUT_MESSAGES], + ); + assert.strictEqual(outputMessages.length, 1); + assert.strictEqual(outputMessages[0].role, "assistant"); + assert.strictEqual(outputMessages[0].parts.length, 1); + assert.strictEqual(outputMessages[0].parts[0].type, "text"); + assert.strictEqual( + outputMessages[0].parts[0].content, + JSON.stringify(objectResponse), + ); + }); + + it("should handle complex multi-turn conversation with mixed content types", () => { + const complexMessages = [ + { + role: "system", + content: "You are an AI assistant that can analyze images and text.", + }, + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { type: "image", url: "data:image/jpeg;base64,..." }, + ], + }, + { + role: "assistant", + content: "I can see a beautiful sunset over a mountain landscape.", + }, + { + role: "user", + content: + "Can you get the weather for this location using your tools?", + }, + ]; + + const attributes = { + "ai.prompt.messages": JSON.stringify(complexMessages), + }; + + transformAiSdkAttributes(attributes); + + // Check input messages transformation + const inputMessages = JSON.parse( + attributes[SpanAttributes.LLM_INPUT_MESSAGES], + ); + assert.strictEqual(inputMessages.length, 4); + + // System message should be preserved + assert.strictEqual(inputMessages[0].role, "system"); + assert.strictEqual( + inputMessages[0].parts[0].content, + "You are an AI assistant that can analyze images and text.", + ); + + // Complex content should be flattened to text parts only + assert.strictEqual(inputMessages[1].role, "user"); + assert.strictEqual( + inputMessages[1].parts[0].content, + "What's in this image?", + ); + + // Assistant response should be preserved + assert.strictEqual(inputMessages[2].role, "assistant"); + assert.strictEqual( + inputMessages[2].parts[0].content, + "I can see a beautiful sunset over a mountain landscape.", + ); + + // User follow-up should be preserved + assert.strictEqual(inputMessages[3].role, "user"); + assert.strictEqual( + inputMessages[3].parts[0].content, + "Can you get the weather for this location using your tools?", + ); + }); + }); + describe("transformAiSdkSpan", () => { it("should transform both span name and attributes", () => { const span = createMockSpan("ai.generateText.doGenerate", {