Skip to content

Commit e26aaf5

Browse files
PROFeNoMPROFeNoM
authored
feat(openai): instrument openai reusable prompts (#6941)
* draft implementation * (drafty) ensure input messages are captured * clean things up * streamline the flow into: check for instructions -> gen template -> normalize -> tag * wtf was that merge * refactor(llmobs): move extractChatTemplateFromInstructions to utils and update OpenAI plugin integration - Moved the extractChatTemplateFromInstructions function to a new utils file for better organization. - Updated the OpenAiLLMObsPlugin to use the new utility function for extracting chat templates from response instructions. - Removed the function from the util.js file and adjusted related tests accordingly. * lint * delete file * handle ResponseInputImage & ResponseInputFile * refactor normalization into a util fn * lint * move ternary check into fn * lint * OpenAI strips image_url from response.prompt.variables fix * lint * lint * Update OpenAI response handling to support multiple image inputs and refine variable extraction logic. Adjusted test cases to reflect changes in prompt structure and variable names. Removed outdated cassette file. * refactor(llmobs): improve chat template extraction for images and files - Updated `_extract_chat_template_from_instructions` to clarify handling of text variables and generic markers for images and files. - Adjusted docstring to specify the use of placeholders for text and generic markers for images and files. - Modified test cases to reflect changes in prompt structure, ensuring consistency in template generation. * simplify some bits * refactor(llmobs): enhance chat template extraction and update test cases - Improved the extraction logic for chat templates, ensuring accurate handling of image and file references. - Updated docstrings for clarity on the use of placeholders and fallback markers. - Added a new test cassette for OpenAI responses and removed an outdated one to maintain consistency in testing. * add a test case * code review * revert
1 parent 930155a commit e26aaf5

File tree

11 files changed

+1467
-3
lines changed

11 files changed

+1467
-3
lines changed

‎packages/dd-trace/src/llmobs/plugins/openai.js‎ renamed to ‎packages/dd-trace/src/llmobs/plugins/openai/index.js‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
'use strict'
22

3-
const LLMObsPlugin = require('./base')
3+
const LLMObsPlugin = require('../base')
4+
const { extractChatTemplateFromInstructions, normalizePromptVariables, extractTextFromContentItem } = require('./utils')
45

56
const allowedParamKeys = new Set([
67
'max_output_tokens',
@@ -221,7 +222,8 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
221222
#tagResponse (span, inputs, response, error) {
222223
// Tag metadata - use allowlist approach for request parameters
223224

224-
const { input, model, ...parameters } = inputs
225+
const { model, ...parameters } = inputs
226+
let input = inputs.input
225227

226228
// Create input messages
227229
const inputMessages = []
@@ -231,10 +233,33 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
231233
inputMessages.push({ role: 'system', content: inputs.instructions })
232234
}
233235

236+
// For reusable prompts, use response.instructions if no explicit input is provided
237+
if (!input && inputs.prompt && response?.instructions) {
238+
input = response.instructions
239+
}
240+
234241
// Handle input - can be string or array of mixed messages
235242
if (Array.isArray(input)) {
236243
for (const item of input) {
237-
if (item.type === 'function_call') {
244+
if (item.type === 'message') {
245+
// Handle instruction messages (from response.instructions for reusable prompts)
246+
const role = item.role
247+
if (!role) continue
248+
249+
let content = ''
250+
if (Array.isArray(item.content)) {
251+
const textParts = item.content
252+
.map(extractTextFromContentItem)
253+
.filter(Boolean)
254+
content = textParts.join('')
255+
} else if (typeof item.content === 'string') {
256+
content = item.content
257+
}
258+
259+
if (content) {
260+
inputMessages.push({ role, content })
261+
}
262+
} else if (item.type === 'function_call') {
238263
// Function call: convert to message with tool_calls
239264
// Parse arguments if it's a JSON string
240265
let parsedArgs = item.arguments
@@ -380,6 +405,22 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
380405

381406
this._tagger.tagLLMIO(span, inputMessages, outputMessages)
382407

408+
// Handle prompt tracking for reusable prompts
409+
if (inputs.prompt && response?.prompt) {
410+
const { id, version } = response.prompt // ResponsePrompt
411+
// TODO: Add proper tagger API for prompt metadata
412+
if (id && version) {
413+
const normalizedVariables = normalizePromptVariables(inputs.prompt.variables)
414+
const chatTemplate = extractChatTemplateFromInstructions(response.instructions, normalizedVariables)
415+
this._tagger._setTag(span, '_ml_obs.meta.input.prompt', {
416+
id,
417+
version,
418+
variables: normalizedVariables,
419+
chat_template: chatTemplate
420+
})
421+
}
422+
}
423+
383424
const outputMetadata = {}
384425

385426
// Add fields from response object (convert numbers to floats)

‎packages/dd-trace/src/llmobs/plugins/openai/utils.js‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
'use strict'
2+
3+
const IMAGE_FALLBACK = '[image]'
4+
const FILE_FALLBACK = '[file]'
5+
6+
const REGEX_SPECIAL_CHARS = /[.*+?^${}()|[\]\\]/g
7+
8+
/**
9+
* Extracts chat templates from OpenAI response instructions by replacing variable values with placeholders.
10+
*
11+
* Performs reverse templating: reconstructs the template by replacing actual values with {{variable_name}}.
12+
* For images/files: uses {{variable_name}} when values are available, falls back to [image]/[file] when stripped.
13+
*
14+
* @param {Array<Object>} instructions - From Response.instructions (array of ResponseInputMessageItem)
15+
* @param {Object<string, string>} variables - Normalized variables (output of normalizePromptVariables)
16+
* @returns {Array<{role: string, content: string}>} Chat template with placeholders
17+
*/
18+
function extractChatTemplateFromInstructions (instructions, variables) {
19+
if (!Array.isArray(instructions) || !variables) return []
20+
21+
const chatTemplate = []
22+
23+
// Build map of values to placeholders - exclude fallback markers so they remain as-is
24+
const valueToPlaceholder = {}
25+
for (const [varName, varValue] of Object.entries(variables)) {
26+
// Exclude fallback markers - they should remain as [image]/[file] in the template
27+
if (varValue && varValue !== IMAGE_FALLBACK && varValue !== FILE_FALLBACK) {
28+
valueToPlaceholder[varValue] = `{{${varName}}}`
29+
}
30+
}
31+
32+
// Sort values by length (longest first) to handle overlapping values correctly
33+
const sortedValues = Object.keys(valueToPlaceholder).sort((a, b) => b.length - a.length)
34+
35+
for (const instruction of instructions) {
36+
const role = instruction.role
37+
if (!role) continue
38+
39+
const contentItems = instruction.content
40+
if (!Array.isArray(contentItems)) continue
41+
42+
// Extract text from all content items (uses actual values for images/files when available)
43+
const textParts = contentItems
44+
.map(extractTextFromContentItem)
45+
.filter(Boolean)
46+
47+
if (textParts.length === 0) continue
48+
49+
// Combine text and replace variable values with placeholders (longest first)
50+
let fullText = textParts.join('')
51+
for (const valueStr of sortedValues) {
52+
const placeholder = valueToPlaceholder[valueStr]
53+
const escapedValue = valueStr.replaceAll(REGEX_SPECIAL_CHARS, String.raw`\$&`)
54+
fullText = fullText.replaceAll(new RegExp(escapedValue, 'g'), placeholder)
55+
}
56+
57+
chatTemplate.push({ role, content: fullText })
58+
}
59+
60+
return chatTemplate
61+
}
62+
63+
/**
64+
* Extracts text content from a content item, using actual image_url/file_id values when available.
65+
*
66+
* Used for both input messages and chat template extraction. Falls back to [image]/[file] markers
67+
* when the actual values are stripped (e.g., by OpenAI's default URL stripping behavior).
68+
*
69+
* @param {Object} contentItem - Content item from Response.instructions[].content (ResponseInputContentItem)
70+
* @returns {string|null} Text content, URL/file reference, or [image]/[file] fallback marker
71+
*/
72+
function extractTextFromContentItem (contentItem) {
73+
if (!contentItem) return null
74+
75+
if (contentItem.text) {
76+
return contentItem.text
77+
}
78+
79+
// For image/file items, extract the actual reference value
80+
if (contentItem.type === 'input_image') {
81+
return contentItem.image_url || contentItem.file_id || IMAGE_FALLBACK
82+
}
83+
84+
if (contentItem.type === 'input_file') {
85+
return contentItem.file_id || contentItem.file_url || contentItem.filename || FILE_FALLBACK
86+
}
87+
88+
return null
89+
}
90+
91+
/**
92+
* Normalizes prompt variables by extracting meaningful values from OpenAI SDK response objects.
93+
*
94+
* Converts ResponseInputText, ResponseInputImage, and ResponseInputFile objects to simple string values.
95+
*
96+
* @param {Object<string, string|Object>} variables - From ResponsePrompt.variables
97+
* @returns {Object<string, string>} Normalized variables with simple string values
98+
*/
99+
function normalizePromptVariables (variables) {
100+
if (!variables) return {}
101+
102+
return Object.fromEntries(
103+
Object.entries(variables).map(([key, value]) => [
104+
key,
105+
extractTextFromContentItem(value) ?? String(value ?? '')
106+
])
107+
)
108+
}
109+
110+
module.exports = {
111+
extractChatTemplateFromInstructions,
112+
normalizePromptVariables,
113+
extractTextFromContentItem
114+
}

‎packages/dd-trace/src/llmobs/span_processor.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,11 @@ class LLMObsSpanProcessor {
126126
inputType = 'value'
127127
}
128128

129+
// Handle prompt metadata for reusable prompts
130+
if (mlObsTags['_ml_obs.meta.input.prompt']) {
131+
input.prompt = mlObsTags['_ml_obs.meta.input.prompt']
132+
}
133+
129134
if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
130135
llmObsSpan.output = mlObsTags[OUTPUT_MESSAGES]
131136
outputType = 'messages'

‎packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_2f619a5a.yaml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
interactions:
2+
- request:
3+
body: "{\n \"prompt\": {\n \"id\": \"pmpt_6911a8b8f7648197b39bd62127a696910d4a05830d5ba1e6\",\n
4+
\ \"version\": \"1\",\n \"variables\": {\n \"phrase\": \"cat in the
5+
hat\",\n \"word\": \"\"\n }\n }\n}"
6+
headers:
7+
? !!python/object/apply:multidict._multidict.istr
8+
- Accept
9+
: - application/json
10+
? !!python/object/apply:multidict._multidict.istr
11+
- Accept-Encoding
12+
: - gzip,deflate
13+
? !!python/object/apply:multidict._multidict.istr
14+
- Connection
15+
: - keep-alive
16+
Content-Length:
17+
- '184'
18+
? !!python/object/apply:multidict._multidict.istr
19+
- Content-Type
20+
: - application/json
21+
? !!python/object/apply:multidict._multidict.istr
22+
- User-Agent
23+
: - OpenAI/JS 4.87.0
24+
? !!python/object/apply:multidict._multidict.istr
25+
- x-stainless-arch
26+
: - arm64
27+
? !!python/object/apply:multidict._multidict.istr
28+
- x-stainless-lang
29+
: - js
30+
? !!python/object/apply:multidict._multidict.istr
31+
- x-stainless-os
32+
: - MacOS
33+
? !!python/object/apply:multidict._multidict.istr
34+
- x-stainless-package-version
35+
: - 4.87.0
36+
? !!python/object/apply:multidict._multidict.istr
37+
- x-stainless-retry-count
38+
: - '0'
39+
? !!python/object/apply:multidict._multidict.istr
40+
- x-stainless-runtime
41+
: - node
42+
? !!python/object/apply:multidict._multidict.istr
43+
- x-stainless-runtime-version
44+
: - v22.17.0
45+
? !!python/object/apply:multidict._multidict.istr
46+
- x-stainless-timeout
47+
: - '600000'
48+
method: POST
49+
uri: https://api.openai.com/v1/responses
50+
response:
51+
body:
52+
string: "{\n \"id\": \"resp_0c6ecbbba70df92401692080e02b98819598a8b9b42f190477\",\n
53+
\ \"object\": \"response\",\n \"created_at\": 1763737824,\n \"status\":
54+
\"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\":
55+
\"developer\"\n },\n \"error\": null,\n \"incomplete_details\": null,\n
56+
\ \"instructions\": [\n {\n \"type\": \"message\",\n \"content\":
57+
[],\n \"role\": \"developer\"\n },\n {\n \"type\": \"message\",\n
58+
\ \"content\": [\n {\n \"type\": \"input_text\",\n \"text\":
59+
\"I saw a cat in the hat and another \"\n }\n ],\n \"role\":
60+
\"user\"\n }\n ],\n \"max_output_tokens\": null,\n \"max_tool_calls\":
61+
null,\n \"model\": \"o4-mini-2025-04-16\",\n \"output\": [\n {\n \"id\":
62+
\"rs_0c6ecbbba70df92401692080e090988195905c3ae3100aec76\",\n \"type\":
63+
\"reasoning\",\n \"summary\": []\n },\n {\n \"id\": \"msg_0c6ecbbba70df92401692080e592408195ad7e5e5a3b30a60f\",\n
64+
\ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\":
65+
[\n {\n \"type\": \"output_text\",\n \"annotations\":
66+
[],\n \"logprobs\": [],\n \"text\": \"It sounds like you\\u2019re
67+
starting a playful rhyme. Would you like help finishing the line? Here are
68+
a few options\\u2014feel free to pick one, tweak it, or tell me what mood
69+
or style you\\u2019d prefer:\\n\\n1. \\u201cI saw a cat in the hat and another
70+
on a mat, \\n Both dancing \\u2019round the room in a colorful cravat.\\u201d
71+
\ \\n\\n2. \\u201cI saw a cat in the hat and another wearing boots, \\n One
72+
chased rolling marbles, the other chased small brutes.\\u201d \\n\\n3. \\u201cI
73+
saw a cat in the hat and another with a gnat, \\n They argued who was bigger
74+
\\u2019til they both grew quite flat.\\u201d \\n\\n4. \\u201cI saw a cat
75+
in the hat and another with a bat, \\n They juggled up some popcorn and
76+
then sat on a splat.\\u201d \\n\\nLet me know which you like best or what
77+
tone you\\u2019re aiming for, and we can build it out into a full little poem!\"\n
78+
\ }\n ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\":
79+
true,\n \"previous_response_id\": null,\n \"prompt\": {\n \"id\": \"pmpt_6911a8b8f7648197b39bd62127a696910d4a05830d5ba1e6\",\n
80+
\ \"variables\": {\n \"phrase\": {\n \"type\": \"input_text\",\n
81+
\ \"text\": \"cat in the hat\"\n },\n \"word\": {\n \"type\":
82+
\"input_text\",\n \"text\": \"\"\n }\n },\n \"version\":
83+
\"1\"\n },\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\":
84+
null,\n \"reasoning\": {\n \"effort\": \"medium\",\n \"summary\": null\n
85+
\ },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\":
86+
false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\":
87+
\"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\":
88+
\"auto\",\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\":
89+
\"disabled\",\n \"usage\": {\n \"input_tokens\": 16,\n \"input_tokens_details\":
90+
{\n \"cached_tokens\": 0\n },\n \"output_tokens\": 718,\n \"output_tokens_details\":
91+
{\n \"reasoning_tokens\": 512\n },\n \"total_tokens\": 734\n },\n
92+
\ \"user\": null,\n \"metadata\": {}\n}"
93+
headers:
94+
CF-RAY:
95+
- 9a211d165c1503f3-CDG
96+
Connection:
97+
- keep-alive
98+
Content-Encoding:
99+
- gzip
100+
Content-Type:
101+
- application/json
102+
Date:
103+
- Fri, 21 Nov 2025 15:10:31 GMT
104+
Server:
105+
- cloudflare
106+
Set-Cookie:
107+
- __cf_bm=0HpM_foHywpoW6._EX94ALugVRAidmAfgAlSmdasPuM-1763737831-1.0.1.1-ql.pMhbt7pntz_jeV3gMMnZNoy4JdwriHUWnKLf92h4zZNipzYhrn_2OMXIoy4QWAlaB7xnYs9a56Pl.tMuxkGawmHJAldmrcbVQak._5sQ;
108+
path=/; expires=Fri, 21-Nov-25 15:40:31 GMT; domain=.api.openai.com; HttpOnly;
109+
Secure; SameSite=None
110+
- _cfuvid=jk3d8alrSLimfBjsF6JI6CStjpMxdjpRCHxHMmGWfmI-1763737831436-0.0.1.1-604800000;
111+
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
112+
Strict-Transport-Security:
113+
- max-age=31536000; includeSubDomains; preload
114+
Transfer-Encoding:
115+
- chunked
116+
X-Content-Type-Options:
117+
- nosniff
118+
alt-svc:
119+
- h3=":443"; ma=86400
120+
cf-cache-status:
121+
- DYNAMIC
122+
openai-organization:
123+
- datadog-staging
124+
openai-processing-ms:
125+
- '7210'
126+
openai-project:
127+
- proj_gt6TQZPRbZfoY2J9AQlEJMpd
128+
openai-version:
129+
- '2020-10-01'
130+
x-envoy-upstream-service-time:
131+
- '7213'
132+
x-ratelimit-limit-requests:
133+
- '30000'
134+
x-ratelimit-limit-tokens:
135+
- '150000000'
136+
x-ratelimit-remaining-requests:
137+
- '29999'
138+
x-ratelimit-remaining-tokens:
139+
- '149999777'
140+
x-ratelimit-reset-requests:
141+
- 2ms
142+
x-ratelimit-reset-tokens:
143+
- 0s
144+
x-request-id:
145+
- req_48665d653be046319c1a4f6de91b72fe
146+
status:
147+
code: 200
148+
message: OK
149+
version: 1

0 commit comments

Comments
 (0)