Skip to content

Commit d976c55

Browse files
sxlijinsxlijin
authored
fix: use roles correctly with openai-responses (#2392)
#2327 only fixed roles correctly when using multimodal inputs <!-- ELLIPSIS_HIDDEN --> ---- > [!IMPORTANT] > Fixes role handling in OpenAI responses, adds tests for multimodal inputs, updates dependencies, and modifies version bump script. > > - **Behavior**: > - Fixes role handling in `openai_client.rs` by removing conditional logic for text-only content, ensuring structured messages are used consistently. > - **Testing**: > - Adds `test_openai_responses.py` for Python to test various OpenAI response scenarios including multimodal inputs. > - Adds `openai-responses.test.ts` for TypeScript to test OpenAI response handling. > - **Dependencies**: > - Replaces `assertpy` with `pyhamcrest` in `pyproject.toml` and `uv.lock`. > - **Scripts**: > - Modifies `bump-version.py` to use `git fetch origin --tags` instead of `git pull --tags`. > - Comments out Ruby dependency installation in `bump-version.py`. > > <sup>This description was created by </sup>[<img alt="Ellipsis" src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup> for 8133354. You can [customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this summary. It will automatically update as commits are pushed.</sup> <!-- ELLIPSIS_HIDDEN -->
1 parent 3950a49 commit d976c55

7 files changed

Lines changed: 416 additions & 148 deletions

File tree

‎engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -138,16 +138,7 @@ impl ProviderStrategy {
138138
json!(prompt)
139139
}
140140
either::Either::Right(messages) => {
141-
// Check if we have any non-text content (multimodal)
142-
let has_media = messages.iter().any(|msg| {
143-
msg.parts
144-
.iter()
145-
.any(|part| !matches!(part, ChatMessagePart::Text(_)))
146-
});
147-
148-
if has_media {
149-
// Use structured array format for multimodal content
150-
let structured_messages: Result<Vec<_>> = messages
141+
let structured_messages: Result<Vec<_>> = messages
151142
.iter()
152143
.map(|msg| {
153144
// Convert message parts to Responses API format
@@ -268,28 +259,7 @@ impl ProviderStrategy {
268259
}))
269260
})
270261
.collect();
271-
json!(structured_messages?)
272-
} else {
273-
// For text-only content, we can use either string or structured format
274-
// Let's use string format for simplicity when there's only text
275-
let text_content = messages
276-
.iter()
277-
.map(|msg| {
278-
let content_text = msg
279-
.parts
280-
.iter()
281-
.filter_map(|part| match part {
282-
ChatMessagePart::Text(text) => Some(text.as_str()),
283-
_ => None,
284-
})
285-
.collect::<Vec<_>>()
286-
.join(" ");
287-
format!("{}: {}", msg.role, content_text)
288-
})
289-
.collect::<Vec<_>>()
290-
.join("\n");
291-
json!(text_content)
292-
}
262+
json!(structured_messages?)
293263
}
294264
};
295265
body.insert("input".into(), input);

‎integ-tests/python/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ authors = [{ name = "None" }]
66
requires-python = "~=3.9"
77
readme = "README.md"
88
dependencies = [
9-
"assertpy",
109
"flask-httpauth>=4.8.0",
1110
"flask>=3.0.3",
1211
"maturin",
@@ -22,6 +21,7 @@ dependencies = [
2221
"google-genai (>=1.5.0,<2.0.0)",
2322
"boto3>=1.37.37",
2423
"pip>=25.1.1",
24+
"pyhamcrest>=2.1.0",
2525
]
2626

2727
[dependency-groups]

‎integ-tests/python/tests/providers/test_openai_responses.py‎

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
import pytest
2+
from hamcrest import assert_that, equal_to
3+
from baml_py import Image, Audio, Pdf
4+
from baml_client import b
5+
6+
7+
@pytest.mark.asyncio
8+
async def test_expose_request_openai_responses_multimodal():
9+
test_image = Image.from_url(
10+
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
11+
)
12+
request = await b.request.TestOpenAIResponsesImageInput(test_image)
13+
14+
assert_that(
15+
request.body.json(),
16+
equal_to(
17+
{
18+
"model": "o1-mini",
19+
"input": [
20+
{
21+
"role": "user",
22+
"content": [
23+
{"type": "input_text", "text": "what is in this content?"},
24+
{
25+
"type": "input_image",
26+
"image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
27+
},
28+
],
29+
}
30+
],
31+
}
32+
),
33+
)
34+
35+
36+
@pytest.mark.asyncio
37+
async def test_expose_request_openai_responses_audio():
38+
test_audio_data = "UklGRnoGAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQoGAACBhYqFbF1fdJivrJBhNjVgodDbq2EcBj+a2/LDciUFLIHO8tiJNwgZaLvt559NEAxQp+PwtmMcBjiR1/LMeSwFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoFJHfH8N2QQAoUYrTp66hVFApGn+DyvmEaBC2Bye/OcyoF"
39+
test_audio = Audio.from_base64("audio/wav", test_audio_data)
40+
request = await b.request.TestOpenAIResponsesImageInput(test_audio)
41+
42+
assert_that(
43+
request.body.json(),
44+
equal_to(
45+
{
46+
"model": "o1-mini",
47+
"input": [
48+
{
49+
"role": "user",
50+
"content": [
51+
{"type": "input_text", "text": "what is in this content?"},
52+
{
53+
"type": "input_audio",
54+
"input_audio": {
55+
"data": test_audio_data,
56+
"format": "wav",
57+
},
58+
},
59+
],
60+
}
61+
],
62+
}
63+
),
64+
)
65+
66+
67+
@pytest.mark.asyncio
68+
async def test_expose_request_openai_responses_pdf_base64():
69+
# Test that base64 PDFs are converted to data URLs
70+
test_pdf_b64 = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK"
71+
test_pdf = Pdf.from_base64(test_pdf_b64)
72+
request = await b.request.TestOpenAIResponsesImageInput(test_pdf)
73+
74+
assert_that(
75+
request.body.json(),
76+
equal_to(
77+
{
78+
"model": "o1-mini",
79+
"input": [
80+
{
81+
"role": "user",
82+
"content": [
83+
{"type": "input_text", "text": "what is in this content?"},
84+
{
85+
"type": "input_file",
86+
"file_url": f"data:application/pdf;base64,{test_pdf_b64}",
87+
},
88+
],
89+
}
90+
],
91+
}
92+
),
93+
)
94+
95+
96+
@pytest.mark.asyncio
97+
async def test_openai_responses_basic():
98+
request = await b.request.TestOpenAIResponses("lorem ipsum")
99+
100+
assert_that(
101+
request.body.json(),
102+
equal_to(
103+
{
104+
"model": "gpt-4.1",
105+
"input": [
106+
{
107+
"role": "user",
108+
"content": [
109+
{
110+
"type": "input_text",
111+
"text": "Write a short haiku about lorem ipsum. Make it simple and beautiful.",
112+
},
113+
],
114+
},
115+
],
116+
}
117+
),
118+
)
119+
120+
121+
@pytest.mark.asyncio
122+
async def test_openai_responses_explicit():
123+
request = await b.request.TestOpenAIResponsesExplicit("lorem ipsum")
124+
125+
assert_that(
126+
request.body.json(),
127+
equal_to(
128+
{
129+
"model": "gpt-4.1",
130+
"input": [
131+
{
132+
"role": "user",
133+
"content": [
134+
{
135+
"type": "input_text",
136+
"text": "Create a brief poem about lorem ipsum. Keep it under 50 words.",
137+
},
138+
],
139+
},
140+
],
141+
}
142+
),
143+
)
144+
145+
146+
@pytest.mark.asyncio
147+
async def test_openai_responses_custom_url():
148+
request = await b.request.TestOpenAIResponsesCustomURL("lorem ipsum")
149+
150+
assert_that(
151+
request.body.json(),
152+
equal_to(
153+
{
154+
"model": "gpt-4.1",
155+
"input": [
156+
{
157+
"role": "user",
158+
"content": [
159+
{
160+
"type": "input_text",
161+
"text": "Tell me an interesting fact about lorem ipsum.",
162+
},
163+
],
164+
},
165+
],
166+
}
167+
),
168+
)
169+
170+
171+
@pytest.mark.asyncio
172+
async def test_openai_responses_conversation():
173+
request = await b.request.TestOpenAIResponsesConversation("lorem ipsum")
174+
175+
assert_that(
176+
request.body.json(),
177+
equal_to(
178+
{
179+
"model": "gpt-4.1",
180+
"input": [
181+
{
182+
"role": "system",
183+
"content": [
184+
{
185+
"type": "input_text",
186+
"text": "You are a helpful assistant that provides concise answers.",
187+
},
188+
],
189+
},
190+
{
191+
"role": "user",
192+
"content": [
193+
{
194+
"type": "input_text",
195+
"text": "What is lorem ipsum?",
196+
},
197+
],
198+
},
199+
{
200+
"role": "assistant",
201+
"content": [
202+
{
203+
"type": "input_text",
204+
"text": "lorem ipsum is a fascinating subject. Let me explain briefly.",
205+
},
206+
],
207+
},
208+
{
209+
"role": "user",
210+
"content": [
211+
{
212+
"type": "input_text",
213+
"text": "Can you give me a simple example?",
214+
},
215+
],
216+
},
217+
],
218+
}
219+
),
220+
)
221+
222+
223+
@pytest.mark.asyncio
224+
async def test_openai_responses_different_model():
225+
request = await b.request.TestOpenAIResponsesDifferentModel("lorem ipsum")
226+
227+
assert_that(
228+
request.body.json(),
229+
equal_to(
230+
{
231+
"model": "gpt-4",
232+
"input": [
233+
{
234+
"role": "user",
235+
"content": [
236+
{
237+
"type": "input_text",
238+
"text": "Explain lorem ipsum in one sentence.",
239+
},
240+
],
241+
},
242+
],
243+
}
244+
),
245+
)
246+
247+
248+
@pytest.mark.asyncio
249+
async def test_expose_request_openai_responses_pdf_url():
250+
# Test that PDF URLs are preserved as URLs (OpenAI Responses API supports file_url with URLs)
251+
test_pdf = Pdf.from_url(
252+
"https://www.usenix.org/system/files/conference/nsdi13/nsdi13-final85.pdf"
253+
)
254+
request = await b.request.TestOpenAIResponsesImageInput(test_pdf)
255+
256+
assert_that(
257+
request.body.json(),
258+
equal_to(
259+
{
260+
"model": "o1-mini",
261+
"input": [
262+
{
263+
"role": "user",
264+
"content": [
265+
{"type": "input_text", "text": "what is in this content?"},
266+
{
267+
"type": "input_file",
268+
"file_url": "https://www.usenix.org/system/files/conference/nsdi13/nsdi13-final85.pdf",
269+
},
270+
],
271+
}
272+
],
273+
}
274+
),
275+
)

0 commit comments

Comments
 (0)