Skip to content

Commit 4bb2f33

Browse files
authored
streamfix (#2530)
- **checkpoint** - **Fix streaming not running the parser, vscode image rendering issue, openai responses bug, and vertex anthropic streaming** <!-- CURSOR_SUMMARY --> --- > [!NOTE] > Refactors streaming to parse/deliver partials reliably, corrects OpenAI Responses message/media formatting, enables Anthropic-on-Vertex streaming, and improves playground cURL preview/copy behavior. > > - **Runtime/Streaming**: > - Decouple SSE consumption from parsing with a throttled parser loop and shared `ParserState`; emit deduped partials via `watch` snapshots. > - Improve final fallback/error (`ErrorCode::Other(2)`, message) and cancellation handling. > - **Providers**: > - **OpenAI Responses**: Normalize content parts (`input_text`, `input_image` with `detail: auto`, `input_file` with `filename`), disallow assistant media, and add unit test for mixed text+file. > - **Vertex (Anthropic on Vertex)**: Route streaming via `ResponseType::Anthropic` when `anthropic_version` is set and set `stream: true` in request body. > - **Streaming request**: Raise SSE event log level to `debug`. > - **Tracing**: > - Downgrade BAML source upload check failures from `error` to `warn`. > - **Playground UI**: > - Stabilize cURL preview: export `curlAtom`, memoize highlighter, retain last cURL during loading, and support copying cURL when cURL tab is active. > - VSCode API wrapper: minor API cleanups for media file loading. > - **Tests**: > - Update/expand integration tests for Responses API (image detail, PDF filename), Anthropic/Vertex streaming, and timing/format tweaks. > > <sup>Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit 2181a28. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot).</sup> <!-- /CURSOR_SUMMARY -->
1 parent 6a5fa73 commit 4bb2f33

11 files changed

Lines changed: 433 additions & 184 deletions

File tree

‎engine/baml-runtime/src/internal/llm_client/orchestrator/stream.rs‎

Lines changed: 171 additions & 117 deletions
Large diffs are not rendered by default.

‎engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs‎

Lines changed: 134 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,23 @@ impl ProviderStrategy {
147147
.iter()
148148
.map(|part| match part {
149149
ChatMessagePart::Text(text) => {
150+
let content_type = if msg.role == "assistant" {
151+
"output_text"
152+
} else {
153+
"input_text"
154+
};
150155
Ok(json!({
151-
"type": if msg.role == "assistant" { "output_text" } else { "input_text" },
156+
"type": content_type,
152157
"text": text
153158
}))
154159
}
155160
ChatMessagePart::Media(media) => {
161+
// For assistant role, we only support text outputs in Responses API
162+
if msg.role == "assistant" {
163+
anyhow::bail!(
164+
"BAML internal error (openai-responses): assistant messages must be text; media not supported for assistant in Responses API"
165+
);
166+
}
156167
match media.media_type {
157168
baml_types::BamlMediaType::Image => {
158169
let image_url = match &media.content {
@@ -165,7 +176,8 @@ impl ProviderStrategy {
165176
}
166177
};
167178
Ok(json!({
168-
"type": if msg.role == "assistant" { "output_image" } else { "input_image" },
179+
"type": "input_image",
180+
"detail": "auto",
169181
"image_url": image_url
170182
}))
171183
}
@@ -177,7 +189,7 @@ impl ProviderStrategy {
177189
.strip_prefix("audio/")
178190
.unwrap_or(&mime_type);
179191
Ok(json!({
180-
"type": if msg.role == "assistant" { "output_audio" } else { "input_audio" },
192+
"type": "input_audio",
181193
"input_audio": {
182194
"data": b64_media.base64,
183195
"format": format
@@ -193,17 +205,19 @@ impl ProviderStrategy {
193205
match &media.content {
194206
baml_types::BamlMediaContent::Url(url_content) => {
195207
Ok(json!({
196-
"type": if msg.role == "assistant" { "output_file" } else { "input_file" },
197-
"file_url": url_content.url
208+
"type": "input_file",
209+
"file_url": url_content.url,
210+
"filename": "document.pdf"
198211
}))
199212
}
200213
baml_types::BamlMediaContent::File(file_content) => {
201214
anyhow::bail!("BAML internal error (openai-responses): Local PDF files are not supported by OpenAI Responses API - use file_url for remote files or upload file and use file_id. File path: {:?}", file_content.relpath);
202215
}
203216
baml_types::BamlMediaContent::Base64(b64_media) => {
204217
Ok(json!({
205-
"type": if msg.role == "assistant" { "output_file" } else { "input_file" },
206-
"file_data": format!("data:{};base64,{}", media.mime_type_as_ok()?, b64_media.base64)
218+
"type": "input_file",
219+
"file_data": format!("data:{};base64,{}", media.mime_type_as_ok()?, b64_media.base64),
220+
"filename": "document.pdf"
207221
}))
208222
}
209223
}
@@ -217,13 +231,23 @@ impl ProviderStrategy {
217231
// Recursively handle the inner part, ignoring metadata for now
218232
match inner_part.as_ref() {
219233
ChatMessagePart::Text(text) => {
234+
let content_type = if msg.role == "assistant" {
235+
"output_text"
236+
} else {
237+
"input_text"
238+
};
220239
Ok(json!({
221-
"type": if msg.role == "assistant" { "output_text" } else { "input_text" },
240+
"type": content_type,
222241
"text": text
223242
}))
224243
}
225244
ChatMessagePart::Media(media) => {
226245
// Handle media same as above - could refactor into helper function
246+
if msg.role == "assistant" {
247+
anyhow::bail!(
248+
"BAML internal error (openai-responses): assistant messages must be text; media not supported for assistant in Responses API"
249+
);
250+
}
227251
match media.media_type {
228252
baml_types::BamlMediaType::Image => {
229253
let image_url = match &media.content {
@@ -236,7 +260,8 @@ impl ProviderStrategy {
236260
}
237261
};
238262
Ok(json!({
239-
"type": if msg.role == "assistant" { "output_image" } else { "input_image" },
263+
"type": "input_image",
264+
"detail": "auto",
240265
"image_url": image_url
241266
}))
242267
}
@@ -698,7 +723,8 @@ impl ToProviderMessage for OpenAIClient {
698723
payload_key.into(),
699724
json!({
700725
"type": "input_file",
701-
"file_url": url_content.url
726+
"file_url": url_content.url,
727+
"filename": "document.pdf"
702728
}),
703729
);
704730
}
@@ -916,4 +942,102 @@ mod tests {
916942
let endpoint = strategy.get_endpoint("https://api.openai.com/v1", true);
917943
assert_eq!(endpoint, "https://api.openai.com/v1/completions");
918944
}
945+
946+
#[test]
947+
fn test_responses_api_builds_input_message_with_text_and_file() {
948+
let strategy = ProviderStrategy::ResponsesApi;
949+
950+
// Properties include model
951+
let mut props = BamlMap::new();
952+
props.insert("model".into(), json!("gpt-5-mini"));
953+
954+
// Build a user message with text and file (PDF url)
955+
let msg = RenderedChatMessage {
956+
role: "user".to_string(),
957+
allow_duplicate_role: false,
958+
parts: vec![
959+
ChatMessagePart::Text("what is in this file?".to_string()),
960+
ChatMessagePart::Media(baml_types::BamlMedia::url(
961+
BamlMediaType::Pdf,
962+
"https://www.berkshirehathaway.com/letters/2024ltr.pdf".to_string(),
963+
Some("application/pdf".to_string()),
964+
)),
965+
],
966+
};
967+
968+
// chat_converter is not used in ResponsesApi branch; construct a minimal client
969+
let responses_client = OpenAIClient {
970+
name: "test".to_string(),
971+
provider: "openai-responses".to_string(),
972+
retry_policy: None,
973+
context: RenderContext_Client {
974+
name: "test".to_string(),
975+
provider: "openai-responses".to_string(),
976+
default_role: "user".to_string(),
977+
allowed_roles: vec!["user".to_string(), "assistant".to_string()],
978+
remap_role: HashMap::new(),
979+
options: IndexMap::new(),
980+
},
981+
features: ModelFeatures {
982+
chat: true,
983+
completion: false,
984+
max_one_system_prompt: false,
985+
resolve_audio_urls: ResolveMediaUrls::Always,
986+
resolve_image_urls: ResolveMediaUrls::Never,
987+
resolve_pdf_urls: ResolveMediaUrls::Never,
988+
resolve_video_urls: ResolveMediaUrls::Never,
989+
allowed_metadata: AllowedRoleMetadata::All,
990+
},
991+
properties: ResolvedOpenAI {
992+
base_url: "https://api.openai.com/v1".to_string(),
993+
api_key: None,
994+
role_selection: RolesSelection::default(),
995+
allowed_metadata: AllowedRoleMetadata::All,
996+
supported_request_modes: SupportedRequestModes::default(),
997+
headers: IndexMap::new(),
998+
properties: BamlMap::new(),
999+
query_params: IndexMap::new(),
1000+
proxy_url: None,
1001+
finish_reason_filter: FinishReasonFilter::All,
1002+
client_response_type: ResponseType::OpenAIResponses,
1003+
},
1004+
client: reqwest::Client::new(),
1005+
};
1006+
1007+
let body_value = strategy
1008+
.build_body(either::Either::Right(&[msg]), &props, &responses_client)
1009+
.expect("should build body");
1010+
1011+
let obj = body_value.as_object().expect("body should be an object");
1012+
assert_eq!(obj.get("model"), Some(&json!("gpt-5-mini")));
1013+
1014+
let input = obj
1015+
.get("input")
1016+
.and_then(|v| v.as_array())
1017+
.expect("input should be array");
1018+
assert_eq!(input.len(), 1);
1019+
1020+
let first_msg = input[0].as_object().expect("message should be object");
1021+
assert_eq!(first_msg.get("role"), Some(&json!("user")));
1022+
let content = first_msg
1023+
.get("content")
1024+
.and_then(|v| v.as_array())
1025+
.expect("content should be array");
1026+
assert_eq!(content.len(), 2);
1027+
1028+
// Validate text part
1029+
let t = content[0].as_object().expect("text part object");
1030+
assert_eq!(t.get("type"), Some(&json!("input_text")));
1031+
assert_eq!(t.get("text"), Some(&json!("what is in this file?")));
1032+
1033+
// Validate file part
1034+
let f = content[1].as_object().expect("file part object");
1035+
assert_eq!(f.get("type"), Some(&json!("input_file")));
1036+
assert_eq!(
1037+
f.get("file_url"),
1038+
Some(&json!(
1039+
"https://www.berkshirehathaway.com/letters/2024ltr.pdf"
1040+
))
1041+
);
1042+
}
9191043
}

‎engine/baml-runtime/src/internal/llm_client/primitive/stream_request.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ pub async fn make_stream_request(
9292
std::future::ready(event.as_ref().is_ok_and(|e| e.data != "[DONE]"))
9393
})
9494
.map(|event| -> Result<serde_json::Value> { Ok(serde_json::from_str(&event?.data)?) })
95-
.inspect(|event| log::trace!("{event:#?}"))
95+
.inspect(|event| log::debug!("{event:#?}"))
9696
.scan(
9797
Ok(LLMCompleteResponse {
9898
client: client_name.clone(),

‎engine/baml-runtime/src/internal/llm_client/primitive/vertex/response_handler.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,9 @@ pub fn scan_vertex_response_stream(
125125
let inner = match accumulated {
126126
Ok(accumulated) => accumulated,
127127
// We'll just keep the first error and return it
128-
Err(e) => return Ok(()),
128+
Err(e) => {
129+
return Ok(());
130+
}
129131
};
130132

131133
let event = VertexResponse::deserialize(&event_body)

‎engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,10 @@ impl WithStreamChat for VertexClient {
132132
self,
133133
either::Either::Right(prompt),
134134
Some(self.properties.model.clone()),
135-
ResponseType::Vertex,
135+
match self.properties.anthropic_version {
136+
Some(ref anthropic_version) => ResponseType::Anthropic,
137+
None => ResponseType::Vertex,
138+
},
136139
ctx,
137140
)
138141
.await
@@ -350,6 +353,13 @@ impl RequestBuilder for VertexClient {
350353
}
351354
}
352355

356+
// If this is an Anthropic-on-Vertex request and streaming is enabled, add `stream: true`
357+
// to the JSON body to mirror Anthropic API behavior.
358+
// See docs here: https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-5-sonnet?authuser=1&hl=en&project=gloo-ai
359+
if stream && self.properties.anthropic_version.is_some() {
360+
json_body.insert("stream".into(), json!(true));
361+
}
362+
353363
let req = req.json(&json_body);
354364

355365
Ok(req)

‎engine/baml-runtime/src/tracingv2/publisher/publisher.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ impl TracePublisher {
547547
{
548548
Ok(response) => response,
549549
Err(e) => {
550-
tracing::error!("Failed to check BAML source upload status: {}", e);
550+
tracing::warn!("Failed to check BAML source upload status: {}", e);
551551
return Err(e.into());
552552
}
553553
};

‎integ-tests/python/tests/providers/test_openai_responses.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ async def test_expose_request_openai_responses_multimodal():
2424
{
2525
"type": "input_image",
2626
"image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
27+
"detail": "auto",
2728
},
2829
],
2930
}
@@ -84,6 +85,7 @@ async def test_expose_request_openai_responses_pdf_base64():
8485
{
8586
"type": "input_file",
8687
"file_data": f"data:application/pdf;base64,{test_pdf_b64}",
88+
"filename": "document.pdf",
8789
},
8890
],
8991
}
@@ -266,6 +268,7 @@ async def test_expose_request_openai_responses_pdf_url():
266268
{
267269
"type": "input_file",
268270
"file_url": "https://www.usenix.org/system/files/conference/nsdi13/nsdi13-final85.pdf",
271+
"filename": "document.pdf",
269272
},
270273
],
271274
}

0 commit comments

Comments
 (0)