Skip to content

Commit 87ed2b1

Browse files
aaronvgaaronvg
authored
Support openai audio input (#1940)
- Fix bug where after LSP restart the playground wouldnt load unless you typed something into the editor - Fix telemetry not working on Cursor <!-- ELLIPSIS_HIDDEN --> > [!IMPORTANT] > Add support for handling audio input URLs in OpenAI client and improve telemetry and logging. > > - **Behavior**: > - Add `resolve_audio_urls` and `resolve_image_urls` to `ModelFeatures` in `mod.rs`. > - Update `OpenAIClient`, `AnthropicClient`, `AwsClient`, `GoogleAIClient`, and `VertexClient` to handle audio and image URLs separately. > - Modify `process_media_urls()` in `traits/mod.rs` to use separate resolution modes for audio and image. > - **Telemetry**: > - Add telemetry logging for test runs in `test-runner.ts`. > - Update telemetry settings handling in `telemetryReporter.ts`. > - **Misc**: > - Add version logging in `server.rs`. > - Remove `restartClient()` function from `util.ts` and update client restart logic in `index.ts`. > > <sup>This description was created by </sup>[<img alt="Ellipsis" src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup> for 2f85961. You can [customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this summary. It will automatically update as commits are pushed.</sup> <!-- ELLIPSIS_HIDDEN -->
1 parent 6d4eb2b commit 87ed2b1

13 files changed

Lines changed: 170 additions & 107 deletions

File tree

‎engine/baml-runtime/src/internal/llm_client/mod.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ pub fn parsed_value_to_response(
7272
Ok(ResponseBamlValue(response_value))
7373
}
7474

75+
// Whether we should download a url into a base64 (resolving it if necessary), as well as
76+
// whether we should add the mime type, etc.
7577
#[derive(Clone, Copy, PartialEq)]
7678
pub enum ResolveMediaUrls {
7779
// there are 5 input formats:
@@ -94,7 +96,9 @@ pub enum ResolveMediaUrls {
9496
// vertex: supports URLs w/ mime, b64 w/ mime
9597
Always,
9698
IfMatchesGoogleFileUri,
99+
// EnsureMime: always add the mime type to the request (which means if it's a url, we may need to resolve it to find the mime type)
97100
EnsureMime,
101+
// Never: don't resolve media urls
98102
Never,
99103
}
100104

@@ -103,7 +107,10 @@ pub struct ModelFeatures {
103107
pub completion: bool,
104108
pub chat: bool,
105109
pub max_one_system_prompt: bool,
106-
pub resolve_media_urls: ResolveMediaUrls,
110+
/// Controls how audio URLs are resolved for this model/provider
111+
pub resolve_audio_urls: ResolveMediaUrls,
112+
/// Controls how image URLs are resolved for this model/provider
113+
pub resolve_image_urls: ResolveMediaUrls,
107114
pub allowed_metadata: AllowedRoleMetadata,
108115
}
109116

‎engine/baml-runtime/src/internal/llm_client/primitive/anthropic/anthropic_client.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ impl AnthropicClient {
157157
chat: true,
158158
completion: false,
159159
max_one_system_prompt: true,
160-
resolve_media_urls: ResolveMediaUrls::Never,
160+
resolve_audio_urls: ResolveMediaUrls::Never,
161+
resolve_image_urls: ResolveMediaUrls::Never,
161162
allowed_metadata: properties.allowed_metadata.clone(),
162163
},
163164
retry_policy: client.retry_policy.clone(),
@@ -180,7 +181,8 @@ impl AnthropicClient {
180181
chat: true,
181182
completion: false,
182183
max_one_system_prompt: true,
183-
resolve_media_urls: ResolveMediaUrls::Never,
184+
resolve_audio_urls: ResolveMediaUrls::Never,
185+
resolve_image_urls: ResolveMediaUrls::Never,
184186
allowed_metadata: properties.allowed_metadata.clone(),
185187
},
186188
retry_policy: client
@@ -213,7 +215,8 @@ impl AnthropicClient {
213215
chat: true,
214216
completion: false,
215217
max_one_system_prompt: true,
216-
resolve_media_urls: ResolveMediaUrls::Never,
218+
resolve_audio_urls: ResolveMediaUrls::Never,
219+
resolve_image_urls: ResolveMediaUrls::Never,
217220
allowed_metadata: AllowedRoleMetadata::None,
218221
},
219222
client: create_client()?,
@@ -325,6 +328,10 @@ impl ToProviderMessage for AnthropicClient {
325328
mut content: serde_json::Map<String, serde_json::Value>,
326329
media: &baml_types::BamlMedia,
327330
) -> Result<serde_json::Map<String, serde_json::Value>> {
331+
let resolve_mode = match media.media_type {
332+
baml_types::BamlMediaType::Audio => self.features.resolve_audio_urls,
333+
baml_types::BamlMediaType::Image => self.features.resolve_image_urls,
334+
};
328335
match &media.content {
329336
BamlMediaContent::Base64(data) => {
330337
content.insert("type".into(), media.media_type.to_string().into());

‎engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,8 @@ impl AwsClient {
289289
chat: true,
290290
completion: false,
291291
max_one_system_prompt: true,
292-
resolve_media_urls: ResolveMediaUrls::Always,
292+
resolve_audio_urls: ResolveMediaUrls::Always,
293+
resolve_image_urls: ResolveMediaUrls::Always,
293294
allowed_metadata: properties.allowed_role_metadata.clone(),
294295
},
295296
retry_policy: client.retry_policy.as_ref().map(|s| s.to_string()),
@@ -312,7 +313,8 @@ impl AwsClient {
312313
chat: true,
313314
completion: false,
314315
max_one_system_prompt: true,
315-
resolve_media_urls: ResolveMediaUrls::Always,
316+
resolve_audio_urls: ResolveMediaUrls::Always,
317+
resolve_image_urls: ResolveMediaUrls::Always,
316318
allowed_metadata: properties.allowed_role_metadata.clone(),
317319
},
318320
retry_policy: client

‎engine/baml-runtime/src/internal/llm_client/primitive/google/googleai_client.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,8 @@ impl GoogleAIClient {
138138
chat: true,
139139
completion: false,
140140
max_one_system_prompt: true,
141-
resolve_media_urls: ResolveMediaUrls::IfMatchesGoogleFileUri,
141+
resolve_audio_urls: ResolveMediaUrls::IfMatchesGoogleFileUri,
142+
resolve_image_urls: ResolveMediaUrls::IfMatchesGoogleFileUri,
142143
allowed_metadata: properties.allowed_metadata.clone(),
143144
},
144145
retry_policy: client
@@ -166,7 +167,8 @@ impl GoogleAIClient {
166167
chat: true,
167168
completion: false,
168169
max_one_system_prompt: true,
169-
resolve_media_urls: ResolveMediaUrls::Always,
170+
resolve_audio_urls: ResolveMediaUrls::Always,
171+
resolve_image_urls: ResolveMediaUrls::Always,
170172
allowed_metadata: properties.allowed_metadata.clone(),
171173
},
172174
retry_policy: client.retry_policy.clone(),

‎engine/baml-runtime/src/internal/llm_client/primitive/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,8 @@ mod tests {
372372
completion: false,
373373
chat: false,
374374
max_one_system_prompt: false,
375-
resolve_media_urls: crate::internal::llm_client::ResolveMediaUrls::Always,
375+
resolve_audio_urls: crate::internal::llm_client::ResolveMediaUrls::Always,
376+
resolve_image_urls: crate::internal::llm_client::ResolveMediaUrls::Always,
376377
allowed_metadata: crate::internal::llm_client::AllowedRoleMetadata::All,
377378
},
378379
context: internal_baml_jinja::RenderContext_Client {

‎engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs‎

Lines changed: 64 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,8 @@ macro_rules! make_openai_client {
227227
chat: true,
228228
completion: false,
229229
max_one_system_prompt: false,
230-
resolve_media_urls: ResolveMediaUrls::Never,
230+
resolve_audio_urls: ResolveMediaUrls::Always,
231+
resolve_image_urls: ResolveMediaUrls::Never,
231232
allowed_metadata: $properties.allowed_metadata.clone(),
232233
},
233234
properties: $properties,
@@ -249,7 +250,8 @@ macro_rules! make_openai_client {
249250
chat: true,
250251
completion: false,
251252
max_one_system_prompt: false,
252-
resolve_media_urls: ResolveMediaUrls::Never,
253+
resolve_audio_urls: ResolveMediaUrls::Always,
254+
resolve_image_urls: ResolveMediaUrls::Never,
253255
allowed_metadata: $properties.allowed_metadata.clone(),
254256
},
255257
properties: $properties,
@@ -338,34 +340,67 @@ impl ToProviderMessage for OpenAIClient {
338340
mut content: serde_json::Map<String, serde_json::Value>,
339341
media: &baml_types::BamlMedia,
340342
) -> Result<serde_json::Map<String, serde_json::Value>> {
341-
let media_type = match media.media_type {
342-
BamlMediaType::Image => "image",
343-
BamlMediaType::Audio => "audio",
344-
};
345-
let media_type = format!("{}_url", media_type);
346-
match &media.content {
347-
BamlMediaContent::Url(media) => {
348-
content.insert("type".into(), json!(media_type));
349-
content.insert(
350-
media_type,
351-
json!({
352-
"url": media.url
353-
}),
354-
);
355-
}
356-
BamlMediaContent::Base64(b64_media) => {
357-
content.insert("type".into(), json!(media_type));
358-
content.insert(
359-
media_type,
360-
json!({
361-
"url": format!("data:{};base64,{}", media.mime_type_as_ok()?, b64_media.base64)
362-
}),
363-
);
343+
match media.media_type {
344+
BamlMediaType::Image => {
345+
let type_value = "image_url";
346+
let payload_key = "image_url";
347+
content.insert("type".into(), json!(type_value));
348+
349+
match &media.content {
350+
BamlMediaContent::Url(url_content) => {
351+
content.insert(payload_key.into(), json!({ "url": url_content.url }));
352+
}
353+
BamlMediaContent::Base64(b64_media) => {
354+
content.insert(
355+
payload_key.into(),
356+
json!({
357+
"url": format!("data:{};base64,{}", media.mime_type_as_ok()?, b64_media.base64)
358+
}),
359+
);
360+
}
361+
BamlMediaContent::File(_) => {
362+
anyhow::bail!("BAML internal error (openai): image file should have been resolved, not processed directly.");
363+
}
364+
}
364365
}
365-
BamlMediaContent::File(_) => {
366-
anyhow::bail!(
367-
"BAML internal error (openai): file should have been resolved to base64"
368-
)
366+
BamlMediaType::Audio => {
367+
let type_value = "input_audio";
368+
let payload_key = "input_audio";
369+
content.insert("type".into(), json!(type_value));
370+
371+
match &media.content {
372+
BamlMediaContent::Base64(b64_media) => {
373+
let mime_type_str = media.mime_type_as_ok()?;
374+
let format_str = match mime_type_str.as_str() {
375+
"audio/wav" | "wav" => "wav",
376+
"audio/mp3" | "mp3" => "mp3",
377+
// Add other supported formats here if needed in the future
378+
_ => anyhow::bail!(
379+
"Unsupported audio format for input_audio: '{}'. OpenAI currently supports 'wav' and 'mp3' for this structure.",
380+
mime_type_str
381+
),
382+
};
383+
384+
content.insert(
385+
payload_key.into(),
386+
json!({
387+
"data": b64_media.base64,
388+
"format": format_str
389+
}),
390+
);
391+
}
392+
BamlMediaContent::Url(_) => {
393+
anyhow::bail!(
394+
"BAML internal error (openai): Audio content is a URL. Expected Base64 for '{}' type due to client's ResolveMediaUrls::Always setting. The URL should have been resolved to base64 before this stage.",
395+
type_value
396+
);
397+
}
398+
BamlMediaContent::File(_) => {
399+
anyhow::bail!(
400+
"BAML internal error (openai): audio file should have been resolved to base64, not processed directly."
401+
);
402+
}
403+
}
369404
}
370405
}
371406
Ok(content)

‎engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ impl VertexClient {
162162
chat: true,
163163
completion: false,
164164
max_one_system_prompt: true,
165-
resolve_media_urls: ResolveMediaUrls::EnsureMime,
165+
resolve_audio_urls: ResolveMediaUrls::EnsureMime,
166+
resolve_image_urls: ResolveMediaUrls::EnsureMime,
166167
allowed_metadata: properties.allowed_metadata.clone(),
167168
},
168169
retry_policy: client
@@ -190,7 +191,8 @@ impl VertexClient {
190191
chat: true,
191192
completion: false,
192193
max_one_system_prompt: true,
193-
resolve_media_urls: ResolveMediaUrls::EnsureMime,
194+
resolve_audio_urls: ResolveMediaUrls::EnsureMime,
195+
resolve_image_urls: ResolveMediaUrls::EnsureMime,
194196
allowed_metadata: properties.allowed_metadata.clone(),
195197
},
196198
retry_policy: client.retry_policy.clone(),

‎engine/baml-runtime/src/internal/llm_client/traits/mod.rs‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,8 @@ where
172172
) -> LLMResponse {
173173
match prompt {
174174
RenderedPrompt::Chat(chat) => match process_media_urls(
175-
self.model_features().resolve_media_urls,
175+
self.model_features().resolve_audio_urls,
176+
self.model_features().resolve_image_urls,
176177
true,
177178
None,
178179
ctx,
@@ -238,8 +239,15 @@ where
238239
RenderedPrompt::Chat(chat) => {
239240
let chat = merge_messages(&chat);
240241
// We never need to resolve media URLs here: webview rendering understands how to handle URLs and file refs
241-
let chat =
242-
process_media_urls(ResolveMediaUrls::Never, true, None, ctx, &chat).await?;
242+
let chat = process_media_urls(
243+
features.resolve_audio_urls,
244+
features.resolve_image_urls,
245+
true,
246+
None,
247+
ctx,
248+
&chat,
249+
)
250+
.await?;
243251
RenderedPrompt::Chat(chat)
244252
}
245253
};
@@ -290,7 +298,8 @@ where
290298
render_settings: RenderCurlSettings,
291299
) -> Result<String> {
292300
let chat_messages: Vec<RenderedChatMessage> = process_media_urls(
293-
self.model_features().resolve_media_urls,
301+
self.model_features().resolve_audio_urls,
302+
self.model_features().resolve_image_urls,
294303
true,
295304
Some(render_settings),
296305
ctx,
@@ -377,7 +386,8 @@ where
377386
let prompt = {
378387
if let RenderedPrompt::Chat(ref chat) = prompt {
379388
match process_media_urls(
380-
self.model_features().resolve_media_urls,
389+
self.model_features().resolve_audio_urls,
390+
self.model_features().resolve_image_urls,
381391
true,
382392
None,
383393
ctx,
@@ -423,7 +433,8 @@ where
423433
/// request. Other formats will be converted into that, depending on what
424434
/// formats are allowed according to supported_media_formats.
425435
async fn process_media_urls(
426-
resolve_media_urls: ResolveMediaUrls,
436+
resolve_audio_urls: ResolveMediaUrls,
437+
resolve_image_urls: ResolveMediaUrls,
427438
resolve_files: bool,
428439
render_settings: Option<RenderCurlSettings>,
429440
ctx: &RuntimeContext,
@@ -443,15 +454,13 @@ async fn process_media_urls(
443454
let Some(part) = any_part.as_media() else {
444455
return Ok::<ChatMessagePart, anyhow::Error>(any_part.clone());
445456
};
446-
let media = process_media(
447-
resolve_media_urls,
448-
resolve_files,
449-
render_settings,
450-
ctx,
451-
part,
452-
)
453-
.await
454-
.map(ChatMessagePart::Media)?;
457+
let resolve_mode = match part.media_type {
458+
BamlMediaType::Audio => resolve_audio_urls,
459+
BamlMediaType::Image => resolve_image_urls,
460+
};
461+
let media = process_media(resolve_mode, resolve_files, render_settings, ctx, part)
462+
.await
463+
.map(ChatMessagePart::Media)?;
455464

456465
if let Some(meta) = any_part.meta() {
457466
Ok(media.with_meta(meta.clone()))
@@ -483,7 +492,7 @@ async fn process_media_urls(
483492
}
484493

485494
async fn process_media(
486-
resolve_media_urls: ResolveMediaUrls,
495+
resolve_mode: ResolveMediaUrls,
487496
resolve_files: bool,
488497
render_settings: RenderCurlSettings,
489498
ctx: &RuntimeContext,
@@ -566,14 +575,14 @@ async fn process_media(
566575

567576
// Currently:
568577
// - Vertex is ResolveMediaUrls::EnsureMime and is the only one that supports URLs w/ mime-type
569-
// - OpenAI is ResolveMediaUrls::Never and allows passing in URLs with optionally specified mime-type
578+
// - OpenAI is ResolveMediaUrls::Never and allows passing in URLs with optionally specified mime-type (but it has different behavior for audio inputs)
570579

571580
// NOTE(sam): if a provider accepts URLs but requires mime-type
572581
// (i.e. Vertex), we currently send it to them as b64. This
573582
// is how it was implemented originally, and while that could be
574583
// problematic in theory, I'm not going to change it until a
575584
// customer complains.
576-
match (resolve_media_urls, part.mime_type.as_deref()) {
585+
match (resolve_mode, part.mime_type.as_deref()) {
577586
(ResolveMediaUrls::Always, _) => {}
578587
(ResolveMediaUrls::EnsureMime, Some("")) | (ResolveMediaUrls::EnsureMime, None) => {
579588
}

‎engine/language_server/src/server.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ impl Server {
135135

136136
// for some reason tracing logs are not available before this point
137137
tracing::info!("Starting server with {} worker threads", worker_threads);
138+
tracing::info!("-------- Version: {}", env!("CARGO_PKG_VERSION"));
138139

139140
let mut session = Session::new(
140141
&client_capabilities,

0 commit comments

Comments
 (0)