Skip to content

Add Nemotron-ASR streaming inference to Rust SDK #613

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rui-ren wants to merge 13 commits into
base: main
Choose a base branch
from
Open

Add Nemotron-ASR streaming inference to Rust SDK #613

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
cb89d83
Add live audio transcription streaming support to Rust SDK
Apr 8, 2026
d90e6e5
Fix FFI null pointer and native session leak in Drop
Apr 8, 2026
5aafded
Improve API parity with C# LiveAudioTranscription
Apr 8, 2026
7fb48dd
Update codex-feedback.md: mark parity gaps as resolved
Apr 8, 2026
3292cb3
Fix CI: update download callback to f64 and apply cargo fmt
Apr 8, 2026
4f748df
Fix CI: remove unused setup_audio_client from live_audio_test
Apr 8, 2026
1945c9e
Address PR review feedback
Apr 13, 2026
e50a889
Fix clippy needless_return for Rust 1.94
Apr 13, 2026
fbd9d89
Fix cargo fmt: collapse single-arm match block
Apr 13, 2026
9f1d031
Address new review: remove session factory from Model/ModelVariant, s…
Apr 24, 2026
14e9af8
Fix start() cancellation leak: await and cleanup native session
Apr 24, 2026
34e1089
Fix error parsing and propagate stream errors in test
Apr 24, 2026
05500c3
Simplify from_json to use map chain per review
Apr 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sdk/rust/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ serde_json = "1"
thiserror = "2"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] }
tokio-stream = "0.1"
tokio-util = "0.7"
futures-core = "0.3"
reqwest = { version = "0.12", features = ["json"] }
urlencoding = "2"
Expand Down
84 changes: 84 additions & 0 deletions sdk/rust/src/detail/core_interop.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ impl ResponseBuffer {
}
}

/// Request buffer with binary payload for `execute_command_with_binary`.
///
/// Used for audio streaming — carries both JSON params and raw PCM bytes.
#[repr(C)]
struct StreamingRequestBuffer {
command: *const i8,
command_length: i32,
data: *const i8,
data_length: i32,
binary_data: *const u8,
binary_data_length: i32,
}

/// Signature for `execute_command`.
type ExecuteCommandFn = unsafe extern "C" fn(*const RequestBuffer, *mut ResponseBuffer);

Expand All @@ -63,6 +76,10 @@ type ExecuteCommandWithCallbackFn = unsafe extern "C" fn(
*mut std::ffi::c_void,
);

/// Signature for `execute_command_with_binary`.
type ExecuteCommandWithBinaryFn =
unsafe extern "C" fn(*const StreamingRequestBuffer, *mut ResponseBuffer);

// ── Library name helpers ─────────────────────────────────────────────────────

#[cfg(target_os = "windows")]
Expand Down Expand Up @@ -237,6 +254,8 @@ pub(crate) struct CoreInterop {
CallbackFn,
*mut std::ffi::c_void,
),
execute_command_with_binary:
Option<unsafe extern "C" fn(*const StreamingRequestBuffer, *mut ResponseBuffer)>,
}

impl std::fmt::Debug for CoreInterop {
Expand Down Expand Up @@ -307,12 +326,22 @@ impl CoreInterop {
*sym
};

// SAFETY: Same as above — symbol must match `ExecuteCommandWithBinaryFn`.
// Optional: older native cores may not export this symbol (used for audio streaming).
let execute_command_with_binary: Option<ExecuteCommandWithBinaryFn> = unsafe {
library
.get::<ExecuteCommandWithBinaryFn>(b"execute_command_with_binary\0")
.ok()
.map(|sym| *sym)
};

Ok(Self {
_library: library,
#[cfg(target_os = "windows")]
_dependency_libs,
execute_command,
execute_command_with_callback,
execute_command_with_binary,
})
}

Expand Down Expand Up @@ -354,6 +383,61 @@ impl CoreInterop {
Self::process_response(response)
}

/// Execute a command with an additional binary payload.
///
/// Used for audio streaming — `binary_data` carries raw PCM bytes
/// alongside the JSON parameters.
pub fn execute_command_with_binary(
&self,
command: &str,
params: Option<&Value>,
binary_data: &[u8],
) -> Result<String> {
let native_fn = self.execute_command_with_binary.ok_or_else(|| {
FoundryLocalError::CommandExecution {
reason: "execute_command_with_binary is not supported by this native core \
(symbol not found)"
.into(),
}
})?;

let cmd = CString::new(command).map_err(|e| FoundryLocalError::CommandExecution {
reason: format!("Invalid command string: {e}"),
})?;

let data_json = match params {
Some(v) => serde_json::to_string(v)?,
None => String::new(),
};
let data_cstr =
CString::new(data_json.as_str()).map_err(|e| FoundryLocalError::CommandExecution {
reason: format!("Invalid data string: {e}"),
})?;

let request = StreamingRequestBuffer {
command: cmd.as_ptr(),
command_length: cmd.as_bytes().len() as i32,
data: data_cstr.as_ptr(),
data_length: data_cstr.as_bytes().len() as i32,
binary_data: if binary_data.is_empty() {
std::ptr::null()
} else {
binary_data.as_ptr()
},
binary_data_length: binary_data.len() as i32,
};

let mut response = ResponseBuffer::new();

// SAFETY: `request` fields point into `cmd`, `data_cstr`, and
// `binary_data` which are all alive for the duration of this call.
unsafe {
(native_fn)(&request, &mut response);
}

Self::process_response(response)
}

/// Execute a command that streams results back via `callback`.
///
/// Each chunk delivered by the native library is decoded as UTF-8 and
Expand Down
6 changes: 4 additions & 2 deletions sdk/rust/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ pub use async_openai::types::chat::{

// Re-export OpenAI response types for convenience.
pub use crate::openai::{
AudioTranscriptionResponse, AudioTranscriptionStream, ChatCompletionStream,
TranscriptionSegment, TranscriptionWord,
AudioTranscriptionResponse, AudioTranscriptionStream, ChatCompletionStream, ContentPart,
CoreErrorResponse, LiveAudioTranscriptionOptions, LiveAudioTranscriptionResponse,
LiveAudioTranscriptionSession, LiveAudioTranscriptionStream, TranscriptionSegment,
TranscriptionWord,
};
pub use async_openai::types::chat::{
ChatChoice, ChatChoiceStream, ChatCompletionMessageToolCall,
Expand Down
10 changes: 10 additions & 0 deletions sdk/rust/src/openai/audio_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use crate::detail::core_interop::CoreInterop;
use crate::error::{FoundryLocalError, Result};

use super::json_stream::JsonStream;
use super::live_audio_client::LiveAudioTranscriptionSession;

/// A segment of a transcription, as returned by the OpenAI-compatible API.
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
Expand Down Expand Up @@ -196,6 +197,15 @@ impl AudioClient {
Ok(AudioTranscriptionStream::new(rx))
}

/// Create a [`LiveAudioTranscriptionSession`] for real-time audio
/// streaming transcription.
///
/// Configure the session's [`settings`](LiveAudioTranscriptionSession::settings)
/// before calling [`start`](LiveAudioTranscriptionSession::start).
pub fn create_live_transcription_session(&self) -> LiveAudioTranscriptionSession {
LiveAudioTranscriptionSession::new(&self.model_id, Arc::clone(&self.core))
}

fn validate_path(path: &str) -> Result<()> {
if path.trim().is_empty() {
return Err(FoundryLocalError::Validation {
Expand Down
Loading
Loading