diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 66553ef595..fbc1fef628 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1024,7 +1024,6 @@ dependencies = [ "buzz-persona", "buzz-sdk", "bytes", - "bzip2 0.6.1", "chrono", "ctrlc", "dirs", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 324218a49d..1ab70ece79 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -101,7 +101,6 @@ mesh-llm-events = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "f45 base64 = "0.22" sha2 = "0.11" tar = "0.4" -bzip2 = "0.6" chrono = { version = "0.4", features = ["serde"] } tauri-plugin-global-shortcut = "2" tauri-plugin-notification = "2.3.3" diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index a815bf2d06..891b1c32b3 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -33,6 +33,7 @@ pub mod pocket; pub mod preprocessing; pub mod reconnect; pub mod relay_api; +pub mod settings; pub mod state; pub mod stt; pub mod transcription; @@ -61,7 +62,8 @@ pub(super) fn drain_until_shutdown( // ── Re-exports ──────────────────────────────────────────────────────────────── -pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; +pub use settings::{get_transcription_language, set_transcription_language}; +pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, TranscriptionLanguage, VoiceInputMode}; pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline}; // ── Imports ─────────────────────────────────────────────────────────────────── @@ -792,7 +794,7 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S Ok(()) } -/// Trigger a background download of voice models (Parakeet STT + Pocket TTS). +/// Trigger a background download of voice models (multilingual Whisper STT + Pocket TTS). /// /// Returns immediately — downloads run in tokio background tasks. /// Poll `get_model_status` to track progress. diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 169ddf66c0..da76223257 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -1,7 +1,7 @@ -//! Model download manager for STT (Parakeet TDT-CTC 110M) and TTS (Pocket TTS) models. +//! Model download manager for STT (Whisper Tiny multilingual) and TTS (Pocket TTS) models. //! //! Mental model: -//! app launch → start_stt_download (background) → ~/.buzz/models/parakeet-tdt-ctc-110m-en/ +//! app launch → start_stt_download (background) → ~/.buzz/models/whisper-tiny-multilingual/ //! app launch → start_tts_download (background) → ~/.buzz/models/pocket-tts/ //! STT pipeline → is_stt_ready() → stt_model_dir() → run inference //! TTS pipeline → is_tts_ready() → tts_model_dir() → run synthesis @@ -10,12 +10,10 @@ //! is written alongside model files — if the on-disk version doesn't match the //! compiled-in version, the model is re-downloaded. //! -//! Upgrade note: an older Moonshine STT model directory at -//! `~/.buzz/models/moonshine-tiny/` is removed best-effort once the new STT -//! model finishes installing successfully. Cleanup is gated on the new model -//! being Ready, so a failed download never removes the previous on-disk model -//! during migration. If removal fails (permissions, etc.) the leftover is -//! harmless and can be removed by hand. +//! Upgrade note: older Moonshine and English-only Parakeet STT directories are +//! removed best-effort once the multilingual model finishes installing. Cleanup +//! is gated on the new model being Ready, so a failed download never removes +//! the previous on-disk model during migration. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -33,10 +31,24 @@ use sha2::{Digest, Sha256}; // To recompute hashes: download each file, run `shasum -a 256 `, and // update the corresponding constant. -/// SHA-256 hash of the STT archive -/// (sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2). -/// Computed from a known-good download. Update when upgrading model versions. -const STT_ARCHIVE_SHA256: &str = "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b"; +/// HuggingFace base URL for sherpa-onnx's multilingual Whisper Tiny export. +/// +/// Pinned to commit 65176e2deb88badc814a94058666cadccc29b61c +/// (2024-07-13) for reproducible downloads. +const STT_HF_BASE: &str = "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-tiny/resolve/\ + 65176e2deb88badc814a94058666cadccc29b61c"; + +/// SHA-256 hashes for the int8 Whisper sessions and tokenizer. +pub(super) const STT_ENCODER_FILENAME: &str = "tiny-encoder.int8.onnx"; +pub(super) const STT_DECODER_FILENAME: &str = "tiny-decoder.int8.onnx"; +pub(super) const STT_TOKENS_FILENAME: &str = "tiny-tokens.txt"; + +#[rustfmt::skip] +const STT_FILE_HASHES: &[(&str, &str)] = &[ + (STT_ENCODER_FILENAME, "d24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434"), + (STT_DECODER_FILENAME, "d2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925"), + (STT_TOKENS_FILENAME, "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126"), +]; /// HuggingFace base URL for the sherpa-onnx Pocket TTS fp32 repackage. /// @@ -86,11 +98,10 @@ const TTS_FILE_HASHES: &[(&str, &str)] = &[ // considered stale and re-downloaded. Increment when upgrading model files. /// Model manifest version for the STT model. Increment when upgrading model files. -/// Bumped from "1" → "2" alongside the migration from Moonshine Tiny to -/// Parakeet TDT-CTC 110M — the model directory name also changed, so this -/// is technically belt-and-suspenders, but it keeps the manifest semantics -/// honest (each version tag identifies one specific set of model bytes). -const STT_MODEL_VERSION: &str = "2"; +/// Bumped from "2" → "3" for the English-only Parakeet → multilingual Whisper +/// migration. The directory name also changes, keeping each version bound to +/// one exact set of model bytes. +const STT_MODEL_VERSION: &str = "3"; /// Model manifest version for Pocket TTS. Increment when upgrading model files. /// Bumped "1" → "2" when the bundled reference voice changed from KevinAHM's @@ -107,57 +118,41 @@ const MANIFEST_FILENAME: &str = ".buzz-model-manifest"; // ── Constants ───────────────────────────────────────────────────────────────── -/// Maximum expected STT archive size (200 MB — actual is ~100 MB). -const MAX_STT_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; +/// Maximum expected size per Whisper file (110 MB; decoder is ~90 MB). +const MAX_STT_FILE_BYTES: u64 = 110 * 1024 * 1024; /// Maximum expected Pocket TTS file size (400 MB per file — largest is /// `lm_main.onnx` at ~303 MB fp32). const MAX_TTS_FILE_BYTES: u64 = 400 * 1024 * 1024; -/// NVIDIA Parakeet TDT-CTC 110M (English, int8) — packaged for sherpa-onnx by -/// k2-fsa. Single ONNX file (CTC head) + tokens.txt. Avg WER ~7.5% across -/// the OpenASR-style benchmarks; ~half the WER of Moonshine Tiny at ~2× the -/// disk footprint. CTC blank-token decoding eliminates the silence/cut-audio -/// hallucination class that hurts encoder-decoder models on noisy huddle audio. -/// License: CC-BY-4.0 (attribution required — see About dialog). -const STT_DOWNLOAD_URL: &str = - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ - sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2"; - -/// Subdirectory name produced by `tar xjf` on the archive. -const STT_ARCHIVE_SUBDIR: &str = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8"; - /// Final directory name under `~/.buzz/models/`. -const STT_MODEL_DIR_NAME: &str = "parakeet-tdt-ctc-110m-en"; +const STT_MODEL_DIR_NAME: &str = "whisper-tiny-multilingual"; /// All files that must be present for the model to be considered ready. /// -/// Includes the attribution sidecar written by Buzz during install. The -/// upstream archive does not ship a license file, so readiness should require -/// the local CC-BY-4.0 attribution to travel with the cached model bytes. -const STT_EXPECTED_FILES: &[&str] = &["model.int8.onnx", "tokens.txt", STT_LICENSE_FILE_NAME]; - -/// CC-BY-4.0 §3(a)(1) attribution block written next to the STT model files -/// after install. Travels with the bytes — if a user copies the model -/// directory, the attribution comes with it. Mirrored in About/Credits. -/// -/// Covers all five §3(a)(1) bullets: creator, copyright notice, license -/// notice, warranty disclaimer reference, and URI to the source material. +/// Includes the attribution sidecar written by Buzz during install. +const STT_EXPECTED_FILES: &[&str] = &[ + STT_ENCODER_FILENAME, + STT_DECODER_FILENAME, + STT_TOKENS_FILENAME, + STT_LICENSE_FILE_NAME, +]; + +/// Attribution and license sidecar written next to the STT model files. const STT_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt"; const STT_LICENSE_TEXT: &str = "\ -NVIDIA Parakeet TDT-CTC 110M (English) -© NVIDIA Corporation. +OpenAI Whisper Tiny (multilingual) +Copyright (c) 2022 OpenAI. -Licensed under the Creative Commons Attribution 4.0 International License -(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/ +Licensed under the MIT License: +https://github.com/openai/whisper/blob/main/LICENSE -Original model: https://huggingface.co/nvidia/parakeet-tdt_ctc-110m -Converted to ONNX with int8 quantization by the sherpa-onnx project -(https://github.com/k2-fsa/sherpa-onnx); Buzz ships this conversion -unmodified. +Original model: https://huggingface.co/openai/whisper-tiny +ONNX conversion and int8 quantization: +https://huggingface.co/csukuangfj/sherpa-onnx-whisper-tiny -Provided \"AS IS\", without warranty of any kind, express or implied. See the -license text for full warranty disclaimer. +The software is provided \"AS IS\", without warranty of any kind, express or +implied. See the MIT License for the complete terms. "; // ── Pocket TTS model ────────────────────────────────────────────────────────── @@ -229,72 +224,14 @@ pub enum ModelStatus { /// Combined status for all voice models (returned to the frontend). /// -/// `stt` is the speech-to-text model status (currently Parakeet TDT-CTC 110M; -/// historically Moonshine Tiny). The field name describes the role, not the -/// specific model, so future model swaps don't ripple into the API surface. +/// `stt` is the speech-to-text model status. The field name describes the +/// role, not the specific model, so model swaps don't ripple into the API. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VoiceModelStatus { pub stt: ModelStatus, pub tts: ModelStatus, } -// ── Safe archive extraction ─────────────────────────────────────────────────── - -/// Extract a .tar.bz2 archive safely using Rust-native crates. -/// -/// The `tar` crate rejects path traversal (absolute paths, `..` components) -/// by default in `unpack()`. We add an explicit pre-check as defense-in-depth. -fn extract_archive(archive_path: &Path, dest_dir: &Path) -> Result<(), String> { - use bzip2::read::BzDecoder; - use std::fs::File; - use tar::Archive; - - let file = File::open(archive_path).map_err(|e| format!("open archive: {e}"))?; - let decoder = BzDecoder::new(file); - let mut archive = Archive::new(decoder); - - // Pre-validate: check all entries for path safety before extracting anything. - // This is defense-in-depth — the tar crate also rejects traversal in unpack(). - { - let file2 = - File::open(archive_path).map_err(|e| format!("open archive for validation: {e}"))?; - let decoder2 = BzDecoder::new(file2); - let mut check_archive = Archive::new(decoder2); - for entry in check_archive - .entries() - .map_err(|e| format!("read archive entries: {e}"))? - { - let entry = entry.map_err(|e| format!("archive entry: {e}"))?; - let path = entry.path().map_err(|e| format!("entry path: {e}"))?; - let path_str = path.to_string_lossy(); - - // Reject absolute paths. - if path.is_absolute() { - return Err(format!("archive contains absolute path: {path_str}")); - } - // Reject path traversal. - for component in path.components() { - if matches!(component, std::path::Component::ParentDir) { - return Err(format!("archive contains path traversal: {path_str}")); - } - } - // Reject symlinks. - if entry.header().entry_type().is_symlink() - || entry.header().entry_type().is_hard_link() - { - return Err(format!("archive contains symlink/hardlink: {path_str}")); - } - } - } - - // Safe to extract — all entries validated. - archive - .unpack(dest_dir) - .map_err(|e| format!("extract archive: {e}"))?; - - Ok(()) -} - // ── Hash verification ───────────────────────────────────────────────────────── /// Compute SHA-256 hash of a file. Returns lowercase hex string. @@ -610,15 +547,9 @@ impl ModelManager { /// Start a background STT model download. No-op if already ready or downloading. /// - /// Also schedules a best-effort cleanup of the legacy Moonshine model - /// directory — but **only when the new STT model is already on disk and - /// Ready**. This covers the "fast-path" upgrade scenario (new model - /// installed by a previous build, `download_stt_model` short-circuits, the - /// post-install cleanup never runs). For users mid-migration (old model - /// present, new model still downloading) we keep the old files until the - /// Parakeet install finishes, avoiding unnecessary data loss if the - /// ~100 MB download fails. The post-install path inside - /// `download_stt_model` handles cleanup once the new install reaches Ready. + /// Also schedules best-effort cleanup of legacy STT model directories, but + /// only when multilingual Whisper is Ready. This covers the fast-path where + /// a previous build already installed the new model. pub fn start_stt_download(&self, http_client: reqwest::Client) { let manager = self.clone(); self.stt.start_download( @@ -630,10 +561,10 @@ impl ModelManager { if self.stt.is_ready(&self.models_dir) { // Detached cleanup task — must not block startup. Gated above on // the new model being Ready, so a mid-migration user keeps their - // existing moonshine-tiny files until Parakeet install completes. + // previous model files until the Whisper install completes. let models_dir = self.models_dir.clone(); tauri::async_runtime::spawn(async move { - cleanup_legacy_moonshine_dir(&models_dir).await; + cleanup_legacy_stt_dirs(&models_dir).await; }); } } @@ -651,98 +582,84 @@ impl ModelManager { // ── Private download implementations ───────────────────────────────────── - /// Download, extract, and verify the STT model archive. + /// Download and verify the multilingual Whisper STT files. async fn download_stt_model(&self, http_client: reqwest::Client) -> Result<(), String> { tokio::fs::create_dir_all(&self.models_dir) .await .map_err(|e| format!("create models dir: {e}"))?; - // Temp filenames derive from the final directory name to avoid colliding - // with leftovers from any previous STT model (e.g. moonshine-tiny.*). - let archive_path = self - .models_dir - .join(format!("{STT_MODEL_DIR_NAME}.tar.bz2")); let temp_dir = self.models_dir.join(format!("{STT_MODEL_DIR_NAME}.tmp")); - - eprintln!("buzz-desktop: downloading STT model from {STT_DOWNLOAD_URL}"); - let response = fetch_url(&http_client, STT_DOWNLOAD_URL, "stt archive").await?; - - let slot = self.stt.clone(); - let bytes = download_file( - response, - &archive_path, - MAX_STT_DOWNLOAD_BYTES, - "stt archive", - |downloaded, content_length| { - if let Some(pct) = - content_length.and_then(|total| (downloaded * 89).checked_div(total)) - { - slot.set_status(ModelStatus::Downloading { - progress_percent: pct.min(89) as u8, - }); - } - }, - ) - .await?; - eprintln!("buzz-desktop: downloaded {bytes} bytes, wrote to disk"); - - // Verify archive integrity before extraction. - let hash = sha256_file(&archive_path).await?; - if hash != STT_ARCHIVE_SHA256 { - let _ = tokio::fs::remove_file(&archive_path).await; - return Err(format!( - "STT archive integrity check failed: expected {STT_ARCHIVE_SHA256}, got {hash}" - )); - } - - self.stt.set_status(ModelStatus::Downloading { - progress_percent: 90, - }); fresh_temp_dir(&temp_dir).await?; - eprintln!("buzz-desktop: extracting STT archive…"); - let (ap, td) = (archive_path.clone(), temp_dir.clone()); - tokio::task::spawn_blocking(move || extract_archive(&ap, &td)) + let total_files = STT_FILE_HASHES.len() as u32; + for (i, (filename, expected)) in STT_FILE_HASHES.iter().enumerate() { + let url = format!("{STT_HF_BASE}/{filename}"); + eprintln!("buzz-desktop: downloading Whisper STT {filename} from {url}"); + let response = fetch_url(&http_client, &url, filename) + .await + .inspect_err(|_| { + let _ = std::fs::remove_dir_all(&temp_dir); + })?; + + let dest = temp_dir.join(filename); + let slot = self.stt.clone(); + let file_index = i as u32; + let bytes = download_file( + response, + &dest, + MAX_STT_FILE_BYTES, + filename, + |downloaded, content_length| { + if let Some(total) = content_length { + if total > 0 { + let file_frac = downloaded as f64 / total as f64; + let base = (file_index as f64 / total_files as f64) * 89.0; + let span = 89.0 / total_files as f64; + let pct = (base + span * file_frac).min(89.0) as u8; + slot.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + } + }, + ) .await - .map_err(|e| format!("tar task panicked: {e}"))??; + .inspect_err(|_| { + let _ = std::fs::remove_dir_all(&temp_dir); + })?; + eprintln!("buzz-desktop: downloaded {bytes} bytes ({filename}), wrote to disk"); - let extracted_subdir = temp_dir.join(STT_ARCHIVE_SUBDIR); - if !extracted_subdir.is_dir() { - let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "expected subdir '{STT_ARCHIVE_SUBDIR}' not found after extraction" - )); - } + let actual = sha256_file(&dest).await?; + if actual != *expected { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "Whisper STT {filename} integrity check failed: expected {expected}, got {actual}" + )); + } - // Write the CC-BY-4.0 attribution sidecar before the atomic install, - // so it lands in the final model dir as part of the same rename. The - // upstream tarball ships no LICENSE/NOTICE, so we provide it ourselves - // per §3(a)(1) (license must travel with Shared material). - let license_path = extracted_subdir.join(STT_LICENSE_FILE_NAME); - if let Err(e) = tokio::fs::write(&license_path, STT_LICENSE_TEXT).await { - let _ = tokio::fs::remove_dir_all(&temp_dir).await; - let _ = tokio::fs::remove_file(&archive_path).await; - return Err(format!("write model license sidecar: {e}")); + let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; + self.stt.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); } - // verify_and_install takes the subdir (actual model files); temp_cleanup removes outer dir. + tokio::fs::write(temp_dir.join(STT_LICENSE_FILE_NAME), STT_LICENSE_TEXT) + .await + .map_err(|e| format!("write STT model license sidecar: {e}"))?; + self.stt.set_status(ModelStatus::Downloading { + progress_percent: 90, + }); + if let Err(e) = self .stt - .verify_and_install(&self.models_dir, &extracted_subdir, Some(&temp_dir)) + .verify_and_install(&self.models_dir, &temp_dir, None) .await { let _ = tokio::fs::remove_dir_all(&temp_dir).await; - let _ = tokio::fs::remove_file(&archive_path).await; return Err(e); } - let _ = tokio::fs::remove_file(&archive_path).await; - // Best-effort cleanup of the previous default STT model dir (Moonshine - // Tiny, ~70 MB). Runs only after the new install reaches Ready, so a - // failed download never removes the previous on-disk model during - // migration. The same cleanup also runs from `start_stt_download` to - // cover users who already have the new model installed. - cleanup_legacy_moonshine_dir(&self.models_dir).await; + cleanup_legacy_stt_dirs(&self.models_dir).await; eprintln!( "buzz-desktop: STT model ready at {}", @@ -890,31 +807,29 @@ pub fn is_stt_ready() -> bool { .unwrap_or(false) } -/// Best-effort cleanup of the legacy Moonshine STT model directory. -/// -/// Removes `~/.buzz/models/moonshine-tiny/` if present (~70 MB on disk). -/// Idempotent — no-op if the directory is absent. Errors are logged and -/// swallowed; the leftover is harmless and the user can remove it manually. +/// Best-effort cleanup of legacy STT model directories. /// /// This is intentionally a free function rather than a method: it has no /// dependency on `ModelManager` state, runs from both pre- and post-install /// code paths, and the call site is meant to be easy to delete in a future /// release once we're confident no users are still on the old model dir. -async fn cleanup_legacy_moonshine_dir(models_dir: &Path) { - let legacy = models_dir.join("moonshine-tiny"); - if !legacy.exists() { - return; - } - match tokio::fs::remove_dir_all(&legacy).await { - Ok(()) => eprintln!( - "buzz-desktop: removed legacy STT model dir {}", - legacy.display() - ), - Err(e) => eprintln!( - "buzz-desktop: could not remove legacy STT model dir {}: {e} \ - (harmless — remove manually to reclaim disk space)", - legacy.display() - ), +async fn cleanup_legacy_stt_dirs(models_dir: &Path) { + for dir_name in ["moonshine-tiny", "parakeet-tdt-ctc-110m-en"] { + let legacy = models_dir.join(dir_name); + if !legacy.exists() { + continue; + } + match tokio::fs::remove_dir_all(&legacy).await { + Ok(()) => eprintln!( + "buzz-desktop: removed legacy STT model dir {}", + legacy.display() + ), + Err(e) => eprintln!( + "buzz-desktop: could not remove legacy STT model dir {}: {e} \ + (harmless — remove manually to reclaim disk space)", + legacy.display() + ), + } } } @@ -934,6 +849,24 @@ pub fn is_tts_ready() -> bool { mod tests { use super::*; + #[test] + fn stt_readiness_requires_all_whisper_files_and_license() { + let temp = tempfile::tempdir().expect("tempdir"); + let slot = ModelSlot::new(STT_MODEL_DIR_NAME, STT_EXPECTED_FILES, STT_MODEL_VERSION); + let model_dir = temp.path().join(STT_MODEL_DIR_NAME); + std::fs::create_dir_all(&model_dir).expect("create model dir"); + + for file in STT_EXPECTED_FILES { + std::fs::write(model_dir.join(file), b"test").expect("write expected file"); + } + std::fs::write(model_dir.join(MANIFEST_FILENAME), STT_MODEL_VERSION).expect("manifest"); + + assert!(slot.is_ready(temp.path())); + + std::fs::remove_file(model_dir.join(STT_DECODER_FILENAME)).expect("remove decoder"); + assert!(!slot.is_ready(temp.path())); + } + #[test] fn tts_readiness_requires_license_sidecar() { let temp = tempfile::tempdir().expect("tempdir"); diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index ceccedd8b6..03b8b30916 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -110,7 +110,15 @@ pub(crate) async fn maybe_start_stt_pipeline( // transcription task's next POST sees a stale generation and exits. // Take the old pipeline OUT of the lock before dropping — Drop joins // the worker thread (~200ms) and must not block under the mutex. - let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt, old_stt) = { + let ( + tts_active, + tts_cancel, + agent_pubkeys_arc, + session_gen, + ptt_active_for_stt, + transcription_language, + old_stt, + ) = { let mut hs = state.huddle()?; // Invalidate any existing transcription task before replacing the pipeline. if hs.stt_pipeline.is_some() { @@ -131,6 +139,7 @@ pub(crate) async fn maybe_start_stt_pipeline( Arc::clone(&hs.agent_pubkeys), Arc::clone(&hs.session_generation), ptt, + hs.transcription_language, old, ) }; @@ -138,7 +147,13 @@ pub(crate) async fn maybe_start_stt_pipeline( drop(old_stt); let constructed = tokio::task::spawn_blocking(move || { - stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt) + stt::SttPipeline::new( + model_dir, + transcription_language, + tts_active, + tts_cancel, + ptt_active_for_stt, + ) }) .await; let (pipeline, text_rx) = match constructed { diff --git a/desktop/src-tauri/src/huddle/settings.rs b/desktop/src-tauri/src/huddle/settings.rs new file mode 100644 index 0000000000..cd4be9fc1b --- /dev/null +++ b/desktop/src-tauri/src/huddle/settings.rs @@ -0,0 +1,56 @@ +//! User-selectable Huddle speech settings. + +use tauri::State; + +use crate::app_state::AppState; + +use super::{pipeline::maybe_start_stt_pipeline, HuddlePhase, TranscriptionLanguage}; + +/// Set the language used by the local multilingual Whisper recognizer. +/// +/// The recognizer captures its language at construction time, so changing the +/// selection during an active transcribed huddle restarts only the STT pipeline. +#[tauri::command] +pub async fn set_transcription_language( + language: TranscriptionLanguage, + state: State<'_, AppState>, +) -> Result<(), String> { + let needs_restart = { + let mut hs = state.huddle()?; + let old_language = hs.transcription_language; + hs.transcription_language = language; + old_language != language + && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) + && hs.stt_pipeline.is_some() + && hs.transcription_enabled + }; + + let restart_result = if needs_restart { + let eph_id = { + let hs = state.huddle()?; + hs.ephemeral_channel_id.clone() + }; + if let Some(eph_id) = eph_id { + maybe_start_stt_pipeline(&state, &eph_id) + .await + .map(|_| ()) + .map_err(|e| format!("restart transcription for language change: {e}")) + } else { + Ok(()) + } + } else { + Ok(()) + }; + + state.emit_huddle_state_changed(); + restart_result +} + +/// Return the language used by the local multilingual Whisper recognizer. +#[tauri::command] +pub fn get_transcription_language( + state: State<'_, AppState>, +) -> Result { + let hs = state.huddle()?; + Ok(hs.transcription_language) +} diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index 876c2d688b..a71f8c9e7f 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -28,6 +28,24 @@ pub enum VoiceInputMode { VoiceActivity, } +/// Language passed to the local multilingual Whisper recognizer. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum TranscriptionLanguage { + German, + #[default] + English, +} + +impl TranscriptionLanguage { + pub(crate) const fn whisper_code(self) -> &'static str { + match self { + Self::German => "de", + Self::English => "en", + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum HuddlePhase { @@ -80,6 +98,8 @@ pub struct HuddleState { pub tts_enabled: bool, /// Whether STT transcript posting is enabled for this huddle. pub transcription_enabled: bool, + /// Language used by the local multilingual Whisper recognizer. + pub transcription_language: TranscriptionLanguage, /// Shared flag: true while TTS is playing audio. /// Shared with the STT pipeline for barge-in / echo gating. #[serde(skip)] @@ -157,6 +177,7 @@ impl Clone for HuddleState { is_creator: self.is_creator, tts_enabled: self.tts_enabled, transcription_enabled: self.transcription_enabled, + transcription_language: self.transcription_language, tts_active: Arc::clone(&self.tts_active), tts_cancel: Arc::clone(&self.tts_cancel), tts_starting: Arc::clone(&self.tts_starting), @@ -184,6 +205,7 @@ impl Default for HuddleState { is_creator: false, tts_enabled: true, transcription_enabled: false, + transcription_language: TranscriptionLanguage::default(), tts_active: Arc::new(AtomicBool::new(false)), tts_cancel: Arc::new(AtomicBool::new(false)), tts_starting: Arc::new(AtomicBool::new(false)), @@ -197,13 +219,16 @@ impl Default for HuddleState { } impl HuddleState { - /// Reset to default state while preserving the session generation counter. + /// Reset to default state while preserving user voice preferences and the + /// session generation counter. /// Used by start_huddle rollback, join_huddle rollback, and teardown_huddle /// to invalidate in-flight transcription tasks without losing the generation. pub(crate) fn reset_preserving_generation(&mut self) { let gen = Arc::clone(&self.session_generation); + let transcription_language = self.transcription_language; *self = Self::default(); self.session_generation = gen; + self.transcription_language = transcription_language; } } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..959868bf9a 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -9,7 +9,7 @@ //! → stt_worker thread //! rubato: 48 kHz → 16 kHz mono //! earshot VAD: accumulate speech frames -//! sherpa-onnx Parakeet TDT-CTC 110M: transcribe on silence +//! sherpa-onnx Whisper Tiny multilingual: transcribe on silence //! → text_rx [mpsc channel] //! → tokio task (start_stt_pipeline) //! builds kind:9 event → relay @@ -31,6 +31,11 @@ use std::{ use tokio::sync::mpsc as tokio_mpsc; +use super::{ + models::{STT_DECODER_FILENAME, STT_ENCODER_FILENAME, STT_TOKENS_FILENAME}, + TranscriptionLanguage, +}; + // ── Public pipeline handle ──────────────────────────────────────────────────── /// Bounded audio queue capacity. @@ -58,6 +63,14 @@ pub struct SttPipeline { thread: Option>, } +struct SttWorkerConfig { + model_dir: PathBuf, + language: TranscriptionLanguage, + tts_active: Arc, + tts_cancel: Option>, + ptt_active: Option>, +} + impl SttPipeline { /// Spawn the pipeline thread. /// @@ -85,6 +98,7 @@ impl SttPipeline { /// thread on every `recv_timeout` call). pub fn new( model_dir: PathBuf, + language: TranscriptionLanguage, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, @@ -94,21 +108,16 @@ impl SttPipeline { let shutdown = Arc::new(AtomicBool::new(false)); let shutdown_worker = Arc::clone(&shutdown); - let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); - let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); + let worker_config = SttWorkerConfig { + model_dir, + language, + tts_active, + tts_cancel: tts_cancel.as_ref().map(Arc::clone), + ptt_active: ptt_active.as_ref().map(Arc::clone), + }; let handle = thread::Builder::new() .name("stt-worker".into()) - .spawn(move || { - stt_worker( - model_dir, - audio_rx, - text_tx, - shutdown_worker, - tts_active, - tts_cancel_worker, - ptt_active_worker, - ) - }) + .spawn(move || stt_worker(worker_config, audio_rx, text_tx, shutdown_worker)) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; let pipeline = Self { @@ -193,7 +202,7 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); /// Number of ONNX Runtime intra-op threads used by the offline recognizer. /// /// Held at 1 (conservative) until we have a local A/B on real huddle audio. -/// Sherpa-onnx's Parakeet example uses 2 and most published RTF numbers are +/// Sherpa-onnx examples commonly use 2 and most published RTF numbers are /// at 2 threads on x86_64 server class hardware, but the encoder runs only /// on VAD chunk boundaries on a dedicated thread, so the threading knob /// trades worker latency against potential oversubscription with the audio @@ -202,14 +211,19 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(50); const STT_NUM_THREADS: i32 = 1; fn stt_worker( - model_dir: PathBuf, + config: SttWorkerConfig, audio_rx: Receiver>, text_tx: tokio_mpsc::Sender, shutdown: Arc, - tts_active: Arc, - tts_cancel: Option>, - ptt_active: Option>, ) { + let SttWorkerConfig { + model_dir, + language, + tts_active, + tts_cancel, + ptt_active, + } = config; + // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -228,16 +242,14 @@ fn stt_worker( // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── // - // Parakeet TDT-CTC 110M ships as a single `model.int8.onnx` (CTC head) plus - // `tokens.txt`. sherpa-onnx infers the model family from which inner config - // has a `model` path set, so we don't need to set `model_type` explicitly. - // (See rust-api-examples/parakeet_tdt_ctc_simulate_streaming_microphone.rs - // in k2-fsa/sherpa-onnx.) - use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; - - let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); - if !tokens_path.exists() || !model_path.exists() { + // Whisper Tiny ships as separate encoder/decoder sessions plus tokens. + // sherpa-onnx infers the model family from the populated Whisper config. + use sherpa_onnx::OfflineRecognizer; + + let tokens_path = model_dir.join(STT_TOKENS_FILENAME); + let encoder_path = model_dir.join(STT_ENCODER_FILENAME); + let decoder_path = model_dir.join(STT_DECODER_FILENAME); + if !tokens_path.exists() || !encoder_path.exists() || !decoder_path.exists() { eprintln!( "buzz-desktop: STT model not found at {} — STT disabled", model_dir.display() @@ -246,13 +258,7 @@ fn stt_worker( return; } - let mut cfg = OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); - cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); - cfg.model_config.num_threads = STT_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every VAD chunk. - cfg.model_config.debug = false; + let cfg = build_recognizer_config(&model_dir, language); let recognizer = match OfflineRecognizer::create(&cfg) { Some(r) => r, @@ -359,6 +365,38 @@ fn stt_worker( // the user has "left." Losing the last partial utterance is acceptable. } +fn build_recognizer_config( + model_dir: &std::path::Path, + language: TranscriptionLanguage, +) -> sherpa_onnx::OfflineRecognizerConfig { + let mut cfg = sherpa_onnx::OfflineRecognizerConfig::default(); + cfg.model_config.whisper.encoder = Some( + model_dir + .join(STT_ENCODER_FILENAME) + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.whisper.decoder = Some( + model_dir + .join(STT_DECODER_FILENAME) + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.whisper.language = Some(language.whisper_code().to_string()); + cfg.model_config.whisper.task = Some("transcribe".to_string()); + cfg.model_config.tokens = Some( + model_dir + .join(STT_TOKENS_FILENAME) + .to_string_lossy() + .into_owned(), + ); + cfg.model_config.num_threads = STT_NUM_THREADS; + // Explicit — defaults are not part of the API contract, and noisy debug + // logging in release builds would be expensive on every VAD chunk. + cfg.model_config.debug = false; + cfg +} + /// Resample a mono 48 kHz chunk to 16 kHz using rubato. /// Returns the resampled samples (may be empty on error). fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec { @@ -565,3 +603,102 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. use super::drain_until_shutdown; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn whisper_config_uses_selected_language_and_transcribe_task() { + let model_dir = std::path::Path::new("/tmp/buzz-whisper-test"); + + let german = build_recognizer_config(model_dir, TranscriptionLanguage::German); + assert_eq!(german.model_config.whisper.language.as_deref(), Some("de")); + assert_eq!( + german.model_config.whisper.task.as_deref(), + Some("transcribe") + ); + assert_eq!( + german.model_config.whisper.encoder.as_deref(), + Some( + model_dir + .join(STT_ENCODER_FILENAME) + .to_string_lossy() + .as_ref() + ) + ); + assert_eq!( + german.model_config.whisper.decoder.as_deref(), + Some( + model_dir + .join(STT_DECODER_FILENAME) + .to_string_lossy() + .as_ref() + ) + ); + assert_eq!( + german.model_config.tokens.as_deref(), + Some( + model_dir + .join(STT_TOKENS_FILENAME) + .to_string_lossy() + .as_ref() + ) + ); + + let english = build_recognizer_config(model_dir, TranscriptionLanguage::English); + assert_eq!(english.model_config.whisper.language.as_deref(), Some("en")); + } + + #[test] + #[ignore = "requires downloaded Whisper model files and a 16 kHz mono WAV"] + fn german_whisper_smoke_test() { + use rodio::Source; + + let model_dir = std::env::var("BUZZ_STT_SMOKE_MODEL_DIR") + .expect("BUZZ_STT_SMOKE_MODEL_DIR must point to the Whisper model directory"); + let wav_path = std::env::var("BUZZ_STT_SMOKE_WAV") + .expect("BUZZ_STT_SMOKE_WAV must point to a 16 kHz mono WAV"); + let expected = std::env::var("BUZZ_STT_SMOKE_EXPECTED") + .unwrap_or_else(|_| "deutscher".to_string()) + .to_lowercase(); + + let decoder = + rodio::Decoder::try_from(std::fs::File::open(&wav_path).expect("open smoke-test WAV")) + .expect("decode smoke-test WAV"); + assert_eq!(decoder.channels().get(), 1, "smoke-test WAV must be mono"); + assert_eq!( + decoder.sample_rate().get(), + 16_000, + "smoke-test WAV must be 16 kHz" + ); + let samples: Vec = decoder.collect(); + + let cfg = build_recognizer_config( + std::path::Path::new(&model_dir), + TranscriptionLanguage::German, + ); + let recognizer = + sherpa_onnx::OfflineRecognizer::create(&cfg).expect("create Whisper recognizer"); + let stream = recognizer.create_stream(); + stream.accept_waveform(16_000, &samples); + + let started = std::time::Instant::now(); + recognizer.decode(&stream); + let elapsed = started.elapsed(); + let text = stream + .get_result() + .map(|result| result.text) + .unwrap_or_default(); + + eprintln!( + "German Whisper smoke test: {:.2}s audio in {:.2}s ({text})", + samples.len() as f64 / 16_000.0, + elapsed.as_secs_f64() + ); + assert!( + text.to_lowercase().contains(&expected), + "transcript did not contain {expected:?}: {text:?}" + ); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 5346791ccf..be63507bdb 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -42,8 +42,9 @@ use huddle::audio_output::{ use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, - end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, get_voice_input_mode, - join_huddle, leave_huddle, push_audio_pcm, set_huddle_transcription_enabled, set_tts_enabled, + end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, + get_transcription_language, get_voice_input_mode, join_huddle, leave_huddle, push_audio_pcm, + set_huddle_transcription_enabled, set_transcription_language, set_tts_enabled, set_voice_input_mode, speak_agent_message, start_huddle, start_stt_pipeline, }; use managed_agents::{ @@ -885,6 +886,8 @@ pub fn run() { get_huddle_agent_pubkeys, set_voice_input_mode, get_voice_input_mode, + set_transcription_language, + get_transcription_language, list_audio_output_devices, set_audio_output_device, get_audio_output_device, diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 35d660e475..7a584e1b76 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -3,6 +3,8 @@ import { listen } from "@tauri-apps/api/event"; import { Bot, Captions, + Check, + Languages, MessageSquareText, PhoneOff, SmilePlus, @@ -48,6 +50,7 @@ type HuddleState = { agent_pubkeys: string[]; tts_enabled: boolean; transcription_enabled: boolean; + transcription_language: "german" | "english"; is_creator: boolean; voice_input_mode: "push_to_talk" | "voice_activity"; }; @@ -529,6 +532,21 @@ export function HuddleBar({ } } + async function handleTranscriptionLanguage( + language: HuddleState["transcription_language"], + ) { + setTranscriptError(null); + try { + await invoke("set_transcription_language", { language }); + const s = await invoke("get_huddle_state"); + setState(s); + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + setTranscriptError(`Language change failed: ${message}`); + console.error("Failed to change transcription language:", e); + } + } + function handleOpenThread() { if (!parentChannelId || !huddleThreadEventId) return; onOpenThread?.(parentChannelId, huddleThreadEventId); @@ -774,6 +792,60 @@ export function HuddleBar({ + + + + + + + Transcript language + + {( + [ + ["german", "Deutsch"], + ["english", "English"], + ] as const + ).map(([language, label]) => { + const selected = barState.transcription_language === language; + return ( + + ); + })} + + +