From 953b1309441898df08e7f8c26918a67fb1a92028 Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Wed, 8 Jul 2026 14:33:36 +0800 Subject: [PATCH] Fix long Windows local ASR transcription --- .../src/asr/local/foundry_provider.rs | 90 +++++++++++++++---- .../src/asr/local/sherpa_provider.rs | 25 ++++++ openless-all/app/src-tauri/src/asr/whisper.rs | 4 +- openless-all/app/src-tauri/src/coordinator.rs | 62 +++++++++---- .../src-tauri/src/coordinator/dictation.rs | 24 +++-- .../src-tauri/src/coordinator/qa_session.rs | 38 ++++---- 6 files changed, 181 insertions(+), 62 deletions(-) diff --git a/openless-all/app/src-tauri/src/asr/local/foundry_provider.rs b/openless-all/app/src-tauri/src/asr/local/foundry_provider.rs index ef09d88d..bb01846b 100644 --- a/openless-all/app/src-tauri/src/asr/local/foundry_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/foundry_provider.rs @@ -23,6 +23,10 @@ use crate::asr::RawTranscript; #[cfg(target_os = "windows")] use super::foundry_runtime::FoundryLocalRuntime; +/// Foundry Local Whisper 属于 Whisper 系模型，原生解码窗口约 30s。每次 SDK +/// 请求保持在窗口内，再由 OpenLess 合并分片文本，避免长听写只返回第一段。 +const FOUNDRY_WHISPER_CHUNK_LIMIT_MS: u64 = 30_000; + pub struct FoundryLocalWhisperAsr { #[cfg(target_os = "windows")] runtime: Arc, @@ -70,6 +74,12 @@ impl FoundryLocalWhisperAsr { self.language_hint.as_deref() } + /// 当前缓冲音频时长（毫秒）。Coordinator 在 transcribe() 调用前读取， + /// 用来给 Foundry Local Whisper 计算动态超时。不消费缓冲。 + pub fn buffer_duration_ms(&self) -> u64 { + pcm_duration_ms(&self.buffer.lock()) + } + pub async fn transcribe(&self, audio_timeout: std::time::Duration) -> Result { let cancel_generation = self.cancel_generation.load(Ordering::SeqCst); let pcm = self.buffer.lock().clone(); @@ -108,26 +118,45 @@ impl FoundryLocalWhisperAsr { #[cfg(target_os = "windows")] { - let wav_file = TempWavFile::create(pcm)?; - let text = self - .runtime - .transcribe_audio_file( - &self.model_alias, - &self.runtime_source, - self.language_hint(), - wav_file.path(), - audio_timeout, - ) - .await - .with_context(|| { - format!( - "transcribe audio file with Foundry Local Whisper model {}", - self.model_alias + let chunks = crate::asr::whisper::split_pcm_by_duration( + pcm, + Some(FOUNDRY_WHISPER_CHUNK_LIMIT_MS), + ); + if chunks.len() > 1 { + log::info!( + "[foundry-asr] splitting {:.2}s audio into {} chunks (limit={}ms)", + duration_ms as f64 / 1000.0, + chunks.len(), + FOUNDRY_WHISPER_CHUNK_LIMIT_MS + ); + } + + let mut texts = Vec::with_capacity(chunks.len()); + for (index, chunk) in chunks.iter().enumerate() { + let wav_file = TempWavFile::create(chunk)?; + let text = self + .runtime + .transcribe_audio_file( + &self.model_alias, + &self.runtime_source, + self.language_hint(), + wav_file.path(), + audio_timeout, ) - })?; + .await + .with_context(|| { + format!( + "transcribe Foundry Local Whisper chunk {}/{} with model {}", + index + 1, + chunks.len(), + self.model_alias + ) + })?; + texts.push(trim_transcript_text(&text)); + } Ok(RawTranscript { - text: trim_transcript_text(&text), + text: crate::asr::whisper::join_transcript_chunks(&texts), duration_ms, }) } @@ -293,6 +322,33 @@ mod tests { assert_eq!(&wav[44..], &[0x01, 0x00, 0xff, 0x7f]); } + #[test] + fn foundry_provider_splits_long_pcm_at_whisper_window() { + let pcm = vec![0u8; 32_000 * 65]; + let chunks = crate::asr::whisper::split_pcm_by_duration( + &pcm, + Some(super::FOUNDRY_WHISPER_CHUNK_LIMIT_MS), + ); + + assert_eq!(chunks.len(), 3); + assert_eq!(chunks[0].len(), 32_000 * 30); + assert_eq!(chunks[1].len(), 32_000 * 30); + assert_eq!(chunks[2].len(), 32_000 * 5); + } + + #[test] + fn foundry_provider_reports_buffer_duration_without_consuming() { + #[cfg(target_os = "windows")] + let (provider, _) = test_provider(); + #[cfg(not(target_os = "windows"))] + let provider = test_provider(); + + provider.consume_pcm_chunk(&vec![0u8; 32_000]); + + assert_eq!(provider.buffer_duration_ms(), 1000); + assert_eq!(provider.buffer.lock().len(), 32_000); + } + #[cfg(target_os = "windows")] #[test] fn foundry_provider_temp_wav_drop_removes_file() { diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index 210fd2a3..6ea1450b 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -103,6 +103,19 @@ impl SherpaOnnxAsr { self.language_hint.as_deref() } + /// 当前缓冲音频时长（毫秒）。Offline 读 PCM buffer；Online 读 worker 已接收 + /// 的 PCM 字节数。不消费缓冲。 + pub fn buffer_duration_ms(&self) -> u64 { + match &self.mode { + SherpaProviderMode::Offline { buffer } => pcm_duration_ms(&buffer.lock()), + SherpaProviderMode::Online { worker } => worker + .lock() + .as_ref() + .map(|worker| pcm_duration_ms_from_bytes(worker.audio_bytes.load(Ordering::SeqCst))) + .unwrap_or(0), + } + } + pub async fn transcribe(&self, audio_timeout: Duration) -> Result { match &self.mode { SherpaProviderMode::Offline { buffer } => { @@ -390,6 +403,18 @@ mod tests { } } + #[test] + fn offline_buffer_duration_reports_16k_pcm_duration_without_consuming() { + let provider = make_provider(); + provider.consume_pcm_chunk(&vec![0u8; 32_000]); + + assert_eq!(provider.buffer_duration_ms(), 1000); + match &provider.mode { + SherpaProviderMode::Offline { buffer } => assert_eq!(buffer.lock().len(), 32_000), + SherpaProviderMode::Online { .. } => panic!("expected offline provider"), + } + } + #[tokio::test] async fn empty_buffer_transcribe_returns_empty_transcript() { let provider = make_provider(); diff --git a/openless-all/app/src-tauri/src/asr/whisper.rs b/openless-all/app/src-tauri/src/asr/whisper.rs index 06b75a96..05744782 100644 --- a/openless-all/app/src-tauri/src/asr/whisper.rs +++ b/openless-all/app/src-tauri/src/asr/whisper.rs @@ -294,7 +294,7 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 { super::pcm::pcm_duration_ms(pcm) } -fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option) -> Vec<&[u8]> { +pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option) -> Vec<&[u8]> { let Some(max_chunk_duration_ms) = max_chunk_duration_ms else { return vec![pcm]; }; @@ -328,7 +328,7 @@ fn transcription_url(base_url: &str) -> Result { Ok(url.to_string()) } -fn join_transcript_chunks(chunks: &[String]) -> String { +pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String { let mut joined = String::new(); for chunk in chunks.iter().map(|chunk| chunk.trim()) { if chunk.is_empty() { diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 99b07d86..b0e3e56a 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -1575,15 +1575,21 @@ impl Coordinator { .map_err(|_| "重新转录超时".to_string())? .map_err(|e| e.to_string())?, #[cfg(target_os = "windows")] - ActiveAsr::FoundryLocalWhisper(local) => local - .transcribe(foundry_audio_transcribe_timeout_duration()) - .await - .map_err(|e| e.to_string())?, + ActiveAsr::FoundryLocalWhisper(local) => { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + local + .transcribe(foundry_audio_transcribe_timeout(audio_secs)) + .await + .map_err(|e| e.to_string())? + } #[cfg(target_os = "windows")] - ActiveAsr::SherpaOnnxLocal(local) => local - .transcribe(sherpa_audio_transcribe_timeout_duration()) - .await - .map_err(|e| e.to_string())?, + ActiveAsr::SherpaOnnxLocal(local) => { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + local + .transcribe(sherpa_audio_transcribe_timeout(audio_secs)) + .await + .map_err(|e| e.to_string())? + } #[cfg(target_os = "macos")] ActiveAsr::Local(local) => { let dur = @@ -2379,17 +2385,31 @@ mod tests { assert!(!asr_transcribe_uses_global_timeout(&active_asr)); } - #[cfg(target_os = "windows")] #[test] - fn foundry_audio_transcribe_timeout_is_separate_from_prepare() { - let timeout = foundry_audio_transcribe_timeout_duration(); - + fn foundry_timeout_floors_at_global_timeout_for_short_audio() { assert_eq!( - timeout, + foundry_audio_transcribe_timeout(5.0), std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS) ); } + #[test] + fn foundry_timeout_scales_with_audio_duration() { + // 65s 录音：65 × 1.0 = 65，+20 = 85s。长音频不再撞 30s 墙。 + assert_eq!( + foundry_audio_transcribe_timeout(65.0), + std::time::Duration::from_secs(85) + ); + } + + #[test] + fn sherpa_timeout_scales_with_audio_duration() { + assert_eq!( + sherpa_audio_transcribe_timeout(65.0), + std::time::Duration::from_secs(85) + ); + } + #[test] fn local_qwen_timeout_floors_at_global_timeout_for_short_audio() { // 5s 录音：5 × 0.6 = 3, +10 = 13, max(30) = 30。短录音兜底。 @@ -3021,9 +3041,11 @@ const POST_SESSION_COOLDOWN_MS: u64 = 600; /// 网络超时预算；只在 ASR 自身超时机制失效时作为最后的防线触发。 const COORDINATOR_GLOBAL_TIMEOUT_SECS: u64 = 30; -#[cfg(target_os = "windows")] -fn foundry_audio_transcribe_timeout_duration() -> std::time::Duration { - std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS) +fn foundry_audio_transcribe_timeout(audio_secs: f64) -> std::time::Duration { + let secs = ((audio_secs * 1.0).ceil() as u64) + .saturating_add(20) + .max(COORDINATOR_GLOBAL_TIMEOUT_SECS); + std::time::Duration::from_secs(secs) } /// 本地 Qwen3-ASR 的动态转写超时。固定 15 秒在长录音（≥ 30s）+ 慢机器 @@ -3050,9 +3072,11 @@ fn whisper_transcribe_timeout(audio_secs: f64) -> std::time::Duration { /// sherpa-onnx offline batch 暂与 Foundry 同档；后续按 Windows 真机 CPU/模型 /// 实测结果再调整。 -#[cfg(target_os = "windows")] -fn sherpa_audio_transcribe_timeout_duration() -> std::time::Duration { - std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS) +fn sherpa_audio_transcribe_timeout(audio_secs: f64) -> std::time::Duration { + let secs = ((audio_secs * 1.0).ceil() as u64) + .saturating_add(20) + .max(COORDINATOR_GLOBAL_TIMEOUT_SECS); + std::time::Duration::from_secs(secs) } pub(crate) fn validate_llm_endpoint(raw: &str) -> anyhow::Result<()> { diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index 01384017..32913d56 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -2247,10 +2247,14 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(foundry_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = foundry_audio_transcribe_timeout(audio_secs); + log::info!( + "[coord] Foundry Local Whisper transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match local.transcribe(timeout_duration).await { Ok(r) => { schedule_foundry_local_asr_release( inner, @@ -2285,10 +2289,14 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { #[cfg(target_os = "windows")] ActiveAsr::SherpaOnnxLocal(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(sherpa_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs); + log::info!( + "[coord] sherpa-onnx transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match local.transcribe(timeout_duration).await { Ok(r) => { schedule_sherpa_onnx_release( inner, diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index 094a1c6c..34c59be7 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -263,10 +263,9 @@ pub(super) async fn transcribe_overlay_dictation_asr( #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(foundry_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = foundry_audio_transcribe_timeout(audio_secs); + match local.transcribe(timeout_duration).await { Ok(raw) => { schedule_foundry_local_asr_release( _inner, @@ -286,10 +285,9 @@ pub(super) async fn transcribe_overlay_dictation_asr( #[cfg(target_os = "windows")] ActiveAsr::SherpaOnnxLocal(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(sherpa_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs); + match local.transcribe(timeout_duration).await { Ok(raw) => { schedule_sherpa_onnx_release( _inner, @@ -826,10 +824,14 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(foundry_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = foundry_audio_transcribe_timeout(audio_secs); + log::info!( + "[coord] QA Foundry Local Whisper transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match local.transcribe(timeout_duration).await { Ok(r) => { schedule_foundry_local_asr_release(inner, AsrReleaseSession::Qa(qa_session_id)); r @@ -852,10 +854,14 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { #[cfg(target_os = "windows")] ActiveAsr::SherpaOnnxLocal(local) => { debug_assert!(!uses_global_timeout); - match local - .transcribe(sherpa_audio_transcribe_timeout_duration()) - .await - { + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = sherpa_audio_transcribe_timeout(audio_secs); + log::info!( + "[coord] QA sherpa-onnx transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match local.transcribe(timeout_duration).await { Ok(r) => { schedule_sherpa_onnx_release(inner, AsrReleaseSession::Qa(qa_session_id)); r