From 2f9543f9b54afcc454abad972cfed87de5492121 Mon Sep 17 00:00:00 2001 From: yeoleobun Date: Fri, 15 Aug 2025 09:13:02 +0800 Subject: [PATCH] fix interuption --- src/event.rs | 12 +++++------- src/handler/tests/webrtc_test.rs | 3 ++- src/media/track/tts.rs | 27 ++++++++++++++++----------- src/synthesis/aliyun.rs | 4 +--- src/synthesis/mod.rs | 5 +---- src/synthesis/tencent_cloud.rs | 7 +++---- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/event.rs b/src/event.rs index d7ed019b..6dd45caf 100644 --- a/src/event.rs +++ b/src/event.rs @@ -102,13 +102,11 @@ pub enum SessionEvent { #[serde(rename = "trackId")] track_id: String, timestamp: u64, - position: u64, // current playback position at the time of interruption - }, - OnInterrupt { - subtitle: String, - position: u32, - total_duration: u32, // ms - current: u32, // current timestamp of total_duration + subtitle: Option, // current tts text + position: Option, // word index in subtitle + #[serde(rename = "totalDuration")] + total_duration: u32, // whole tts duration + current: u32, // elapsed time since start of tts }, AsrFinal { #[serde(rename = "trackId")] diff --git a/src/handler/tests/webrtc_test.rs b/src/handler/tests/webrtc_test.rs index a9976658..86796871 100644 --- a/src/handler/tests/webrtc_test.rs +++ b/src/handler/tests/webrtc_test.rs @@ -562,11 +562,12 @@ async fn test_tts_interrupt_with_config(tts_config: SynthesisOption, text: &str) .await .unwrap(); } - SessionEvent::OnInterrupt { + SessionEvent::Interruption { subtitle, position, total_duration, current, + .. } => { interrupted_clone.store(true, Ordering::Relaxed); info!( diff --git a/src/media/track/tts.rs b/src/media/track/tts.rs index e09e6ad5..8b7db895 100644 --- a/src/media/track/tts.rs +++ b/src/media/track/tts.rs @@ -173,6 +173,8 @@ impl Track for TtsTrack { let synthesize_done = Arc::new(AtomicBool::new(false)); let synthesize_done_clone = synthesize_done.clone(); let session_id = self.session_id.clone(); + let current_text = Arc::new(Mutex::new(None)); + let current_text_clone = current_text.clone(); let subtitles = Arc::new(Mutex::new(Vec::::new())); let subtitles_clone = subtitles.clone(); let total_audio_len = Arc::new(AtomicUsize::new(0)); @@ -181,6 +183,10 @@ impl Track for TtsTrack { let mut last_play_id = None; while let Some(command) = command_rx.recv().await { let text = command.text; + { + let mut current_text = current_text_clone.lock().await; + *current_text = Some(text.clone()); + } let play_id = command.play_id; let mut option = command.option; if option.speaker.is_none() { @@ -230,7 +236,6 @@ impl Track for TtsTrack { let duration = bytes_size_to_duration(audio.len(), sample_rate); total_audio_len_clone.store(audio.len(), Ordering::Relaxed); subtitles.push(TTSSubtitle::new( - &text, 0, duration, 0, @@ -438,26 +443,26 @@ impl Track for TtsTrack { let remaining_size = buffer.lock().await.len() * 2; debug!(session_id, "total_size: {} remaining_size: {}", total_size, remaining_size); let sended_size = total_size - remaining_size; - let mut text = "".into(); - let mut position = 0; - let mut current = bytes_size_to_duration(sended_size, sample_rate); - let total_duration = bytes_size_to_duration(total_size, sample_rate); + let text = current_text.lock().await.take(); + let mut position = None; + let current = bytes_size_to_duration(sended_size, sample_rate); + let total_duration = bytes_size_to_duration(total_size, sample_rate); let subtitles = subtitles.lock().await; - debug!("subtitles: {:?}", subtitles); for subtitle in subtitles.iter().rev() { if subtitle.begin_time <= current { - text = subtitle.text.clone(); - position = subtitle.begin_index; - current = subtitle.begin_time; + position = Some(subtitle.begin_index); break; } } - let _ = event_sender_clone.send(SessionEvent::OnInterrupt { + let event = SessionEvent::Interruption { + track_id: track_id.clone(), + timestamp: crate::get_timestamp(), subtitle: text, position, total_duration, current, - }); + }; + let _ = event_sender_clone.send(event); } } event_sender_clone diff --git a/src/synthesis/aliyun.rs b/src/synthesis/aliyun.rs index 61039ef0..dea05e3e 100644 --- a/src/synthesis/aliyun.rs +++ b/src/synthesis/aliyun.rs @@ -233,12 +233,11 @@ impl AliyunTTSState { self.text .chars() .enumerate() - .position(|(i, _)| i == usage as usize) + .position(|(i, _)| i >= usage as usize) .unwrap_or(default as usize) as u32 } fn move_forward(&mut self, current_usage: u32) -> Vec { - let subtitle = self.text[self.last_usage as usize..current_usage as usize].to_string(); let begin_index = self.usage_to_index(self.last_usage, 0); let end_index = self.usage_to_index(current_usage, self.text.len() as u32); let begin_time = self.size_to_time(self.last_size); @@ -246,7 +245,6 @@ impl AliyunTTSState { self.last_size = self.current_size; self.last_usage = current_usage; vec![TTSSubtitle::new( - &subtitle, begin_time, end_time, begin_index, diff --git a/src/synthesis/mod.rs b/src/synthesis/mod.rs index 718317af..92c64ff0 100644 --- a/src/synthesis/mod.rs +++ b/src/synthesis/mod.rs @@ -119,23 +119,20 @@ pub enum TTSEvent { #[allow(dead_code)] #[derive(Debug)] pub struct TTSSubtitle { - pub text: String, pub begin_time: u32, pub end_time: u32, pub begin_index: u32, - end_index: u32, + pub end_index: u32, } impl TTSSubtitle { pub fn new( - text: &str, begin_time: u32, end_time: u32, begin_index: u32, end_index: u32, ) -> Self { Self { - text: text.to_string(), begin_time, end_time, begin_index, diff --git a/src/synthesis/tencent_cloud.rs b/src/synthesis/tencent_cloud.rs index b3a98654..318420b4 100644 --- a/src/synthesis/tencent_cloud.rs +++ b/src/synthesis/tencent_cloud.rs @@ -39,7 +39,6 @@ struct WebSocketResult { #[derive(Debug, Deserialize)] #[serde(rename_all = "PascalCase")] struct Subtitle { - text: String, begin_time: u32, end_time: u32, begin_index: u32, @@ -49,7 +48,6 @@ struct Subtitle { impl From<&Subtitle> for TTSSubtitle { fn from(subtitle: &Subtitle) -> Self { TTSSubtitle::new( - &subtitle.text, subtitle.begin_time, subtitle.end_time, subtitle.begin_index, @@ -197,9 +195,10 @@ impl TencentCloudTtsClient { if response.code != 0 { return Some(Err(anyhow!( - "Tencent TTS faild, Session: {}, error: {}", + "Tencent TTS faild, Session: {}, error: {}, text: {}", session_id, - response.message + response.message, + text ))); }