From 9a85d656d6dbd9e3d0d6bdce1e9a7d07a3b63a9d Mon Sep 17 00:00:00 2001 From: Anthony Ronning <101225832+AnthonyRonning@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:35:12 +0000 Subject: [PATCH] fix(tts): restore Supertonic 2 --- frontend/src-tauri/src/lib.rs | 4 - frontend/src-tauri/src/tts.rs | 1251 ++++------------- frontend/src/components/TTSDownloadDialog.tsx | 478 ++----- frontend/src/components/UnifiedChat.tsx | 118 +- frontend/src/services/tts/TTSContext.tsx | 771 ++++------ frontend/src/services/tts/ttsPlayback.test.ts | 99 -- frontend/src/services/tts/ttsPlayback.ts | 58 - 7 files changed, 677 insertions(+), 2102 deletions(-) delete mode 100644 frontend/src/services/tts/ttsPlayback.test.ts delete mode 100644 frontend/src/services/tts/ttsPlayback.ts diff --git a/frontend/src-tauri/src/lib.rs b/frontend/src-tauri/src/lib.rs index a135a38a..d586857a 100644 --- a/frontend/src-tauri/src/lib.rs +++ b/frontend/src-tauri/src/lib.rs @@ -120,9 +120,7 @@ pub fn run() { tts::tts_get_status, tts::tts_download_models, tts::tts_load_models, - tts::tts_chunk_text, tts::tts_synthesize, - tts::tts_synthesize_chunk, tts::tts_unload_models, tts::tts_delete_models, ]) @@ -356,9 +354,7 @@ pub fn run() { tts::tts_get_status, tts::tts_download_models, tts::tts_load_models, - tts::tts_chunk_text, tts::tts_synthesize, - tts::tts_synthesize_chunk, tts::tts_unload_models, tts::tts_delete_models, ]) diff --git a/frontend/src-tauri/src/tts.rs b/frontend/src-tauri/src/tts.rs index 7158764a..8fa0477e 100644 --- a/frontend/src-tauri/src/tts.rs +++ b/frontend/src-tauri/src/tts.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use futures_util::StreamExt; use hound::{SampleFormat, WavSpec, WavWriter}; @@ -11,10 +11,9 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::fs::{self, File}; -use std::io::{BufReader, Cursor, Read, Write}; +use std::io::{BufReader, Cursor, Write}; use std::path::{Path, PathBuf}; use std::sync::Mutex; -use std::time::Instant; use tauri::{AppHandle, Emitter}; use unicode_normalization::UnicodeNormalization; @@ -27,12 +26,10 @@ static RE_STRIKE: Lazy = Lazy::new(|| Regex::new(r"~~([^~]+)~~").unwrap() static RE_CODE: Lazy = Lazy::new(|| Regex::new(r"`([^`]+)`").unwrap()); static RE_CODEBLOCK: Lazy = Lazy::new(|| Regex::new(r"(?s)```[^`]*```").unwrap()); static RE_HEADER: Lazy = Lazy::new(|| Regex::new(r"(?m)^#{1,6}\s*").unwrap()); -static RE_XML_TAG: Lazy = - Lazy::new(|| Regex::new(r"]*)?>").unwrap()); static RE_EMOJI: Lazy = Lazy::new(|| { Regex::new(r"[\x{1F600}-\x{1F64F}\x{1F300}-\x{1F5FF}\x{1F680}-\x{1F6FF}\x{1F700}-\x{1F77F}\x{1F780}-\x{1F7FF}\x{1F800}-\x{1F8FF}\x{1F900}-\x{1F9FF}\x{1FA00}-\x{1FA6F}\x{1FA70}-\x{1FAFF}\x{2600}-\x{26FF}\x{2700}-\x{27BF}\x{1F1E6}-\x{1F1FF}]+").unwrap() }); -static RE_LEGACY_DIACRITICS: Lazy = Lazy::new(|| { +static RE_DIACRITICS: Lazy = Lazy::new(|| { Regex::new(r"[\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{030A}\u{030B}\u{030C}\u{0327}\u{0328}\u{0329}\u{032A}\u{032B}\u{032C}\u{032D}\u{032E}\u{032F}]").unwrap() }); static RE_SPACE_COMMA: Lazy = Lazy::new(|| Regex::new(r" ,").unwrap()); @@ -51,181 +48,85 @@ static RE_ENDS_PUNCT: Lazy = Lazy::new(|| { }); static RE_SENTENCE: Lazy = Lazy::new(|| Regex::new(r"([.!?])\s+").unwrap()); -// Model downloads are pinned to an immutable Hugging Face revision. Keep the -// sizes and hashes in sync with that revision so partial or substituted files -// never become loadable sessions. -const HUGGINGFACE_REVISION: &str = "3cadd1ee6394adea1bd021217a0e650ede09a323"; -const HUGGINGFACE_BASE_URL: &str = "https://huggingface.co/Supertone/supertonic-3/resolve"; -const MODEL_REVISION_FILE: &str = "supertonic_revision.txt"; -const SUPERTONIC3_CACHE_DIR: &str = "supertonic-3"; -const TTS_MODELS_DIR_ENV: &str = "MAPLE_TTS_MODELS_DIR"; -const DEFAULT_LANGUAGE: &str = "na"; -const DEFAULT_VOICE_STYLE: &str = "F2.json"; -const SUPERTONIC3_TOTAL_STEPS: usize = 8; -const LEGACY_TOTAL_STEPS: usize = 10; -const DEFAULT_CHUNK_CHARS: usize = 300; -const CJK_CHUNK_CHARS: usize = 120; -const SUPERTONIC3_DEFAULT_SPEED: f32 = 1.2; -const LEGACY_DEFAULT_SPEED: f32 = 1.2; -const MIN_TTS_SPEED: f32 = 0.5; -const MAX_TTS_SPEED: f32 = 2.0; - -const AVAILABLE_LANGUAGES: &[&str] = &[ - "en", "ko", "ja", "ar", "bg", "cs", "da", "de", "el", "es", "et", "fi", "fr", "hi", "hr", "hu", - "id", "it", "lt", "lv", "nl", "pl", "pt", "ro", "ru", "sk", "sl", "sv", "tr", "uk", "vi", "na", -]; - -#[derive(Clone, Copy, Debug)] -struct ModelFile { - name: &'static str, - url_path: &'static str, - size: u64, - sha256: &'static str, -} - -const SUPERTONIC3_MODEL_FILES: &[ModelFile] = &[ - ModelFile { - name: "duration_predictor.onnx", - url_path: "onnx/duration_predictor.onnx", - size: 3_700_147, - sha256: "c3eb91414d5ff8a7a239b7fe9e34e7e2bf8a8140d8375ffb14718b1c639325db", - }, - ModelFile { - name: "text_encoder.onnx", - url_path: "onnx/text_encoder.onnx", - size: 36_416_150, - sha256: "c7befd5ea8c3119769e8a6c1486c4edc6a3bc8365c67621c881bbb774b9902ff", - }, - ModelFile { - name: "vector_estimator.onnx", - url_path: "onnx/vector_estimator.onnx", - size: 256_534_781, - sha256: "883ac868ea0275ef0e991524dc64f16b3c0376efd7c320af6b53f5b780d7c61c", - }, - ModelFile { - name: "vocoder.onnx", - url_path: "onnx/vocoder.onnx", - size: 101_424_195, - sha256: "085de76dd8e8d5836d6ca66826601f615939218f90e519f70ee8a36ed2a4c4ba", - }, - ModelFile { - name: "tts.json", - url_path: "onnx/tts.json", - size: 8_253, - sha256: "42078d3aef1cd43ab43021f3c54f47d2d75ceb4e75f627f118890128b06a0d09", - }, - ModelFile { - name: "unicode_indexer.json", - url_path: "onnx/unicode_indexer.json", - size: 277_676, - sha256: "9bf7346e43883a81f8645c81224f786d43c5b57f3641f6e7671a7d6c493cb24f", - }, - ModelFile { - name: "F1.json", - url_path: "voice_styles/F1.json", - size: 292_046, - sha256: "bbdec6ee00231c2c742ad05483df5334cab3b52fda3ba38e6a07059c4563dbc2", - }, - ModelFile { - name: "F2.json", - url_path: "voice_styles/F2.json", - size: 292_423, - sha256: "7c722c6a72707b1a77f035d67f0d1351ba187738e06f7683e8c72b1df3477fc6", - }, - ModelFile { - name: "F3.json", - url_path: "voice_styles/F3.json", - size: 290_794, - sha256: "12f6ef2573baa2defa1128069cb59f203e3ab67c92af77b42df8a0e3a2f7c6ab", - }, - ModelFile { - name: "F4.json", - url_path: "voice_styles/F4.json", - size: 291_808, - sha256: "c2fa764c1225a76dfc3e2c73e8aa4f70d9ee48793860eb34c295fff01c2e032b", - }, - ModelFile { - name: "F5.json", - url_path: "voice_styles/F5.json", - size: 291_479, - sha256: "45966e73316415626cf41a7d1c6f3b4c70dbc1ba2bee5c1978ef0ce33244fc8d", - }, - ModelFile { - name: "M1.json", - url_path: "voice_styles/M1.json", - size: 291_748, - sha256: "e35604687f5d23694b8e91593a93eec0e4eca6c0b02bb8ed69139ab2ea6b0a5b", - }, - ModelFile { - name: "M2.json", - url_path: "voice_styles/M2.json", - size: 292_055, - sha256: "b76cbf62bac707c710cf0ae5aba5e31eea1a6339a9734bfae33ab98499534a50", - }, - ModelFile { - name: "M3.json", - url_path: "voice_styles/M3.json", - size: 290_198, - sha256: "ea1ac35ccb91b0d7ecad533a2fbd0eec10c91513d8951e3b25fbba99954e159b", - }, - ModelFile { - name: "M4.json", - url_path: "voice_styles/M4.json", - size: 291_522, - sha256: "ca8eefad4fcd989c9379032ff3e50738adc547eeb5e221b82593a6d7b3bac303", - }, - ModelFile { - name: "M5.json", - url_path: "voice_styles/M5.json", - size: 291_469, - sha256: "dd22b92740314321f8ae11c5e87f8dd60d060f15dd3a632b5adf77f471f77af2", - }, -]; - -const SUPERTONIC3_TOTAL_MODEL_SIZE: u64 = 401_276_744; - -// Supertonic 1 stored its assets directly in the root directory. Existing -// users can keep using those models until they explicitly choose to upgrade. -const LEGACY_MODEL_FILES: &[(&str, u64)] = &[ - ("duration_predictor.onnx", 1_500_789), - ("text_encoder.onnx", 27_348_373), - ("vector_estimator.onnx", 132_471_364), - ("vocoder.onnx", 101_405_066), - ("tts.json", 8_645), - ("unicode_indexer.json", 262_134), - ("F1.json", 420_622), - ("F2.json", 420_905), - ("M1.json", 421_053), - ("M2.json", 421_027), +// Pin model downloads to a specific repo revision to ensure integrity and reproducibility. +const HUGGINGFACE_REVISION: &str = "b6856d033f622c63ea29441795be266a1133e227"; +const HUGGINGFACE_BASE_URL: &str = "https://huggingface.co/Supertone/supertonic/resolve"; + +// (file_name, url_path, expected_size_bytes, expected_sha256_hex) +const MODEL_FILES: &[(&str, &str, u64, &str)] = &[ + ( + "duration_predictor.onnx", + "onnx/duration_predictor.onnx", + 1_500_789, + "b861580c56a0cba2a2b82aa697ecb3c5a163c3240c60a0ddfac369d21d054092", + ), + ( + "text_encoder.onnx", + "onnx/text_encoder.onnx", + 27_348_373, + "ba0c8ea74aeb5df00d21a89b8d47c71317f47120232e3deef95024dba37dbd88", + ), + ( + "vector_estimator.onnx", + "onnx/vector_estimator.onnx", + 132_471_364, + "b3f82ecd2e9decc4e2236048b03628a1c1d5f14a792ba274a59b7325107aa6a6", + ), + ( + "vocoder.onnx", + "onnx/vocoder.onnx", + 101_405_066, + "19bd51f47a186069c752403518a40f7ea4c647455056d2511f7249691ecddf7c", + ), + ( + "tts.json", + "onnx/tts.json", + 8_645, + "4dac5f986698a3ace9a97ea2545d43f6c8ba120d25e005f8c905128281be9b6d", + ), + ( + "unicode_indexer.json", + "onnx/unicode_indexer.json", + 262_134, + "0c3800ba4fb1fc760c9070eb43a0ad5a68279ec165742591a68ea3edca452978", + ), + ( + "F1.json", + "voice_styles/F1.json", + 420_622, + "1450bcad84a2790eaf73f85e763dd5bae7c399f55d692c4835cf4f7686b5a10f", + ), + ( + "F2.json", + "voice_styles/F2.json", + 420_905, + "47c8d44445ef8ac8aae8ef5806feca21903483cbd4f1232e405184a40520a549", + ), + ( + "M1.json", + "voice_styles/M1.json", + 421_053, + "273c9ba6582d2e00383d8fbe2f5d660d86e8fba849c91ff695384d1a6e2e02f1", + ), + ( + "M2.json", + "voice_styles/M2.json", + 421_027, + "26898a9ec3de1b5bf8cc3f6cbf41930543ca0403f2201e12aad849691ff315dd", + ), ]; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ModelVersion { - Supertonic3, - Legacy, -} - -fn default_tts_speed(model_version: ModelVersion) -> f32 { - match model_version { - ModelVersion::Supertonic3 => SUPERTONIC3_DEFAULT_SPEED, - ModelVersion::Legacy => LEGACY_DEFAULT_SPEED, - } -} +const TOTAL_MODEL_SIZE: u64 = 264_679_978; // bytes +const DEFAULT_TTS_SPEED: f32 = 1.2; +const MIN_TTS_SPEED: f32 = 0.5; +const MAX_TTS_SPEED: f32 = 2.0; -fn resolve_tts_speed(model_version: ModelVersion, requested: Option) -> f32 { +fn resolve_tts_speed(requested: Option) -> f32 { match requested { Some(speed) if speed.is_finite() && speed > 0.0 => { speed.clamp(MIN_TTS_SPEED, MAX_TTS_SPEED) } - _ => default_tts_speed(model_version), - } -} - -fn total_steps_for_model(model_version: ModelVersion) -> usize { - match model_version { - ModelVersion::Supertonic3 => SUPERTONIC3_TOTAL_STEPS, - ModelVersion::Legacy => LEGACY_TOTAL_STEPS, + _ => DEFAULT_TTS_SPEED, } } @@ -279,15 +180,11 @@ pub struct Style { struct UnicodeProcessor { indexer: Vec, - unknown_token_id: i64, } impl UnicodeProcessor { - fn new(indexer: Vec, unknown_token_id: i64) -> Self { - UnicodeProcessor { - indexer, - unknown_token_id, - } + fn new(indexer: Vec) -> Self { + UnicodeProcessor { indexer } } fn call(&self, text_list: &[String]) -> (Vec>, Array3) { @@ -303,7 +200,8 @@ impl UnicodeProcessor { if val < self.indexer.len() { row[j] = self.indexer[val]; } else { - row[j] = self.unknown_token_id; + // Use 0 (padding token) for out-of-vocabulary characters + row[j] = 0; } } text_ids.push(row); @@ -314,7 +212,7 @@ impl UnicodeProcessor { } } -fn normalize_text_for_tts(text: &str, strip_legacy_diacritics: bool) -> String { +fn preprocess_text(text: &str) -> String { let mut text: String = text.nfkd().collect(); // Remove markdown formatting (using pre-compiled regexes) @@ -326,10 +224,6 @@ fn normalize_text_for_tts(text: &str, strip_legacy_diacritics: bool) -> String { text = RE_CODE.replace_all(&text, "$1").to_string(); text = RE_CODEBLOCK.replace_all(&text, "").to_string(); text = RE_HEADER.replace_all(&text, "").to_string(); - // Expression tags are deliberately not exposed in Maple's first - // Supertonic 3 iteration. Strip arbitrary XML-like input before adding the - // trusted language wrapper below. - text = RE_XML_TAG.replace_all(&text, " ").to_string(); text = RE_EMOJI.replace_all(&text, "").to_string(); // Replace various dashes and symbols @@ -338,7 +232,6 @@ fn normalize_text_for_tts(text: &str, strip_legacy_diacritics: bool) -> String { ("‑", "-"), ("—", "-"), ("¯", " "), - ("_", " "), ("\u{201C}", "\""), ("\u{201D}", "\""), ("\u{2018}", "'"), @@ -357,9 +250,7 @@ fn normalize_text_for_tts(text: &str, strip_legacy_diacritics: bool) -> String { text = text.replace(from, to); } - if strip_legacy_diacritics { - text = RE_LEGACY_DIACRITICS.replace_all(&text, "").to_string(); - } + text = RE_DIACRITICS.replace_all(&text, "").to_string(); // Remove special symbols for symbol in &["♥", "☆", "♡", "©", "\\"] { @@ -396,33 +287,6 @@ fn normalize_text_for_tts(text: &str, strip_legacy_diacritics: bool) -> String { text } -fn normalize_language(language: Option<&str>) -> Result<&str> { - let language = language.unwrap_or(DEFAULT_LANGUAGE).trim(); - if AVAILABLE_LANGUAGES.contains(&language) { - Ok(language) - } else { - bail!("Invalid TTS language: {language}. Available languages: {AVAILABLE_LANGUAGES:?}") - } -} - -fn prepare_text( - text: &str, - model_version: ModelVersion, - language: Option<&str>, -) -> Result> { - let normalized = normalize_text_for_tts(text, model_version == ModelVersion::Legacy); - if normalized.is_empty() { - return Ok(None); - } - - if model_version == ModelVersion::Legacy { - return Ok(Some(normalized)); - } - - let language = normalize_language(language)?; - Ok(Some(format!("<{language}>{normalized}"))) -} - fn length_to_mask(lengths: &[usize], max_len: Option) -> Array3 { let bsz = lengths.len(); let max_len = max_len.unwrap_or_else(|| *lengths.iter().max().unwrap_or(&0)); @@ -483,38 +347,15 @@ fn sample_noisy_latent( (noisy_latent, latent_mask) } -fn hard_split_chars(text: &str, max_chars: usize) -> Vec { - let mut chunks = Vec::new(); - let mut current = String::new(); - for character in text.chars() { - if current.chars().count() == max_chars { - chunks.push(std::mem::take(&mut current)); - } - current.push(character); - } - if !current.is_empty() { - chunks.push(current); - } - chunks -} - -fn split_by_words(text: &str, max_chars: usize) -> Vec { - let mut chunks = Vec::new(); +/// Split text by words when it exceeds max_len +fn split_by_words(text: &str, max_len: usize) -> Vec { + let mut result = Vec::new(); let mut current = String::new(); for word in text.split_whitespace() { - let word_chars = word.chars().count(); - if word_chars > max_chars { - if !current.is_empty() { - chunks.push(std::mem::take(&mut current)); - } - chunks.extend(hard_split_chars(word, max_chars)); - continue; - } - - let separator = usize::from(!current.is_empty()); - if current.chars().count() + separator + word_chars > max_chars { - chunks.push(std::mem::take(&mut current)); + if current.len() + word.len() + 1 > max_len && !current.is_empty() { + result.push(current.trim().to_string()); + current.clear(); } if !current.is_empty() { current.push(' '); @@ -523,48 +364,20 @@ fn split_by_words(text: &str, max_chars: usize) -> Vec { } if !current.is_empty() { - chunks.push(current); - } - chunks -} - -fn push_chunk_unit(chunks: &mut Vec, current: &mut String, unit: &str, max_chars: usize) { - let unit = unit.trim(); - if unit.is_empty() { - return; - } - - if unit.chars().count() > max_chars { - if !current.is_empty() { - chunks.push(std::mem::take(current)); - } - chunks.extend(split_by_words(unit, max_chars)); - return; - } - - let separator = usize::from(!current.is_empty()); - if current.chars().count() + separator + unit.chars().count() > max_chars { - chunks.push(std::mem::take(current)); - } - if !current.is_empty() { - current.push(' '); + result.push(current.trim().to_string()); } - current.push_str(unit); + result } -fn chunk_text(text: &str, max_chars: usize) -> Vec { - if max_chars == 0 { - return Vec::new(); - } +fn chunk_text(text: &str, max_len: usize) -> Vec { let text = text.trim(); if text.is_empty() { - return Vec::new(); + return vec![String::new()]; } static RE_PARA: Lazy = Lazy::new(|| Regex::new(r"\n\s*\n").unwrap()); let paragraphs: Vec<&str> = RE_PARA.split(text).collect(); let mut chunks = Vec::new(); - let mut current = String::new(); for para in paragraphs { let para = para.trim(); @@ -572,53 +385,89 @@ fn chunk_text(text: &str, max_chars: usize) -> Vec { continue; } - if para.chars().count() <= max_chars { - push_chunk_unit(&mut chunks, &mut current, para, max_chars); + if para.len() <= max_len { + chunks.push(para.to_string()); continue; } + // Split by sentence boundaries, keeping punctuation + let mut current = String::new(); let mut last_end = 0; for m in RE_SENTENCE.find_iter(para) { - let sentence = para[last_end..m.start() + 1].trim(); + let sentence = para[last_end..m.start() + 1].trim(); // +1 to include punctuation last_end = m.end(); - push_chunk_unit(&mut chunks, &mut current, sentence, max_chars); + + if sentence.is_empty() { + continue; + } + + // If single sentence exceeds max_len, split by words + if sentence.len() > max_len { + if !current.is_empty() { + chunks.push(current.trim().to_string()); + current.clear(); + } + chunks.extend(split_by_words(sentence, max_len)); + continue; + } + + if current.len() + sentence.len() + 1 > max_len && !current.is_empty() { + chunks.push(current.trim().to_string()); + current.clear(); + } + + if !current.is_empty() { + current.push(' '); + } + current.push_str(sentence); } + // Remaining text after last sentence boundary let remaining = para[last_end..].trim(); - push_chunk_unit(&mut chunks, &mut current, remaining, max_chars); - } - - if !current.is_empty() { - chunks.push(current); + if !remaining.is_empty() { + // If remaining exceeds max_len, split by words + if remaining.len() > max_len { + if !current.is_empty() { + chunks.push(current.trim().to_string()); + } + chunks.extend(split_by_words(remaining, max_len)); + } else if current.len() + remaining.len() + 1 > max_len && !current.is_empty() { + chunks.push(current.trim().to_string()); + chunks.push(remaining.to_string()); + } else { + if !current.is_empty() { + current.push(' '); + } + current.push_str(remaining); + chunks.push(current.trim().to_string()); + } + } else if !current.is_empty() { + chunks.push(current.trim().to_string()); + } } - chunks -} -fn chunk_size_for_language(language: &str) -> usize { - if matches!(language, "ko" | "ja") { - CJK_CHUNK_CHARS + if chunks.is_empty() { + vec![String::new()] } else { - DEFAULT_CHUNK_CHARS + chunks } } pub struct TTSState { - loaded: Option, + tts: Option, + style: Option