Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 23 additions & 12 deletions openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! 阿里云百炼(DashScope)多模态生成同步接口的批量 ASR 客户端。
//!
//! `fun-asr-flash` 系列是**非实时录音文件识别**模型,走 DashScope 私有的
//! `fun-asr-flash` 与 `qwen-audio-3.0-asr-flash` 系列是**非实时录音文件识别**
//! 模型,走 DashScope 私有的
//! `multimodal-generation/generation` HTTP 接口,既不是实时 WebSocket 双工
//! (见 `bailian.rs`),也不是 OpenAI 兼容的 `/audio/transcriptions`
//! (见 `whisper.rs`)。因此单独成一路批量客户端:录音结束后把整段 PCM 编成
Expand Down Expand Up @@ -28,6 +29,7 @@ pub const PROVIDER_ID: &str = "bailian-fun-asr-flash";
pub const DEFAULT_ENDPOINT: &str =
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation";
pub const DEFAULT_MODEL: &str = "fun-asr-flash-2026-06-15";
pub const QWEN_AUDIO_MODEL: &str = "qwen-audio-3.0-asr-flash";

pub struct DashScopeMultimodalASR {
api_key: String,
Expand Down Expand Up @@ -119,6 +121,11 @@ impl DashScopeMultimodalASR {
}
}

pub fn is_supported_model(model: &str) -> bool {
let model = model.trim();
model == DEFAULT_MODEL || model == QWEN_AUDIO_MODEL
}

impl crate::recorder::AudioConsumer for DashScopeMultimodalASR {
fn consume_pcm_chunk(&self, pcm: &[u8]) {
self.buffer.lock().extend_from_slice(pcm);
Expand Down Expand Up @@ -154,6 +161,8 @@ pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value {
"data:audio/wav;base64,{}",
base64::engine::general_purpose::STANDARD.encode(wav)
);
// qwen-audio-3.0-asr-flash 还支持 vocabulary 与 language_hints;当前批量客户端
// 尚未将这两项设置映射到请求体,暂时保持自动语言检测且不传热词。
serde_json::json!({
"model": model,
"input": {
Expand Down Expand Up @@ -262,17 +271,19 @@ mod tests {

#[test]
fn body_uses_multimodal_generation_shape() {
let body = dashscope_multimodal_body(DEFAULT_MODEL, b"wav");
assert_eq!(body["model"], DEFAULT_MODEL);
let audio = &body["input"]["messages"][0]["content"][0];
assert_eq!(audio["type"], "input_audio");
assert!(audio["input_audio"]["data"]
.as_str()
.unwrap()
.starts_with("data:audio/wav;base64,"));
assert_eq!(body["parameters"]["format"], "wav");
assert_eq!(body["parameters"]["sample_rate"], "16000");
assert!(body["parameters"].get("vocabulary_id").is_none());
for model in [DEFAULT_MODEL, QWEN_AUDIO_MODEL] {
let body = dashscope_multimodal_body(model, b"wav");
assert_eq!(body["model"], model);
let audio = &body["input"]["messages"][0]["content"][0];
assert_eq!(audio["type"], "input_audio");
assert!(audio["input_audio"]["data"]
.as_str()
.unwrap()
.starts_with("data:audio/wav;base64,"));
assert_eq!(body["parameters"]["format"], "wav");
assert_eq!(body["parameters"]["sample_rate"], "16000");
assert!(body["parameters"].get("vocabulary_id").is_none());
}
}

#[test]
Expand Down
9 changes: 7 additions & 2 deletions openless-all/app/src-tauri/src/commands/providers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
crate::asr::qwen_realtime::DEFAULT_MODEL.to_string(),
"qwen3-asr-flash-realtime-2026-02-10".to_string(),
"qwen3-asr-flash-realtime-2025-10-27".to_string(),
crate::asr::dashscope_multimodal::QWEN_AUDIO_MODEL.to_string(),
crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string(),
],
});
Expand Down Expand Up @@ -81,7 +82,10 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
{
// multimodal-generation 无模型列表 HTTP 接口;与 mimo 同,返回静态别名。
return Ok(ProviderModelsResult {
models: vec![crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()],
models: vec![
crate::asr::dashscope_multimodal::QWEN_AUDIO_MODEL.to_string(),
crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string(),
],
});
}
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::elevenlabs::PROVIDER_ID {
Expand Down Expand Up @@ -388,7 +392,8 @@ async fn validate_elevenlabs_asr_provider() -> Result<(), String> {
})
}

/// fun-asr-flash 官方公开示例音频,用于连通性校验。该模型对纯静音会返回
/// DashScope 录音文件 ASR 官方公开示例音频,用于两个已支持模型的连通性校验。
/// 这类模型对纯静音会返回
/// 400("no speech" 类错误),无法像 Whisper/Mimo 那样发静音探活;改用这段
/// 阿里官方文档在案的示例 wav(由 DashScope 侧拉取),key/endpoint/model 有效
/// 即返回 200。
Expand Down
27 changes: 22 additions & 5 deletions openless-all/app/src-tauri/src/coordinator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,8 @@ pub(crate) fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
/// 统一「阿里云百炼」入口的模型 → 底层协议 id 路由。
///
/// 三条百炼协议(fun-asr-realtime 经典实时 / qwen3-asr-flash-realtime Realtime /
/// fun-asr-flash 录音文件)在 UI 上收成一个 provider `bailian`(一把 key),**构建时**
/// fun-asr-flash 与 qwen-audio-3.0-asr-flash 录音文件)在 UI 上收成一个
/// provider `bailian`(一把 key),**构建时**
/// 按所选模型二次路由到具体协议客户端。凭据 / 「已配置」判定仍看真实 active
/// `bailian`(→ ApiKeyOnly,一把 key),只有这里的 build 分发用得上 effective id。
///
Expand Down Expand Up @@ -349,11 +350,11 @@ pub(crate) fn resolve_effective_asr_provider(
Ok(crate::asr::bailian::PROVIDER_ID.to_string())
} else if model.starts_with("qwen3-asr-flash-realtime") {
Ok(crate::asr::qwen_realtime::PROVIDER_ID.to_string())
} else if model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
} else if crate::asr::dashscope_multimodal::is_supported_model(model) {
Ok(crate::asr::dashscope_multimodal::PROVIDER_ID.to_string())
} else {
Err(format!(
"不支持的百炼 ASR 模型:{model}。支持 fun-asr-realtime、paraformer-realtime、sensevoice-realtime、qwen3-asr-flash-realtime 和 fun-asr-flash-2026-06-15"
"不支持的百炼 ASR 模型:{model}。支持 fun-asr-realtime、paraformer-realtime、sensevoice-realtime、qwen3-asr-flash-realtime、qwen-audio-3.0-asr-flash 和 fun-asr-flash-2026-06-15"
))
}
}
Expand All @@ -372,11 +373,11 @@ pub(crate) fn stepfun_model_is_stream(model: &str) -> bool {

pub(crate) fn validate_dashscope_multimodal_model(model: &str) -> Result<(), String> {
let model = model.trim();
if model.is_empty() || model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
if model.is_empty() || crate::asr::dashscope_multimodal::is_supported_model(model) {
return Ok(());
}
Err(format!(
"不支持的 DashScope 录音文件 ASR 模型:{model}。该协议仅支持 fun-asr-flash-2026-06-15;fun-asr-flash-8k-realtime 系列属于 8 kHz 实时 WebSocket 模型,当前尚未支持"
"不支持的 DashScope 录音文件 ASR 模型:{model}。该协议支持 qwen-audio-3.0-asr-flash 和 fun-asr-flash-2026-06-15;fun-asr-flash-8k-realtime 系列属于 8 kHz 实时 WebSocket 模型,当前尚未支持"
))
}

Expand Down Expand Up @@ -3047,6 +3048,10 @@ mod tests {
resolve_effective_asr_provider(bailian, "fun-asr-flash-2026-06-15").unwrap(),
crate::asr::dashscope_multimodal::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "qwen-audio-3.0-asr-flash").unwrap(),
crate::asr::dashscope_multimodal::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "paraformer-realtime-v2").unwrap(),
crate::asr::bailian::PROVIDER_ID
Expand Down Expand Up @@ -3082,6 +3087,18 @@ mod tests {
assert!(error.contains("不支持的百炼 ASR 模型"));
}

#[test]
fn validates_only_supported_dashscope_multimodal_models() {
assert!(validate_dashscope_multimodal_model("").is_ok());
assert!(
validate_dashscope_multimodal_model("fun-asr-flash-2026-06-15").is_ok()
);
assert!(validate_dashscope_multimodal_model("qwen-audio-3.0-asr-flash").is_ok());
assert!(
validate_dashscope_multimodal_model("qwen-audio-3.0-asr-flash-streaming").is_err()
);
}

#[test]
fn derive_bailian_endpoint_preserves_region_host_and_selects_protocol_path() {
let endpoint = "https://workspace.ap-southeast-1.maas.aliyuncs.com/custom?x=1";
Expand Down
6 changes: 4 additions & 2 deletions openless-all/app/src/pages/settings/ProvidersSection.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -554,12 +554,14 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
// 统一「阿里云百炼」下,按模型名判断走哪种协议(与后端
// coordinator::resolve_effective_asr_provider 保持一致):qwen3-asr-flash-realtime* 与
// fun-asr-realtime* 与 fun-asr-flash-8k-realtime* 都是实时模型;当前支持的
// fun-asr-flash-2026-06-15 是「录音文件·说完转写」。
// fun-asr-flash-2026-06-15 / qwen-audio-3.0-asr-flash 是「录音文件·说完转写」。
function bailianModelIsRecordedFile(model: string): boolean {
const m = model.trim();
return m === 'fun-asr-flash-2026-06-15';
return m === 'fun-asr-flash-2026-06-15' || m === 'qwen-audio-3.0-asr-flash';
}

// qwen-audio-3.0-asr-flash 官方支持热词,但批量协议尚未把该设置写入请求体;
// 在后端接入前不展示一个实际不生效的热词输入框。
function bailianModelSupportsVocabulary(model: string): boolean {
const m = model.trim();
return !m
Expand Down