diff --git a/src/vs/workbench/contrib/agentsVoice/browser/agentsVoice.contribution.ts b/src/vs/workbench/contrib/agentsVoice/browser/agentsVoice.contribution.ts index 7644f12f89e88..7bb5eef9c51d2 100644 --- a/src/vs/workbench/contrib/agentsVoice/browser/agentsVoice.contribution.ts +++ b/src/vs/workbench/contrib/agentsVoice/browser/agentsVoice.contribution.ts @@ -677,7 +677,7 @@ configurationRegistry.registerConfiguration({ nls.localize('agents.voice.language.ko', "Korean"), nls.localize('agents.voice.language.zh', "Chinese"), ], - markdownDescription: nls.localize('agents.voice.language', "The language used for speech recognition, dictation, and spoken responses. The selectable languages support native voice output. Automatic follows the system or browser locale for speech recognition and dictation, and uses English voice output when the detected language does not support native voice output. Changing this while voice mode is connected takes effect immediately."), + markdownDescription: nls.localize('agents.voice.language', "The language used for speech recognition, dictation, and spoken responses. The selectable languages support native voice output. Automatic uses the configured display language for speech recognition and dictation when supported; otherwise, it follows the system or browser locale. English voice output is used when the detected language does not support native voice output. Changing this while voice mode is connected takes effect immediately."), default: 'auto', scope: ConfigurationScope.APPLICATION, }, diff --git a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts index 277a275d055b6..7bb3f0d1c8bd1 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts @@ -291,7 +291,7 @@ configurationRegistry.registerConfiguration({ nls.localize('dictation.model.mai.label', "MAI — Cloud"), ], markdownEnumDescriptions: [ - nls.localize('dictation.model.nemotronMultilingual', "NVIDIA Nemotron 3.5 multilingual streaming RNN-T, run on-device through Microsoft Foundry Local. Works offline; no audio leaves the device. Automatic language selection follows the Voice Mode language setting and system or browser locale, with model detection as a fallback. Downloaded on first use and cached on disk."), + nls.localize('dictation.model.nemotronMultilingual', "NVIDIA Nemotron 3.5 multilingual streaming RNN-T, run on-device through Microsoft Foundry Local. Works offline; no audio leaves the device. Automatic language selection follows the Voice Mode language setting; when that setting is Automatic, dictation uses the configured display language when supported, then the system or browser locale, with model detection as a fallback. Downloaded on first use and cached on disk."), nls.localize('dictation.model.mai', "Cloud transcription through the same Microsoft AI voice service used by Voice Mode. Requires a network connection and GitHub sign-in; audio is streamed to the service."), ], markdownDescription: nls.localize('dictation.model', "The model used for dictation. On-device models download on first use and run locally through Microsoft Foundry Local; the cloud option streams audio to the Microsoft AI voice service."), diff --git a/src/vs/workbench/contrib/chat/browser/speechToText/dictationLanguage.ts b/src/vs/workbench/contrib/chat/browser/speechToText/dictationLanguage.ts index 3b1180256ffd1..41499cb9322a3 100644 --- a/src/vs/workbench/contrib/chat/browser/speechToText/dictationLanguage.ts +++ b/src/vs/workbench/contrib/chat/browser/speechToText/dictationLanguage.ts @@ -3,6 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { Language } from '../../../../../base/common/platform.js'; + const NEMOTRON_LOCALES = new Set([ 'ar-AR', 'bg-BG', 'cs-CZ', 'da-DK', 'de-DE', 'en-GB', 'en-US', 'es-ES', 'es-US', 'et-EE', 'fi-FI', 'fr-CA', 'fr-FR', 'el-GR', 'he-IL', 'hi-IN', @@ -50,16 +52,29 @@ const NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE: Readonly> = { zh: 'zh-CN', }; +function getConfiguredDisplayLanguage(): string | undefined { + return Language.value(); +} + /** * Resolve the on-device dictation language using the same setting semantics as - * Voice Mode. Automatic follows the browser locale when Nemotron supports it, - * then falls back to the model's language detection. + * Voice Mode. Automatic follows the configured display language when supported, + * then the system or browser locale, then the model's language detection. */ -export function resolveDictationLanguage(configuredLanguage: unknown, browserLanguage: string | undefined): string { +export function resolveDictationLanguage(configuredLanguage: unknown, browserLanguage: string | undefined, displayLanguage = getConfiguredDisplayLanguage()): string { const configured = typeof configuredLanguage === 'string' ? configuredLanguage.trim() : ''; - const candidate = configured && configured.toLowerCase() !== 'auto' ? configured : browserLanguage; + if (configured && configured.toLowerCase() !== 'auto') { + return resolveSupportedDictationLanguage(configured) ?? 'auto'; + } + + return resolveSupportedDictationLanguage(displayLanguage) + ?? resolveSupportedDictationLanguage(browserLanguage) + ?? 'auto'; +} + +function resolveSupportedDictationLanguage(candidate: string | undefined): string | undefined { if (!candidate || typeof Intl.getCanonicalLocales !== 'function') { - return 'auto'; + return undefined; } try { @@ -67,8 +82,8 @@ export function resolveDictationLanguage(configuredLanguage: unknown, browserLan if (NEMOTRON_LOCALES.has(canonical)) { return canonical; } - return NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE[canonical.split('-')[0]] ?? 'auto'; + return NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE[canonical.split('-')[0]]; } catch { - return 'auto'; + return undefined; } } diff --git a/src/vs/workbench/contrib/chat/browser/voiceClient/voiceClientService.ts b/src/vs/workbench/contrib/chat/browser/voiceClient/voiceClientService.ts index f27c290d007d0..e8039ca5def1f 100644 --- a/src/vs/workbench/contrib/chat/browser/voiceClient/voiceClientService.ts +++ b/src/vs/workbench/contrib/chat/browser/voiceClient/voiceClientService.ts @@ -7,6 +7,7 @@ import { Disposable } from '../../../../../base/common/lifecycle.js'; import { Emitter, Event } from '../../../../../base/common/event.js'; import { generateUuid } from '../../../../../base/common/uuid.js'; import { mainWindow } from '../../../../../base/browser/window.js'; +import { Language } from '../../../../../base/common/platform.js'; import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; import { ILogService } from '../../../../../platform/log/common/log.js'; import { IProductService } from '../../../../../platform/product/common/productService.js'; @@ -59,6 +60,26 @@ function asOptionalNonEmptyString(value: unknown): string | undefined { return result && result.length > 0 ? result : undefined; } +function canonicalizeSupportedLanguage(value: string | undefined, supportedBases: ReadonlySet): string | undefined { + const candidate = value?.trim(); + if (!candidate || typeof Intl.getCanonicalLocales !== 'function') { + return undefined; + } + + try { + const canonical = Intl.getCanonicalLocales(candidate)[0]; + return supportedBases.has(canonical.split('-')[0]) ? canonical : undefined; + } catch { + return undefined; + } +} + +export function resolveAutomaticVoiceLanguage(browserLanguage: string | undefined, displayLanguage: string | undefined): string { + return canonicalizeSupportedLanguage(displayLanguage, ASR_SUPPORTED_LANGUAGE_BASES) + ?? canonicalizeSupportedLanguage(browserLanguage, ASR_SUPPORTED_LANGUAGE_BASES) + ?? DEFAULT_LANGUAGE; +} + function asTranscriptionStatus(value: unknown): IVoiceTranscription['status'] | undefined { return value === 'partial' || value === 'final' ? value : undefined; } @@ -201,7 +222,7 @@ export class VoiceClientService extends Disposable implements IVoiceClientServic private _getLanguage(): string { const configured = this._configurationService.getValue('agents.voice.language'); if (typeof configured === 'string' && configured.trim().toLowerCase() !== 'auto') { - const language = this._canonicalizeSupportedLanguage(configured, TTS_SUPPORTED_LANGUAGE_BASES); + const language = canonicalizeSupportedLanguage(configured, TTS_SUPPORTED_LANGUAGE_BASES); if (language) { return language; } @@ -209,22 +230,7 @@ export class VoiceClientService extends Disposable implements IVoiceClientServic return DEFAULT_LANGUAGE; } - return this._canonicalizeSupportedLanguage(this._window?.navigator.language, ASR_SUPPORTED_LANGUAGE_BASES) - ?? DEFAULT_LANGUAGE; - } - - private _canonicalizeSupportedLanguage(value: string | undefined, supportedBases: ReadonlySet): string | undefined { - const candidate = value?.trim(); - if (!candidate || typeof Intl.getCanonicalLocales !== 'function') { - return undefined; - } - - try { - const canonical = Intl.getCanonicalLocales(candidate)[0]; - return supportedBases.has(canonical.split('-')[0]) ? canonical : undefined; - } catch { - return undefined; - } + return resolveAutomaticVoiceLanguage(this._window?.navigator.language, Language.value()); } private _sendSetLanguage(): void { diff --git a/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts index 876a3a3014013..3db623f34e846 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts @@ -12,20 +12,28 @@ suite('ChatSpeechToTextService', () => { ensureNoDisposablesAreLeakedInTestSuite(); - test('resolves the dictation language from Voice Mode configuration and browser locale', () => { + test('resolves the dictation language from Voice Mode configuration, display language, and browser locale', () => { assert.deepStrictEqual({ explicit: resolveDictationLanguage('fr-FR', 'de-DE'), - automatic: resolveDictationLanguage('auto', 'uk-UA'), - regionalAutomatic: resolveDictationLanguage('auto', 'pt-BR'), - additionalSupportedAutomatic: resolveDictationLanguage('auto', 'he-IL'), - unsupportedRegion: resolveDictationLanguage('auto', 'en-AU'), + explicitWithDisplayLanguage: resolveDictationLanguage('fr-FR', 'de-DE', 'ja'), + displayLanguage: resolveDictationLanguage('auto', 'en-US', 'de'), + englishDisplayLanguage: resolveDictationLanguage('auto', 'de-DE', 'en'), + unsupportedDisplayLanguage: resolveDictationLanguage('auto', 'pt-BR', 'id-ID'), + automatic: resolveDictationLanguage('auto', 'uk-UA', 'id-ID'), + regionalAutomatic: resolveDictationLanguage('auto', 'pt-BR', 'id-ID'), + additionalSupportedAutomatic: resolveDictationLanguage('auto', 'he-IL', 'id-ID'), + unsupportedRegion: resolveDictationLanguage('auto', 'en-AU', 'id-ID'), explicitSpanish: resolveDictationLanguage('es', 'en-US'), explicitAdaptationReady: resolveDictationLanguage('lt', 'en-US'), - regionalPortugueseFallback: resolveDictationLanguage('auto', 'pt-AO'), + regionalPortugueseFallback: resolveDictationLanguage('auto', 'pt-AO', 'id-ID'), invalidExplicit: resolveDictationLanguage('not a locale', 'de-DE'), - missing: resolveDictationLanguage(undefined, undefined), + missing: resolveDictationLanguage(undefined, undefined, 'id-ID'), }, { explicit: 'fr-FR', + explicitWithDisplayLanguage: 'fr-FR', + displayLanguage: 'de-DE', + englishDisplayLanguage: 'en-US', + unsupportedDisplayLanguage: 'pt-BR', automatic: 'uk-UA', regionalAutomatic: 'pt-BR', additionalSupportedAutomatic: 'he-IL', diff --git a/src/vs/workbench/contrib/chat/test/browser/voiceClient/voiceClientService.test.ts b/src/vs/workbench/contrib/chat/test/browser/voiceClient/voiceClientService.test.ts index f3765510417a3..63faae47f2837 100644 --- a/src/vs/workbench/contrib/chat/test/browser/voiceClient/voiceClientService.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/voiceClient/voiceClientService.test.ts @@ -11,7 +11,7 @@ import { TestConfigurationService } from '../../../../../../platform/configurati import { NullLogService } from '../../../../../../platform/log/common/log.js'; import product from '../../../../../../platform/product/common/product.js'; import { IProductService } from '../../../../../../platform/product/common/productService.js'; -import { VoiceClientService } from '../../../browser/voiceClient/voiceClientService.js'; +import { resolveAutomaticVoiceLanguage, VoiceClientService } from '../../../browser/voiceClient/voiceClientService.js'; import { IVoiceAudioResponse, IVoiceBargeIn, IVoiceNarrationAck, IVoiceNarrationSignal, IVoiceSpeechStarted, IVoiceTranscription } from '../../../common/voiceClient/voiceClientService.js'; class TestWebSocket { @@ -560,20 +560,36 @@ suite('VoiceClientService', () => { }]); }); - test('uses browser locale for auto and falls back when unavailable', async () => { + test('uses the display language for auto', async () => { const first = createService({ 'agents.voice.language': 'auto' }); await first.service.connect(createTestWindow('pt-BR')); first.service.sendStartSession({ sessions: [], display_locale: '' }, 'machine'); - const browserLocale = socket().sent[0].session_context; + const withBrowserLocale = socket().sent[0].session_context; const second = createService({ 'agents.voice.language': 'auto' }); await second.service.connect(createTestWindow('')); second.service.sendStartSession({ sessions: [], display_locale: '' }, 'machine'); - const fallbackLocale = socket().sent[0].session_context; + const withoutBrowserLocale = socket().sent[0].session_context; - assert.deepStrictEqual({ browserLocale, fallbackLocale }, { - browserLocale: { sessions: [], display_locale: 'pt-BR' }, - fallbackLocale: { sessions: [], display_locale: 'en-US' }, + assert.deepStrictEqual({ withBrowserLocale, withoutBrowserLocale }, { + withBrowserLocale: { sessions: [], display_locale: 'en' }, + withoutBrowserLocale: { sessions: [], display_locale: 'en' }, + }); + }); + + test('resolves automatic language from display language before browser locale', () => { + assert.deepStrictEqual({ + displayLanguage: resolveAutomaticVoiceLanguage('en-US', 'de'), + englishDisplayLanguage: resolveAutomaticVoiceLanguage('de-DE', 'en'), + browserLocale: resolveAutomaticVoiceLanguage('pt-BR', undefined), + unsupportedDisplayLanguage: resolveAutomaticVoiceLanguage('pt-BR', 'he-IL'), + missing: resolveAutomaticVoiceLanguage(undefined, undefined), + }, { + displayLanguage: 'de', + englishDisplayLanguage: 'en', + browserLocale: 'pt-BR', + unsupportedDisplayLanguage: 'pt-BR', + missing: 'en-US', }); }); @@ -601,7 +617,7 @@ suite('VoiceClientService', () => { }); }); - test('preserves an automatic ASR-only browser locale', async () => { + test('prefers the display language over an ASR-only browser locale', async () => { const { service } = createService({ 'agents.voice.language': 'auto' }); await service.connect(createTestWindow('ar-SA')); @@ -609,11 +625,11 @@ suite('VoiceClientService', () => { assert.deepStrictEqual(socket().sent[0].session_context, { sessions: [], - display_locale: 'ar-SA', + display_locale: 'en', }); }); - test('falls back for an unsupported automatic browser locale', async () => { + test('prefers the display language over an unsupported browser locale', async () => { const { service } = createService({ 'agents.voice.language': 'auto' }); await service.connect(createTestWindow('he-IL')); @@ -621,7 +637,7 @@ suite('VoiceClientService', () => { assert.deepStrictEqual(socket().sent[0].session_context, { sessions: [], - display_locale: 'en-US', + display_locale: 'en', }); }); @@ -643,7 +659,7 @@ suite('VoiceClientService', () => { } : message), [ { type: 'start_session', - session_context: { sessions: [], display_locale: 'en-GB' }, + session_context: { sessions: [], display_locale: 'en' }, voice: 'victoria_neutral', }, { type: 'set_language', language: 'fr-FR' },