Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ configurationRegistry.registerConfiguration({
nls.localize('agents.voice.language.ko', "Korean"),
nls.localize('agents.voice.language.zh', "Chinese"),
],
markdownDescription: nls.localize('agents.voice.language', "The language used for speech recognition, dictation, and spoken responses. The selectable languages support native voice output. Automatic follows the system or browser locale for speech recognition and dictation, and uses English voice output when the detected language does not support native voice output. Changing this while voice mode is connected takes effect immediately."),
markdownDescription: nls.localize('agents.voice.language', "The language used for speech recognition, dictation, and spoken responses. The selectable languages support native voice output. Automatic uses the configured display language for speech recognition and dictation when supported; otherwise, it follows the system or browser locale. English voice output is used when the detected language does not support native voice output. Changing this while voice mode is connected takes effect immediately."),
default: 'auto',
scope: ConfigurationScope.APPLICATION,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ configurationRegistry.registerConfiguration({
nls.localize('dictation.model.mai.label', "MAI — Cloud"),
],
markdownEnumDescriptions: [
nls.localize('dictation.model.nemotronMultilingual', "NVIDIA Nemotron 3.5 multilingual streaming RNN-T, run on-device through Microsoft Foundry Local. Works offline; no audio leaves the device. Automatic language selection follows the Voice Mode language setting and system or browser locale, with model detection as a fallback. Downloaded on first use and cached on disk."),
nls.localize('dictation.model.nemotronMultilingual', "NVIDIA Nemotron 3.5 multilingual streaming RNN-T, run on-device through Microsoft Foundry Local. Works offline; no audio leaves the device. Automatic language selection follows the Voice Mode language setting; when that setting is Automatic, dictation uses the configured display language when supported, then the system or browser locale, with model detection as a fallback. Downloaded on first use and cached on disk."),
nls.localize('dictation.model.mai', "Cloud transcription through the same Microsoft AI voice service used by Voice Mode. Requires a network connection and GitHub sign-in; audio is streamed to the service."),
],
markdownDescription: nls.localize('dictation.model', "The model used for dictation. On-device models download on first use and run locally through Microsoft Foundry Local; the cloud option streams audio to the Microsoft AI voice service."),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { Language } from '../../../../../base/common/platform.js';

const NEMOTRON_LOCALES = new Set([
'ar-AR', 'bg-BG', 'cs-CZ', 'da-DK', 'de-DE', 'en-GB', 'en-US', 'es-ES',
'es-US', 'et-EE', 'fi-FI', 'fr-CA', 'fr-FR', 'el-GR', 'he-IL', 'hi-IN',
Expand Down Expand Up @@ -50,25 +52,38 @@ const NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE: Readonly<Record<string, string>> = {
zh: 'zh-CN',
};

function getConfiguredDisplayLanguage(): string | undefined {
return Language.value();
}

/**
* Resolve the on-device dictation language using the same setting semantics as
* Voice Mode. Automatic follows the browser locale when Nemotron supports it,
* then falls back to the model's language detection.
* Voice Mode. Automatic follows the configured display language when supported,
* then the system or browser locale, then the model's language detection.
*/
export function resolveDictationLanguage(configuredLanguage: unknown, browserLanguage: string | undefined): string {
export function resolveDictationLanguage(configuredLanguage: unknown, browserLanguage: string | undefined, displayLanguage = getConfiguredDisplayLanguage()): string {
const configured = typeof configuredLanguage === 'string' ? configuredLanguage.trim() : '';
const candidate = configured && configured.toLowerCase() !== 'auto' ? configured : browserLanguage;
if (configured && configured.toLowerCase() !== 'auto') {
return resolveSupportedDictationLanguage(configured) ?? 'auto';
}

return resolveSupportedDictationLanguage(displayLanguage)
?? resolveSupportedDictationLanguage(browserLanguage)
?? 'auto';
}

function resolveSupportedDictationLanguage(candidate: string | undefined): string | undefined {
if (!candidate || typeof Intl.getCanonicalLocales !== 'function') {
return 'auto';
return undefined;
}

try {
const canonical = Intl.getCanonicalLocales(candidate)[0];
if (NEMOTRON_LOCALES.has(canonical)) {
return canonical;
}
return NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE[canonical.split('-')[0]] ?? 'auto';
return NEMOTRON_DEFAULT_LOCALE_BY_LANGUAGE[canonical.split('-')[0]];
} catch {
return 'auto';
return undefined;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { Disposable } from '../../../../../base/common/lifecycle.js';
import { Emitter, Event } from '../../../../../base/common/event.js';
import { generateUuid } from '../../../../../base/common/uuid.js';
import { mainWindow } from '../../../../../base/browser/window.js';
import { Language } from '../../../../../base/common/platform.js';
import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
import { ILogService } from '../../../../../platform/log/common/log.js';
import { IProductService } from '../../../../../platform/product/common/productService.js';
Expand Down Expand Up @@ -59,6 +60,26 @@ function asOptionalNonEmptyString(value: unknown): string | undefined {
return result && result.length > 0 ? result : undefined;
}

function canonicalizeSupportedLanguage(value: string | undefined, supportedBases: ReadonlySet<string>): string | undefined {
const candidate = value?.trim();
if (!candidate || typeof Intl.getCanonicalLocales !== 'function') {
return undefined;
}

try {
const canonical = Intl.getCanonicalLocales(candidate)[0];
return supportedBases.has(canonical.split('-')[0]) ? canonical : undefined;
} catch {
return undefined;
}
}

export function resolveAutomaticVoiceLanguage(browserLanguage: string | undefined, displayLanguage: string | undefined): string {
return canonicalizeSupportedLanguage(displayLanguage, ASR_SUPPORTED_LANGUAGE_BASES)
?? canonicalizeSupportedLanguage(browserLanguage, ASR_SUPPORTED_LANGUAGE_BASES)
?? DEFAULT_LANGUAGE;
}

function asTranscriptionStatus(value: unknown): IVoiceTranscription['status'] | undefined {
return value === 'partial' || value === 'final' ? value : undefined;
}
Expand Down Expand Up @@ -201,30 +222,15 @@ export class VoiceClientService extends Disposable implements IVoiceClientServic
private _getLanguage(): string {
const configured = this._configurationService.getValue<string>('agents.voice.language');
if (typeof configured === 'string' && configured.trim().toLowerCase() !== 'auto') {
const language = this._canonicalizeSupportedLanguage(configured, TTS_SUPPORTED_LANGUAGE_BASES);
const language = canonicalizeSupportedLanguage(configured, TTS_SUPPORTED_LANGUAGE_BASES);
if (language) {
return language;
}
this._logService.warn(`[voice] Unsupported agents.voice.language value '${configured}', falling back to ${DEFAULT_LANGUAGE}`);
return DEFAULT_LANGUAGE;
}

return this._canonicalizeSupportedLanguage(this._window?.navigator.language, ASR_SUPPORTED_LANGUAGE_BASES)
?? DEFAULT_LANGUAGE;
}

private _canonicalizeSupportedLanguage(value: string | undefined, supportedBases: ReadonlySet<string>): string | undefined {
const candidate = value?.trim();
if (!candidate || typeof Intl.getCanonicalLocales !== 'function') {
return undefined;
}

try {
const canonical = Intl.getCanonicalLocales(candidate)[0];
return supportedBases.has(canonical.split('-')[0]) ? canonical : undefined;
} catch {
return undefined;
}
return resolveAutomaticVoiceLanguage(this._window?.navigator.language, Language.value());
}

private _sendSetLanguage(): void {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,28 @@ suite('ChatSpeechToTextService', () => {

ensureNoDisposablesAreLeakedInTestSuite();

test('resolves the dictation language from Voice Mode configuration and browser locale', () => {
test('resolves the dictation language from Voice Mode configuration, display language, and browser locale', () => {
assert.deepStrictEqual({
explicit: resolveDictationLanguage('fr-FR', 'de-DE'),
automatic: resolveDictationLanguage('auto', 'uk-UA'),
regionalAutomatic: resolveDictationLanguage('auto', 'pt-BR'),
additionalSupportedAutomatic: resolveDictationLanguage('auto', 'he-IL'),
unsupportedRegion: resolveDictationLanguage('auto', 'en-AU'),
explicitWithDisplayLanguage: resolveDictationLanguage('fr-FR', 'de-DE', 'ja'),
displayLanguage: resolveDictationLanguage('auto', 'en-US', 'de'),
englishDisplayLanguage: resolveDictationLanguage('auto', 'de-DE', 'en'),
unsupportedDisplayLanguage: resolveDictationLanguage('auto', 'pt-BR', 'id-ID'),
automatic: resolveDictationLanguage('auto', 'uk-UA', 'id-ID'),
regionalAutomatic: resolveDictationLanguage('auto', 'pt-BR', 'id-ID'),
additionalSupportedAutomatic: resolveDictationLanguage('auto', 'he-IL', 'id-ID'),
unsupportedRegion: resolveDictationLanguage('auto', 'en-AU', 'id-ID'),
explicitSpanish: resolveDictationLanguage('es', 'en-US'),
explicitAdaptationReady: resolveDictationLanguage('lt', 'en-US'),
regionalPortugueseFallback: resolveDictationLanguage('auto', 'pt-AO'),
regionalPortugueseFallback: resolveDictationLanguage('auto', 'pt-AO', 'id-ID'),
invalidExplicit: resolveDictationLanguage('not a locale', 'de-DE'),
missing: resolveDictationLanguage(undefined, undefined),
missing: resolveDictationLanguage(undefined, undefined, 'id-ID'),
}, {
explicit: 'fr-FR',
explicitWithDisplayLanguage: 'fr-FR',
displayLanguage: 'de-DE',
englishDisplayLanguage: 'en-US',
unsupportedDisplayLanguage: 'pt-BR',
automatic: 'uk-UA',
regionalAutomatic: 'pt-BR',
additionalSupportedAutomatic: 'he-IL',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { TestConfigurationService } from '../../../../../../platform/configurati
import { NullLogService } from '../../../../../../platform/log/common/log.js';
import product from '../../../../../../platform/product/common/product.js';
import { IProductService } from '../../../../../../platform/product/common/productService.js';
import { VoiceClientService } from '../../../browser/voiceClient/voiceClientService.js';
import { resolveAutomaticVoiceLanguage, VoiceClientService } from '../../../browser/voiceClient/voiceClientService.js';
import { IVoiceAudioResponse, IVoiceBargeIn, IVoiceNarrationAck, IVoiceNarrationSignal, IVoiceSpeechStarted, IVoiceTranscription } from '../../../common/voiceClient/voiceClientService.js';

class TestWebSocket {
Expand Down Expand Up @@ -560,20 +560,36 @@ suite('VoiceClientService', () => {
}]);
});

test('uses browser locale for auto and falls back when unavailable', async () => {
test('uses the display language for auto', async () => {
const first = createService({ 'agents.voice.language': 'auto' });
await first.service.connect(createTestWindow('pt-BR'));
first.service.sendStartSession({ sessions: [], display_locale: '' }, 'machine');
const browserLocale = socket().sent[0].session_context;
const withBrowserLocale = socket().sent[0].session_context;

const second = createService({ 'agents.voice.language': 'auto' });
await second.service.connect(createTestWindow(''));
second.service.sendStartSession({ sessions: [], display_locale: '' }, 'machine');
const fallbackLocale = socket().sent[0].session_context;
const withoutBrowserLocale = socket().sent[0].session_context;

assert.deepStrictEqual({ browserLocale, fallbackLocale }, {
browserLocale: { sessions: [], display_locale: 'pt-BR' },
fallbackLocale: { sessions: [], display_locale: 'en-US' },
assert.deepStrictEqual({ withBrowserLocale, withoutBrowserLocale }, {
withBrowserLocale: { sessions: [], display_locale: 'en' },
withoutBrowserLocale: { sessions: [], display_locale: 'en' },
});
});

test('resolves automatic language from display language before browser locale', () => {
assert.deepStrictEqual({
displayLanguage: resolveAutomaticVoiceLanguage('en-US', 'de'),
englishDisplayLanguage: resolveAutomaticVoiceLanguage('de-DE', 'en'),
browserLocale: resolveAutomaticVoiceLanguage('pt-BR', undefined),
unsupportedDisplayLanguage: resolveAutomaticVoiceLanguage('pt-BR', 'he-IL'),
missing: resolveAutomaticVoiceLanguage(undefined, undefined),
}, {
displayLanguage: 'de',
englishDisplayLanguage: 'en',
browserLocale: 'pt-BR',
unsupportedDisplayLanguage: 'pt-BR',
missing: 'en-US',
});
});

Expand Down Expand Up @@ -601,27 +617,27 @@ suite('VoiceClientService', () => {
});
});

test('preserves an automatic ASR-only browser locale', async () => {
test('prefers the display language over an ASR-only browser locale', async () => {
const { service } = createService({ 'agents.voice.language': 'auto' });

await service.connect(createTestWindow('ar-SA'));
service.sendStartSession({ sessions: [], display_locale: '' }, 'machine');

assert.deepStrictEqual(socket().sent[0].session_context, {
sessions: [],
display_locale: 'ar-SA',
display_locale: 'en',
});
});

test('falls back for an unsupported automatic browser locale', async () => {
test('prefers the display language over an unsupported browser locale', async () => {
const { service } = createService({ 'agents.voice.language': 'auto' });

await service.connect(createTestWindow('he-IL'));
service.sendStartSession({ sessions: [], display_locale: '' }, 'machine');

assert.deepStrictEqual(socket().sent[0].session_context, {
sessions: [],
display_locale: 'en-US',
display_locale: 'en',
});
});

Expand All @@ -643,7 +659,7 @@ suite('VoiceClientService', () => {
} : message), [
{
type: 'start_session',
session_context: { sessions: [], display_locale: 'en-GB' },
session_context: { sessions: [], display_locale: 'en' },
voice: 'victoria_neutral',
},
{ type: 'set_language', language: 'fr-FR' },
Expand Down
Loading