feat(elevenlabs): add secondary_languages for Scribe v2 realtime STT

livekit-plugins/livekit-plugins-elevenlabs/livekit/plugins/elevenlabs/stt.py

@@ -95,6 +95,7 @@ class STTOptions:
     sample_rate: STTRealtimeSampleRates
     server_vad: NotGivenOr[VADOptions | None]
     keyterms: NotGivenOr[list[str]]
+    secondary_languages: NotGivenOr[list[str]]
     no_verbatim: bool
     enable_logging: bool
     previous_text: str | None
@@ -116,6 +117,7 @@ def __init__(
         model: NotGivenOr[ElevenLabsSTTModels | str] = NOT_GIVEN,
         model_id: NotGivenOr[ElevenLabsSTTModels | str] = NOT_GIVEN,  # Deprecated
         keyterms: NotGivenOr[list[str]] = NOT_GIVEN,
+        secondary_languages: NotGivenOr[list[str]] = NOT_GIVEN,
         no_verbatim: NotGivenOr[bool] = NOT_GIVEN,
         enable_logging: bool = True,
         previous_text: NotGivenOr[str] = NOT_GIVEN,
@@ -141,6 +143,11 @@ def __init__(
                 Supported for both Scribe v2 (batch) and Scribe v2 realtime. Batch accepts up to
                 1000 keyterms of at most 50 characters each; realtime accepts up to 50 keyterms of
                 at most 20 characters each. Usage incurs additional costs.
+            secondary_languages (NotGivenOr[list[str]]): A list of language codes to constrain
+                speech prediction to, in addition to `language_code`. Useful for bilingual
+                applications where the audio switches between a primary and a limited set of
+                secondary languages. Only supported for Scribe v2 realtime. When omitted, the
+                model predicts from its full set of supported languages.
             no_verbatim (NotGivenOr[bool]): When True, the model removes filler words, false starts
                 and disfluencies from the transcript, producing cleaner output. Supported for both
                 Scribe v2 (batch) and Scribe v2 realtime. Default is False.
@@ -184,6 +191,14 @@ def __init__(
             )
             resolved_previous_text = None
 
+        resolved_secondary_languages = secondary_languages
+        if not use_realtime and is_given(secondary_languages):
+            logger.warning(
+                "`secondary_languages` is only supported for Scribe v2 realtime model "
+                "and will be ignored"
+            )
+            resolved_secondary_languages = NOT_GIVEN
+
         super().__init__(
             capabilities=STTCapabilities(
                 streaming=use_realtime,
@@ -209,6 +224,7 @@ def __init__(
             include_timestamps=include_timestamps,
             model_id=model,
             keyterms=keyterms,
+            secondary_languages=resolved_secondary_languages,
             no_verbatim=no_verbatim if is_given(no_verbatim) else False,
             enable_logging=enable_logging,
             previous_text=resolved_previous_text,
@@ -338,6 +354,7 @@ def update_options(
         tag_audio_events: NotGivenOr[bool] = NOT_GIVEN,
         server_vad: NotGivenOr[VADOptions] = NOT_GIVEN,
         keyterms: NotGivenOr[list[str]] = NOT_GIVEN,
+        secondary_languages: NotGivenOr[list[str]] = NOT_GIVEN,
         no_verbatim: NotGivenOr[bool] = NOT_GIVEN,
     ) -> None:
         if is_given(tag_audio_events):
@@ -349,11 +366,26 @@ def update_options(
         if is_given(keyterms):
             self._opts.keyterms = keyterms
 
+        if is_given(secondary_languages):
+            if self._opts.model_id == "scribe_v2_realtime":
+                self._opts.secondary_languages = secondary_languages
+            else:
+                logger.warning(
+                    "`secondary_languages` is only supported for Scribe v2 realtime model "
+                    "and will be ignored"
+                )
+                secondary_languages = NOT_GIVEN
+
         if is_given(no_verbatim):
             self._opts.no_verbatim = no_verbatim
 
         for stream in self._streams:
-            stream.update_options(server_vad=server_vad, no_verbatim=no_verbatim, keyterms=keyterms)
+            stream.update_options(
+                server_vad=server_vad,
+                no_verbatim=no_verbatim,
+                keyterms=keyterms,
+                secondary_languages=secondary_languages,
+            )
 
     def stream(
         self,
@@ -402,6 +434,7 @@ def update_options(
         server_vad: NotGivenOr[VADOptions] = NOT_GIVEN,
         no_verbatim: NotGivenOr[bool] = NOT_GIVEN,
         keyterms: NotGivenOr[list[str]] = NOT_GIVEN,
+        secondary_languages: NotGivenOr[list[str]] = NOT_GIVEN,
     ) -> None:
         if is_given(server_vad):
             self._opts.server_vad = server_vad
@@ -412,6 +445,9 @@ def update_options(
         if is_given(keyterms):
             self._opts.keyterms = keyterms
             self._reconnect_event.set()
+        if is_given(secondary_languages):
+            self._opts.secondary_languages = secondary_languages
+            self._reconnect_event.set()
 
     def _on_audio_duration_report(self, duration: float) -> None:
         usage_event = stt.SpeechEvent(
@@ -620,6 +656,12 @@ async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
         if is_given(self._opts.keyterms):
             params.extend(f"keyterms={quote(keyterm)}" for keyterm in self._opts.keyterms)
 
+        if is_given(self._opts.secondary_languages):
+            params.extend(
+                f"secondary_languages={quote(language)}"
+                for language in self._opts.secondary_languages
+            )
+
         query_string = "&".join(params)
 
         # Convert HTTPS URL to WSS

tests/test_plugin_elevenlabs_stt.py

@@ -43,6 +43,7 @@ def _new_stream(*, server_vad=NOT_GIVEN) -> elevenlabs_stt.SpeechStream:
         sample_rate=16000,
         server_vad=server_vad,
         keyterms=NOT_GIVEN,
+        secondary_languages=NOT_GIVEN,
         no_verbatim=False,
         enable_logging=True,
         previous_text=None,