Add first interim STT latency metric

livekit-agents/livekit/agents/llm/chat_context.py

@@ -238,6 +238,18 @@ class MetricsReport(TypedDict, total=False):
     User `ChatMessage` only
     """
 
+    first_interim_delay: float
+    """Time from speech onset to the first interim transcript.
+
+    User `ChatMessage` only
+    """
+
+    first_interim_status: Literal["received", "absent"]
+    """Terminal outcome for first-interim STT responsiveness.
+
+    User `ChatMessage` only
+    """
+
     end_of_turn_delay: float
     """Amount of time between the end of speech and the decision to end the user's turn
 

livekit-agents/livekit/agents/metrics/base.py

@@ -103,6 +103,11 @@ class EOUMetrics(_BaseMetrics):
     """Time taken to obtain the transcript after the end of the user's speech.
     Set to 0.0 if the end of speech was not detected.
     """
+    first_interim_delay: float | None
+    """Time from speech onset to the first interim transcript, if one was received."""
+
+    first_interim_status: Literal["received", "absent"] | None = None
+    """Whether the utterance produced an interim transcript before its terminal turn event."""
 
     on_user_turn_completed_delay: float
     """Time taken to invoke the user's `Agent.on_user_turn_completed` callback."""

livekit-agents/livekit/agents/telemetry/otel_metrics.py

@@ -40,6 +40,19 @@
     unit="s",
     description="Time from end of speech to transcript available",
 )
+_turn_first_interim_delay = _meter.create_histogram(
+    "lk.agents.turn.first_interim_delay",
+    unit="s",
+    description="Time from speech onset to first interim transcript",
+)
+_turn_first_interim_received = _meter.create_counter(
+    "lk.agents.turn.first_interim_received",
+    description="Number of utterances that produced a first interim transcript",
+)
+_turn_first_interim_absent = _meter.create_counter(
+    "lk.agents.turn.first_interim_absent",
+    description="Number of completed utterances without an interim transcript",
+)
 _turn_end_of_turn_delay = _meter.create_histogram(
     "lk.agents.turn.end_of_turn_delay",
     unit="s",
@@ -120,6 +133,12 @@ def _record_turn_metrics(report: MetricsReport) -> None:
         _turn_tts_ttfb.record(report["tts_node_ttfb"], attributes=tts_attrs)
     if "transcription_delay" in report:
         _turn_transcription_delay.record(report["transcription_delay"], attributes=stt_attrs)
+    if "first_interim_delay" in report:
+        _turn_first_interim_delay.record(report["first_interim_delay"], attributes=stt_attrs)
+    if report.get("first_interim_status") == "received":
+        _turn_first_interim_received.add(1, attributes=stt_attrs)
+    elif report.get("first_interim_status") == "absent":
+        _turn_first_interim_absent.add(1, attributes=stt_attrs)
     if "end_of_turn_delay" in report:
         _turn_end_of_turn_delay.record(report["end_of_turn_delay"], attributes=stt_attrs)
     if "on_user_turn_completed_delay" in report:

livekit-agents/livekit/agents/voice/agent_activity.py

@@ -2443,6 +2443,8 @@ async def _user_turn_completed_task(
             timestamp=time.time(),
             end_of_utterance_delay=info.metrics.end_of_turn_delay or 0.0,
             transcription_delay=info.metrics.transcription_delay or 0.0,
+            first_interim_delay=info.metrics.first_interim_delay,
+            first_interim_status=info.metrics.first_interim_status,
             on_user_turn_completed_delay=on_user_turn_completed_delay,
             speech_id=speech_handle.id,
             metadata=metadata,
@@ -4310,6 +4312,12 @@ def _init_metrics_from_end_of_turn(self, info: _EndOfTurnInfo) -> llm.MetricsRep
         if info.metrics.transcription_delay is not None:
             metrics_report["transcription_delay"] = info.metrics.transcription_delay
 
+        if info.metrics.first_interim_delay is not None:
+            metrics_report["first_interim_delay"] = info.metrics.first_interim_delay
+
+        if info.metrics.first_interim_status is not None:
+            metrics_report["first_interim_status"] = info.metrics.first_interim_status
+
         if info.metrics.end_of_turn_delay is not None:
             metrics_report["end_of_turn_delay"] = info.metrics.end_of_turn_delay
 

livekit-agents/livekit/agents/voice/audio_recognition.py

@@ -63,6 +63,8 @@ class _EndOfTurnMetrics:
     stopped_speaking_at: float | None
     transcription_delay: float | None
     end_of_turn_delay: float | None
+    first_interim_delay: float | None = None
+    first_interim_status: Literal["received", "absent"] | None = None
 
 
 @dataclass
@@ -82,6 +84,7 @@ def _compute_end_of_turn_metrics(
     last_speaking_time: float | None,
     last_final_transcript_time: float | None,
     now: float,
+    first_interim_time: float | None = None,
 ) -> _EndOfTurnMetrics:
     """Compute the end-of-turn timing metrics from the captured turn anchors.
 
@@ -107,13 +110,21 @@ def _compute_end_of_turn_metrics(
             stopped_speaking_at=None,
             transcription_delay=None,
             end_of_turn_delay=None,
+            first_interim_delay=None,
+            first_interim_status=None,
         )
 
     return _EndOfTurnMetrics(
         started_speaking_at=speech_start_time,
         stopped_speaking_at=last_speaking_time,
         transcription_delay=max(last_final_transcript_time - last_speaking_time, 0),
         end_of_turn_delay=max(now - last_speaking_time, 0),
+        first_interim_delay=(
+            max(first_interim_time - speech_start_time, 0)
+            if first_interim_time is not None
+            else None
+        ),
+        first_interim_status="received" if first_interim_time is not None else "absent",
     )
 
 
@@ -271,6 +282,7 @@ def __init__(
         self._last_final_transcript_time: float | None = None
         self._last_speaking_time: float | None = None
         self._speech_start_time: float | None = None
+        self._first_interim_time: float | None = None
 
         # used for manual commit_user_turn
         self._final_transcript_received = asyncio.Event()
@@ -979,6 +991,7 @@ def _clear_user_turn(self) -> None:
         self._final_transcript_confidence = []
         self._last_final_transcript_time = None
         self._speech_start_time = None
+        self._first_interim_time = None
         self._last_speaking_time = None
         self._vad_speech_started = False
         self._user_turn_committed = False
@@ -1270,6 +1283,12 @@ async def _on_stt_event(self, ev: stt.SpeechEvent) -> None:
                 )
 
         elif ev.type == stt.SpeechEventType.INTERIM_TRANSCRIPT:
+            if (
+                self._first_interim_time is None
+                and self._speech_start_time is not None
+                and ev.alternatives[0].text
+            ):
+                self._first_interim_time = time.time()
             self._hooks.on_interim_transcript(
                 ev,
                 speaking=self._speaking
@@ -1648,6 +1667,7 @@ async def _bounce_eou_task(
                 speech_start_time=speech_start_time,
                 last_speaking_time=last_speaking_time,
                 last_final_transcript_time=last_final_transcript_time,
+                first_interim_time=self._first_interim_time,
                 now=time.time(),
             )
             committed = self._hooks.on_end_of_turn(
@@ -1697,6 +1717,7 @@ async def _bounce_eou_task(
                 # only reset if there is no new speech
                 if self._last_speaking_time == last_speaking_time:
                     self._speech_start_time = None
+                    self._first_interim_time = None
                     self._vad_speech_started = False
                     self._last_speaking_time = None
 

tests/test_stt_first_interim_metric.py

@@ -0,0 +1,65 @@
+"""Unit coverage for first-interim STT responsiveness metrics."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from livekit.agents.stt import SpeechData, SpeechEvent, SpeechEventType
+from livekit.agents.voice.audio_recognition import (
+    AudioRecognition,
+    _compute_end_of_turn_metrics,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_first_interim_delay_uses_speech_onset() -> None:
+    metrics = _compute_end_of_turn_metrics(
+        speech_start_time=10.0,
+        last_speaking_time=14.0,
+        last_final_transcript_time=14.5,
+        first_interim_time=10.25,
+        now=15.0,
+    )
+    assert metrics.first_interim_delay == pytest.approx(0.25)
+    assert metrics.first_interim_status == "received"
+
+
+def test_first_interim_delay_is_unavailable_without_interim() -> None:
+    metrics = _compute_end_of_turn_metrics(
+        speech_start_time=10.0,
+        last_speaking_time=14.0,
+        last_final_transcript_time=14.5,
+        first_interim_time=None,
+        now=15.0,
+    )
+    assert metrics.first_interim_delay is None
+    assert metrics.first_interim_status == "absent"
+
+
+async def test_blank_interim_does_not_capture_first_interim_time() -> None:
+    recognition = AudioRecognition.__new__(AudioRecognition)
+    recognition._turn_detection_mode = "vad"
+    recognition._interruption_enabled = False
+    recognition._first_interim_time = None
+    recognition._speech_start_time = 10.0
+    recognition._hooks = MagicMock()
+    recognition._vad = None
+
+    blank_interim = SpeechEvent(
+        type=SpeechEventType.INTERIM_TRANSCRIPT,
+        alternatives=[SpeechData(language="", text="")],
+    )
+    with patch("livekit.agents.voice.audio_recognition.time.time", return_value=11.0):
+        await recognition._on_stt_event(blank_interim)
+
+    assert recognition._first_interim_time is None
+
+    populated_interim = SpeechEvent(
+        type=SpeechEventType.INTERIM_TRANSCRIPT,
+        alternatives=[SpeechData(language="", text="hello")],
+    )
+    with patch("livekit.agents.voice.audio_recognition.time.time", return_value=12.0):
+        await recognition._on_stt_event(populated_interim)
+
+    assert recognition._first_interim_time == 12.0