Send/receive Voice Assistant audio via ESPHome native API (#114800)

* Protobuf audio test

* Remove extraneous code

* Rework voice assistant pipeline

* Move variables

* Fix reading flags

* Dont directly put to queue

* Bump aioesphomeapi to 24.0.0

* Update tests

- Add more tests for API pipeline
- Convert some udp tests to use api pipeline
- Update fixtures for new device info flags

* Fix bad merge

---------

Co-authored-by: Michael Hansen <mike@rhasspy.org>

homeassistant/components/esphome/binary_sensor.py

@@ -33,7 +33,9 @@ async def async_setup_entry(
 
     entry_data = DomainData.get(hass).get_entry_data(entry)
     assert entry_data.device_info is not None
-    if entry_data.device_info.voice_assistant_version:
+    if entry_data.device_info.voice_assistant_feature_flags_compat(
+        entry_data.api_version
+    ):
         async_add_entities([EsphomeAssistInProgressBinarySensor(entry_data)])
 
 

homeassistant/components/esphome/entry_data.py

@@ -257,7 +257,9 @@ async def async_update_static_infos(
         if async_get_dashboard(hass):
             needed_platforms.add(Platform.UPDATE)
 
-        if self.device_info and self.device_info.voice_assistant_version:
+        if self.device_info and self.device_info.voice_assistant_feature_flags_compat(
+            self.api_version
+        ):
             needed_platforms.add(Platform.BINARY_SENSOR)
             needed_platforms.add(Platform.SELECT)
 

homeassistant/components/esphome/manager.py

@@ -21,6 +21,7 @@
     UserService,
     UserServiceArgType,
     VoiceAssistantAudioSettings,
+    VoiceAssistantFeature,
 )
 from awesomeversion import AwesomeVersion
 import voluptuous as vol
@@ -72,7 +73,11 @@
 
 # Import config flow so that it's added to the registry
 from .entry_data import RuntimeEntryData
-from .voice_assistant import VoiceAssistantUDPServer
+from .voice_assistant import (
+    VoiceAssistantAPIPipeline,
+    VoiceAssistantPipeline,
+    VoiceAssistantUDPPipeline,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -143,7 +148,7 @@ class ESPHomeManager:
         "cli",
         "device_id",
         "domain_data",
-        "voice_assistant_udp_server",
+        "voice_assistant_pipeline",
         "reconnect_logic",
         "zeroconf_instance",
         "entry_data",
@@ -168,7 +173,7 @@ def __init__(
         self.cli = cli
         self.device_id: str | None = None
         self.domain_data = domain_data
-        self.voice_assistant_udp_server: VoiceAssistantUDPServer | None = None
+        self.voice_assistant_pipeline: VoiceAssistantPipeline | None = None
         self.reconnect_logic: ReconnectLogic | None = None
         self.zeroconf_instance = zeroconf_instance
         self.entry_data = entry_data
@@ -327,9 +332,10 @@ def async_on_state_subscription(
     def _handle_pipeline_finished(self) -> None:
         self.entry_data.async_set_assist_pipeline_state(False)
 
-        if self.voice_assistant_udp_server is not None:
-            self.voice_assistant_udp_server.close()
-            self.voice_assistant_udp_server = None
+        if self.voice_assistant_pipeline is not None:
+            if isinstance(self.voice_assistant_pipeline, VoiceAssistantUDPPipeline):
+                self.voice_assistant_pipeline.close()
+            self.voice_assistant_pipeline = None
 
     async def _handle_pipeline_start(
         self,
@@ -339,38 +345,60 @@ async def _handle_pipeline_start(
         wake_word_phrase: str | None,
     ) -> int | None:
         """Start a voice assistant pipeline."""
-        if self.voice_assistant_udp_server is not None:
+        if self.voice_assistant_pipeline is not None:
             _LOGGER.warning("Voice assistant UDP server was not stopped")
-            self.voice_assistant_udp_server.stop()
-            self.voice_assistant_udp_server = None
+            self.voice_assistant_pipeline.stop()
+            self.voice_assistant_pipeline = None
 
         hass = self.hass
-        self.voice_assistant_udp_server = VoiceAssistantUDPServer(
-            hass,
-            self.entry_data,
-            self.cli.send_voice_assistant_event,
-            self._handle_pipeline_finished,
-        )
-        port = await self.voice_assistant_udp_server.start_server()
+        assert self.entry_data.device_info is not None
+        if (
+            self.entry_data.device_info.voice_assistant_feature_flags_compat(
+                self.entry_data.api_version
+            )
+            & VoiceAssistantFeature.API_AUDIO
+        ):
+            self.voice_assistant_pipeline = VoiceAssistantAPIPipeline(
+                hass,
+                self.entry_data,
+                self.cli.send_voice_assistant_event,
+                self._handle_pipeline_finished,
+                self.cli,
+            )
+            port = 0
+        else:
+            self.voice_assistant_pipeline = VoiceAssistantUDPPipeline(
+                hass,
+                self.entry_data,
+                self.cli.send_voice_assistant_event,
+                self._handle_pipeline_finished,
+            )
+            port = await self.voice_assistant_pipeline.start_server()
 
         assert self.device_id is not None, "Device ID must be set"
         hass.async_create_background_task(
-            self.voice_assistant_udp_server.run_pipeline(
+            self.voice_assistant_pipeline.run_pipeline(
                 device_id=self.device_id,
                 conversation_id=conversation_id or None,
                 flags=flags,
                 audio_settings=audio_settings,
                 wake_word_phrase=wake_word_phrase,
             ),
-            "esphome.voice_assistant_udp_server.run_pipeline",
+            "esphome.voice_assistant_pipeline.run_pipeline",
         )
 
         return port
 
     async def _handle_pipeline_stop(self) -> None:
         """Stop a voice assistant pipeline."""
-        if self.voice_assistant_udp_server is not None:
-            self.voice_assistant_udp_server.stop()
+        if self.voice_assistant_pipeline is not None:
+            self.voice_assistant_pipeline.stop()
+
+    async def _handle_audio(self, data: bytes) -> None:
+        if self.voice_assistant_pipeline is None:
+            return
+        assert isinstance(self.voice_assistant_pipeline, VoiceAssistantAPIPipeline)
+        self.voice_assistant_pipeline.receive_audio_bytes(data)
 
     async def on_connect(self) -> None:
         """Subscribe to states and list entities on successful API login."""
@@ -472,13 +500,23 @@ async def _on_connnect(self) -> None:
                 )
             )
 
-        if device_info.voice_assistant_version:
-            entry_data.disconnect_callbacks.add(
-                cli.subscribe_voice_assistant(
-                    self._handle_pipeline_start,
-                    self._handle_pipeline_stop,
+        flags = device_info.voice_assistant_feature_flags_compat(api_version)
+        if flags:
+            if flags & VoiceAssistantFeature.API_AUDIO:
+                entry_data.disconnect_callbacks.add(
+                    cli.subscribe_voice_assistant(
+                        handle_start=self._handle_pipeline_start,
+                        handle_stop=self._handle_pipeline_stop,
+                        handle_audio=self._handle_audio,
+                    )
+                )
+            else:
+                entry_data.disconnect_callbacks.add(
+                    cli.subscribe_voice_assistant(
+                        handle_start=self._handle_pipeline_start,
+                        handle_stop=self._handle_pipeline_stop,
+                    )
                 )
-            )
 
         cli.subscribe_states(entry_data.async_update_state)
         cli.subscribe_service_calls(self.async_on_service_call)

homeassistant/components/esphome/manifest.json

@@ -15,7 +15,7 @@
   "iot_class": "local_push",
   "loggers": ["aioesphomeapi", "noiseprotocol", "bleak_esphome"],
   "requirements": [
-    "aioesphomeapi==23.2.0",
+    "aioesphomeapi==24.0.0",
     "esphome-dashboard-api==1.2.3",
     "bleak-esphome==1.0.0"
   ],

homeassistant/components/esphome/select.py

@@ -42,7 +42,9 @@ async def async_setup_entry(
 
     entry_data = DomainData.get(hass).get_entry_data(entry)
     assert entry_data.device_info is not None
-    if entry_data.device_info.voice_assistant_version:
+    if entry_data.device_info.voice_assistant_feature_flags_compat(
+        entry_data.api_version
+    ):
         async_add_entities(
             [
                 EsphomeAssistPipelineSelect(hass, entry_data),