Add VAD sensitivity to ESPHome (#95283)

* Change to "finished speaking detection"

* Add select entity to ESPHome for finished speaking detection

* Fix entity name

* Use vad select in stt stream

---------

Co-authored-by: J. Nick Koston <nick@koston.org>

homeassistant/components/assist_pipeline/strings.json

@@ -13,7 +13,7 @@
         }
       },
       "vad_sensitivity": {
-        "name": "Silence sensitivity",
+        "name": "Finished speaking detection",
         "state": {
           "default": "Default",
           "aggressive": "Aggressive",

homeassistant/components/esphome/select.py

@@ -3,7 +3,10 @@ from __future__ import annotations
 
 from aioesphomeapi import EntityInfo, SelectInfo, SelectState
 
-from homeassistant.components.assist_pipeline.select import AssistPipelineSelect
+from homeassistant.components.assist_pipeline.select import (
+    AssistPipelineSelect,
+    VadSensitivitySelect,
+)
 from homeassistant.components.select import SelectEntity
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import HomeAssistant, callback
@@ -37,7 +40,12 @@ async def async_setup_entry(
     entry_data = DomainData.get(hass).get_entry_data(entry)
     assert entry_data.device_info is not None
     if entry_data.device_info.voice_assistant_version:
-        async_add_entities([EsphomeAssistPipelineSelect(hass, entry_data)])
+        async_add_entities(
+            [
+                EsphomeAssistPipelineSelect(hass, entry_data),
+                EsphomeVadSensitivitySelect(hass, entry_data),
+            ]
+        )
 
 
 class EsphomeSelect(EsphomeEntity[SelectInfo, SelectState], SelectEntity):
@@ -68,3 +76,12 @@ class EsphomeAssistPipelineSelect(EsphomeAssistEntity, AssistPipelineSelect):
         """Initialize a pipeline selector."""
         EsphomeAssistEntity.__init__(self, entry_data)
         AssistPipelineSelect.__init__(self, hass, self._device_info.mac_address)
+
+
+class EsphomeVadSensitivitySelect(EsphomeAssistEntity, VadSensitivitySelect):
+    """VAD sensitivity selector for VoIP devices."""
+
+    def __init__(self, hass: HomeAssistant, entry_data: RuntimeEntryData) -> None:
+        """Initialize a VAD sensitivity selector."""
+        EsphomeAssistEntity.__init__(self, entry_data)
+        VadSensitivitySelect.__init__(self, hass, self._device_info.mac_address)

homeassistant/components/esphome/strings.json

@@ -67,6 +67,14 @@
         "state": {
           "preferred": "[%key:component::assist_pipeline::entity::select::pipeline::state::preferred%]"
         }
+      },
+      "vad_sensitivity": {
+        "name": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::name%]",
+        "state": {
+          "default": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::default%]",
+          "aggressive": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::aggressive%]",
+          "relaxed": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::relaxed%]"
+        }
       }
     }
   },

homeassistant/components/esphome/voice_assistant.py

@@ -19,7 +19,10 @@ from homeassistant.components.assist_pipeline import (
     async_pipeline_from_audio_stream,
     select as pipeline_select,
 )
-from homeassistant.components.assist_pipeline.vad import VoiceCommandSegmenter
+from homeassistant.components.assist_pipeline.vad import (
+    VadSensitivity,
+    VoiceCommandSegmenter,
+)
 from homeassistant.components.media_player import async_process_play_media_url
 from homeassistant.core import Context, HomeAssistant, callback
 
@@ -251,9 +254,9 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
                 chunk = await self.queue.get()
 
     async def _iterate_packets_with_vad(
-        self, pipeline_timeout: float
+        self, pipeline_timeout: float, silence_seconds: float
     ) -> Callable[[], AsyncIterable[bytes]] | None:
-        segmenter = VoiceCommandSegmenter()
+        segmenter = VoiceCommandSegmenter(silence_seconds=silence_seconds)
         chunk_buffer: deque[bytes] = deque(maxlen=100)
         try:
             async with async_timeout.timeout(pipeline_timeout):
@@ -305,7 +308,16 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
         )
 
         if use_vad:
-            stt_stream = await self._iterate_packets_with_vad(pipeline_timeout)
+            stt_stream = await self._iterate_packets_with_vad(
+                pipeline_timeout,
+                silence_seconds=VadSensitivity.to_seconds(
+                    pipeline_select.get_vad_sensitivity(
+                        self.hass,
+                        DOMAIN,
+                        self.device_info.mac_address,
+                    )
+                ),
+            )
             # Error or timeout occurred and was handled already
             if stt_stream is None:
                 return

tests/components/esphome/test_select.py

@@ -25,6 +25,20 @@ async def test_pipeline_selector(
     assert state.state == "preferred"
 
 
+async def test_vad_sensitivity_select(
+    hass: HomeAssistant,
+    mock_voice_assistant_v1_entry,
+) -> None:
+    """Test VAD sensitivity select.
+
+    Functionality is tested in assist_pipeline/test_select.py.
+    This test is only to ensure it is set up.
+    """
+    state = hass.states.get("select.test_finished_speaking_detection")
+    assert state is not None
+    assert state.state == "default"
+
+
 async def test_select_generic_entity(
     hass: HomeAssistant, mock_client: APIClient, mock_generic_device_entry
 ) -> None:

tests/components/voip/test_select.py

@@ -29,6 +29,6 @@ async def test_vad_sensitivity_select(
     Functionality is tested in assist_pipeline/test_select.py.
     This test is only to ensure it is set up.
     """
-    state = hass.states.get("select.192_168_1_210_silence_sensitivity")
+    state = hass.states.get("select.192_168_1_210_finished_speaking_detection")
     assert state is not None
     assert state.state == "default"