From 0f08e6699c39419446ab3b1bf7a08d48efa9f6ff Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 26 Jun 2023 15:47:32 -0500 Subject: [PATCH] Add VAD sensitivity to ESPHome (#95283) * Change to "finished speaking detection" * Add select entity to ESPHome for finished speaking detection * Fix entity name * Use vad select in stt stream --------- Co-authored-by: J. Nick Koston --- .../components/assist_pipeline/strings.json | 2 +- homeassistant/components/esphome/select.py | 21 +++++++++++++++++-- homeassistant/components/esphome/strings.json | 8 +++++++ .../components/esphome/voice_assistant.py | 20 ++++++++++++++---- tests/components/esphome/test_select.py | 14 +++++++++++++ tests/components/voip/test_select.py | 2 +- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/assist_pipeline/strings.json b/homeassistant/components/assist_pipeline/strings.json index edcdff752f6..8fa67879fc3 100644 --- a/homeassistant/components/assist_pipeline/strings.json +++ b/homeassistant/components/assist_pipeline/strings.json @@ -13,7 +13,7 @@ } }, "vad_sensitivity": { - "name": "Silence sensitivity", + "name": "Finished speaking detection", "state": { "default": "Default", "aggressive": "Aggressive", diff --git a/homeassistant/components/esphome/select.py b/homeassistant/components/esphome/select.py index 9849f7cded8..a3464b137dc 100644 --- a/homeassistant/components/esphome/select.py +++ b/homeassistant/components/esphome/select.py @@ -3,7 +3,10 @@ from __future__ import annotations from aioesphomeapi import EntityInfo, SelectInfo, SelectState -from homeassistant.components.assist_pipeline.select import AssistPipelineSelect +from homeassistant.components.assist_pipeline.select import ( + AssistPipelineSelect, + VadSensitivitySelect, +) from homeassistant.components.select import SelectEntity from homeassistant.config_entries import ConfigEntry from homeassistant.core import HomeAssistant, callback @@ -37,7 +40,12 @@ async def async_setup_entry( entry_data = DomainData.get(hass).get_entry_data(entry) assert entry_data.device_info is not None if entry_data.device_info.voice_assistant_version: - async_add_entities([EsphomeAssistPipelineSelect(hass, entry_data)]) + async_add_entities( + [ + EsphomeAssistPipelineSelect(hass, entry_data), + EsphomeVadSensitivitySelect(hass, entry_data), + ] + ) class EsphomeSelect(EsphomeEntity[SelectInfo, SelectState], SelectEntity): @@ -68,3 +76,12 @@ class EsphomeAssistPipelineSelect(EsphomeAssistEntity, AssistPipelineSelect): """Initialize a pipeline selector.""" EsphomeAssistEntity.__init__(self, entry_data) AssistPipelineSelect.__init__(self, hass, self._device_info.mac_address) + + +class EsphomeVadSensitivitySelect(EsphomeAssistEntity, VadSensitivitySelect): + """VAD sensitivity selector for VoIP devices.""" + + def __init__(self, hass: HomeAssistant, entry_data: RuntimeEntryData) -> None: + """Initialize a VAD sensitivity selector.""" + EsphomeAssistEntity.__init__(self, entry_data) + VadSensitivitySelect.__init__(self, hass, self._device_info.mac_address) diff --git a/homeassistant/components/esphome/strings.json b/homeassistant/components/esphome/strings.json index 915e55fde32..2ec1fe1bc41 100644 --- a/homeassistant/components/esphome/strings.json +++ b/homeassistant/components/esphome/strings.json @@ -67,6 +67,14 @@ "state": { "preferred": "[%key:component::assist_pipeline::entity::select::pipeline::state::preferred%]" } + }, + "vad_sensitivity": { + "name": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::name%]", + "state": { + "default": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::default%]", + "aggressive": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::aggressive%]", + "relaxed": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::relaxed%]" + } } } }, diff --git a/homeassistant/components/esphome/voice_assistant.py b/homeassistant/components/esphome/voice_assistant.py index 4f6131f449b..6b49549d812 100644 --- a/homeassistant/components/esphome/voice_assistant.py +++ b/homeassistant/components/esphome/voice_assistant.py @@ -19,7 +19,10 @@ from homeassistant.components.assist_pipeline import ( async_pipeline_from_audio_stream, select as pipeline_select, ) -from homeassistant.components.assist_pipeline.vad import VoiceCommandSegmenter +from homeassistant.components.assist_pipeline.vad import ( + VadSensitivity, + VoiceCommandSegmenter, +) from homeassistant.components.media_player import async_process_play_media_url from homeassistant.core import Context, HomeAssistant, callback @@ -251,9 +254,9 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol): chunk = await self.queue.get() async def _iterate_packets_with_vad( - self, pipeline_timeout: float + self, pipeline_timeout: float, silence_seconds: float ) -> Callable[[], AsyncIterable[bytes]] | None: - segmenter = VoiceCommandSegmenter() + segmenter = VoiceCommandSegmenter(silence_seconds=silence_seconds) chunk_buffer: deque[bytes] = deque(maxlen=100) try: async with async_timeout.timeout(pipeline_timeout): @@ -305,7 +308,16 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol): ) if use_vad: - stt_stream = await self._iterate_packets_with_vad(pipeline_timeout) + stt_stream = await self._iterate_packets_with_vad( + pipeline_timeout, + silence_seconds=VadSensitivity.to_seconds( + pipeline_select.get_vad_sensitivity( + self.hass, + DOMAIN, + self.device_info.mac_address, + ) + ), + ) # Error or timeout occurred and was handled already if stt_stream is None: return diff --git a/tests/components/esphome/test_select.py b/tests/components/esphome/test_select.py index 5f6974ec035..8d17276c304 100644 --- a/tests/components/esphome/test_select.py +++ b/tests/components/esphome/test_select.py @@ -25,6 +25,20 @@ async def test_pipeline_selector( assert state.state == "preferred" +async def test_vad_sensitivity_select( + hass: HomeAssistant, + mock_voice_assistant_v1_entry, +) -> None: + """Test VAD sensitivity select. + + Functionality is tested in assist_pipeline/test_select.py. + This test is only to ensure it is set up. + """ + state = hass.states.get("select.test_finished_speaking_detection") + assert state is not None + assert state.state == "default" + + async def test_select_generic_entity( hass: HomeAssistant, mock_client: APIClient, mock_generic_device_entry ) -> None: diff --git a/tests/components/voip/test_select.py b/tests/components/voip/test_select.py index 9d45477a429..7dd041a6866 100644 --- a/tests/components/voip/test_select.py +++ b/tests/components/voip/test_select.py @@ -29,6 +29,6 @@ async def test_vad_sensitivity_select( Functionality is tested in assist_pipeline/test_select.py. This test is only to ensure it is set up. """ - state = hass.states.get("select.192_168_1_210_silence_sensitivity") + state = hass.states.get("select.192_168_1_210_finished_speaking_detection") assert state is not None assert state.state == "default"