Add VAD sensitivity to ESPHome (#95283)

* Change to "finished speaking detection"

* Add select entity to ESPHome for finished speaking detection

* Fix entity name

* Use vad select in stt stream

---------

Co-authored-by: J. Nick Koston <nick@koston.org>
This commit is contained in:
Michael Hansen 2023-06-26 15:47:32 -05:00 committed by GitHub
parent c6775920f5
commit 0f08e6699c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 59 additions and 8 deletions

View file

@ -13,7 +13,7 @@
}
},
"vad_sensitivity": {
"name": "Silence sensitivity",
"name": "Finished speaking detection",
"state": {
"default": "Default",
"aggressive": "Aggressive",

View file

@ -3,7 +3,10 @@ from __future__ import annotations
from aioesphomeapi import EntityInfo, SelectInfo, SelectState
from homeassistant.components.assist_pipeline.select import AssistPipelineSelect
from homeassistant.components.assist_pipeline.select import (
AssistPipelineSelect,
VadSensitivitySelect,
)
from homeassistant.components.select import SelectEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant, callback
@ -37,7 +40,12 @@ async def async_setup_entry(
entry_data = DomainData.get(hass).get_entry_data(entry)
assert entry_data.device_info is not None
if entry_data.device_info.voice_assistant_version:
async_add_entities([EsphomeAssistPipelineSelect(hass, entry_data)])
async_add_entities(
[
EsphomeAssistPipelineSelect(hass, entry_data),
EsphomeVadSensitivitySelect(hass, entry_data),
]
)
class EsphomeSelect(EsphomeEntity[SelectInfo, SelectState], SelectEntity):
@ -68,3 +76,12 @@ class EsphomeAssistPipelineSelect(EsphomeAssistEntity, AssistPipelineSelect):
"""Initialize a pipeline selector."""
EsphomeAssistEntity.__init__(self, entry_data)
AssistPipelineSelect.__init__(self, hass, self._device_info.mac_address)
class EsphomeVadSensitivitySelect(EsphomeAssistEntity, VadSensitivitySelect):
"""VAD sensitivity selector for VoIP devices."""
def __init__(self, hass: HomeAssistant, entry_data: RuntimeEntryData) -> None:
"""Initialize a VAD sensitivity selector."""
EsphomeAssistEntity.__init__(self, entry_data)
VadSensitivitySelect.__init__(self, hass, self._device_info.mac_address)

View file

@ -67,6 +67,14 @@
"state": {
"preferred": "[%key:component::assist_pipeline::entity::select::pipeline::state::preferred%]"
}
},
"vad_sensitivity": {
"name": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::name%]",
"state": {
"default": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::default%]",
"aggressive": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::aggressive%]",
"relaxed": "[%key:component::assist_pipeline::entity::select::vad_sensitivity::state::relaxed%]"
}
}
}
},

View file

@ -19,7 +19,10 @@ from homeassistant.components.assist_pipeline import (
async_pipeline_from_audio_stream,
select as pipeline_select,
)
from homeassistant.components.assist_pipeline.vad import VoiceCommandSegmenter
from homeassistant.components.assist_pipeline.vad import (
VadSensitivity,
VoiceCommandSegmenter,
)
from homeassistant.components.media_player import async_process_play_media_url
from homeassistant.core import Context, HomeAssistant, callback
@ -251,9 +254,9 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
chunk = await self.queue.get()
async def _iterate_packets_with_vad(
self, pipeline_timeout: float
self, pipeline_timeout: float, silence_seconds: float
) -> Callable[[], AsyncIterable[bytes]] | None:
segmenter = VoiceCommandSegmenter()
segmenter = VoiceCommandSegmenter(silence_seconds=silence_seconds)
chunk_buffer: deque[bytes] = deque(maxlen=100)
try:
async with async_timeout.timeout(pipeline_timeout):
@ -305,7 +308,16 @@ class VoiceAssistantUDPServer(asyncio.DatagramProtocol):
)
if use_vad:
stt_stream = await self._iterate_packets_with_vad(pipeline_timeout)
stt_stream = await self._iterate_packets_with_vad(
pipeline_timeout,
silence_seconds=VadSensitivity.to_seconds(
pipeline_select.get_vad_sensitivity(
self.hass,
DOMAIN,
self.device_info.mac_address,
)
),
)
# Error or timeout occurred and was handled already
if stt_stream is None:
return

View file

@ -25,6 +25,20 @@ async def test_pipeline_selector(
assert state.state == "preferred"
async def test_vad_sensitivity_select(
hass: HomeAssistant,
mock_voice_assistant_v1_entry,
) -> None:
"""Test VAD sensitivity select.
Functionality is tested in assist_pipeline/test_select.py.
This test is only to ensure it is set up.
"""
state = hass.states.get("select.test_finished_speaking_detection")
assert state is not None
assert state.state == "default"
async def test_select_generic_entity(
hass: HomeAssistant, mock_client: APIClient, mock_generic_device_entry
) -> None:

View file

@ -29,6 +29,6 @@ async def test_vad_sensitivity_select(
Functionality is tested in assist_pipeline/test_select.py.
This test is only to ensure it is set up.
"""
state = hass.states.get("select.192_168_1_210_silence_sensitivity")
state = hass.states.get("select.192_168_1_210_finished_speaking_detection")
assert state is not None
assert state.state == "default"