Reduce overhead to save the larger registries (#113462)
We save the device and entity registry to disk quite often, and the cost of serializing them to the storage can block the event loop for >100ms. Add a cache to reduce the change the loop is blocked at an inopportune time at run time. The first write after startup will still be a little slow but we do have to serialize the bulk of it at least once as there is no way to avoid this ``` 2024-03-14 11:28:19.765 WARNING (MainThread) [homeassistant.helpers.storage] Writing data with data_func: core.device_registry 2024-03-14 11:28:20.020 WARNING (MainThread) [homeassistant.helpers.storage] Writing data with data_func: core.entity_registry 2024-03-14 11:28:20.178 WARNING (MainThread) [asyncio] Executing <TimerHandle cancelled when=2319925.760294916 Store._async_schedule_callback_delayed_write() created at /Users/bdraco/home-assistant/homeassistant/helpers/storage.py:328> took 0.159 seconds ```
This commit is contained in:
J. Nick Koston
GitHub
parent
28836be3eb
commit
5b80eb4c3d
2 changed files with 110 additions and 86 deletions
|
|
@ -31,7 +31,7 @@ from .deprecation import (
|
|||
dir_with_deprecated_constants,
|
||||
)
|
||||
from .frame import report
|
||||
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes
|
||||
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment
|
||||
from .registry import BaseRegistry
|
||||
from .typing import UNDEFINED, UndefinedType
|
||||
|
||||
|
|
@ -301,8 +301,35 @@ class DeviceEntry:
|
|||
)
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def as_storage_fragment(self) -> json_fragment:
|
||||
"""Return a json fragment for storage."""
|
||||
return json_fragment(
|
||||
json_bytes(
|
||||
{
|
||||
"area_id": self.area_id,
|
||||
"config_entries": list(self.config_entries),
|
||||
"configuration_url": self.configuration_url,
|
||||
"connections": list(self.connections),
|
||||
"disabled_by": self.disabled_by,
|
||||
"entry_type": self.entry_type,
|
||||
"hw_version": self.hw_version,
|
||||
"id": self.id,
|
||||
"identifiers": list(self.identifiers),
|
||||
"labels": list(self.labels),
|
||||
"manufacturer": self.manufacturer,
|
||||
"model": self.model,
|
||||
"name_by_user": self.name_by_user,
|
||||
"name": self.name,
|
||||
"serial_number": self.serial_number,
|
||||
"sw_version": self.sw_version,
|
||||
"via_device_id": self.via_device_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@attr.s(slots=True, frozen=True)
|
||||
|
||||
@attr.s(frozen=True)
|
||||
class DeletedDeviceEntry:
|
||||
"""Deleted Device Registry Entry."""
|
||||
|
||||
|
|
@ -328,6 +355,21 @@ class DeletedDeviceEntry:
|
|||
is_new=True,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def as_storage_fragment(self) -> json_fragment:
|
||||
"""Return a json fragment for storage."""
|
||||
return json_fragment(
|
||||
json_bytes(
|
||||
{
|
||||
"config_entries": list(self.config_entries),
|
||||
"connections": list(self.connections),
|
||||
"identifiers": list(self.identifiers),
|
||||
"id": self.id,
|
||||
"orphaned_timestamp": self.orphaned_timestamp,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def format_mac(mac: str) -> str:
|
||||
|
|
@ -904,44 +946,14 @@ class DeviceRegistry(BaseRegistry):
|
|||
self._device_data = devices.data
|
||||
|
||||
@callback
|
||||
def _data_to_save(self) -> dict[str, list[dict[str, Any]]]:
|
||||
def _data_to_save(self) -> dict[str, Any]:
|
||||
"""Return data of device registry to store in a file."""
|
||||
data: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
data["devices"] = [
|
||||
{
|
||||
"area_id": entry.area_id,
|
||||
"config_entries": list(entry.config_entries),
|
||||
"configuration_url": entry.configuration_url,
|
||||
"connections": list(entry.connections),
|
||||
"disabled_by": entry.disabled_by,
|
||||
"entry_type": entry.entry_type,
|
||||
"hw_version": entry.hw_version,
|
||||
"id": entry.id,
|
||||
"identifiers": list(entry.identifiers),
|
||||
"labels": list(entry.labels),
|
||||
"manufacturer": entry.manufacturer,
|
||||
"model": entry.model,
|
||||
"name_by_user": entry.name_by_user,
|
||||
"name": entry.name,
|
||||
"serial_number": entry.serial_number,
|
||||
"sw_version": entry.sw_version,
|
||||
"via_device_id": entry.via_device_id,
|
||||
return {
|
||||
"devices": [entry.as_storage_fragment for entry in self.devices.values()],
|
||||
"deleted_devices": [
|
||||
entry.as_storage_fragment for entry in self.deleted_devices.values()
|
||||
],
|
||||
}
|
||||
for entry in self.devices.values()
|
||||
]
|
||||
data["deleted_devices"] = [
|
||||
{
|
||||
"config_entries": list(entry.config_entries),
|
||||
"connections": list(entry.connections),
|
||||
"identifiers": list(entry.identifiers),
|
||||
"id": entry.id,
|
||||
"orphaned_timestamp": entry.orphaned_timestamp,
|
||||
}
|
||||
for entry in self.deleted_devices.values()
|
||||
]
|
||||
|
||||
return data
|
||||
|
||||
@callback
|
||||
def async_clear_config_entry(self, config_entry_id: str) -> None:
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from homeassistant.util.read_only_dict import ReadOnlyDict
|
|||
|
||||
from . import device_registry as dr, storage
|
||||
from .device_registry import EVENT_DEVICE_REGISTRY_UPDATED
|
||||
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes
|
||||
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment
|
||||
from .registry import BaseRegistry
|
||||
from .typing import UNDEFINED, UndefinedType
|
||||
|
||||
|
|
@ -311,6 +311,41 @@ class RegistryEntry:
|
|||
)
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def as_storage_fragment(self) -> json_fragment:
|
||||
"""Return a json fragment for storage."""
|
||||
return json_fragment(
|
||||
json_bytes(
|
||||
{
|
||||
"aliases": list(self.aliases),
|
||||
"area_id": self.area_id,
|
||||
"capabilities": self.capabilities,
|
||||
"config_entry_id": self.config_entry_id,
|
||||
"device_class": self.device_class,
|
||||
"device_id": self.device_id,
|
||||
"disabled_by": self.disabled_by,
|
||||
"entity_category": self.entity_category,
|
||||
"entity_id": self.entity_id,
|
||||
"hidden_by": self.hidden_by,
|
||||
"icon": self.icon,
|
||||
"id": self.id,
|
||||
"has_entity_name": self.has_entity_name,
|
||||
"labels": list(self.labels),
|
||||
"name": self.name,
|
||||
"options": self.options,
|
||||
"original_device_class": self.original_device_class,
|
||||
"original_icon": self.original_icon,
|
||||
"original_name": self.original_name,
|
||||
"platform": self.platform,
|
||||
"supported_features": self.supported_features,
|
||||
"translation_key": self.translation_key,
|
||||
"unique_id": self.unique_id,
|
||||
"previous_unique_id": self.previous_unique_id,
|
||||
"unit_of_measurement": self.unit_of_measurement,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@callback
|
||||
def write_unavailable_state(self, hass: HomeAssistant) -> None:
|
||||
"""Write the unavailable state to the state machine."""
|
||||
|
|
@ -340,7 +375,7 @@ class RegistryEntry:
|
|||
hass.states.async_set(self.entity_id, STATE_UNAVAILABLE, attrs)
|
||||
|
||||
|
||||
@attr.s(slots=True, frozen=True)
|
||||
@attr.s(frozen=True)
|
||||
class DeletedRegistryEntry:
|
||||
"""Deleted Entity Registry Entry."""
|
||||
|
||||
|
|
@ -357,6 +392,22 @@ class DeletedRegistryEntry:
|
|||
"""Compute domain value."""
|
||||
return split_entity_id(self.entity_id)[0]
|
||||
|
||||
@cached_property
|
||||
def as_storage_fragment(self) -> json_fragment:
|
||||
"""Return a json fragment for storage."""
|
||||
return json_fragment(
|
||||
json_bytes(
|
||||
{
|
||||
"config_entry_id": self.config_entry_id,
|
||||
"entity_id": self.entity_id,
|
||||
"id": self.id,
|
||||
"orphaned_timestamp": self.orphaned_timestamp,
|
||||
"platform": self.platform,
|
||||
"unique_id": self.unique_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class EntityRegistryStore(storage.Store[dict[str, list[dict[str, Any]]]]):
|
||||
"""Store entity registry data."""
|
||||
|
|
@ -1197,51 +1248,12 @@ class EntityRegistry(BaseRegistry):
|
|||
@callback
|
||||
def _data_to_save(self) -> dict[str, Any]:
|
||||
"""Return data of entity registry to store in a file."""
|
||||
data: dict[str, Any] = {}
|
||||
|
||||
data["entities"] = [
|
||||
{
|
||||
"aliases": list(entry.aliases),
|
||||
"area_id": entry.area_id,
|
||||
"capabilities": entry.capabilities,
|
||||
"config_entry_id": entry.config_entry_id,
|
||||
"device_class": entry.device_class,
|
||||
"device_id": entry.device_id,
|
||||
"disabled_by": entry.disabled_by,
|
||||
"entity_category": entry.entity_category,
|
||||
"entity_id": entry.entity_id,
|
||||
"hidden_by": entry.hidden_by,
|
||||
"icon": entry.icon,
|
||||
"id": entry.id,
|
||||
"has_entity_name": entry.has_entity_name,
|
||||
"labels": list(entry.labels),
|
||||
"name": entry.name,
|
||||
"options": entry.options,
|
||||
"original_device_class": entry.original_device_class,
|
||||
"original_icon": entry.original_icon,
|
||||
"original_name": entry.original_name,
|
||||
"platform": entry.platform,
|
||||
"supported_features": entry.supported_features,
|
||||
"translation_key": entry.translation_key,
|
||||
"unique_id": entry.unique_id,
|
||||
"previous_unique_id": entry.previous_unique_id,
|
||||
"unit_of_measurement": entry.unit_of_measurement,
|
||||
return {
|
||||
"entities": [entry.as_storage_fragment for entry in self.entities.values()],
|
||||
"deleted_entities": [
|
||||
entry.as_storage_fragment for entry in self.deleted_entities.values()
|
||||
],
|
||||
}
|
||||
for entry in self.entities.values()
|
||||
]
|
||||
data["deleted_entities"] = [
|
||||
{
|
||||
"config_entry_id": entry.config_entry_id,
|
||||
"entity_id": entry.entity_id,
|
||||
"id": entry.id,
|
||||
"orphaned_timestamp": entry.orphaned_timestamp,
|
||||
"platform": entry.platform,
|
||||
"unique_id": entry.unique_id,
|
||||
}
|
||||
for entry in self.deleted_entities.values()
|
||||
]
|
||||
|
||||
return data
|
||||
|
||||
@callback
|
||||
def async_clear_label_id(self, label_id: str) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue