evlli/hass-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reduce number of columns when selecting attributes for history (#91717)

Browse source
This commit is contained in:
J. Nick Koston 2023-04-22 07:21:08 -05:00 • committed by GitHub
parent 6e628d2f06
commit 34b824a27b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 61 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
homeassistant/components/recorder/db_schema.py
View file

@ -25,6 +25,7 @@ from sqlalchemy import (
SmallInteger,
String,
Text,
case,
type_coerce,
)
from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
@ -821,3 +822,11 @@ ENTITY_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["entity_id"]
OLD_ENTITY_ID_IN_EVENT: ColumnElement = OLD_FORMAT_EVENT_DATA_JSON["entity_id"]
DEVICE_ID_IN_EVENT: ColumnElement = EVENT_DATA_JSON["device_id"]
OLD_STATE = aliased(States, name="old_state")
SHARED_ATTR_OR_LEGACY_ATTRIBUTES = case(
(StateAttributes.shared_attrs.is_(None), States.attributes),
else_=StateAttributes.shared_attrs,
).label("attributes")
SHARED_DATA_OR_LEGACY_EVENT_DATA = case(
(EventData.shared_data.is_(None), Events.event_data), else_=EventData.shared_data
).label("event_data")

8
homeassistant/components/recorder/history/modern.py
View file

@ -28,7 +28,7 @@ from homeassistant.core import HomeAssistant, State, split_entity_id
import homeassistant.util.dt as dt_util
from ... import recorder
from ..db_schema import StateAttributes, States
from ..db_schema import SHARED_ATTR_OR_LEGACY_ATTRIBUTES, StateAttributes, States
from ..filters import Filters
from ..models import (
LazyState,
@ -70,7 +70,7 @@ def _stmt_and_join_attributes(
if include_last_changed:
_select = _select.add_columns(States.last_changed_ts)
if not no_attributes:
_select = _select.add_columns(States.attributes, StateAttributes.shared_attrs)
_select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES)
return _select
@ -87,7 +87,7 @@ def _stmt_and_join_attributes_for_start_state(
literal(value=None).label("last_changed_ts").cast(CASTABLE_DOUBLE_TYPE)
)
if not no_attributes:
_select = _select.add_columns(States.attributes, StateAttributes.shared_attrs)
_select = _select.add_columns(SHARED_ATTR_OR_LEGACY_ATTRIBUTES)
return _select
@ -104,7 +104,7 @@ def _select_from_subquery(
base_select = base_select.add_columns(subquery.c.last_changed_ts)
if no_attributes:
return base_select
return base_select.add_columns(subquery.c.attributes, subquery.c.shared_attrs)
return base_select.add_columns(subquery.c.attributes)
def get_significant_states(

24
homeassistant/components/recorder/models/legacy.py
View file

@ -15,7 +15,7 @@ from homeassistant.const import (
from homeassistant.core import Context, State
import homeassistant.util.dt as dt_util
from .state_attributes import decode_attributes_from_row
from .state_attributes import decode_attributes_from_source
from .time import (
process_datetime_to_timestamp,
process_timestamp,
@ -57,7 +57,9 @@ class LegacyLazyStatePreSchema31(State):
def attributes(self) -> dict[str, Any]:
"""State attributes."""
if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache)
self._attributes = decode_attributes_from_row_legacy(
self._row, self.attr_cache
)
return self._attributes
@attributes.setter
@ -147,7 +149,7 @@ def legacy_row_to_compressed_state_pre_schema_31(
"""Convert a database row to a compressed state before schema 31."""
comp_state = {
COMPRESSED_STATE_STATE: row.state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache),
}
if start_time:
comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time.timestamp()
@ -202,7 +204,9 @@ class LegacyLazyState(State):
def attributes(self) -> dict[str, Any]:
"""State attributes."""
if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache)
self._attributes = decode_attributes_from_row_legacy(
self._row, self.attr_cache
)
return self._attributes
@attributes.setter
@ -273,7 +277,7 @@ def legacy_row_to_compressed_state(
"""Convert a database row to a compressed state schema 31 and later."""
comp_state = {
COMPRESSED_STATE_STATE: row.state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row_legacy(row, attr_cache),
}
if start_time:
comp_state[COMPRESSED_STATE_LAST_UPDATED] = dt_util.utc_to_timestamp(start_time)
@ -285,3 +289,13 @@ def legacy_row_to_compressed_state(
) and row_last_updated_ts != row_last_changed_ts:
comp_state[COMPRESSED_STATE_LAST_CHANGED] = row_last_changed_ts
return comp_state
def decode_attributes_from_row_legacy(
row: Row, attr_cache: dict[str, dict[str, Any]]
) -> dict[str, Any]:
"""Decode attributes from a database row."""
return decode_attributes_from_source(
getattr(row, "shared_attrs", None) or getattr(row, "attributes", None),
attr_cache,
)

12
homeassistant/components/recorder/models/state.py
View file

@ -16,7 +16,7 @@ from homeassistant.const import (
from homeassistant.core import Context, State
import homeassistant.util.dt as dt_util
from .state_attributes import decode_attributes_from_row
from .state_attributes import decode_attributes_from_source
from .time import process_timestamp
# pylint: disable=invalid-name
@ -70,7 +70,9 @@ class LazyState(State):
def attributes(self) -> dict[str, Any]:
"""State attributes."""
if self._attributes is None:
self._attributes = decode_attributes_from_row(self._row, self.attr_cache)
self._attributes = decode_attributes_from_source(
getattr(self._row, "attributes", None), self.attr_cache
)
return self._attributes
@attributes.setter
@ -144,10 +146,12 @@ def row_to_compressed_state(
state: str,
last_updated_ts: float | None,
) -> dict[str, Any]:
"""Convert a database row to a compressed state schema 31 and later."""
"""Convert a database row to a compressed state schema 41 and later."""
comp_state: dict[str, Any] = {
COMPRESSED_STATE_STATE: state,
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),
COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_source(
getattr(row, "attributes", None), attr_cache
),
}
row_last_updated_ts: float = last_updated_ts or start_time_ts # type: ignore[assignment]
comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts

11
homeassistant/components/recorder/models/state_attributes.py
View file

@ -5,21 +5,16 @@ from __future__ import annotations
import logging
from typing import Any
from sqlalchemy.engine.row import Row
from homeassistant.util.json import json_loads_object
EMPTY_JSON_OBJECT = "{}"
_LOGGER = logging.getLogger(__name__)
def decode_attributes_from_row(
row: Row, attr_cache: dict[str, dict[str, Any]]
def decode_attributes_from_source(
source: Any, attr_cache: dict[str, dict[str, Any]]
) -> dict[str, Any]:
"""Decode attributes from a database row."""
source: str | None = getattr(row, "shared_attrs", None) or getattr(
row, "attributes", None
)
"""Decode attributes from a row source."""
if not source or source == EMPTY_JSON_OBJECT:
return {}
if (attributes := attr_cache.get(source)) is not None:

24
tests/components/recorder/test_history.py
View file

@ -23,8 +23,11 @@ from homeassistant.components.recorder.db_schema import (
)
from homeassistant.components.recorder.filters import Filters
from homeassistant.components.recorder.history import legacy
from homeassistant.components.recorder.models import LazyState, process_timestamp
from homeassistant.components.recorder.models.legacy import LegacyLazyStatePreSchema31
from homeassistant.components.recorder.models import process_timestamp
from homeassistant.components.recorder.models.legacy import (
LegacyLazyState,
LegacyLazyStatePreSchema31,
)
from homeassistant.components.recorder.util import session_scope
import homeassistant.core as ha
from homeassistant.core import HomeAssistant, State
@ -60,13 +63,11 @@ async def _async_get_states(
return [
LegacyLazyStatePreSchema31(row, attr_cache, None)
if pre_31_schema
else LazyState(
else LegacyLazyState(
row,
attr_cache,
None,
row.entity_id,
row.state,
getattr(row, "last_updated_ts", None),
)
for row in legacy._get_rows_with_session(
hass,
@ -903,6 +904,7 @@ async def test_state_changes_during_period_query_during_migration_to_schema_25(
conn.commit()
with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True
hist = history.state_changes_during_period(
hass,
@ -944,9 +946,8 @@ async def test_get_states_query_during_migration_to_schema_25(
point = start + timedelta(seconds=1)
end = point + timedelta(seconds=1)
entity_id = "light.test"
await recorder.get_instance(hass).async_add_executor_job(
_add_db_entries, hass, point, [entity_id]
)
await instance.async_add_executor_job(_add_db_entries, hass, point, [entity_id])
assert instance.states_meta_manager.active
no_attributes = True
hist = await _async_get_states(hass, end, [entity_id], no_attributes=no_attributes)
@ -964,6 +965,7 @@ async def test_get_states_query_during_migration_to_schema_25(
conn.commit()
with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True
hist = await _async_get_states(
hass, end, [entity_id], no_attributes=no_attributes
@ -998,9 +1000,8 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities(
entity_id_2 = "switch.test"
entity_ids = [entity_id_1, entity_id_2]
await recorder.get_instance(hass).async_add_executor_job(
_add_db_entries, hass, point, entity_ids
)
await instance.async_add_executor_job(_add_db_entries, hass, point, entity_ids)
assert instance.states_meta_manager.active
no_attributes = True
hist = await _async_get_states(hass, end, entity_ids, no_attributes=no_attributes)
@ -1018,6 +1019,7 @@ async def test_get_states_query_during_migration_to_schema_25_multiple_entities(
conn.commit()
with patch.object(instance, "schema_version", 24):
instance.states_meta_manager.active = False
no_attributes = True
hist = await _async_get_states(
hass, end, entity_ids, no_attributes=no_attributes

11
tests/components/recorder/test_models.py
View file

@ -273,14 +273,13 @@ async def test_lazy_state_handles_include_json(
assert "Error converting row to state attributes" in caplog.text
async def test_lazy_state_prefers_shared_attrs_over_attrs(
async def test_lazy_state_can_decode_attributes(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Test that the LazyState prefers shared_attrs over attributes."""
"""Test that the LazyState prefers can decode attributes."""
row = PropertyMock(
entity_id="sensor.invalid",
shared_attrs='{"shared":true}',
attributes='{"shared":false}',
attributes='{"shared":true}',
)
assert LazyState(row, {}, None, row.entity_id, "", 1).attributes == {"shared": True}
@ -293,7 +292,7 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed(
row = PropertyMock(
entity_id="sensor.valid",
state="off",
shared_attrs='{"shared":true}',
attributes='{"shared":true}',
last_updated_ts=now.timestamp(),
last_changed_ts=(now - timedelta(seconds=60)).timestamp(),
)
@ -324,7 +323,7 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed(
row = PropertyMock(
entity_id="sensor.valid",
state="off",
shared_attrs='{"shared":true}',
attributes='{"shared":true}',
last_updated_ts=now.timestamp(),
last_changed_ts=now.timestamp(),
)