268425b5e3
* Handle statistics columns being unmigrated from previous downgrades If the user downgraded HA from 2023.3.x to an older version without restoring the database and they upgrade again with the same database they will have unmigrated statistics columns since we only migrate them once. As its expensive to check, we do not want to check every time at startup, so we will only do this one more time since the risk that someone will downgrade to an older version is very low at this point. * add guard to sqlite to prevent re-migrate * test * move test to insert with old schema * use helper * normalize timestamps * remove * add check * add fallback migration * add fallback migration * commit * remove useless logging * remove useless logging * do the other columns at the same time * coverage * dry * comment * Update tests/components/recorder/test_migration_from_schema_32.py
956 lines
29 KiB
Python
956 lines
29 KiB
Python
"""Queries for the recorder."""
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import delete, distinct, func, lambda_stmt, select, union_all, update
|
|
from sqlalchemy.sql.lambdas import StatementLambdaElement
|
|
from sqlalchemy.sql.selectable import Select
|
|
|
|
from .db_schema import (
|
|
EventData,
|
|
Events,
|
|
EventTypes,
|
|
RecorderRuns,
|
|
StateAttributes,
|
|
States,
|
|
StatesMeta,
|
|
Statistics,
|
|
StatisticsRuns,
|
|
StatisticsShortTerm,
|
|
)
|
|
|
|
|
|
def select_event_type_ids(event_types: tuple[str, ...]) -> Select:
|
|
"""Generate a select for event type ids.
|
|
|
|
This query is intentionally not a lambda statement as it is used inside
|
|
other lambda statements.
|
|
"""
|
|
return select(EventTypes.event_type_id).where(
|
|
EventTypes.event_type.in_(event_types)
|
|
)
|
|
|
|
|
|
def get_shared_attributes(hashes: list[int]) -> StatementLambdaElement:
|
|
"""Load shared attributes from the database."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
StateAttributes.attributes_id, StateAttributes.shared_attrs
|
|
).where(StateAttributes.hash.in_(hashes))
|
|
)
|
|
|
|
|
|
def get_shared_event_datas(hashes: list[int]) -> StatementLambdaElement:
|
|
"""Load shared event data from the database."""
|
|
return lambda_stmt(
|
|
lambda: select(EventData.data_id, EventData.shared_data).where(
|
|
EventData.hash.in_(hashes)
|
|
)
|
|
)
|
|
|
|
|
|
def find_event_type_ids(event_types: Iterable[str]) -> StatementLambdaElement:
|
|
"""Find an event_type id by event_type."""
|
|
return lambda_stmt(
|
|
lambda: select(EventTypes.event_type_id, EventTypes.event_type).filter(
|
|
EventTypes.event_type.in_(event_types)
|
|
)
|
|
)
|
|
|
|
|
|
def find_all_states_metadata_ids() -> StatementLambdaElement:
|
|
"""Find all metadata_ids and entity_ids."""
|
|
return lambda_stmt(lambda: select(StatesMeta.metadata_id, StatesMeta.entity_id))
|
|
|
|
|
|
def find_states_metadata_ids(entity_ids: Iterable[str]) -> StatementLambdaElement:
|
|
"""Find metadata_ids by entity_ids."""
|
|
return lambda_stmt(
|
|
lambda: select(StatesMeta.metadata_id, StatesMeta.entity_id).filter(
|
|
StatesMeta.entity_id.in_(entity_ids)
|
|
)
|
|
)
|
|
|
|
|
|
def _state_attrs_exist(attr: int | None) -> Select:
|
|
"""Check if a state attributes id exists in the states table."""
|
|
return select(func.min(States.attributes_id)).where(States.attributes_id == attr)
|
|
|
|
|
|
def attributes_ids_exist_in_states_with_fast_in_distinct(
|
|
attributes_ids: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Find attributes ids that exist in the states table."""
|
|
return lambda_stmt(
|
|
lambda: select(distinct(States.attributes_id)).filter(
|
|
States.attributes_id.in_(attributes_ids)
|
|
)
|
|
)
|
|
|
|
|
|
def attributes_ids_exist_in_states(
|
|
attr1: int,
|
|
attr2: int | None,
|
|
attr3: int | None,
|
|
attr4: int | None,
|
|
attr5: int | None,
|
|
attr6: int | None,
|
|
attr7: int | None,
|
|
attr8: int | None,
|
|
attr9: int | None,
|
|
attr10: int | None,
|
|
attr11: int | None,
|
|
attr12: int | None,
|
|
attr13: int | None,
|
|
attr14: int | None,
|
|
attr15: int | None,
|
|
attr16: int | None,
|
|
attr17: int | None,
|
|
attr18: int | None,
|
|
attr19: int | None,
|
|
attr20: int | None,
|
|
attr21: int | None,
|
|
attr22: int | None,
|
|
attr23: int | None,
|
|
attr24: int | None,
|
|
attr25: int | None,
|
|
attr26: int | None,
|
|
attr27: int | None,
|
|
attr28: int | None,
|
|
attr29: int | None,
|
|
attr30: int | None,
|
|
attr31: int | None,
|
|
attr32: int | None,
|
|
attr33: int | None,
|
|
attr34: int | None,
|
|
attr35: int | None,
|
|
attr36: int | None,
|
|
attr37: int | None,
|
|
attr38: int | None,
|
|
attr39: int | None,
|
|
attr40: int | None,
|
|
attr41: int | None,
|
|
attr42: int | None,
|
|
attr43: int | None,
|
|
attr44: int | None,
|
|
attr45: int | None,
|
|
attr46: int | None,
|
|
attr47: int | None,
|
|
attr48: int | None,
|
|
attr49: int | None,
|
|
attr50: int | None,
|
|
attr51: int | None,
|
|
attr52: int | None,
|
|
attr53: int | None,
|
|
attr54: int | None,
|
|
attr55: int | None,
|
|
attr56: int | None,
|
|
attr57: int | None,
|
|
attr58: int | None,
|
|
attr59: int | None,
|
|
attr60: int | None,
|
|
attr61: int | None,
|
|
attr62: int | None,
|
|
attr63: int | None,
|
|
attr64: int | None,
|
|
attr65: int | None,
|
|
attr66: int | None,
|
|
attr67: int | None,
|
|
attr68: int | None,
|
|
attr69: int | None,
|
|
attr70: int | None,
|
|
attr71: int | None,
|
|
attr72: int | None,
|
|
attr73: int | None,
|
|
attr74: int | None,
|
|
attr75: int | None,
|
|
attr76: int | None,
|
|
attr77: int | None,
|
|
attr78: int | None,
|
|
attr79: int | None,
|
|
attr80: int | None,
|
|
attr81: int | None,
|
|
attr82: int | None,
|
|
attr83: int | None,
|
|
attr84: int | None,
|
|
attr85: int | None,
|
|
attr86: int | None,
|
|
attr87: int | None,
|
|
attr88: int | None,
|
|
attr89: int | None,
|
|
attr90: int | None,
|
|
attr91: int | None,
|
|
attr92: int | None,
|
|
attr93: int | None,
|
|
attr94: int | None,
|
|
attr95: int | None,
|
|
attr96: int | None,
|
|
attr97: int | None,
|
|
attr98: int | None,
|
|
attr99: int | None,
|
|
attr100: int | None,
|
|
) -> StatementLambdaElement:
|
|
"""Generate the find attributes select only once.
|
|
|
|
https://docs.sqlalchemy.org/en/14/core/connections.html#quick-guidelines-for-lambdas
|
|
"""
|
|
return lambda_stmt(
|
|
lambda: union_all(
|
|
_state_attrs_exist(attr1),
|
|
_state_attrs_exist(attr2),
|
|
_state_attrs_exist(attr3),
|
|
_state_attrs_exist(attr4),
|
|
_state_attrs_exist(attr5),
|
|
_state_attrs_exist(attr6),
|
|
_state_attrs_exist(attr7),
|
|
_state_attrs_exist(attr8),
|
|
_state_attrs_exist(attr9),
|
|
_state_attrs_exist(attr10),
|
|
_state_attrs_exist(attr11),
|
|
_state_attrs_exist(attr12),
|
|
_state_attrs_exist(attr13),
|
|
_state_attrs_exist(attr14),
|
|
_state_attrs_exist(attr15),
|
|
_state_attrs_exist(attr16),
|
|
_state_attrs_exist(attr17),
|
|
_state_attrs_exist(attr18),
|
|
_state_attrs_exist(attr19),
|
|
_state_attrs_exist(attr20),
|
|
_state_attrs_exist(attr21),
|
|
_state_attrs_exist(attr22),
|
|
_state_attrs_exist(attr23),
|
|
_state_attrs_exist(attr24),
|
|
_state_attrs_exist(attr25),
|
|
_state_attrs_exist(attr26),
|
|
_state_attrs_exist(attr27),
|
|
_state_attrs_exist(attr28),
|
|
_state_attrs_exist(attr29),
|
|
_state_attrs_exist(attr30),
|
|
_state_attrs_exist(attr31),
|
|
_state_attrs_exist(attr32),
|
|
_state_attrs_exist(attr33),
|
|
_state_attrs_exist(attr34),
|
|
_state_attrs_exist(attr35),
|
|
_state_attrs_exist(attr36),
|
|
_state_attrs_exist(attr37),
|
|
_state_attrs_exist(attr38),
|
|
_state_attrs_exist(attr39),
|
|
_state_attrs_exist(attr40),
|
|
_state_attrs_exist(attr41),
|
|
_state_attrs_exist(attr42),
|
|
_state_attrs_exist(attr43),
|
|
_state_attrs_exist(attr44),
|
|
_state_attrs_exist(attr45),
|
|
_state_attrs_exist(attr46),
|
|
_state_attrs_exist(attr47),
|
|
_state_attrs_exist(attr48),
|
|
_state_attrs_exist(attr49),
|
|
_state_attrs_exist(attr50),
|
|
_state_attrs_exist(attr51),
|
|
_state_attrs_exist(attr52),
|
|
_state_attrs_exist(attr53),
|
|
_state_attrs_exist(attr54),
|
|
_state_attrs_exist(attr55),
|
|
_state_attrs_exist(attr56),
|
|
_state_attrs_exist(attr57),
|
|
_state_attrs_exist(attr58),
|
|
_state_attrs_exist(attr59),
|
|
_state_attrs_exist(attr60),
|
|
_state_attrs_exist(attr61),
|
|
_state_attrs_exist(attr62),
|
|
_state_attrs_exist(attr63),
|
|
_state_attrs_exist(attr64),
|
|
_state_attrs_exist(attr65),
|
|
_state_attrs_exist(attr66),
|
|
_state_attrs_exist(attr67),
|
|
_state_attrs_exist(attr68),
|
|
_state_attrs_exist(attr69),
|
|
_state_attrs_exist(attr70),
|
|
_state_attrs_exist(attr71),
|
|
_state_attrs_exist(attr72),
|
|
_state_attrs_exist(attr73),
|
|
_state_attrs_exist(attr74),
|
|
_state_attrs_exist(attr75),
|
|
_state_attrs_exist(attr76),
|
|
_state_attrs_exist(attr77),
|
|
_state_attrs_exist(attr78),
|
|
_state_attrs_exist(attr79),
|
|
_state_attrs_exist(attr80),
|
|
_state_attrs_exist(attr81),
|
|
_state_attrs_exist(attr82),
|
|
_state_attrs_exist(attr83),
|
|
_state_attrs_exist(attr84),
|
|
_state_attrs_exist(attr85),
|
|
_state_attrs_exist(attr86),
|
|
_state_attrs_exist(attr87),
|
|
_state_attrs_exist(attr88),
|
|
_state_attrs_exist(attr89),
|
|
_state_attrs_exist(attr90),
|
|
_state_attrs_exist(attr91),
|
|
_state_attrs_exist(attr92),
|
|
_state_attrs_exist(attr93),
|
|
_state_attrs_exist(attr94),
|
|
_state_attrs_exist(attr95),
|
|
_state_attrs_exist(attr96),
|
|
_state_attrs_exist(attr97),
|
|
_state_attrs_exist(attr98),
|
|
_state_attrs_exist(attr99),
|
|
_state_attrs_exist(attr100),
|
|
)
|
|
)
|
|
|
|
|
|
def data_ids_exist_in_events_with_fast_in_distinct(
|
|
data_ids: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Find data ids that exist in the events table."""
|
|
return lambda_stmt(
|
|
lambda: select(distinct(Events.data_id)).filter(Events.data_id.in_(data_ids))
|
|
)
|
|
|
|
|
|
def _event_data_id_exist(data_id: int | None) -> Select:
|
|
"""Check if a event data id exists in the events table."""
|
|
return select(func.min(Events.data_id)).where(Events.data_id == data_id)
|
|
|
|
|
|
def data_ids_exist_in_events(
|
|
id1: int,
|
|
id2: int | None,
|
|
id3: int | None,
|
|
id4: int | None,
|
|
id5: int | None,
|
|
id6: int | None,
|
|
id7: int | None,
|
|
id8: int | None,
|
|
id9: int | None,
|
|
id10: int | None,
|
|
id11: int | None,
|
|
id12: int | None,
|
|
id13: int | None,
|
|
id14: int | None,
|
|
id15: int | None,
|
|
id16: int | None,
|
|
id17: int | None,
|
|
id18: int | None,
|
|
id19: int | None,
|
|
id20: int | None,
|
|
id21: int | None,
|
|
id22: int | None,
|
|
id23: int | None,
|
|
id24: int | None,
|
|
id25: int | None,
|
|
id26: int | None,
|
|
id27: int | None,
|
|
id28: int | None,
|
|
id29: int | None,
|
|
id30: int | None,
|
|
id31: int | None,
|
|
id32: int | None,
|
|
id33: int | None,
|
|
id34: int | None,
|
|
id35: int | None,
|
|
id36: int | None,
|
|
id37: int | None,
|
|
id38: int | None,
|
|
id39: int | None,
|
|
id40: int | None,
|
|
id41: int | None,
|
|
id42: int | None,
|
|
id43: int | None,
|
|
id44: int | None,
|
|
id45: int | None,
|
|
id46: int | None,
|
|
id47: int | None,
|
|
id48: int | None,
|
|
id49: int | None,
|
|
id50: int | None,
|
|
id51: int | None,
|
|
id52: int | None,
|
|
id53: int | None,
|
|
id54: int | None,
|
|
id55: int | None,
|
|
id56: int | None,
|
|
id57: int | None,
|
|
id58: int | None,
|
|
id59: int | None,
|
|
id60: int | None,
|
|
id61: int | None,
|
|
id62: int | None,
|
|
id63: int | None,
|
|
id64: int | None,
|
|
id65: int | None,
|
|
id66: int | None,
|
|
id67: int | None,
|
|
id68: int | None,
|
|
id69: int | None,
|
|
id70: int | None,
|
|
id71: int | None,
|
|
id72: int | None,
|
|
id73: int | None,
|
|
id74: int | None,
|
|
id75: int | None,
|
|
id76: int | None,
|
|
id77: int | None,
|
|
id78: int | None,
|
|
id79: int | None,
|
|
id80: int | None,
|
|
id81: int | None,
|
|
id82: int | None,
|
|
id83: int | None,
|
|
id84: int | None,
|
|
id85: int | None,
|
|
id86: int | None,
|
|
id87: int | None,
|
|
id88: int | None,
|
|
id89: int | None,
|
|
id90: int | None,
|
|
id91: int | None,
|
|
id92: int | None,
|
|
id93: int | None,
|
|
id94: int | None,
|
|
id95: int | None,
|
|
id96: int | None,
|
|
id97: int | None,
|
|
id98: int | None,
|
|
id99: int | None,
|
|
id100: int | None,
|
|
) -> StatementLambdaElement:
|
|
"""Generate the find event data select only once.
|
|
|
|
https://docs.sqlalchemy.org/en/14/core/connections.html#quick-guidelines-for-lambdas
|
|
"""
|
|
return lambda_stmt(
|
|
lambda: union_all(
|
|
_event_data_id_exist(id1),
|
|
_event_data_id_exist(id2),
|
|
_event_data_id_exist(id3),
|
|
_event_data_id_exist(id4),
|
|
_event_data_id_exist(id5),
|
|
_event_data_id_exist(id6),
|
|
_event_data_id_exist(id7),
|
|
_event_data_id_exist(id8),
|
|
_event_data_id_exist(id9),
|
|
_event_data_id_exist(id10),
|
|
_event_data_id_exist(id11),
|
|
_event_data_id_exist(id12),
|
|
_event_data_id_exist(id13),
|
|
_event_data_id_exist(id14),
|
|
_event_data_id_exist(id15),
|
|
_event_data_id_exist(id16),
|
|
_event_data_id_exist(id17),
|
|
_event_data_id_exist(id18),
|
|
_event_data_id_exist(id19),
|
|
_event_data_id_exist(id20),
|
|
_event_data_id_exist(id21),
|
|
_event_data_id_exist(id22),
|
|
_event_data_id_exist(id23),
|
|
_event_data_id_exist(id24),
|
|
_event_data_id_exist(id25),
|
|
_event_data_id_exist(id26),
|
|
_event_data_id_exist(id27),
|
|
_event_data_id_exist(id28),
|
|
_event_data_id_exist(id29),
|
|
_event_data_id_exist(id30),
|
|
_event_data_id_exist(id31),
|
|
_event_data_id_exist(id32),
|
|
_event_data_id_exist(id33),
|
|
_event_data_id_exist(id34),
|
|
_event_data_id_exist(id35),
|
|
_event_data_id_exist(id36),
|
|
_event_data_id_exist(id37),
|
|
_event_data_id_exist(id38),
|
|
_event_data_id_exist(id39),
|
|
_event_data_id_exist(id40),
|
|
_event_data_id_exist(id41),
|
|
_event_data_id_exist(id42),
|
|
_event_data_id_exist(id43),
|
|
_event_data_id_exist(id44),
|
|
_event_data_id_exist(id45),
|
|
_event_data_id_exist(id46),
|
|
_event_data_id_exist(id47),
|
|
_event_data_id_exist(id48),
|
|
_event_data_id_exist(id49),
|
|
_event_data_id_exist(id50),
|
|
_event_data_id_exist(id51),
|
|
_event_data_id_exist(id52),
|
|
_event_data_id_exist(id53),
|
|
_event_data_id_exist(id54),
|
|
_event_data_id_exist(id55),
|
|
_event_data_id_exist(id56),
|
|
_event_data_id_exist(id57),
|
|
_event_data_id_exist(id58),
|
|
_event_data_id_exist(id59),
|
|
_event_data_id_exist(id60),
|
|
_event_data_id_exist(id61),
|
|
_event_data_id_exist(id62),
|
|
_event_data_id_exist(id63),
|
|
_event_data_id_exist(id64),
|
|
_event_data_id_exist(id65),
|
|
_event_data_id_exist(id66),
|
|
_event_data_id_exist(id67),
|
|
_event_data_id_exist(id68),
|
|
_event_data_id_exist(id69),
|
|
_event_data_id_exist(id70),
|
|
_event_data_id_exist(id71),
|
|
_event_data_id_exist(id72),
|
|
_event_data_id_exist(id73),
|
|
_event_data_id_exist(id74),
|
|
_event_data_id_exist(id75),
|
|
_event_data_id_exist(id76),
|
|
_event_data_id_exist(id77),
|
|
_event_data_id_exist(id78),
|
|
_event_data_id_exist(id79),
|
|
_event_data_id_exist(id80),
|
|
_event_data_id_exist(id81),
|
|
_event_data_id_exist(id82),
|
|
_event_data_id_exist(id83),
|
|
_event_data_id_exist(id84),
|
|
_event_data_id_exist(id85),
|
|
_event_data_id_exist(id86),
|
|
_event_data_id_exist(id87),
|
|
_event_data_id_exist(id88),
|
|
_event_data_id_exist(id89),
|
|
_event_data_id_exist(id90),
|
|
_event_data_id_exist(id91),
|
|
_event_data_id_exist(id92),
|
|
_event_data_id_exist(id93),
|
|
_event_data_id_exist(id94),
|
|
_event_data_id_exist(id95),
|
|
_event_data_id_exist(id96),
|
|
_event_data_id_exist(id97),
|
|
_event_data_id_exist(id98),
|
|
_event_data_id_exist(id99),
|
|
_event_data_id_exist(id100),
|
|
)
|
|
)
|
|
|
|
|
|
def disconnect_states_rows(state_ids: Iterable[int]) -> StatementLambdaElement:
|
|
"""Disconnect states rows."""
|
|
return lambda_stmt(
|
|
lambda: update(States)
|
|
.where(States.old_state_id.in_(state_ids))
|
|
.values(old_state_id=None)
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_states_rows(state_ids: Iterable[int]) -> StatementLambdaElement:
|
|
"""Delete states rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(States)
|
|
.where(States.state_id.in_(state_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_event_data_rows(data_ids: Iterable[int]) -> StatementLambdaElement:
|
|
"""Delete event_data rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(EventData)
|
|
.where(EventData.data_id.in_(data_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_states_attributes_rows(
|
|
attributes_ids: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Delete states_attributes rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(StateAttributes)
|
|
.where(StateAttributes.attributes_id.in_(attributes_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_statistics_runs_rows(
|
|
statistics_runs: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Delete statistics_runs rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(StatisticsRuns)
|
|
.where(StatisticsRuns.run_id.in_(statistics_runs))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_statistics_short_term_rows(
|
|
short_term_statistics: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Delete statistics_short_term rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(StatisticsShortTerm)
|
|
.where(StatisticsShortTerm.id.in_(short_term_statistics))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_event_rows(
|
|
event_ids: Iterable[int],
|
|
) -> StatementLambdaElement:
|
|
"""Delete statistics_short_term rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(Events)
|
|
.where(Events.event_id.in_(event_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_recorder_runs_rows(
|
|
purge_before: datetime, current_run_id: int
|
|
) -> StatementLambdaElement:
|
|
"""Delete recorder_runs rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(RecorderRuns)
|
|
.filter(RecorderRuns.start < purge_before)
|
|
.filter(RecorderRuns.run_id != current_run_id)
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def find_events_to_purge(
|
|
purge_before: float, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find events to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(Events.event_id, Events.data_id)
|
|
.filter(Events.time_fired_ts < purge_before)
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_states_to_purge(
|
|
purge_before: float, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find states to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(States.state_id, States.attributes_id)
|
|
.filter(States.last_updated_ts < purge_before)
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_short_term_statistics_to_purge(
|
|
purge_before: datetime, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find short term statistics to purge."""
|
|
purge_before_ts = purge_before.timestamp()
|
|
return lambda_stmt(
|
|
lambda: select(StatisticsShortTerm.id)
|
|
.filter(StatisticsShortTerm.start_ts < purge_before_ts)
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_statistics_runs_to_purge(
|
|
purge_before: datetime, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find statistics_runs to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(StatisticsRuns.run_id)
|
|
.filter(StatisticsRuns.start < purge_before)
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_latest_statistics_runs_run_id() -> StatementLambdaElement:
|
|
"""Find the latest statistics_runs run_id."""
|
|
return lambda_stmt(lambda: select(func.max(StatisticsRuns.run_id)))
|
|
|
|
|
|
def find_legacy_event_state_and_attributes_and_data_ids_to_purge(
|
|
purge_before: float, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find the latest row in the legacy format to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
Events.event_id, Events.data_id, States.state_id, States.attributes_id
|
|
)
|
|
.outerjoin(States, Events.event_id == States.event_id)
|
|
.filter(Events.time_fired_ts < purge_before)
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_legacy_detached_states_and_attributes_to_purge(
|
|
purge_before: float, max_bind_vars: int
|
|
) -> StatementLambdaElement:
|
|
"""Find states rows with event_id set but not linked event_id in Events."""
|
|
return lambda_stmt(
|
|
lambda: select(States.state_id, States.attributes_id)
|
|
.outerjoin(Events, States.event_id == Events.event_id)
|
|
.filter(States.event_id.isnot(None))
|
|
.filter(
|
|
(States.last_updated_ts < purge_before) | States.last_updated_ts.is_(None)
|
|
)
|
|
.filter(Events.event_id.is_(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_legacy_row() -> StatementLambdaElement:
|
|
"""Check if there are still states in the table with an event_id."""
|
|
return lambda_stmt(lambda: select(func.max(States.event_id)))
|
|
|
|
|
|
def find_events_context_ids_to_migrate(max_bind_vars: int) -> StatementLambdaElement:
|
|
"""Find events context_ids to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
Events.event_id,
|
|
Events.time_fired_ts,
|
|
Events.context_id,
|
|
Events.context_user_id,
|
|
Events.context_parent_id,
|
|
)
|
|
.filter(Events.context_id_bin.is_(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_event_type_to_migrate(max_bind_vars: int) -> StatementLambdaElement:
|
|
"""Find events event_type to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
Events.event_id,
|
|
Events.event_type,
|
|
)
|
|
.filter(Events.event_type_id.is_(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_entity_ids_to_migrate(max_bind_vars: int) -> StatementLambdaElement:
|
|
"""Find entity_id to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
States.state_id,
|
|
States.entity_id,
|
|
)
|
|
.filter(States.metadata_id.is_(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def batch_cleanup_entity_ids() -> StatementLambdaElement:
|
|
"""Find entity_id to cleanup."""
|
|
# Self join because This version of MariaDB doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'
|
|
return lambda_stmt(
|
|
lambda: update(States)
|
|
.where(
|
|
States.state_id.in_(
|
|
select(States.state_id)
|
|
.join(
|
|
states_with_entity_ids := select(
|
|
States.state_id.label("state_id_with_entity_id")
|
|
)
|
|
.filter(States.entity_id.is_not(None))
|
|
.limit(5000)
|
|
.subquery(),
|
|
States.state_id == states_with_entity_ids.c.state_id_with_entity_id,
|
|
)
|
|
.alias("states_with_entity_ids")
|
|
.select()
|
|
)
|
|
)
|
|
.values(entity_id=None)
|
|
)
|
|
|
|
|
|
def has_used_states_event_ids() -> StatementLambdaElement:
|
|
"""Check if there are used event_ids in the states table."""
|
|
return lambda_stmt(
|
|
lambda: select(States.state_id).filter(States.event_id.isnot(None)).limit(1)
|
|
)
|
|
|
|
|
|
def has_events_context_ids_to_migrate() -> StatementLambdaElement:
|
|
"""Check if there are events context ids to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(Events.event_id).filter(Events.context_id_bin.is_(None)).limit(1)
|
|
)
|
|
|
|
|
|
def has_states_context_ids_to_migrate() -> StatementLambdaElement:
|
|
"""Check if there are states context ids to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(States.state_id).filter(States.context_id_bin.is_(None)).limit(1)
|
|
)
|
|
|
|
|
|
def has_event_type_to_migrate() -> StatementLambdaElement:
|
|
"""Check if there are event_types to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(Events.event_id).filter(Events.event_type_id.is_(None)).limit(1)
|
|
)
|
|
|
|
|
|
def has_entity_ids_to_migrate() -> StatementLambdaElement:
|
|
"""Check if there are entity_id to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(States.state_id).filter(States.metadata_id.is_(None)).limit(1)
|
|
)
|
|
|
|
|
|
def find_states_context_ids_to_migrate(max_bind_vars: int) -> StatementLambdaElement:
|
|
"""Find events context_ids to migrate."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
States.state_id,
|
|
States.last_updated_ts,
|
|
States.context_id,
|
|
States.context_user_id,
|
|
States.context_parent_id,
|
|
)
|
|
.filter(States.context_id_bin.is_(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_event_types_to_purge() -> StatementLambdaElement:
|
|
"""Find event_type_ids to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(EventTypes.event_type_id, EventTypes.event_type).where(
|
|
EventTypes.event_type_id.not_in(
|
|
select(EventTypes.event_type_id).join(
|
|
used_event_type_ids := select(
|
|
distinct(Events.event_type_id).label("used_event_type_id")
|
|
).subquery(),
|
|
EventTypes.event_type_id
|
|
== used_event_type_ids.c.used_event_type_id,
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
def find_entity_ids_to_purge() -> StatementLambdaElement:
|
|
"""Find entity_ids to purge."""
|
|
return lambda_stmt(
|
|
lambda: select(StatesMeta.metadata_id, StatesMeta.entity_id).where(
|
|
StatesMeta.metadata_id.not_in(
|
|
select(StatesMeta.metadata_id).join(
|
|
used_states_metadata_id := select(
|
|
distinct(States.metadata_id).label("used_states_metadata_id")
|
|
).subquery(),
|
|
StatesMeta.metadata_id
|
|
== used_states_metadata_id.c.used_states_metadata_id,
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
def delete_event_types_rows(event_type_ids: Iterable[int]) -> StatementLambdaElement:
|
|
"""Delete EventTypes rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(EventTypes)
|
|
.where(EventTypes.event_type_id.in_(event_type_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_states_meta_rows(metadata_ids: Iterable[int]) -> StatementLambdaElement:
|
|
"""Delete StatesMeta rows."""
|
|
return lambda_stmt(
|
|
lambda: delete(StatesMeta)
|
|
.where(StatesMeta.metadata_id.in_(metadata_ids))
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def find_unmigrated_short_term_statistics_rows(
|
|
max_bind_vars: int,
|
|
) -> StatementLambdaElement:
|
|
"""Find unmigrated short term statistics rows."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
StatisticsShortTerm.id,
|
|
StatisticsShortTerm.start,
|
|
StatisticsShortTerm.created,
|
|
StatisticsShortTerm.last_reset,
|
|
)
|
|
.filter(StatisticsShortTerm.start_ts.is_(None))
|
|
.filter(StatisticsShortTerm.start.isnot(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def find_unmigrated_statistics_rows(max_bind_vars: int) -> StatementLambdaElement:
|
|
"""Find unmigrated statistics rows."""
|
|
return lambda_stmt(
|
|
lambda: select(
|
|
Statistics.id, Statistics.start, Statistics.created, Statistics.last_reset
|
|
)
|
|
.filter(Statistics.start_ts.is_(None))
|
|
.filter(Statistics.start.isnot(None))
|
|
.limit(max_bind_vars)
|
|
)
|
|
|
|
|
|
def migrate_single_short_term_statistics_row_to_timestamp(
|
|
statistic_id: int,
|
|
start_ts: float | None,
|
|
created_ts: float | None,
|
|
last_reset_ts: float | None,
|
|
) -> StatementLambdaElement:
|
|
"""Migrate a single short term statistics row to timestamp."""
|
|
return lambda_stmt(
|
|
lambda: update(StatisticsShortTerm)
|
|
.where(StatisticsShortTerm.id == statistic_id)
|
|
.values(
|
|
start_ts=start_ts,
|
|
start=None,
|
|
created_ts=created_ts,
|
|
created=None,
|
|
last_reset_ts=last_reset_ts,
|
|
last_reset=None,
|
|
)
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def migrate_single_statistics_row_to_timestamp(
|
|
statistic_id: int,
|
|
start_ts: float | None,
|
|
created_ts: float | None,
|
|
last_reset_ts: float | None,
|
|
) -> StatementLambdaElement:
|
|
"""Migrate a single statistics row to timestamp."""
|
|
return lambda_stmt(
|
|
lambda: update(Statistics)
|
|
.where(Statistics.id == statistic_id)
|
|
.values(
|
|
start_ts=start_ts,
|
|
start=None,
|
|
created_ts=created_ts,
|
|
created=None,
|
|
last_reset_ts=last_reset_ts,
|
|
last_reset=None,
|
|
)
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_duplicate_short_term_statistics_row(
|
|
statistic_id: int,
|
|
) -> StatementLambdaElement:
|
|
"""Delete a single duplicate short term statistics row."""
|
|
return lambda_stmt(
|
|
lambda: delete(StatisticsShortTerm)
|
|
.where(StatisticsShortTerm.id == statistic_id)
|
|
.execution_options(synchronize_session=False)
|
|
)
|
|
|
|
|
|
def delete_duplicate_statistics_row(statistic_id: int) -> StatementLambdaElement:
|
|
"""Delete a single duplicate statistics row."""
|
|
return lambda_stmt(
|
|
lambda: delete(Statistics)
|
|
.where(Statistics.id == statistic_id)
|
|
.execution_options(synchronize_session=False)
|
|
)
|