From 80fd33047988f61c652dd765237873278fdf6e5a Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Thu, 9 Sep 2021 08:35:53 +0200 Subject: [PATCH] Add sum_decrease and sum_increase statistics (#55850) --- .../components/recorder/migration.py | 6 ++ homeassistant/components/recorder/models.py | 4 +- .../components/recorder/statistics.py | 5 +- homeassistant/components/sensor/recorder.py | 23 +++++- tests/components/history/test_init.py | 2 + tests/components/recorder/test_statistics.py | 10 +++ tests/components/sensor/test_recorder.py | 80 +++++++++++++++++++ 7 files changed, 124 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index 4a5c456df28..c694aa678f0 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -501,6 +501,12 @@ def _apply_update(engine, session, new_version, old_version): # noqa: C901 "sum DOUBLE PRECISION", ], ) + elif new_version == 21: + _add_columns( + connection, + "statistics", + ["sum_increase DOUBLE PRECISION"], + ) else: raise ValueError(f"No schema migration defined for version {new_version}") diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 28eff4d9d95..11c614c9ea1 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -39,7 +39,7 @@ import homeassistant.util.dt as dt_util # pylint: disable=invalid-name Base = declarative_base() -SCHEMA_VERSION = 20 +SCHEMA_VERSION = 21 _LOGGER = logging.getLogger(__name__) @@ -229,6 +229,7 @@ class StatisticData(TypedDict, total=False): last_reset: datetime | None state: float sum: float + sum_increase: float class Statistics(Base): # type: ignore @@ -253,6 +254,7 @@ class Statistics(Base): # type: ignore last_reset = Column(DATETIME_TYPE) state = Column(DOUBLE_TYPE) sum = Column(DOUBLE_TYPE) + sum_increase = Column(DOUBLE_TYPE) @staticmethod def from_stats(metadata_id: str, start: datetime, stats: StatisticData): diff --git a/homeassistant/components/recorder/statistics.py b/homeassistant/components/recorder/statistics.py index db82eb1ee39..c8f4e48563c 100644 --- a/homeassistant/components/recorder/statistics.py +++ b/homeassistant/components/recorder/statistics.py @@ -48,6 +48,7 @@ QUERY_STATISTICS = [ Statistics.last_reset, Statistics.state, Statistics.sum, + Statistics.sum_increase, ] QUERY_STATISTIC_META = [ @@ -458,7 +459,9 @@ def _sorted_statistics_to_dict( "max": convert(db_state.max, units), "last_reset": _process_timestamp_to_utc_isoformat(db_state.last_reset), "state": convert(db_state.state, units), - "sum": convert(db_state.sum, units), + "sum": (_sum := convert(db_state.sum, units)), + "sum_increase": (inc := convert(db_state.sum_increase, units)), + "sum_decrease": None if _sum is None or inc is None else inc - _sum, } for db_state in group ) diff --git a/homeassistant/components/sensor/recorder.py b/homeassistant/components/sensor/recorder.py index e78f9a942c6..d38ae589ef0 100644 --- a/homeassistant/components/sensor/recorder.py +++ b/homeassistant/components/sensor/recorder.py @@ -138,7 +138,7 @@ def _time_weighted_average( ) -> float: """Calculate a time weighted average. - The average is calculated by, weighting the states by duration in seconds between + The average is calculated by weighting the states by duration in seconds between state changes. Note: there's no interpolation of values between state changes. """ @@ -342,7 +342,11 @@ def compile_statistics( # noqa: C901 ) history_list = {**history_list, **_history_list} - for entity_id, state_class, device_class in entities: + for ( # pylint: disable=too-many-nested-blocks + entity_id, + state_class, + device_class, + ) in entities: if entity_id not in history_list: continue @@ -392,13 +396,16 @@ def compile_statistics( # noqa: C901 if "sum" in wanted_statistics[entity_id]: last_reset = old_last_reset = None new_state = old_state = None - _sum = 0 + _sum = 0.0 + sum_increase = 0.0 + sum_increase_tmp = 0.0 last_stats = statistics.get_last_statistics(hass, 1, entity_id, False) if entity_id in last_stats: # We have compiled history for this sensor before, use that as a starting point last_reset = old_last_reset = last_stats[entity_id][0]["last_reset"] new_state = old_state = last_stats[entity_id][0]["state"] - _sum = last_stats[entity_id][0]["sum"] or 0 + _sum = last_stats[entity_id][0]["sum"] or 0.0 + sum_increase = last_stats[entity_id][0]["sum_increase"] or 0.0 for fstate, state in fstates: @@ -452,6 +459,10 @@ def compile_statistics( # noqa: C901 # The sensor has been reset, update the sum if old_state is not None: _sum += new_state - old_state + sum_increase += sum_increase_tmp + sum_increase_tmp = 0.0 + if fstate > 0: + sum_increase_tmp += fstate # ..and update the starting point new_state = fstate old_last_reset = last_reset @@ -461,6 +472,8 @@ def compile_statistics( # noqa: C901 else: old_state = new_state else: + if new_state is not None and fstate > new_state: + sum_increase_tmp += fstate - new_state new_state = fstate # Deprecated, will be removed in Home Assistant 2021.11 @@ -476,9 +489,11 @@ def compile_statistics( # noqa: C901 # Update the sum with the last state _sum += new_state - old_state + sum_increase += sum_increase_tmp if last_reset is not None: stat["last_reset"] = dt_util.parse_datetime(last_reset) stat["sum"] = _sum + stat["sum_increase"] = sum_increase stat["state"] = new_state result[entity_id]["stat"] = stat diff --git a/tests/components/history/test_init.py b/tests/components/history/test_init.py index 7909d8f0239..27c2024750c 100644 --- a/tests/components/history/test_init.py +++ b/tests/components/history/test_init.py @@ -914,6 +914,8 @@ async def test_statistics_during_period( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } diff --git a/tests/components/recorder/test_statistics.py b/tests/components/recorder/test_statistics.py index 0580460a537..2434f8b4703 100644 --- a/tests/components/recorder/test_statistics.py +++ b/tests/components/recorder/test_statistics.py @@ -51,6 +51,8 @@ def test_compile_hourly_statistics(hass_recorder): "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } expected_2 = { "statistic_id": "sensor.test1", @@ -61,6 +63,8 @@ def test_compile_hourly_statistics(hass_recorder): "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } expected_stats1 = [ {**expected_1, "statistic_id": "sensor.test1"}, @@ -166,6 +170,8 @@ def test_compile_hourly_statistics_exception( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } expected_2 = { "statistic_id": "sensor.test1", @@ -176,6 +182,8 @@ def test_compile_hourly_statistics_exception( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } expected_stats1 = [ {**expected_1, "statistic_id": "sensor.test1"}, @@ -233,6 +241,8 @@ def test_rename_entity(hass_recorder): "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } expected_stats1 = [ {**expected_1, "statistic_id": "sensor.test1"}, diff --git a/tests/components/sensor/test_recorder.py b/tests/components/sensor/test_recorder.py index 84b683cf3c3..6108f4a7ef8 100644 --- a/tests/components/sensor/test_recorder.py +++ b/tests/components/sensor/test_recorder.py @@ -101,6 +101,8 @@ def test_compile_hourly_statistics( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -163,6 +165,8 @@ def test_compile_hourly_statistics_unsupported(hass_recorder, caplog, attributes "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ], "sensor.test6": [ @@ -175,6 +179,8 @@ def test_compile_hourly_statistics_unsupported(hass_recorder, caplog, attributes "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ], "sensor.test7": [ @@ -187,6 +193,8 @@ def test_compile_hourly_statistics_unsupported(hass_recorder, caplog, attributes "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ], } @@ -258,6 +266,8 @@ def test_compile_hourly_sum_statistics_amount( "last_reset": process_timestamp_to_utc_isoformat(zero), "state": approx(factor * seq[2]), "sum": approx(factor * 10.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 10.0), }, { "statistic_id": "sensor.test1", @@ -268,6 +278,8 @@ def test_compile_hourly_sum_statistics_amount( "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(factor * seq[5]), "sum": approx(factor * 40.0), + "sum_decrease": approx(factor * 10.0), + "sum_increase": approx(factor * 50.0), }, { "statistic_id": "sensor.test1", @@ -278,6 +290,8 @@ def test_compile_hourly_sum_statistics_amount( "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(factor * seq[8]), "sum": approx(factor * 70.0), + "sum_decrease": approx(factor * 10.0), + "sum_increase": approx(factor * 80.0), }, ] } @@ -352,6 +366,8 @@ def test_compile_hourly_sum_statistics_amount_reset_every_state_change( "last_reset": process_timestamp_to_utc_isoformat(one), "state": approx(factor * seq[7]), "sum": approx(factor * (sum(seq) - seq[0])), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * (sum(seq) - seq[0])), }, ] } @@ -416,6 +432,10 @@ def test_compile_hourly_sum_statistics_nan_inf_state( "last_reset": process_timestamp_to_utc_isoformat(one), "state": approx(factor * seq[7]), "sum": approx(factor * (seq[2] + seq[3] + seq[4] + seq[6] + seq[7])), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx( + factor * (seq[2] + seq[3] + seq[4] + seq[6] + seq[7]) + ), }, ] } @@ -478,6 +498,8 @@ def test_compile_hourly_sum_statistics_total_no_reset( "last_reset": None, "state": approx(factor * seq[2]), "sum": approx(factor * 10.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 10.0), }, { "statistic_id": "sensor.test1", @@ -488,6 +510,8 @@ def test_compile_hourly_sum_statistics_total_no_reset( "last_reset": None, "state": approx(factor * seq[5]), "sum": approx(factor * 30.0), + "sum_decrease": approx(factor * 10.0), + "sum_increase": approx(factor * 40.0), }, { "statistic_id": "sensor.test1", @@ -498,6 +522,8 @@ def test_compile_hourly_sum_statistics_total_no_reset( "last_reset": None, "state": approx(factor * seq[8]), "sum": approx(factor * 60.0), + "sum_decrease": approx(factor * 10.0), + "sum_increase": approx(factor * 70.0), }, ] } @@ -558,6 +584,8 @@ def test_compile_hourly_sum_statistics_total_increasing( "last_reset": None, "state": approx(factor * seq[2]), "sum": approx(factor * 10.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 10.0), }, { "statistic_id": "sensor.test1", @@ -568,6 +596,8 @@ def test_compile_hourly_sum_statistics_total_increasing( "last_reset": None, "state": approx(factor * seq[5]), "sum": approx(factor * 50.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 50.0), }, { "statistic_id": "sensor.test1", @@ -578,6 +608,8 @@ def test_compile_hourly_sum_statistics_total_increasing( "last_reset": None, "state": approx(factor * seq[8]), "sum": approx(factor * 80.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 80.0), }, ] } @@ -648,6 +680,8 @@ def test_compile_hourly_sum_statistics_total_increasing_small_dip( "min": None, "state": approx(factor * seq[2]), "sum": approx(factor * 10.0), + "sum_decrease": approx(factor * 0.0), + "sum_increase": approx(factor * 10.0), }, { "last_reset": None, @@ -658,6 +692,8 @@ def test_compile_hourly_sum_statistics_total_increasing_small_dip( "min": None, "state": approx(factor * seq[5]), "sum": approx(factor * 30.0), + "sum_decrease": approx(factor * 1.0), + "sum_increase": approx(factor * 31.0), }, { "last_reset": None, @@ -668,6 +704,8 @@ def test_compile_hourly_sum_statistics_total_increasing_small_dip( "min": None, "state": approx(factor * seq[8]), "sum": approx(factor * 60.0), + "sum_decrease": approx(factor * 2.0), + "sum_increase": approx(factor * 62.0), }, ] } @@ -735,6 +773,8 @@ def test_compile_hourly_energy_statistics_unsupported(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(zero), "state": approx(20.0), "sum": approx(10.0), + "sum_decrease": approx(0.0), + "sum_increase": approx(10.0), }, { "statistic_id": "sensor.test1", @@ -745,6 +785,8 @@ def test_compile_hourly_energy_statistics_unsupported(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(40.0), "sum": approx(40.0), + "sum_decrease": approx(10.0), + "sum_increase": approx(50.0), }, { "statistic_id": "sensor.test1", @@ -755,6 +797,8 @@ def test_compile_hourly_energy_statistics_unsupported(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(70.0), "sum": approx(70.0), + "sum_decrease": approx(10.0), + "sum_increase": approx(80.0), }, ] } @@ -818,6 +862,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(zero), "state": approx(20.0), "sum": approx(10.0), + "sum_decrease": approx(0.0), + "sum_increase": approx(10.0), }, { "statistic_id": "sensor.test1", @@ -828,6 +874,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(40.0), "sum": approx(40.0), + "sum_decrease": approx(10.0), + "sum_increase": approx(50.0), }, { "statistic_id": "sensor.test1", @@ -838,6 +886,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(70.0), "sum": approx(70.0), + "sum_decrease": approx(10.0), + "sum_increase": approx(80.0), }, ], "sensor.test2": [ @@ -850,6 +900,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(zero), "state": approx(130.0), "sum": approx(20.0), + "sum_decrease": approx(0.0), + "sum_increase": approx(20.0), }, { "statistic_id": "sensor.test2", @@ -860,6 +912,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(45.0), "sum": approx(-65.0), + "sum_decrease": approx(130.0), + "sum_increase": approx(65.0), }, { "statistic_id": "sensor.test2", @@ -870,6 +924,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(75.0), "sum": approx(-35.0), + "sum_decrease": approx(130.0), + "sum_increase": approx(95.0), }, ], "sensor.test3": [ @@ -882,6 +938,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(zero), "state": approx(5.0 / 1000), "sum": approx(5.0 / 1000), + "sum_decrease": approx(0.0 / 1000), + "sum_increase": approx(5.0 / 1000), }, { "statistic_id": "sensor.test3", @@ -892,6 +950,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(50.0 / 1000), "sum": approx(60.0 / 1000), + "sum_decrease": approx(0.0 / 1000), + "sum_increase": approx(60.0 / 1000), }, { "statistic_id": "sensor.test3", @@ -902,6 +962,8 @@ def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog): "last_reset": process_timestamp_to_utc_isoformat(four), "state": approx(90.0 / 1000), "sum": approx(100.0 / 1000), + "sum_decrease": approx(0.0 / 1000), + "sum_increase": approx(100.0 / 1000), }, ], } @@ -955,6 +1017,8 @@ def test_compile_hourly_statistics_unchanged( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -987,6 +1051,8 @@ def test_compile_hourly_statistics_partially_unavailable(hass_recorder, caplog): "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1044,6 +1110,8 @@ def test_compile_hourly_statistics_unavailable( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1194,6 +1262,8 @@ def test_compile_hourly_statistics_changing_units_1( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1220,6 +1290,8 @@ def test_compile_hourly_statistics_changing_units_1( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1325,6 +1397,8 @@ def test_compile_hourly_statistics_changing_units_3( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1349,6 +1423,8 @@ def test_compile_hourly_statistics_changing_units_3( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, } ] } @@ -1427,6 +1503,8 @@ def test_compile_hourly_statistics_changing_statistics( "last_reset": None, "state": None, "sum": None, + "sum_decrease": None, + "sum_increase": None, }, { "statistic_id": "sensor.test1", @@ -1437,6 +1515,8 @@ def test_compile_hourly_statistics_changing_statistics( "last_reset": None, "state": approx(30.0), "sum": approx(30.0), + "sum_decrease": approx(10.0), + "sum_increase": approx(40.0), }, ] }