evlli/hass-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Memory optimization for logbook (#21549)

Browse source
This commit is contained in:
Anders Melchiorsen 2019-03-02 08:23:45 +01:00 • committed by Paulus Schoutsen
parent f1b867dccb
commit 0c8a31b8ec
3 changed files with 125 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

111
homeassistant/components/logbook/__init__.py
View file

@ -146,8 +146,8 @@ class LogbookView(HomeAssistantView):
def json_events(): def json_events():
"""Fetch events and generate JSON.""" """Fetch events and generate JSON."""
return self.json(list( return self.json(
_get_events(hass, self.config, start_day, end_day, entity_id))) _get_events(hass, self.config, start_day, end_day, entity_id))
return await hass.async_add_job(json_events) return await hass.async_add_job(json_events)
@ -393,11 +393,17 @@ def _generate_filter_from_config(config):
def _get_events(hass, config, start_day, end_day, entity_id=None): def _get_events(hass, config, start_day, end_day, entity_id=None):
"""Get events for a period of time.""" """Get events for a period of time."""
from homeassistant.components.recorder.models import Events, States from homeassistant.components.recorder.models import Events, States
from homeassistant.components.recorder.util import ( from homeassistant.components.recorder.util import session_scope
execute, session_scope)
entities_filter = _generate_filter_from_config(config) entities_filter = _generate_filter_from_config(config)
def yield_events(query):
"""Yield Events that are not filtered away."""
for row in query.yield_per(500):
event = row.to_native()
if _keep_event(event, entities_filter):
yield event
with session_scope(hass=hass) as session: with session_scope(hass=hass) as session:
if entity_id is not None: if entity_id is not None:
entity_ids = [entity_id.lower()] entity_ids = [entity_id.lower()]
@ -413,77 +419,70 @@ def _get_events(hass, config, start_day, end_day, entity_id=None):
States.entity_id.in_(entity_ids)) States.entity_id.in_(entity_ids))
| (States.state_id.is_(None))) | (States.state_id.is_(None)))
events = execute(query) return list(humanify(hass, yield_events(query)))
return humanify(hass, _exclude_events(events, entities_filter))
def _exclude_events(events, entities_filter): def _keep_event(event, entities_filter):
filtered_events = [] domain, entity_id = None, None
for event in events:
domain, entity_id = None, None
if event.event_type == EVENT_STATE_CHANGED: if event.event_type == EVENT_STATE_CHANGED:
entity_id = event.data.get('entity_id') entity_id = event.data.get('entity_id')
if entity_id is None: if entity_id is None:
continue return False
# Do not report on new entities # Do not report on new entities
if event.data.get('old_state') is None: if event.data.get('old_state') is None:
continue return False
new_state = event.data.get('new_state') new_state = event.data.get('new_state')
# Do not report on entity removal # Do not report on entity removal
if not new_state: if not new_state:
continue return False
attributes = new_state.get('attributes', {}) attributes = new_state.get('attributes', {})
# If last_changed != last_updated only attributes have changed # If last_changed != last_updated only attributes have changed
# we do not report on that yet. # we do not report on that yet.
last_changed = new_state.get('last_changed') last_changed = new_state.get('last_changed')
last_updated = new_state.get('last_updated') last_updated = new_state.get('last_updated')
if last_changed != last_updated: if last_changed != last_updated:
continue return False
domain = split_entity_id(entity_id)[0] domain = split_entity_id(entity_id)[0]
# Also filter auto groups. # Also filter auto groups.
if domain == 'group' and attributes.get('auto', False): if domain == 'group' and attributes.get('auto', False):
continue return False
# exclude entities which are customized hidden # exclude entities which are customized hidden
hidden = attributes.get(ATTR_HIDDEN, False) hidden = attributes.get(ATTR_HIDDEN, False)
if hidden: if hidden:
continue return False
elif event.event_type == EVENT_LOGBOOK_ENTRY: elif event.event_type == EVENT_LOGBOOK_ENTRY:
domain = event.data.get(ATTR_DOMAIN) domain = event.data.get(ATTR_DOMAIN)
entity_id = event.data.get(ATTR_ENTITY_ID) entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_AUTOMATION_TRIGGERED: elif event.event_type == EVENT_AUTOMATION_TRIGGERED:
domain = 'automation' domain = 'automation'
entity_id = event.data.get(ATTR_ENTITY_ID) entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_SCRIPT_STARTED: elif event.event_type == EVENT_SCRIPT_STARTED:
domain = 'script' domain = 'script'
entity_id = event.data.get(ATTR_ENTITY_ID) entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_ALEXA_SMART_HOME: elif event.event_type == EVENT_ALEXA_SMART_HOME:
domain = 'alexa' domain = 'alexa'
elif event.event_type == EVENT_HOMEKIT_CHANGED: elif event.event_type == EVENT_HOMEKIT_CHANGED:
domain = DOMAIN_HOMEKIT domain = DOMAIN_HOMEKIT
if not entity_id and domain: if not entity_id and domain:
entity_id = "%s." % (domain, ) entity_id = "%s." % (domain, )
if not entity_id or entities_filter(entity_id): return not entity_id or entities_filter(entity_id)
filtered_events.append(event)
return filtered_events
def _entry_message_from_state(domain, state): def _entry_message_from_state(domain, state):

11
homeassistant/scripts/benchmark/__init__.py
View file

@ -180,12 +180,15 @@ def _logbook_filtering(hass, last_changed, last_updated):
'new_state': new_state 'new_state': new_state
}) })
events = [event] * 10**5 def yield_events(event):
# pylint: disable=protected-access
entities_filter = logbook._generate_filter_from_config({})
for _ in range(10**5):
if logbook._keep_event(event, entities_filter):
yield event
start = timer() start = timer()
# pylint: disable=protected-access list(logbook.humanify(None, yield_events(event)))
events = logbook._exclude_events(events, {})
list(logbook.humanify(None, events))
return timer() - start return timer() - start

106
tests/components/logbook/test_init.py
View file

@ -148,10 +148,11 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointB, entity_id2, 20) eventB = self.create_state_changed_event(pointB, entity_id2, 20)
eventA.data['old_state'] = None eventA.data['old_state'] = None
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config({})
(ha.Event(EVENT_HOMEASSISTANT_STOP), events = [e for e in
eventA, eventB), (ha.Event(EVENT_HOMEASSISTANT_STOP),
logbook._generate_filter_from_config({})) eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -172,10 +173,11 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointB, entity_id2, 20) eventB = self.create_state_changed_event(pointB, entity_id2, 20)
eventA.data['new_state'] = None eventA.data['new_state'] = None
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config({})
(ha.Event(EVENT_HOMEASSISTANT_STOP), events = [e for e in
eventA, eventB), (ha.Event(EVENT_HOMEASSISTANT_STOP),
logbook._generate_filter_from_config({})) eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -196,10 +198,11 @@ class TestComponentLogbook(unittest.TestCase):
{ATTR_HIDDEN: 'true'}) {ATTR_HIDDEN: 'true'})
eventB = self.create_state_changed_event(pointB, entity_id2, 20) eventB = self.create_state_changed_event(pointB, entity_id2, 20)
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config({})
(ha.Event(EVENT_HOMEASSISTANT_STOP), events = [e for e in
eventA, eventB), (ha.Event(EVENT_HOMEASSISTANT_STOP),
logbook._generate_filter_from_config({})) eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -223,9 +226,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}}) logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), config[logbook.DOMAIN])
logbook._generate_filter_from_config(config[logbook.DOMAIN])) events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -249,11 +254,13 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_DOMAINS: ['switch', 'alexa', DOMAIN_HOMEKIT]}}}) logbook.CONF_DOMAINS: ['switch', 'alexa', DOMAIN_HOMEKIT]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_START), config[logbook.DOMAIN])
ha.Event(EVENT_ALEXA_SMART_HOME), events = [e for e in
ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB), (ha.Event(EVENT_HOMEASSISTANT_START),
logbook._generate_filter_from_config(config[logbook.DOMAIN])) ha.Event(EVENT_ALEXA_SMART_HOME),
ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -283,9 +290,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}}) logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), config[logbook.DOMAIN])
logbook._generate_filter_from_config(config[logbook.DOMAIN])) events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -316,9 +325,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}}) logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), config[logbook.DOMAIN])
logbook._generate_filter_from_config(config[logbook.DOMAIN])) events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -342,9 +353,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_INCLUDE: { logbook.DOMAIN: {logbook.CONF_INCLUDE: {
logbook.CONF_ENTITIES: [entity_id2, ]}}}) logbook.CONF_ENTITIES: [entity_id2, ]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), config[logbook.DOMAIN])
logbook._generate_filter_from_config(config[logbook.DOMAIN])) events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries) assert 2 == len(entries)
@ -378,10 +391,12 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {}, ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_INCLUDE: { logbook.DOMAIN: {logbook.CONF_INCLUDE: {
logbook.CONF_DOMAINS: ['sensor', 'alexa', DOMAIN_HOMEKIT]}}}) logbook.CONF_DOMAINS: ['sensor', 'alexa', DOMAIN_HOMEKIT]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_START), config[logbook.DOMAIN])
event_alexa, event_homekit, eventA, eventB), events = [e for e in
logbook._generate_filter_from_config(config[logbook.DOMAIN])) (ha.Event(EVENT_HOMEASSISTANT_START),
event_alexa, event_homekit, eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 4 == len(entries) assert 4 == len(entries)
@ -415,10 +430,13 @@ class TestComponentLogbook(unittest.TestCase):
logbook.CONF_EXCLUDE: { logbook.CONF_EXCLUDE: {
logbook.CONF_DOMAINS: ['switch', ], logbook.CONF_DOMAINS: ['switch', ],
logbook.CONF_ENTITIES: ['sensor.bli', ]}}}) logbook.CONF_ENTITIES: ['sensor.bli', ]}}})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config(
(ha.Event(EVENT_HOMEASSISTANT_START), eventA1, eventA2, eventA3, config[logbook.DOMAIN])
eventB1, eventB2), events = [e for e in
logbook._generate_filter_from_config(config[logbook.DOMAIN])) (ha.Event(EVENT_HOMEASSISTANT_START),
eventA1, eventA2, eventA3,
eventB1, eventB2)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 5 == len(entries) assert 5 == len(entries)
@ -443,9 +461,10 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointA, entity_id2, 20, eventB = self.create_state_changed_event(pointA, entity_id2, 20,
{'auto': True}) {'auto': True})
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config({})
(eventA, eventB), events = [e for e in
logbook._generate_filter_from_config({})) (eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 1 == len(entries) assert 1 == len(entries)
@ -463,9 +482,10 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event( eventB = self.create_state_changed_event(
pointA, entity_id2, 20, last_changed=pointA, last_updated=pointB) pointA, entity_id2, 20, last_changed=pointA, last_updated=pointB)
events = logbook._exclude_events( entities_filter = logbook._generate_filter_from_config({})
(eventA, eventB), events = [e for e in
logbook._generate_filter_from_config({})) (eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events)) entries = list(logbook.humanify(self.hass, events))
assert 1 == len(entries) assert 1 == len(entries)