Bootstrap / Component setup async (#6264)

* Bootstrap / Entiy setup async

* Cleanup add_job stuff / return task/future object

* Address paulus comments / part 1

* fix install pip

* Cleanup bootstrap / move config stuff to config.py

* Make demo async

* Further bootstrap improvement

* Address Martin's comments

* Fix initial tests

* Fix final tests

* Fix bug with prepare loader

* Remove no longer needed things

* Log error when invalid config

* More cleanup

* Cleanups platform events & fix lint

* Use a non blocking add_entities callback for platform

* Fix Autoamtion is setup befor entity is ready

* Better automation fix

* Address paulus comments

* Typo

* fix lint

* rename functions

* fix tests

* fix test

* change exceptions

* fix spell
This commit is contained in:
Pascal Vizeli 2017-03-01 05:33:19 +01:00 committed by Paulus Schoutsen
parent 383b0914b3
commit 41f558b181
109 changed files with 764 additions and 848 deletions

View file

@ -4,38 +4,41 @@ import logging
import logging.handlers
import os
import sys
from time import time
from collections import OrderedDict
from types import ModuleType
from typing import Any, Optional, Dict
import voluptuous as vol
from voluptuous.humanize import humanize_error
import homeassistant.components as core_components
from homeassistant.components import persistent_notification
import homeassistant.config as conf_util
from homeassistant.config import async_notify_setup_error
import homeassistant.core as core
from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE
import homeassistant.loader as loader
import homeassistant.util.package as pkg_util
from homeassistant.util.async import (
run_coroutine_threadsafe, run_callback_threadsafe)
from homeassistant.util.async import run_coroutine_threadsafe
from homeassistant.util.logging import AsyncHandler
from homeassistant.util.yaml import clear_secret_cache
from homeassistant.const import EVENT_COMPONENT_LOADED, PLATFORM_FORMAT
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers import (
event_decorators, service, config_per_platform, extract_domain_configs)
from homeassistant.helpers import event_decorators, service
from homeassistant.helpers.signal import async_register_signal_handling
_LOGGER = logging.getLogger(__name__)
ATTR_COMPONENT = 'component'
DATA_SETUP = 'setup_tasks'
DATA_PIP_LOCK = 'pip_lock'
ERROR_LOG_FILENAME = 'home-assistant.log'
DATA_PERSISTENT_ERRORS = 'bootstrap_persistent_errors'
HA_COMPONENT_URL = '[{}](https://home-assistant.io/components/{}/)'
FIRST_INIT_COMPONENT = set((
'recorder', 'mqtt', 'mqtt_eventstream', 'logger', 'introduction'))
def setup_component(hass: core.HomeAssistant, domain: str,
@ -52,49 +55,82 @@ def async_setup_component(hass: core.HomeAssistant, domain: str,
This method is a coroutine.
"""
if domain in hass.config.components:
_LOGGER.debug('Component %s already set up.', domain)
return True
setup_tasks = hass.data.get(DATA_SETUP)
if not loader.PREPARED:
yield from hass.loop.run_in_executor(None, loader.prepare, hass)
if setup_tasks is not None and domain in setup_tasks:
return (yield from setup_tasks[domain])
if config is None:
config = {}
components = loader.load_order_component(domain)
if setup_tasks is None:
setup_tasks = hass.data[DATA_SETUP] = {}
# OrderedSet is empty if component or dependencies could not be resolved
if not components:
_async_persistent_notification(hass, domain, True)
return False
task = setup_tasks[domain] = hass.async_add_job(
_async_setup_component(hass, domain, config))
for component in components:
res = yield from _async_setup_component(hass, component, config)
if not res:
_LOGGER.error('Component %s failed to setup', component)
_async_persistent_notification(hass, component, True)
return False
return (yield from task)
@asyncio.coroutine
def _async_process_requirements(hass: core.HomeAssistant, name: str,
requirements) -> bool:
"""Install the requirements for a component.
This method is a coroutine.
"""
if hass.config.skip_pip:
return True
pip_lock = hass.data.get(DATA_PIP_LOCK)
if pip_lock is None:
pip_lock = hass.data[DATA_PIP_LOCK] = asyncio.Lock(loop=hass.loop)
def pip_install(mod):
"""Install packages."""
return pkg_util.install_package(mod, target=hass.config.path('deps'))
with (yield from pip_lock):
for req in requirements:
ret = yield from hass.loop.run_in_executor(None, pip_install, req)
if not ret:
_LOGGER.error('Not initializing %s because could not install '
'dependency %s', name, req)
async_notify_setup_error(hass, name)
return False
return True
def _handle_requirements(hass: core.HomeAssistant, component,
name: str) -> bool:
"""Install the requirements for a component.
@asyncio.coroutine
def _async_process_dependencies(hass, config, name, dependencies):
"""Ensure all dependencies are set up."""
blacklisted = [dep for dep in dependencies
if dep in loader.DEPENDENCY_BLACKLIST]
This method needs to run in an executor.
"""
if hass.config.skip_pip or not hasattr(component, 'REQUIREMENTS'):
if blacklisted:
_LOGGER.error('Unable to setup dependencies of %s: '
'found blacklisted dependencies: %s',
name, ', '.join(blacklisted))
return False
tasks = [async_setup_component(hass, dep, config) for dep
in dependencies]
if not tasks:
return True
for req in component.REQUIREMENTS:
if not pkg_util.install_package(req, target=hass.config.path('deps')):
_LOGGER.error('Not initializing %s because could not install '
'dependency %s', name, req)
_async_persistent_notification(hass, name)
return False
results = yield from asyncio.gather(*tasks, loop=hass.loop)
failed = [dependencies[idx] for idx, res
in enumerate(results) if not res]
if failed:
_LOGGER.error('Unable to setup dependencies of %s. '
'Setup failed for dependencies: %s',
name, ', '.join(failed))
return False
return True
@ -104,172 +140,78 @@ def _async_setup_component(hass: core.HomeAssistant,
"""Setup a component for Home Assistant.
This method is a coroutine.
hass: Home Assistant instance.
domain: Domain of component to setup.
config: The Home Assistant configuration.
"""
# pylint: disable=too-many-return-statements
if domain in hass.config.components:
return True
def log_error(msg):
"""Log helper."""
_LOGGER.error('Setup failed for %s: %s', domain, msg)
async_notify_setup_error(hass, domain, True)
setup_lock = hass.data.get('setup_lock')
if setup_lock is None:
setup_lock = hass.data['setup_lock'] = asyncio.Lock(loop=hass.loop)
# Validate no circular dependencies
components = loader.load_order_component(domain)
setup_progress = hass.data.get('setup_progress')
if setup_progress is None:
setup_progress = hass.data['setup_progress'] = []
if domain in setup_progress:
_LOGGER.error('Attempt made to setup %s during setup of %s',
domain, domain)
_async_persistent_notification(hass, domain, True)
# OrderedSet is empty if component or dependencies could not be resolved
if not components:
log_error('Unable to resolve component or dependencies')
return False
try:
# Used to indicate to discovery that a setup is ongoing and allow it
# to wait till it is done.
did_lock = False
if not setup_lock.locked():
yield from setup_lock.acquire()
did_lock = True
setup_progress.append(domain)
config = yield from async_prepare_setup_component(hass, config, domain)
if config is None:
return False
component = loader.get_component(domain)
if component is None:
_async_persistent_notification(hass, domain)
return False
async_comp = hasattr(component, 'async_setup')
try:
_LOGGER.info("Setting up %s", domain)
if async_comp:
result = yield from component.async_setup(hass, config)
else:
result = yield from hass.loop.run_in_executor(
None, component.setup, hass, config)
except Exception: # pylint: disable=broad-except
_LOGGER.exception('Error during setup of component %s', domain)
_async_persistent_notification(hass, domain, True)
return False
if result is False:
_LOGGER.error('component %s failed to initialize', domain)
_async_persistent_notification(hass, domain, True)
return False
elif result is not True:
_LOGGER.error('component %s did not return boolean if setup '
'was successful. Disabling component.', domain)
_async_persistent_notification(hass, domain, True)
loader.set_component(domain, None)
return False
hass.config.components.add(component.DOMAIN)
hass.bus.async_fire(
EVENT_COMPONENT_LOADED, {ATTR_COMPONENT: component.DOMAIN}
)
return True
finally:
setup_progress.remove(domain)
if did_lock:
setup_lock.release()
def prepare_setup_component(hass: core.HomeAssistant, config: dict,
domain: str):
"""Prepare setup of a component and return processed config."""
return run_coroutine_threadsafe(
async_prepare_setup_component(hass, config, domain), loop=hass.loop
).result()
@asyncio.coroutine
def async_prepare_setup_component(hass: core.HomeAssistant, config: dict,
domain: str):
"""Prepare setup of a component and return processed config.
This method is a coroutine.
"""
# pylint: disable=too-many-return-statements
component = loader.get_component(domain)
missing_deps = [dep for dep in getattr(component, 'DEPENDENCIES', [])
if dep not in hass.config.components]
if missing_deps:
_LOGGER.error(
'Not initializing %s because not all dependencies loaded: %s',
domain, ", ".join(missing_deps))
return None
processed_config = \
conf_util.async_process_component_config(hass, config, domain)
if hasattr(component, 'CONFIG_SCHEMA'):
try:
config = component.CONFIG_SCHEMA(config)
except vol.Invalid as ex:
async_log_exception(ex, domain, config, hass)
return None
if processed_config is None:
log_error('Invalid config')
return False
elif hasattr(component, 'PLATFORM_SCHEMA'):
platforms = []
for p_name, p_config in config_per_platform(config, domain):
# Validate component specific platform schema
try:
p_validated = component.PLATFORM_SCHEMA(p_config)
except vol.Invalid as ex:
async_log_exception(ex, domain, config, hass)
continue
if not hass.config.skip_pip and hasattr(component, 'REQUIREMENTS'):
req_success = yield from _async_process_requirements(
hass, domain, component.REQUIREMENTS)
if not req_success:
log_error('Could not install all requirements.')
return False
# Not all platform components follow same pattern for platforms
# So if p_name is None we are not going to validate platform
# (the automation component is one of them)
if p_name is None:
platforms.append(p_validated)
continue
if hasattr(component, 'DEPENDENCIES'):
dep_success = yield from _async_process_dependencies(
hass, config, domain, component.DEPENDENCIES)
platform = yield from async_prepare_setup_platform(
hass, config, domain, p_name)
if not dep_success:
log_error('Could not setup all dependencies.')
return False
if platform is None:
continue
async_comp = hasattr(component, 'async_setup')
# Validate platform specific schema
if hasattr(platform, 'PLATFORM_SCHEMA'):
try:
# pylint: disable=no-member
p_validated = platform.PLATFORM_SCHEMA(p_validated)
except vol.Invalid as ex:
async_log_exception(ex, '{}.{}'.format(domain, p_name),
p_validated, hass)
continue
try:
_LOGGER.info("Setting up %s", domain)
if async_comp:
result = yield from component.async_setup(hass, processed_config)
else:
result = yield from hass.loop.run_in_executor(
None, component.setup, hass, processed_config)
except Exception: # pylint: disable=broad-except
_LOGGER.exception('Error during setup of component %s', domain)
async_notify_setup_error(hass, domain, True)
return False
platforms.append(p_validated)
if result is False:
log_error('Component failed to initialize.')
return False
elif result is not True:
log_error('Component did not return boolean if setup was successful. '
'Disabling component.')
loader.set_component(domain, None)
return False
# Create a copy of the configuration with all config for current
# component removed and add validated config back in.
filter_keys = extract_domain_configs(config, domain)
config = {key: value for key, value in config.items()
if key not in filter_keys}
config[domain] = platforms
hass.config.components.add(component.DOMAIN)
res = yield from hass.loop.run_in_executor(
None, _handle_requirements, hass, component, domain)
if not res:
return None
hass.bus.async_fire(
EVENT_COMPONENT_LOADED, {ATTR_COMPONENT: component.DOMAIN}
)
return config
def prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
platform_name: str) -> Optional[ModuleType]:
"""Load a platform and makes sure dependencies are setup."""
return run_coroutine_threadsafe(
async_prepare_setup_platform(hass, config, domain, platform_name),
loop=hass.loop
).result()
return True
@asyncio.coroutine
@ -280,17 +222,19 @@ def async_prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
This method is a coroutine.
"""
if not loader.PREPARED:
yield from hass.loop.run_in_executor(None, loader.prepare, hass)
platform_path = PLATFORM_FORMAT.format(domain, platform_name)
def log_error(msg):
"""Log helper."""
_LOGGER.error('Unable to prepare setup for platform %s: %s',
platform_path, msg)
async_notify_setup_error(hass, platform_path)
platform = loader.get_platform(domain, platform_name)
# Not found
if platform is None:
_LOGGER.error('Unable to find platform %s', platform_path)
_async_persistent_notification(hass, platform_path)
log_error('Unable to find platform')
return None
# Already loaded
@ -298,25 +242,22 @@ def async_prepare_setup_platform(hass: core.HomeAssistant, config, domain: str,
return platform
# Load dependencies
for component in getattr(platform, 'DEPENDENCIES', []):
if component in loader.DEPENDENCY_BLACKLIST:
raise HomeAssistantError(
'{} is not allowed to be a dependency.'.format(component))
if hasattr(platform, 'DEPENDENCIES'):
dep_success = yield from _async_process_dependencies(
hass, config, platform_path, platform.DEPENDENCIES)
res = yield from async_setup_component(hass, component, config)
if not res:
_LOGGER.error(
'Unable to prepare setup for platform %s because '
'dependency %s could not be initialized', platform_path,
component)
_async_persistent_notification(hass, platform_path, True)
if not dep_success:
log_error('Could not setup all dependencies.')
return False
if not hass.config.skip_pip and hasattr(platform, 'REQUIREMENTS'):
req_success = yield from _async_process_requirements(
hass, platform_path, platform.REQUIREMENTS)
if not req_success:
log_error('Could not install all requirements.')
return None
res = yield from hass.loop.run_in_executor(
None, _handle_requirements, hass, platform, platform_path)
if not res:
return None
return platform
@ -339,23 +280,14 @@ def from_config_dict(config: Dict[str, Any],
hass.config.config_dir = config_dir
mount_local_lib_path(config_dir)
@asyncio.coroutine
def _async_init_from_config_dict(future):
try:
re_hass = yield from async_from_config_dict(
config, hass, config_dir, enable_log, verbose, skip_pip,
log_rotate_days)
future.set_result(re_hass)
# pylint: disable=broad-except
except Exception as exc:
future.set_exception(exc)
# run task
future = asyncio.Future(loop=hass.loop)
hass.async_add_job(_async_init_from_config_dict(future))
hass.loop.run_until_complete(future)
hass = hass.loop.run_until_complete(
async_from_config_dict(
config, hass, config_dir, enable_log, verbose, skip_pip,
log_rotate_days)
)
return future.result()
return hass
@asyncio.coroutine
@ -372,19 +304,15 @@ def async_from_config_dict(config: Dict[str, Any],
Dynamically loads required components and its dependencies.
This method is a coroutine.
"""
start = time()
hass.async_track_tasks()
setup_lock = hass.data.get('setup_lock')
if setup_lock is None:
setup_lock = hass.data['setup_lock'] = asyncio.Lock(loop=hass.loop)
yield from setup_lock.acquire()
core_config = config.get(core.DOMAIN, {})
try:
yield from conf_util.async_process_ha_core_config(hass, core_config)
except vol.Invalid as ex:
async_log_exception(ex, 'homeassistant', core_config, hass)
conf_util.async_log_exception(ex, 'homeassistant', core_config, hass)
return None
yield from hass.loop.run_in_executor(
@ -433,20 +361,25 @@ def async_from_config_dict(config: Dict[str, Any],
event_decorators.HASS = hass
service.HASS = hass
# Setup the components
dependency_blacklist = loader.DEPENDENCY_BLACKLIST - set(components)
# stage 1
for component in components:
if component not in FIRST_INIT_COMPONENT:
continue
hass.async_add_job(async_setup_component(hass, component, config))
for domain in loader.load_order_components(components):
if domain in dependency_blacklist:
raise HomeAssistantError(
'{} is not allowed to be a dependency'.format(domain))
yield from hass.async_block_till_done()
yield from _async_setup_component(hass, domain, config)
setup_lock.release()
# stage 2
for component in components:
if component in FIRST_INIT_COMPONENT:
continue
hass.async_add_job(async_setup_component(hass, component, config))
yield from hass.async_stop_track_tasks()
stop = time()
_LOGGER.info('Home Assistant initialized in %ss', round(stop-start, 2))
async_register_signal_handling(hass)
return hass
@ -464,22 +397,13 @@ def from_config_file(config_path: str,
if hass is None:
hass = core.HomeAssistant()
@asyncio.coroutine
def _async_init_from_config_file(future):
try:
re_hass = yield from async_from_config_file(
config_path, hass, verbose, skip_pip, log_rotate_days)
future.set_result(re_hass)
# pylint: disable=broad-except
except Exception as exc:
future.set_exception(exc)
# run task
future = asyncio.Future(loop=hass.loop)
hass.loop.create_task(_async_init_from_config_file(future))
hass.loop.run_until_complete(future)
hass = hass.loop.run_until_complete(
async_from_config_file(
config_path, hass, verbose, skip_pip, log_rotate_days)
)
return future.result()
return hass
@asyncio.coroutine
@ -588,62 +512,6 @@ def async_enable_logging(hass: core.HomeAssistant, verbose: bool=False,
'Unable to setup error log %s (access denied)', err_log_path)
def log_exception(ex, domain, config, hass):
"""Generate log exception for config validation."""
run_callback_threadsafe(
hass.loop, async_log_exception, ex, domain, config, hass).result()
@core.callback
def _async_persistent_notification(hass: core.HomeAssistant, component: str,
link: Optional[bool]=False):
"""Print a persistent notification.
This method must be run in the event loop.
"""
errors = hass.data.get(DATA_PERSISTENT_ERRORS)
if errors is None:
errors = hass.data[DATA_PERSISTENT_ERRORS] = {}
errors[component] = errors.get(component) or link
_lst = [HA_COMPONENT_URL.format(name.replace('_', '-'), name)
if link else name for name, link in errors.items()]
message = ('The following components and platforms could not be set up:\n'
'* ' + '\n* '.join(list(_lst)) + '\nPlease check your config')
persistent_notification.async_create(
hass, message, 'Invalid config', 'invalid_config')
@core.callback
def async_log_exception(ex, domain, config, hass):
"""Generate log exception for config validation.
This method must be run in the event loop.
"""
message = 'Invalid config for [{}]: '.format(domain)
if hass is not None:
_async_persistent_notification(hass, domain, True)
if 'extra keys not allowed' in ex.error_message:
message += '[{}] is an invalid option for [{}]. Check: {}->{}.'\
.format(ex.path[-1], domain, domain,
'->'.join(str(m) for m in ex.path))
else:
message += '{}.'.format(humanize_error(config, ex))
domain_config = config.get(domain, config)
message += " (See {}, line {}). ".format(
getattr(domain_config, '__config_file__', '?'),
getattr(domain_config, '__line__', '?'))
if domain != 'homeassistant':
message += ('Please check the docs at '
'https://home-assistant.io/components/{}/'.format(domain))
_LOGGER.error(message)
def mount_local_lib_path(config_dir: str) -> str:
"""Add local library to Python Path.