Add support for after_dependencies (#23148)

* Add support for after_dependencies

* Remove assert false"

* Fix types

homeassistant/bootstrap.py

@@ -26,8 +26,8 @@ ERROR_LOG_FILENAME = 'home-assistant.log'
 # hass.data key for logging information.
 DATA_LOGGING = 'logging'
 
+CORE_INTEGRATIONS = ('homeassistant', 'persistent_notification')
 LOGGING_INTEGRATIONS = {'logger', 'system_log'}
-
 STAGE_1_INTEGRATIONS = {
     # To record data
     'recorder',
@@ -91,60 +91,7 @@ async def async_from_config_dict(config: Dict[str, Any],
     hass.config_entries = config_entries.ConfigEntries(hass, config)
     await hass.config_entries.async_initialize()
 
-    domains = _get_domains(hass, config)
+    await _async_set_up_integrations(hass, config)
-
-    # Resolve all dependencies of all components so we can find the logging
-    # and integrations that need faster initialization.
-    resolved_domains_task = asyncio.gather(*[
-        loader.async_component_dependencies(hass, domain)
-        for domain in domains
-    ], return_exceptions=True)
-
-    # Set up core.
-    if not all(await asyncio.gather(
-            async_setup_component(hass, 'homeassistant', config),
-            async_setup_component(hass, 'persistent_notification', config),
-    )):
-        _LOGGER.error("Home Assistant core failed to initialize. "
-                      "Further initialization aborted")
-        return hass
-
-    _LOGGER.debug("Home Assistant core initialized")
-
-    # Finish resolving domains
-    for dep_domains in await resolved_domains_task:
-        # Result is either a set or an exception. We ignore exceptions
-        # It will be properly handled during setup of the domain.
-        if isinstance(dep_domains, set):
-            domains.update(dep_domains)
-
-    # setup components
-    logging_domains = domains & LOGGING_INTEGRATIONS
-    stage_1_domains = domains & STAGE_1_INTEGRATIONS
-    stage_2_domains = domains - logging_domains - stage_1_domains
-
-    await asyncio.gather(*[
-        async_setup_component(hass, domain, config)
-        for domain in logging_domains
-    ])
-
-    # Kick off loading the registries. They don't need to be awaited.
-    asyncio.gather(
-        hass.helpers.device_registry.async_get_registry(),
-        hass.helpers.entity_registry.async_get_registry(),
-        hass.helpers.area_registry.async_get_registry())
-
-    # Continue setting up the components
-    for to_load in (stage_1_domains, stage_2_domains):
-        if not to_load:
-            continue
-
-        await asyncio.gather(*[
-            async_setup_component(hass, domain, config)
-            for domain in to_load
-        ])
-
-    await hass.async_block_till_done()
 
     stop = time()
     _LOGGER.info("Home Assistant initialized in %.2fs", stop-start)
@@ -352,3 +299,113 @@ def _get_domains(hass: core.HomeAssistant, config: Dict[str, Any]) -> Set[str]:
         domains.add('hassio')
 
     return domains
+
+
+async def _async_set_up_integrations(
+        hass: core.HomeAssistant, config: Dict[str, Any]) -> None:
+    """Set up all the integrations."""
+    domains = _get_domains(hass, config)
+
+    # Resolve all dependencies of all components so we can find the logging
+    # and integrations that need faster initialization.
+    resolved_domains_task = asyncio.gather(*[
+        loader.async_component_dependencies(hass, domain)
+        for domain in domains
+    ], return_exceptions=True)
+
+    # Set up core.
+    _LOGGER.debug("Setting up %s", CORE_INTEGRATIONS)
+
+    if not all(await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in CORE_INTEGRATIONS
+    ])):
+        _LOGGER.error("Home Assistant core failed to initialize. "
+                      "Further initialization aborted")
+        return
+
+    _LOGGER.debug("Home Assistant core initialized")
+
+    # Finish resolving domains
+    for dep_domains in await resolved_domains_task:
+        # Result is either a set or an exception. We ignore exceptions
+        # It will be properly handled during setup of the domain.
+        if isinstance(dep_domains, set):
+            domains.update(dep_domains)
+
+    # setup components
+    logging_domains = domains & LOGGING_INTEGRATIONS
+    stage_1_domains = domains & STAGE_1_INTEGRATIONS
+    stage_2_domains = domains - logging_domains - stage_1_domains
+
+    if logging_domains:
+        _LOGGER.debug("Setting up %s", logging_domains)
+
+        await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in logging_domains
+        ])
+
+    # Kick off loading the registries. They don't need to be awaited.
+    asyncio.gather(
+        hass.helpers.device_registry.async_get_registry(),
+        hass.helpers.entity_registry.async_get_registry(),
+        hass.helpers.area_registry.async_get_registry())
+
+    if stage_1_domains:
+        await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in logging_domains
+        ])
+
+    # Load all integrations
+    after_dependencies = {}  # type: Dict[str, Set[str]]
+
+    for int_or_exc in await asyncio.gather(*[
+            loader.async_get_integration(hass, domain)
+            for domain in stage_2_domains
+    ], return_exceptions=True):
+        # Exceptions are handled in async_setup_component.
+        if (isinstance(int_or_exc, loader.Integration) and
+                int_or_exc.after_dependencies):
+            after_dependencies[int_or_exc.domain] = set(
+                int_or_exc.after_dependencies
+            )
+
+    last_load = None
+    while stage_2_domains:
+        domains_to_load = set()
+
+        for domain in stage_2_domains:
+            after_deps = after_dependencies.get(domain)
+            # Load if integration has no after_dependencies or they are
+            # all loaded
+            if (not after_deps or
+                    not after_deps-hass.config.components):
+                domains_to_load.add(domain)
+
+        if not domains_to_load or domains_to_load == last_load:
+            break
+
+        _LOGGER.debug("Setting up %s", domains_to_load)
+
+        await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in domains_to_load
+        ])
+
+        last_load = domains_to_load
+        stage_2_domains -= domains_to_load
+
+    # These are stage 2 domains that never have their after_dependencies
+    # satisfied.
+    if stage_2_domains:
+        _LOGGER.debug("Final set up: %s", stage_2_domains)
+
+        await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in stage_2_domains
+        ])
+
+    # Wrap up startup
+    await hass.async_block_till_done()

homeassistant/components/camera/manifest.json

@@ -6,5 +6,8 @@
   "dependencies": [
     "http"
   ],
+  "after_dependencies": [
+    "stream"
+  ],
   "codeowners": []
 }

homeassistant/components/mysensors/manifest.json

@@ -6,5 +6,8 @@
     "pymysensors==0.18.0"
   ],
   "dependencies": [],
+  "after_dependencies": [
+    "mqtt"
+  ],
   "codeowners": []
 }

homeassistant/components/owntracks/manifest.json

@@ -8,5 +8,8 @@
   "dependencies": [
     "webhook"
   ],
+  "after_dependencies": [
+    "mqtt"
+  ],
   "codeowners": []
 }

homeassistant/loader.py

@@ -21,6 +21,8 @@ from typing import (
     TypeVar,
     List,
     Dict,
+    Union,
+    cast,
 )
 
 # Typing imports that create a circular dependency
@@ -116,6 +118,8 @@ class Integration:
         self.name = manifest['name']  # type: str
         self.domain = manifest['domain']  # type: str
         self.dependencies = manifest['dependencies']  # type: List[str]
+        self.after_dependencies = manifest.get(
+            'after_dependencies')  # type: Optional[List[str]]
         self.requirements = manifest['requirements']  # type: List[str]
         _LOGGER.info("Loaded %s from %s", self.domain, pkg_path)
 
@@ -150,7 +154,8 @@ async def async_get_integration(hass: 'HomeAssistant', domain: str)\
             raise IntegrationNotFound(domain)
         cache = hass.data[DATA_INTEGRATIONS] = {}
 
-    int_or_evt = cache.get(domain, _UNDEF)  # type: Optional[Integration]
+    int_or_evt = cache.get(
+        domain, _UNDEF)  # type: Optional[Union[Integration, asyncio.Event]]
 
     if isinstance(int_or_evt, asyncio.Event):
         await int_or_evt.wait()
@@ -161,7 +166,7 @@ async def async_get_integration(hass: 'HomeAssistant', domain: str)\
     elif int_or_evt is None:
         raise IntegrationNotFound(domain)
     else:
-        return int_or_evt
+        return cast(Integration, int_or_evt)
 
     event = cache[domain] = asyncio.Event()
 

script/hassfest/dependencies.py

@@ -35,6 +35,7 @@ def validate_dependencies(integration: Integration):
                           r"hass\.components\.(\w+)")
     referenced -= ALLOWED_USED_COMPONENTS
     referenced -= set(integration.manifest['dependencies'])
+    referenced -= set(integration.manifest.get('after_dependencies', []))
 
     if referenced:
         for domain in sorted(referenced):

script/hassfest/manifest.py

@@ -13,6 +13,7 @@ MANIFEST_SCHEMA = vol.Schema({
     vol.Required('documentation'): str,
     vol.Required('requirements'): [str],
     vol.Required('dependencies'): [str],
+    vol.Optional('after_dependencies'): [str],
     vol.Required('codeowners'): [str],
 })
 

tests/common.py

@@ -442,13 +442,16 @@ class MockModule:
                  requirements=None, config_schema=None, platform_schema=None,
                  platform_schema_base=None, async_setup=None,
                  async_setup_entry=None, async_unload_entry=None,
-                 async_migrate_entry=None, async_remove_entry=None):
+                 async_migrate_entry=None, async_remove_entry=None,
+                 partial_manifest=None):
         """Initialize the mock module."""
         self.__name__ = 'homeassistant.components.{}'.format(domain)
         self.__file__ = 'homeassistant/components/{}'.format(domain)
         self.DOMAIN = domain
         self.DEPENDENCIES = dependencies or []
         self.REQUIREMENTS = requirements or []
+        # Overlay to be used when generating manifest from this module
+        self._partial_manifest = partial_manifest
 
         if config_schema is not None:
             self.CONFIG_SCHEMA = config_schema
@@ -481,6 +484,13 @@ class MockModule:
         if async_remove_entry is not None:
             self.async_remove_entry = async_remove_entry
 
+    def mock_manifest(self):
+        """Generate a mock manifest to represent this module."""
+        return {
+            **loader.manifest_from_legacy_module(self),
+            **(self._partial_manifest or {})
+        }
+
 
 class MockPlatform:
     """Provide a fake platform."""
@@ -906,7 +916,7 @@ def mock_integration(hass, module):
     """Mock an integration."""
     integration = loader.Integration(
         hass, 'homeassisant.components.{}'.format(module.DOMAIN), None,
-        loader.manifest_from_legacy_module(module))
+        module.mock_manifest())
 
     _LOGGER.info("Adding mock integration: %s", module.DOMAIN)
     hass.data.setdefault(

tests/test_bootstrap.py

@@ -9,7 +9,9 @@ import homeassistant.config as config_util
 from homeassistant import bootstrap
 import homeassistant.util.dt as dt_util
 
-from tests.common import patch_yaml_files, get_test_config_dir, mock_coro
+from tests.common import (
+    patch_yaml_files, get_test_config_dir, mock_coro, mock_integration,
+    MockModule)
 
 ORIG_TIMEZONE = dt_util.DEFAULT_TIME_ZONE
 VERSION_PATH = os.path.join(get_test_config_dir(), config_util.VERSION_FILE)
@@ -87,3 +89,154 @@ async def test_load_hassio(hass):
 
     with patch.dict(os.environ, {'HASSIO': '1'}):
         assert bootstrap._get_domains(hass, {}) == {'hassio'}
+
+
+async def test_empty_setup(hass):
+    """Test an empty set up loads the core."""
+    await bootstrap._async_set_up_integrations(hass, {})
+    for domain in bootstrap.CORE_INTEGRATIONS:
+        assert domain in hass.config.components, domain
+
+
+async def test_core_failure_aborts(hass, caplog):
+    """Test failing core setup aborts further setup."""
+    with patch('homeassistant.components.homeassistant.async_setup',
+               return_value=mock_coro(False)):
+        await bootstrap._async_set_up_integrations(hass, {
+            'group': {}
+        })
+
+    assert 'core failed to initialize' in caplog.text
+    # We aborted early, group not set up
+    assert 'group' not in hass.config.components
+
+
+async def test_setting_up_config(hass, caplog):
+    """Test we set up domains in config."""
+    await bootstrap._async_set_up_integrations(hass, {
+        'group hello': {},
+        'homeassistant': {}
+    })
+
+    assert 'group' in hass.config.components
+
+
+async def test_setup_after_deps_all_present(hass, caplog):
+    """Test after_dependencies when all present."""
+    caplog.set_level(logging.DEBUG)
+    order = []
+
+    def gen_domain_setup(domain):
+        async def async_setup(hass, config):
+            order.append(domain)
+            return True
+
+        return async_setup
+
+    mock_integration(hass, MockModule(
+        domain='root',
+        async_setup=gen_domain_setup('root')
+    ))
+    mock_integration(hass, MockModule(
+        domain='first_dep',
+        async_setup=gen_domain_setup('first_dep'),
+        partial_manifest={
+            'after_dependencies': ['root']
+        }
+    ))
+    mock_integration(hass, MockModule(
+        domain='second_dep',
+        async_setup=gen_domain_setup('second_dep'),
+        partial_manifest={
+            'after_dependencies': ['first_dep']
+        }
+    ))
+
+    await bootstrap._async_set_up_integrations(hass, {
+        'root': {},
+        'first_dep': {},
+        'second_dep': {},
+    })
+
+    assert 'root' in hass.config.components
+    assert 'first_dep' in hass.config.components
+    assert 'second_dep' in hass.config.components
+    assert order == ['root', 'first_dep', 'second_dep']
+
+
+async def test_setup_after_deps_not_trigger_load(hass, caplog):
+    """Test after_dependencies does not trigger loading it."""
+    caplog.set_level(logging.DEBUG)
+    order = []
+
+    def gen_domain_setup(domain):
+        async def async_setup(hass, config):
+            order.append(domain)
+            return True
+
+        return async_setup
+
+    mock_integration(hass, MockModule(
+        domain='root',
+        async_setup=gen_domain_setup('root')
+    ))
+    mock_integration(hass, MockModule(
+        domain='first_dep',
+        async_setup=gen_domain_setup('first_dep'),
+        partial_manifest={
+            'after_dependencies': ['root']
+        }
+    ))
+    mock_integration(hass, MockModule(
+        domain='second_dep',
+        async_setup=gen_domain_setup('second_dep'),
+        partial_manifest={
+            'after_dependencies': ['first_dep']
+        }
+    ))
+
+    await bootstrap._async_set_up_integrations(hass, {
+        'root': {},
+        'second_dep': {},
+    })
+
+    assert 'root' in hass.config.components
+    assert 'first_dep' not in hass.config.components
+    assert 'second_dep' in hass.config.components
+    assert order == ['root', 'second_dep']
+
+
+async def test_setup_after_deps_not_present(hass, caplog):
+    """Test after_dependencies when referenced integration doesn't exist."""
+    caplog.set_level(logging.DEBUG)
+    order = []
+
+    def gen_domain_setup(domain):
+        async def async_setup(hass, config):
+            order.append(domain)
+            return True
+
+        return async_setup
+
+    mock_integration(hass, MockModule(
+        domain='root',
+        async_setup=gen_domain_setup('root')
+    ))
+    mock_integration(hass, MockModule(
+        domain='second_dep',
+        async_setup=gen_domain_setup('second_dep'),
+        partial_manifest={
+            'after_dependencies': ['first_dep']
+        }
+    ))
+
+    await bootstrap._async_set_up_integrations(hass, {
+        'root': {},
+        'first_dep': {},
+        'second_dep': {},
+    })
+
+    assert 'root' in hass.config.components
+    assert 'first_dep' not in hass.config.components
+    assert 'second_dep' in hass.config.components
+    assert order == ['root', 'second_dep']