Load as many components in parallel as possible (#20806)

* Load as many components in parallel as possible

* Lint

homeassistant/bootstrap.py

@@ -10,7 +10,8 @@ from typing import Any, Optional, Dict
 import voluptuous as vol
 
 from homeassistant import (
-    core, config as conf_util, config_entries, components as core_components)
+    core, config as conf_util, config_entries, components as core_components,
+    loader)
 from homeassistant.components import persistent_notification
 from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE
 from homeassistant.setup import async_setup_component
@@ -124,6 +125,15 @@ async def async_from_config_dict(config: Dict[str, Any],
                      if key != core.DOMAIN)
     components.update(hass.config_entries.async_domains())
 
+    # Resolve all dependencies of all components.
+    for component in list(components):
+        try:
+            components.update(loader.component_dependencies(hass, component))
+        except loader.LoaderError:
+            # Ignore it, or we'll break startup
+            # It will be properly handled during setup.
+            pass
+
     # setup components
     res = await core_components.async_setup(hass, config)
     if not res:

homeassistant/loader.py

@@ -18,7 +18,6 @@ from types import ModuleType
 from typing import Optional, Set, TYPE_CHECKING, Callable, Any, TypeVar  # noqa pylint: disable=unused-import
 
 from homeassistant.const import PLATFORM_FORMAT
-from homeassistant.util import OrderedSet
 
 # Typing imports that create a circular dependency
 # pylint: disable=using-constant-test,unused-import
@@ -39,6 +38,30 @@ PATH_CUSTOM_COMPONENTS = 'custom_components'
 PACKAGE_COMPONENTS = 'homeassistant.components'
 
 
+class LoaderError(Exception):
+    """Loader base error."""
+
+
+class ComponentNotFound(LoaderError):
+    """Raised when a component is not found."""
+
+    def __init__(self, domain: str) -> None:
+        """Initialize a component not found error."""
+        super().__init__("Component {} not found.".format(domain))
+        self.domain = domain
+
+
+class CircularDependency(LoaderError):
+    """Raised when a circular dependency is found when resolving components."""
+
+    def __init__(self, from_domain: str, to_domain: str) -> None:
+        """Initialize circular dependency error."""
+        super().__init__("Circular dependency detected: {} -> {}.".format(
+            from_domain, to_domain))
+        self.from_domain = from_domain
+        self.to_domain = to_domain
+
+
 def set_component(hass,  # type: HomeAssistant
                   comp_name: str, component: Optional[ModuleType]) -> None:
     """Set a component in the cache.
@@ -235,57 +258,46 @@ def bind_hass(func: CALLABLE_T) -> CALLABLE_T:
     return func
 
 
-def load_order_component(hass,  # type: HomeAssistant
+def component_dependencies(hass,  # type: HomeAssistant
-                         comp_name: str) -> OrderedSet:
+                           comp_name: str) -> Set[str]:
-    """Return an OrderedSet of components in the correct order of loading.
+    """Return all dependencies and subdependencies of components.
 
-    Returns an empty list if a circular dependency is detected
+    Raises CircularDependency if a circular dependency is found.
-    or the component could not be loaded. In both cases, the error is
-    logged.
 
     Async friendly.
     """
-    return _load_order_component(hass, comp_name, OrderedSet(), set())
+    return _component_dependencies(hass, comp_name, set(), set())
 
 
-def _load_order_component(hass,  # type: HomeAssistant
+def _component_dependencies(hass,  # type: HomeAssistant
-                          comp_name: str, load_order: OrderedSet,
+                            comp_name: str, loaded: Set[str],
-                          loading: Set) -> OrderedSet:
+                            loading: Set) -> Set[str]:
-    """Recursive function to get load order of components.
+    """Recursive function to get component dependencies.
 
     Async friendly.
     """
     component = get_component(hass, comp_name)
 
-    # If None it does not exist, error already thrown by get_component.
     if component is None:
-        return OrderedSet()
+        raise ComponentNotFound(comp_name)
 
     loading.add(comp_name)
 
     for dependency in getattr(component, 'DEPENDENCIES', []):
         # Check not already loaded
-        if dependency in load_order:
+        if dependency in loaded:
             continue
 
         # If we are already loading it, we have a circular dependency.
         if dependency in loading:
-            _LOGGER.error("Circular dependency detected: %s -> %s",
+            raise CircularDependency(comp_name, dependency)
-                          comp_name, dependency)
-            return OrderedSet()
 
-        dep_load_order = _load_order_component(
+        dep_loaded = _component_dependencies(
-            hass, dependency, load_order, loading)
+            hass, dependency, loaded, loading)
 
-        # length == 0 means error loading dependency or children
+        loaded.update(dep_loaded)
-        if not dep_load_order:
-            _LOGGER.error("Error loading %s dependency: %s",
-                          comp_name, dependency)
-            return OrderedSet()
 
-        load_order.update(dep_load_order)
+    loaded.add(comp_name)
-
-    load_order.add(comp_name)
     loading.remove(comp_name)
 
-    return load_order
+    return loaded

homeassistant/setup.py

@@ -106,12 +106,18 @@ async def _async_setup_component(hass: core.HomeAssistant,
         log_error("Component not found.", False)
         return False
 
-    # Validate no circular dependencies
+    # Validate all dependencies exist and there are no circular dependencies
-    components = loader.load_order_component(hass, domain)
+    try:
-
+        loader.component_dependencies(hass, domain)
-    # OrderedSet is empty if component or dependencies could not be resolved
+    except loader.ComponentNotFound as err:
-    if not components:
+        _LOGGER.error(
-        log_error("Unable to resolve component or dependencies.")
+            "Not setting up %s because we are unable to resolve "
+            "(sub)dependency %s", domain, err.domain)
+        return False
+    except loader.CircularDependency as err:
+        _LOGGER.error(
+            "Not setting up %s because it contains a circular dependency: "
+            "%s -> %s", domain, err.from_domain, err.to_domain)
         return False
 
     processed_config = \

homeassistant/util/__init__.py

@@ -1,7 +1,6 @@
 """Helper methods for various modules."""
 import asyncio
 from datetime import datetime, timedelta
-from itertools import chain
 import threading
 import re
 import enum
@@ -141,96 +140,6 @@ class OrderedEnum(enum.Enum):
         return NotImplemented
 
 
-class OrderedSet(MutableSet[T]):
-    """Ordered set taken from http://code.activestate.com/recipes/576694/."""
-
-    def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
-        """Initialize the set."""
-        self.end = end = []  # type: List[Any]
-        end += [None, end, end]  # sentinel node for doubly linked list
-        self.map = {}  # type: Dict[T, List] # key --> [key, prev, next]
-        if iterable is not None:
-            self |= iterable  # type: ignore
-
-    def __len__(self) -> int:
-        """Return the length of the set."""
-        return len(self.map)
-
-    def __contains__(self, key: T) -> bool:  # type: ignore
-        """Check if key is in set."""
-        return key in self.map
-
-    # pylint: disable=arguments-differ
-    def add(self, key: T) -> None:
-        """Add an element to the end of the set."""
-        if key not in self.map:
-            end = self.end
-            curr = end[1]
-            curr[2] = end[1] = self.map[key] = [key, curr, end]
-
-    def promote(self, key: T) -> None:
-        """Promote element to beginning of the set, add if not there."""
-        if key in self.map:
-            self.discard(key)
-
-        begin = self.end[2]
-        curr = begin[1]
-        curr[2] = begin[1] = self.map[key] = [key, curr, begin]
-
-    # pylint: disable=arguments-differ
-    def discard(self, key: T) -> None:
-        """Discard an element from the set."""
-        if key in self.map:
-            key, prev_item, next_item = self.map.pop(key)
-            prev_item[2] = next_item
-            next_item[1] = prev_item
-
-    def __iter__(self) -> Iterator[T]:
-        """Iterate of the set."""
-        end = self.end
-        curr = end[2]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[2]
-
-    def __reversed__(self) -> Iterator[T]:
-        """Reverse the ordering."""
-        end = self.end
-        curr = end[1]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[1]
-
-    # pylint: disable=arguments-differ
-    def pop(self, last: bool = True) -> T:
-        """Pop element of the end of the set.
-
-        Set last=False to pop from the beginning.
-        """
-        if not self:
-            raise KeyError('set is empty')
-        key = self.end[1][0] if last else self.end[2][0]
-        self.discard(key)
-        return key  # type: ignore
-
-    def update(self, *args: Any) -> None:
-        """Add elements from args to the set."""
-        for item in chain(*args):
-            self.add(item)
-
-    def __repr__(self) -> str:
-        """Return the representation."""
-        if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self))
-
-    def __eq__(self, other: Any) -> bool:
-        """Return the comparison."""
-        if isinstance(other, OrderedSet):
-            return len(self) == len(other) and list(self) == list(other)
-        return set(self) == set(other)
-
-
 class Throttle:
     """A class for throttling the execution of tasks.
 

tests/test_loader.py

@@ -1,63 +1,52 @@
 """Test to verify that we can load components."""
-# pylint: disable=protected-access
 import asyncio
-import unittest
 
 import pytest
 
 import homeassistant.loader as loader
 import homeassistant.components.http as http
 
-from tests.common import (
+from tests.common import MockModule, async_mock_service
-    get_test_home_assistant, MockModule, async_mock_service)
 
 
-class TestLoader(unittest.TestCase):
+def test_set_component(hass):
-    """Test the loader module."""
-
-    # pylint: disable=invalid-name
-    def setUp(self):
-        """Set up tests."""
-        self.hass = get_test_home_assistant()
-
-    # pylint: disable=invalid-name
-    def tearDown(self):
-        """Stop everything that was started."""
-        self.hass.stop()
-
-    def test_set_component(self):
     """Test if set_component works."""
     comp = object()
-        loader.set_component(self.hass, 'switch.test_set', comp)
+    loader.set_component(hass, 'switch.test_set', comp)
 
-        assert loader.get_component(self.hass, 'switch.test_set') is comp
+    assert loader.get_component(hass, 'switch.test_set') is comp
 
-    def test_get_component(self):
+
+def test_get_component(hass):
     """Test if get_component works."""
-        assert http == loader.get_component(self.hass, 'http')
+    assert http == loader.get_component(hass, 'http')
 
-    def test_load_order_component(self):
+
+def test_component_dependencies(hass):
     """Test if we can get the proper load order of components."""
-        loader.set_component(self.hass, 'mod1', MockModule('mod1'))
+    loader.set_component(hass, 'mod1', MockModule('mod1'))
-        loader.set_component(self.hass, 'mod2', MockModule('mod2', ['mod1']))
+    loader.set_component(hass, 'mod2', MockModule('mod2', ['mod1']))
-        loader.set_component(self.hass, 'mod3', MockModule('mod3', ['mod2']))
+    loader.set_component(hass, 'mod3', MockModule('mod3', ['mod2']))
 
-        assert ['mod1', 'mod2', 'mod3'] == \
+    assert {'mod1', 'mod2', 'mod3'} == \
-            loader.load_order_component(self.hass, 'mod3')
+        loader.component_dependencies(hass, 'mod3')
 
     # Create circular dependency
-        loader.set_component(self.hass, 'mod1', MockModule('mod1', ['mod3']))
+    loader.set_component(hass, 'mod1', MockModule('mod1', ['mod3']))
 
-        assert [] == loader.load_order_component(self.hass, 'mod3')
+    with pytest.raises(loader.CircularDependency):
+        print(loader.component_dependencies(hass, 'mod3'))
 
     # Depend on non-existing component
-        loader.set_component(self.hass, 'mod1',
+    loader.set_component(hass, 'mod1',
                          MockModule('mod1', ['nonexisting']))
 
-        assert [] == loader.load_order_component(self.hass, 'mod1')
+    with pytest.raises(loader.ComponentNotFound):
+        print(loader.component_dependencies(hass, 'mod1'))
 
-        # Try to get load order for non-existing component
+    # Try to get dependencies for non-existing component
-        assert [] == loader.load_order_component(self.hass, 'mod1')
+    with pytest.raises(loader.ComponentNotFound):
+        print(loader.component_dependencies(hass, 'nonexisting'))
 
 
 def test_component_loader(hass):

tests/util/test_init.py

@@ -105,67 +105,6 @@ class TestUtil(unittest.TestCase):
         with pytest.raises(TypeError):
             TestEnum.FIRST >= 1
 
-    def test_ordered_set(self):
-        """Test ordering of set."""
-        set1 = util.OrderedSet([1, 2, 3, 4])
-        set2 = util.OrderedSet([3, 4, 5])
-
-        assert 4 == len(set1)
-        assert 3 == len(set2)
-
-        assert 1 in set1
-        assert 2 in set1
-        assert 3 in set1
-        assert 4 in set1
-        assert 5 not in set1
-
-        assert 1 not in set2
-        assert 2 not in set2
-        assert 3 in set2
-        assert 4 in set2
-        assert 5 in set2
-
-        set1.add(5)
-        assert 5 in set1
-
-        set1.discard(5)
-        assert 5 not in set1
-
-        # Try again while key is not in
-        set1.discard(5)
-        assert 5 not in set1
-
-        assert [1, 2, 3, 4] == list(set1)
-        assert [4, 3, 2, 1] == list(reversed(set1))
-
-        assert 1 == set1.pop(False)
-        assert [2, 3, 4] == list(set1)
-
-        assert 4 == set1.pop()
-        assert [2, 3] == list(set1)
-
-        assert 'OrderedSet()' == str(util.OrderedSet())
-        assert 'OrderedSet([2, 3])' == str(set1)
-
-        assert set1 == util.OrderedSet([2, 3])
-        assert set1 != util.OrderedSet([3, 2])
-        assert set1 == set([2, 3])
-        assert set1 == {3, 2}
-        assert set1 == [2, 3]
-        assert set1 == [3, 2]
-        assert set1 != {2}
-
-        set3 = util.OrderedSet(set1)
-        set3.update(set2)
-
-        assert [3, 4, 5, 2] == set3
-        assert [3, 4, 5, 2] == set1 | set2
-        assert [3] == set1 & set2
-        assert [2] == set1 - set2
-
-        set1.update([1, 2], [5, 6])
-        assert [2, 3, 1, 5, 6] == set1
-
     def test_throttle(self):
         """Test the add cooldown decorator."""
         calls1 = []