load cleanups (#23112)

* load cleanups

* Remove unused methods

* Allow importing requirements at the top of a file

* Fix test

* Lint

* Install reqs ASAP when loading platforms

homeassistant/bootstrap.py

@@ -26,49 +26,16 @@ ERROR_LOG_FILENAME = 'home-assistant.log'
 # hass.data key for logging information.
 DATA_LOGGING = 'logging'
 
-LOGGING_COMPONENT = {'logger', 'system_log'}
+LOGGING_INTEGRATIONS = {'logger', 'system_log'}
 
-FIRST_INIT_COMPONENT = {
+STAGE_1_INTEGRATIONS = {
+    # To record data
     'recorder',
-    'mqtt',
+    # To make sure we forward data to other instances
     'mqtt_eventstream',
-    'frontend',
-    'history',
 }
 
 
-def from_config_dict(config: Dict[str, Any],
-                     hass: Optional[core.HomeAssistant] = None,
-                     config_dir: Optional[str] = None,
-                     enable_log: bool = True,
-                     verbose: bool = False,
-                     skip_pip: bool = False,
-                     log_rotate_days: Any = None,
-                     log_file: Any = None,
-                     log_no_color: bool = False) \
-                     -> Optional[core.HomeAssistant]:
-    """Try to configure Home Assistant from a configuration dictionary.
-
-    Dynamically loads required components and its dependencies.
-    """
-    if hass is None:
-        hass = core.HomeAssistant()
-        if config_dir is not None:
-            config_dir = os.path.abspath(config_dir)
-            hass.config.config_dir = config_dir
-            if not is_virtual_env():
-                hass.loop.run_until_complete(
-                    async_mount_local_lib_path(config_dir))
-
-    # run task
-    hass = hass.loop.run_until_complete(
-        async_from_config_dict(
-            config, hass, config_dir, enable_log, verbose, skip_pip,
-            log_rotate_days, log_file, log_no_color)
-    )
-    return hass
-
-
 async def async_from_config_dict(config: Dict[str, Any],
                                  hass: core.HomeAssistant,
                                  config_dir: Optional[str] = None,
@@ -126,15 +93,12 @@ async def async_from_config_dict(config: Dict[str, Any],
 
     domains = _get_domains(hass, config)
 
-    # Resolve all dependencies of all components.
+    # Resolve all dependencies of all components so we can find the logging
-    for dep_domains in await asyncio.gather(*[
+    # and integrations that need faster initialization.
-            loader.async_component_dependencies(hass, domain)
+    resolved_domains_task = asyncio.gather(*[
-            for domain in domains
+        loader.async_component_dependencies(hass, domain)
-    ], return_exceptions=True):
+        for domain in domains
-        # Result is either a set or an exception. We ignore exceptions
+    ], return_exceptions=True)
-        # It will be properly handled during setup of the domain.
-        if isinstance(dep_domains, set):
-            domains.update(dep_domains)
 
     # Set up core.
     if not all(await asyncio.gather(
@@ -147,14 +111,22 @@ async def async_from_config_dict(config: Dict[str, Any],
 
     _LOGGER.debug("Home Assistant core initialized")
 
-    # setup components
+    # Finish resolving domains
-    # stage 0, load logging components
+    for dep_domains in await resolved_domains_task:
-    for domain in domains:
+        # Result is either a set or an exception. We ignore exceptions
-        if domain in LOGGING_COMPONENT:
+        # It will be properly handled during setup of the domain.
-            hass.async_create_task(
+        if isinstance(dep_domains, set):
-                async_setup_component(hass, domain, config))
+            domains.update(dep_domains)
 
-    await hass.async_block_till_done()
+    # setup components
+    logging_domains = domains & LOGGING_INTEGRATIONS
+    stage_1_domains = domains & STAGE_1_INTEGRATIONS
+    stage_2_domains = domains - logging_domains - stage_1_domains
+
+    await asyncio.gather(*[
+        async_setup_component(hass, domain, config)
+        for domain in logging_domains
+    ])
 
     # Kick off loading the registries. They don't need to be awaited.
     asyncio.gather(
@@ -162,19 +134,15 @@ async def async_from_config_dict(config: Dict[str, Any],
         hass.helpers.entity_registry.async_get_registry(),
         hass.helpers.area_registry.async_get_registry())
 
-    # stage 1
+    # Continue setting up the components
-    for domain in domains:
+    for to_load in (stage_1_domains, stage_2_domains):
-        if domain in FIRST_INIT_COMPONENT:
+        if not to_load:
-            hass.async_create_task(
-                async_setup_component(hass, domain, config))
-
-    await hass.async_block_till_done()
-
-    # stage 2
-    for domain in domains:
-        if domain in FIRST_INIT_COMPONENT or domain in LOGGING_COMPONENT:
             continue
-        hass.async_create_task(async_setup_component(hass, domain, config))
+
+        await asyncio.gather(*[
+            async_setup_component(hass, domain, config)
+            for domain in to_load
+        ])
 
     await hass.async_block_till_done()
 
@@ -229,32 +197,6 @@ async def async_from_config_dict(config: Dict[str, Any],
     return hass
 
 
-def from_config_file(config_path: str,
-                     hass: Optional[core.HomeAssistant] = None,
-                     verbose: bool = False,
-                     skip_pip: bool = True,
-                     log_rotate_days: Any = None,
-                     log_file: Any = None,
-                     log_no_color: bool = False)\
-        -> Optional[core.HomeAssistant]:
-    """Read the configuration file and try to start all the functionality.
-
-    Will add functionality to 'hass' parameter if given,
-    instantiates a new Home Assistant object if 'hass' is not given.
-    """
-    if hass is None:
-        hass = core.HomeAssistant()
-
-    # run task
-    hass = hass.loop.run_until_complete(
-        async_from_config_file(
-            config_path, hass, verbose, skip_pip,
-            log_rotate_days, log_file, log_no_color)
-    )
-
-    return hass
-
-
 async def async_from_config_file(config_path: str,
                                  hass: core.HomeAssistant,
                                  verbose: bool = False,

homeassistant/components/hue/bridge.py

@@ -1,6 +1,7 @@
 """Code to handle a Hue bridge."""
 import asyncio
 
+import aiohue
 import async_timeout
 import voluptuous as vol
 
@@ -133,8 +134,6 @@ class HueBridge:
 
 async def get_bridge(hass, host, username=None):
     """Create a bridge object and verify authentication."""
-    import aiohue
-
     bridge = aiohue.Bridge(
         host, username=username,
         websession=aiohttp_client.async_get_clientsession(hass)

homeassistant/components/hue/config_flow.py

@@ -3,6 +3,7 @@ import asyncio
 import json
 import os
 
+from aiohue.discovery import discover_nupnp
 import async_timeout
 import voluptuous as vol
 
@@ -57,8 +58,6 @@ class HueFlowHandler(config_entries.ConfigFlow):
 
     async def async_step_init(self, user_input=None):
         """Handle a flow start."""
-        from aiohue.discovery import discover_nupnp
-
         if user_input is not None:
             self.host = user_input['host']
             return await self.async_step_link()

homeassistant/components/hue/light.py

@@ -5,6 +5,7 @@ import logging
 from time import monotonic
 import random
 
+import aiohue
 import async_timeout
 
 from homeassistant.components import hue
@@ -152,8 +153,6 @@ async def async_update_items(hass, bridge, async_add_entities,
                              request_bridge_update, is_group, current,
                              progress_waiting):
     """Update either groups or lights from the bridge."""
-    import aiohue
-
     if is_group:
         api_type = 'group'
         api = bridge.api.groups

homeassistant/loader.py

@@ -29,8 +29,6 @@ if TYPE_CHECKING:
 
 CALLABLE_T = TypeVar('CALLABLE_T', bound=Callable)  # noqa pylint: disable=invalid-name
 
-PREPARED = False
-
 DEPENDENCY_BLACKLIST = {'config'}
 
 _LOGGER = logging.getLogger(__name__)
@@ -170,6 +168,7 @@ async def async_get_integration(hass: 'HomeAssistant', domain: str)\
             return integration
 
     except ImportError:
+        # Import error if "custom_components" doesn't exist
         pass
 
     from homeassistant import components
@@ -376,9 +375,6 @@ async def _async_component_dependencies(hass,  # type: HomeAssistant
     """
     integration = await async_get_integration(hass, domain)
 
-    if integration is None:
-        raise IntegrationNotFound(domain)
-
     loading.add(domain)
 
     for dependency_domain in integration.dependencies:

homeassistant/setup.py

@@ -100,12 +100,6 @@ async def _async_setup_component(hass: core.HomeAssistant,
         log_error("Integration not found.", False)
         return False
 
-    try:
-        component = integration.get_component()
-    except ImportError:
-        log_error("Unable to import component", False)
-        return False
-
     # Validate all dependencies exist and there are no circular dependencies
     try:
         await loader.async_component_dependencies(hass, domain)
@@ -120,6 +114,14 @@ async def _async_setup_component(hass: core.HomeAssistant,
             "%s -> %s", domain, err.from_domain, err.to_domain)
         return False
 
+    # Process requirements as soon as possible, so we can import the component
+    # without requiring imports to be in functions.
+    try:
+        await async_process_deps_reqs(hass, config, integration)
+    except HomeAssistantError as err:
+        log_error(str(err))
+        return False
+
     processed_config = await conf_util.async_process_component_config(
         hass, config, integration)
 
@@ -127,15 +129,15 @@ async def _async_setup_component(hass: core.HomeAssistant,
         log_error("Invalid config.")
         return False
 
-    try:
-        await async_process_deps_reqs(hass, config, integration)
-    except HomeAssistantError as err:
-        log_error(str(err))
-        return False
-
     start = timer()
     _LOGGER.info("Setting up %s", domain)
 
+    try:
+        component = integration.get_component()
+    except ImportError:
+        log_error("Unable to import component", False)
+        return False
+
     if hasattr(component, 'PLATFORM_SCHEMA'):
         # Entity components have their own warning
         warn_task = None
@@ -211,6 +213,14 @@ async def async_prepare_setup_platform(hass: core.HomeAssistant,
         log_error("Integration not found")
         return None
 
+    # Process deps and reqs as soon as possible, so that requirements are
+    # available when we import the platform.
+    try:
+        await async_process_deps_reqs(hass, hass_config, integration)
+    except HomeAssistantError as err:
+        log_error(str(err))
+        return None
+
     try:
         platform = integration.get_platform(domain)
     except ImportError:
@@ -238,12 +248,6 @@ async def async_prepare_setup_platform(hass: core.HomeAssistant,
                 log_error("Unable to set up component.")
                 return None
 
-    try:
-        await async_process_deps_reqs(hass, hass_config, integration)
-    except HomeAssistantError as err:
-        log_error(str(err))
-        return None
-
     return platform
 
 

tests/components/panel_custom/test_init.py

@@ -25,7 +25,11 @@ async def test_webcomponent_custom_path_not_found(hass):
             hass, 'panel_custom', config
         )
         assert not result
-        assert len(hass.data.get(frontend.DATA_PANELS, {})) == 0
+
+        panels = hass.data.get(frontend.DATA_PANELS, [])
+
+        assert panels
+        assert 'nice_url' not in panels
 
 
 async def test_webcomponent_custom_path(hass):

tests/test_bootstrap.py

@@ -52,31 +52,6 @@ def test_home_assistant_core_config_validation(hass):
     assert result is None
 
 
-def test_from_config_dict_not_mount_deps_folder(loop):
-    """Test that we do not mount the deps folder inside from_config_dict."""
-    with patch('homeassistant.bootstrap.is_virtual_env', return_value=False), \
-        patch('homeassistant.core.HomeAssistant',
-              return_value=Mock(loop=loop)), \
-        patch('homeassistant.bootstrap.async_mount_local_lib_path',
-              return_value=mock_coro()) as mock_mount, \
-        patch('homeassistant.bootstrap.async_from_config_dict',
-              return_value=mock_coro()):
-
-        bootstrap.from_config_dict({}, config_dir='.')
-        assert len(mock_mount.mock_calls) == 1
-
-    with patch('homeassistant.bootstrap.is_virtual_env', return_value=True), \
-        patch('homeassistant.core.HomeAssistant',
-              return_value=Mock(loop=loop)), \
-        patch('homeassistant.bootstrap.async_mount_local_lib_path',
-              return_value=mock_coro()) as mock_mount, \
-        patch('homeassistant.bootstrap.async_from_config_dict',
-              return_value=mock_coro()):
-
-        bootstrap.from_config_dict({}, config_dir='.')
-        assert len(mock_mount.mock_calls) == 0
-
-
 async def test_async_from_config_file_not_mount_deps_folder(loop):
     """Test that we not mount the deps folder inside async_from_config_file."""
     hass = Mock(