Ensure frontend is available if integrations fail to start - Part 1 of 2 (#36093)

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>

homeassistant/bootstrap.py

@@ -29,6 +29,9 @@ _LOGGER = logging.getLogger(__name__)
 
 ERROR_LOG_FILENAME = "home-assistant.log"
 
+# How long to wait until things that run on bootstrap have to finish.
+TIMEOUT_EVENT_BOOTSTRAP = 15
+
 # hass.data key for logging information.
 DATA_LOGGING = "logging"
 
@@ -44,6 +47,13 @@ STAGE_1_INTEGRATIONS = {
     "mqtt_eventstream",
     # To provide account link implementations
     "cloud",
+    # Ensure supervisor is available
+    "hassio",
+    # Get the frontend up and running as soon
+    # as possible so problem integrations can
+    # be removed
+    "frontend",
+    "config",
 }
 
 
@@ -399,6 +409,8 @@ async def _async_set_up_integrations(
     )
 
     if stage_1_domains:
+        _LOGGER.info("Setting up %s", stage_1_domains)
+
         await async_setup_multi_components(stage_1_domains)
 
     # Load all integrations
@@ -442,4 +454,11 @@ async def _async_set_up_integrations(
 
     # Wrap up startup
     _LOGGER.debug("Waiting for startup to wrap up")
+    try:
+        async with timeout(TIMEOUT_EVENT_BOOTSTRAP):
             await hass.async_block_till_done()
+    except asyncio.TimeoutError:
+        _LOGGER.warning(
+            "Something is blocking Home Assistant from wrapping up the "
+            "bootstrap phase. We're going to continue anyway."
+        )

homeassistant/components/http/__init__.py

@@ -4,7 +4,7 @@ import logging
 import os
 import ssl
 from traceback import extract_stack
-from typing import Optional, cast
+from typing import Dict, Optional, cast
 
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPMovedPermanently
@@ -15,7 +15,7 @@ from homeassistant.const import (
     EVENT_HOMEASSISTANT_STOP,
     SERVER_PORT,
 )
-from homeassistant.core import HomeAssistant
+from homeassistant.core import Event, HomeAssistant
 from homeassistant.helpers import storage
 import homeassistant.helpers.config_validation as cv
 from homeassistant.loader import bind_hass
@@ -216,29 +216,25 @@ async def async_setup(hass, config):
         ssl_profile=ssl_profile,
     )
 
-    async def stop_server(event):
+    startup_listeners = []
+
+    async def stop_server(event: Event) -> None:
         """Stop the server."""
         await server.stop()
 
-    async def start_server(event):
+    async def start_server(event: Event) -> None:
         """Start the server."""
+
+        for listener in startup_listeners:
+            listener()
+
         hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_server)
-        await server.start()
 
-        # If we are set up successful, we store the HTTP settings for safe mode.
+        await start_http_server_and_save_config(hass, dict(conf), server)
-        store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
 
-        if CONF_TRUSTED_PROXIES in conf:
+    startup_listeners.append(
-            conf_to_save = dict(conf)
+        hass.bus.async_listen(EVENT_HOMEASSISTANT_START, start_server)
-            conf_to_save[CONF_TRUSTED_PROXIES] = [
+    )
-                str(ip.network_address) for ip in conf_to_save[CONF_TRUSTED_PROXIES]
-            ]
-        else:
-            conf_to_save = conf
-
-        await store.async_save(conf_to_save)
-
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_START, start_server)
 
     hass.http = server
 
@@ -418,3 +414,20 @@ class HomeAssistantHTTP:
         """Stop the aiohttp server."""
         await self.site.stop()
         await self.runner.cleanup()
+
+
+async def start_http_server_and_save_config(
+    hass: HomeAssistant, conf: Dict, server: HomeAssistantHTTP
+) -> None:
+    """Startup the http server and save the config."""
+    await server.start()  # type: ignore
+
+    # If we are set up successful, we store the HTTP settings for safe mode.
+    store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
+
+    if CONF_TRUSTED_PROXIES in conf:
+        conf[CONF_TRUSTED_PROXIES] = [
+            str(ip.network_address) for ip in conf[CONF_TRUSTED_PROXIES]
+        ]
+
+    await store.async_save(conf)

homeassistant/components/http/view.py

@@ -14,7 +14,7 @@ from aiohttp.web_exceptions import (
 import voluptuous as vol
 
 from homeassistant import exceptions
-from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK
+from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK, HTTP_SERVICE_UNAVAILABLE
 from homeassistant.core import Context, is_callback
 from homeassistant.helpers.json import JSONEncoder
 
@@ -107,8 +107,8 @@ def request_handler_factory(view: HomeAssistantView, handler: Callable) -> Calla
 
     async def handle(request: web.Request) -> web.StreamResponse:
         """Handle incoming request."""
-        if not request.app[KEY_HASS].is_running:
+        if request.app[KEY_HASS].is_stopping:
-            return web.Response(status=503)
+            return web.Response(status=HTTP_SERVICE_UNAVAILABLE)
 
         authenticated = request.get(KEY_AUTHENTICATED, False)
 

homeassistant/components/websocket_api/decorators.py

@@ -31,7 +31,9 @@ def async_response(
     @wraps(func)
     def schedule_handler(hass, connection, msg):
         """Schedule the handler."""
-        hass.async_create_task(_handle_async_response(func, hass, connection, msg))
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responders
+        hass.loop.create_task(_handle_async_response(func, hass, connection, msg))
 
     return schedule_handler
 

homeassistant/components/websocket_api/http.py

@@ -165,7 +165,9 @@ class WebSocketHandler:
             EVENT_HOMEASSISTANT_STOP, handle_hass_stop
         )
 
-        self._writer_task = self.hass.async_create_task(self._writer())
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responses
+        self._writer_task = self.hass.loop.create_task(self._writer())
 
         auth = AuthPhase(self._logger, self.hass, self._send_message, request)
         connection = None

homeassistant/core.py

@@ -209,6 +209,11 @@ class HomeAssistant:
         """Return if Home Assistant is running."""
         return self.state in (CoreState.starting, CoreState.running)
 
+    @property
+    def is_stopping(self) -> bool:
+        """Return if Home Assistant is stopping."""
+        return self.state in (CoreState.stopping, CoreState.final_write)
+
     def start(self) -> int:
         """Start Home Assistant.
 
@@ -260,6 +265,7 @@ class HomeAssistant:
 
         setattr(self.loop, "_thread_ident", threading.get_ident())
         self.bus.async_fire(EVENT_HOMEASSISTANT_START)
+        self.bus.async_fire(EVENT_CORE_CONFIG_UPDATE)
 
         try:
             # Only block for EVENT_HOMEASSISTANT_START listener
@@ -1391,6 +1397,7 @@ class Config:
             "version": __version__,
             "config_source": self.config_source,
             "safe_mode": self.safe_mode,
+            "state": self.hass.state.value,
             "external_url": self.external_url,
             "internal_url": self.internal_url,
         }

tests/components/hassio/test_discovery.py

@@ -60,6 +60,9 @@ async def test_hassio_discovery_startup(hass, aioclient_mock, hassio_client):
 
 async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client):
     """Test startup and discovery with hass discovery."""
+    aioclient_mock.post(
+        "http://127.0.0.1/supervisor/options", json={"result": "ok", "data": {}},
+    )
     aioclient_mock.get(
         "http://127.0.0.1/discovery",
         json={
@@ -101,7 +104,7 @@ async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client
         await async_setup_component(hass, "hassio", {})
         await hass.async_block_till_done()
 
-        assert aioclient_mock.call_count == 2
+        assert aioclient_mock.call_count == 3
         assert mock_mqtt.called
         mock_mqtt.assert_called_with(
             {

tests/components/http/test_data_validator.py

@@ -11,7 +11,7 @@ from tests.async_mock import Mock
 async def get_client(aiohttp_client, validator):
     """Generate a client that hits a view decorated with validator."""
     app = web.Application()
-    app["hass"] = Mock(is_running=True)
+    app["hass"] = Mock(is_stopping=False)
 
     class TestView(HomeAssistantView):
         url = "/"

tests/components/http/test_view.py

@@ -19,7 +19,13 @@ from tests.async_mock import AsyncMock, Mock
 @pytest.fixture
 def mock_request():
     """Mock a request."""
-    return Mock(app={"hass": Mock(is_running=True)}, match_info={})
+    return Mock(app={"hass": Mock(is_stopping=False)}, match_info={})
+
+
+@pytest.fixture
+def mock_request_with_stopping():
+    """Mock a request."""
+    return Mock(app={"hass": Mock(is_stopping=True)}, match_info={})
 
 
 async def test_invalid_json(caplog):
@@ -55,3 +61,11 @@ async def test_handling_service_not_found(mock_request):
             Mock(requires_auth=False),
             AsyncMock(side_effect=ServiceNotFound("test", "test")),
         )(mock_request)
+
+
+async def test_not_running(mock_request_with_stopping):
+    """Test we get a 503 when not running."""
+    response = await request_handler_factory(
+        Mock(requires_auth=False), AsyncMock(side_effect=Unauthorized)
+    )(mock_request_with_stopping)
+    assert response.status == 503

tests/components/logbook/test_init.py

@@ -43,6 +43,7 @@ class TestComponentLogbook(unittest.TestCase):
         """Set up things to be run when tests are started."""
         self.hass = get_test_home_assistant()
         init_recorder_component(self.hass)  # Force an in memory DB
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
             assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG)
 
     def tearDown(self):

tests/components/panel_iframe/test_init.py

@@ -4,6 +4,7 @@ import unittest
 from homeassistant import setup
 from homeassistant.components import frontend
 
+from tests.async_mock import patch
 from tests.common import get_test_home_assistant
 
 
@@ -26,12 +27,16 @@ class TestPanelIframe(unittest.TestCase):
         ]
 
         for conf in to_try:
+            with patch(
+                "homeassistant.components.http.start_http_server_and_save_config"
+            ):
                 assert not setup.setup_component(
                     self.hass, "panel_iframe", {"panel_iframe": conf}
                 )
 
     def test_correct_config(self):
         """Test correct config."""
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
             assert setup.setup_component(
                 self.hass,
                 "panel_iframe",

tests/test_bootstrap.py

@@ -201,6 +201,43 @@ async def test_setup_after_deps_not_present(hass, caplog):
     assert order == ["root", "second_dep"]
 
 
+async def test_setup_continues_if_blocked(hass, caplog):
+    """Test we continue after timeout if blocked."""
+    caplog.set_level(logging.DEBUG)
+    order = []
+
+    def gen_domain_setup(domain):
+        async def async_setup(hass, config):
+            order.append(domain)
+            return True
+
+        return async_setup
+
+    mock_integration(
+        hass, MockModule(domain="root", async_setup=gen_domain_setup("root"))
+    )
+    mock_integration(
+        hass,
+        MockModule(
+            domain="second_dep",
+            async_setup=gen_domain_setup("second_dep"),
+            partial_manifest={"after_dependencies": ["first_dep"]},
+        ),
+    )
+
+    with patch.object(bootstrap, "TIMEOUT_EVENT_BOOTSTRAP", 0):
+        hass.async_create_task(asyncio.sleep(2))
+        await bootstrap._async_set_up_integrations(
+            hass, {"root": {}, "first_dep": {}, "second_dep": {}}
+        )
+
+    assert "root" in hass.config.components
+    assert "first_dep" not in hass.config.components
+    assert "second_dep" in hass.config.components
+    assert order == ["root", "second_dep"]
+    assert "blocking Home Assistant from wrapping up" in caplog.text
+
+
 @pytest.fixture
 def mock_is_virtual_env():
     """Mock enable logging."""
@@ -261,7 +298,9 @@ async def test_setup_hass(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"browser": {}, "frontend": {}},
-    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000):
+    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000), patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,
@@ -338,7 +377,7 @@ async def test_setup_hass_invalid_yaml(
     """Test it works."""
     with patch(
         "homeassistant.config.async_hass_config_yaml", side_effect=HomeAssistantError
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -391,7 +430,9 @@ async def test_setup_hass_safe_mode(
     hass.config_entries._async_schedule_save()
     await flush_store(hass.config_entries._store)
 
-    with patch("homeassistant.components.browser.setup") as browser_setup:
+    with patch("homeassistant.components.browser.setup") as browser_setup, patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -421,7 +462,7 @@ async def test_setup_hass_invalid_core_config(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"homeassistant": {"non-existing": 1}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -451,7 +492,7 @@ async def test_setup_safe_mode_if_no_frontend(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"map": {}, "person": {"invalid": True}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,

tests/test_core.py

@@ -35,7 +35,7 @@ from homeassistant.exceptions import InvalidEntityFormatError, InvalidStateError
 import homeassistant.util.dt as dt_util
 from homeassistant.util.unit_system import METRIC_SYSTEM
 
-from tests.async_mock import MagicMock, Mock, patch
+from tests.async_mock import MagicMock, Mock, PropertyMock, patch
 from tests.common import async_mock_service, get_test_home_assistant
 
 PST = pytz.timezone("America/Los_Angeles")
@@ -901,6 +901,8 @@ class TestConfig(unittest.TestCase):
     def test_as_dict(self):
         """Test as dict."""
         self.config.config_dir = "/test/ha-config"
+        self.config.hass = MagicMock()
+        type(self.config.hass.state).value = PropertyMock(return_value="RUNNING")
         expected = {
             "latitude": 0,
             "longitude": 0,
@@ -914,6 +916,7 @@ class TestConfig(unittest.TestCase):
             "version": __version__,
             "config_source": "default",
             "safe_mode": False,
+            "state": "RUNNING",
             "external_url": None,
             "internal_url": None,
         }