Detect lingering threads after tests (#37270)

* Detect lingering threads after tests

* Make sure cast is setup before checking state

* Make sure we ask executors of old hass to shutdown

We are not waiting here, just hoping for the best

* Make sure all instances of hass and executors is stopped.

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

* Also apply hass stopping to scripts

* Adjust to changes how we set up executor

* Add new CoreState.stopped

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

homeassistant/bootstrap.py

@@ -141,6 +141,7 @@ async def async_setup_hass(
 
         safe_mode = True
         old_config = hass.config
+
         hass = core.HomeAssistant()
         hass.config.skip_pip = old_config.skip_pip
         hass.config.internal_url = old_config.internal_url

homeassistant/core.py

@@ -152,6 +152,7 @@ class CoreState(enum.Enum):
     running = "RUNNING"
     stopping = "STOPPING"
     final_write = "FINAL_WRITE"
+    stopped = "STOPPED"
 
     def __str__(self) -> str:
         """Return the event."""
@@ -443,6 +444,7 @@ class HomeAssistant:
         await self.loop.shutdown_default_executor()  # type: ignore
 
         self.exit_code = exit_code
+        self.state = CoreState.stopped
 
         if self._stopped is not None:
             self._stopped.set()

tests/components/cast/test_media_player.py

@@ -552,6 +552,7 @@ async def test_disconnect_on_stop(hass: HomeAssistantType):
 async def test_entry_setup_no_config(hass: HomeAssistantType):
     """Test setting up entry with no config.."""
     await async_setup_component(hass, "cast", {})
+    await hass.async_block_till_done()
 
     with patch(
         "homeassistant.components.cast.media_player._async_setup_platform",
@@ -567,6 +568,7 @@ async def test_entry_setup_single_config(hass: HomeAssistantType):
     await async_setup_component(
         hass, "cast", {"cast": {"media_player": {"host": "bla"}}}
     )
+    await hass.async_block_till_done()
 
     with patch(
         "homeassistant.components.cast.media_player._async_setup_platform",
@@ -582,6 +584,7 @@ async def test_entry_setup_list_config(hass: HomeAssistantType):
     await async_setup_component(
         hass, "cast", {"cast": {"media_player": [{"host": "bla"}, {"host": "blu"}]}}
     )
+    await hass.async_block_till_done()
 
     with patch(
         "homeassistant.components.cast.media_player._async_setup_platform",
@@ -598,6 +601,7 @@ async def test_entry_setup_platform_not_ready(hass: HomeAssistantType):
     await async_setup_component(
         hass, "cast", {"cast": {"media_player": {"host": "bla"}}}
     )
+    await hass.async_block_till_done()
 
     with patch(
         "homeassistant.components.cast.media_player._async_setup_platform",

tests/conftest.py

@@ -2,6 +2,7 @@
 import asyncio
 import functools
 import logging
+import threading
 
 import pytest
 import requests_mock as _requests_mock
@@ -78,6 +79,8 @@ util.get_local_ip = lambda: "127.0.0.1"
 @pytest.fixture(autouse=True)
 def verify_cleanup():
     """Verify that the test has cleaned up resources correctly."""
+    threads_before = frozenset(threading.enumerate())
+
     yield
 
     if len(INSTANCES) >= 2:
@@ -86,6 +89,9 @@ def verify_cleanup():
             inst.stop()
         pytest.exit(f"Detected non stopped instances ({count}), aborting test run")
 
+    threads = frozenset(threading.enumerate()) - threads_before
+    assert not threads
+
 
 @pytest.fixture
 def hass_storage():
@@ -122,6 +128,30 @@ def hass(loop, hass_storage, request):
         raise ex
 
 
+@pytest.fixture
+async def stop_hass():
+    """Make sure all hass are stopped."""
+    orig_hass = ha.HomeAssistant
+
+    created = []
+
+    def mock_hass():
+        hass_inst = orig_hass()
+        created.append(hass_inst)
+        return hass_inst
+
+    with patch("homeassistant.core.HomeAssistant", mock_hass):
+        yield
+
+    for hass_inst in created:
+        if hass_inst.state == ha.CoreState.stopped:
+            continue
+
+        with patch.object(hass_inst.loop, "stop"):
+            await hass_inst.async_block_till_done()
+            await hass_inst.async_stop(force=True)
+
+
 @pytest.fixture
 def requests_mock():
     """Fixture to provide a requests mocker."""

tests/scripts/test_check_config.py

@@ -1,6 +1,8 @@
 """Test check_config script."""
 import logging
 
+import pytest
+
 from homeassistant.config import YAML_CONFIG_FILE
 import homeassistant.scripts.check_config as check_config
 
@@ -23,6 +25,11 @@ BASE_CONFIG = (
 BAD_CORE_CONFIG = "homeassistant:\n  unit_system: bad\n\n\n"
 
 
+@pytest.fixture(autouse=True)
+async def apply_stop_hass(stop_hass):
+    """Make sure all hass are stopped."""
+
+
 def normalize_yaml_files(check_dict):
     """Remove configuration path from ['yaml_files']."""
     root = get_test_config_dir()

tests/test_bootstrap.py

@@ -33,6 +33,20 @@ def apply_mock_storage(hass_storage):
     """Apply the storage mock."""
 
 
+@pytest.fixture(autouse=True)
+async def apply_stop_hass(stop_hass):
+    """Make sure all hass are stopped."""
+
+
+@pytest.fixture(autouse=True)
+def mock_http_start_stop():
+    """Mock HTTP start and stop."""
+    with patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ), patch("homeassistant.components.http.HomeAssistantHTTP.stop"):
+        yield
+
+
 @patch("homeassistant.bootstrap.async_enable_logging", Mock())
 async def test_home_assistant_core_config_validation(hass):
     """Test if we pass in wrong information for HA conf."""
@@ -360,9 +374,7 @@ async def test_setup_hass(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"browser": {}, "frontend": {}},
-    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000), patch(
+    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000):
-        "homeassistant.components.http.start_http_server_and_save_config"
-    ):
         hass = await bootstrap.async_setup_hass(
             runner.RuntimeConfig(
                 config_dir=get_test_config_dir(),
@@ -418,8 +430,6 @@ async def test_setup_hass_takes_longer_than_log_slow_startup(
     ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 0.3), patch(
         "homeassistant.components.frontend.async_setup",
         side_effect=_async_setup_that_blocks_startup,
-    ), patch(
-        "homeassistant.components.http.start_http_server_and_save_config"
     ):
         await bootstrap.async_setup_hass(
             runner.RuntimeConfig(
@@ -447,7 +457,7 @@ async def test_setup_hass_invalid_yaml(
     """Test it works."""
     with patch(
         "homeassistant.config.async_hass_config_yaml", side_effect=HomeAssistantError
-    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
+    ):
         hass = await bootstrap.async_setup_hass(
             runner.RuntimeConfig(
                 config_dir=get_test_config_dir(),
@@ -501,8 +511,6 @@ async def test_setup_hass_safe_mode(
 ):
     """Test it works."""
     with patch("homeassistant.components.browser.setup") as browser_setup, patch(
-        "homeassistant.components.http.start_http_server_and_save_config"
-    ), patch(
         "homeassistant.config_entries.ConfigEntries.async_domains",
         return_value=["browser"],
     ):
@@ -538,7 +546,7 @@ async def test_setup_hass_invalid_core_config(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"homeassistant": {"non-existing": 1}},
-    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
+    ):
         hass = await bootstrap.async_setup_hass(
             runner.RuntimeConfig(
                 config_dir=get_test_config_dir(),
@@ -578,7 +586,7 @@ async def test_setup_safe_mode_if_no_frontend(
             "map": {},
             "person": {"invalid": True},
         },
-    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
+    ):
         hass = await bootstrap.async_setup_hass(
             runner.RuntimeConfig(
                 config_dir=get_test_config_dir(),

tests/test_core.py

@@ -1119,7 +1119,7 @@ async def test_hass_start_starts_the_timer(loop):
 
     finally:
         await hass.async_stop()
-        assert hass.state == ha.CoreState.not_running
+        assert hass.state == ha.CoreState.stopped
 
 
 async def test_start_taking_too_long(loop, caplog):
@@ -1140,7 +1140,7 @@ async def test_start_taking_too_long(loop, caplog):
 
     finally:
         await hass.async_stop()
-        assert hass.state == ha.CoreState.not_running
+        assert hass.state == ha.CoreState.stopped
 
 
 async def test_track_task_functions(loop):