Make hassfest.dependencies faster with multiprocessing (#81486)

* hassfest.dependencies: split to two loops

* hassfest.dependencies: use multiprocessing for import scan

script/hassfest/dependencies.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import ast
 from collections import deque
+import multiprocessing
 from pathlib import Path
 
 from homeassistant.const import Platform
@@ -227,35 +228,49 @@ def find_non_referenced_integrations(
     return referenced
 
 
-def validate_dependencies(
-    integrations: dict[str, Integration],
+def _compute_integration_dependencies(
     integration: Integration,
-    check_dependencies: bool,
-) -> None:
-    """Validate all dependencies."""
+) -> tuple[str, dict[Path, set[str]] | None]:
+    """Compute integration dependencies."""
     # Some integrations are allowed to have violations.
     if integration.domain in IGNORE_VIOLATIONS:
-        return
+        return (integration.domain, None)
 
     # Find usage of hass.components
     collector = ImportCollector(integration)
     collector.collect()
+    return (integration.domain, collector.referenced)
 
-    for domain in sorted(
-        find_non_referenced_integrations(
-            integrations, integration, collector.referenced
-        )
-    ):
-        integration.add_error(
-            "dependencies",
-            f"Using component {domain} but it's not in 'dependencies' "
-            "or 'after_dependencies'",
+
+def _validate_dependency_imports(
+    integrations: dict[str, Integration],
+) -> None:
+    """Validate all dependencies."""
+
+    # Find integration dependencies with multiprocessing
+    # (because it takes some time to parse thousands of files)
+    with multiprocessing.Pool() as pool:
+        integration_imports = dict(
+            pool.imap_unordered(
+                _compute_integration_dependencies,
+                integrations.values(),
+                chunksize=10,
+            )
         )
 
-    if check_dependencies:
-        _check_circular_deps(
-            integrations, integration.domain, integration, set(), deque()
-        )
+    for integration in integrations.values():
+        referenced = integration_imports[integration.domain]
+        if not referenced:  # Either ignored or has no references
+            continue
+
+        for domain in sorted(
+            find_non_referenced_integrations(integrations, integration, referenced)
+        ):
+            integration.add_error(
+                "dependencies",
+                f"Using component {domain} but it's not in 'dependencies' "
+                "or 'after_dependencies'",
+            )
 
 
 def _check_circular_deps(
@@ -266,6 +281,7 @@ def _check_circular_deps(
     checking: deque[str],
 ) -> None:
     """Check for circular dependencies pointing at starting_domain."""
+
     if integration.domain in checked or integration.domain in checking:
         return
 
@@ -297,17 +313,21 @@ def _check_circular_deps(
     checking.remove(integration.domain)
 
 
-def validate(integrations: dict[str, Integration], config: Config) -> None:
-    """Handle dependencies for integrations."""
-    # check for non-existing dependencies
+def _validate_circular_dependencies(integrations: dict[str, Integration]) -> None:
     for integration in integrations.values():
-        validate_dependencies(
-            integrations,
-            integration,
-            check_dependencies=not config.specific_integrations,
+        if integration.domain in IGNORE_VIOLATIONS:
+            continue
+
+        _check_circular_deps(
+            integrations, integration.domain, integration, set(), deque()
         )
 
-        if config.specific_integrations:
+
+def _validate_dependencies_exist(
+    integrations: dict[str, Integration],
+) -> None:
+    for integration in integrations.values():
+        if not integration.manifest:
             continue
 
         # check that all referenced dependencies exist
@@ -323,3 +343,15 @@ def validate(integrations: dict[str, Integration], config: Config) -> None:
                 integration.add_error(
                     "dependencies", f"Dependency {dep} does not exist"
                 )
+
+
+def validate(
+    integrations: dict[str, Integration],
+    config: Config,
+) -> None:
+    """Handle dependencies for integrations."""
+    _validate_dependency_imports(integrations)
+
+    if not config.specific_integrations:
+        _validate_dependencies_exist(integrations)
+        _validate_circular_dependencies(integrations)