Use importlib metadata to check installed packages (#24114)

* Use importlib metadata

* Fix script

* Remove unused import

* Update requirements"

homeassistant/package_constraints.txt

@@ -4,6 +4,7 @@ async_timeout==3.0.1
 attrs==19.1.0
 bcrypt==3.1.6
 certifi>=2018.04.16
+importlib-metadata==0.15
 jinja2>=2.10
 PyJWT==1.7.1
 cryptography==2.6.1

homeassistant/requirements.py

@@ -3,11 +3,7 @@ import asyncio
 from functools import partial
 import logging
 import os
-import sys
 from typing import Any, Dict, List, Optional
-from urllib.parse import urlparse
-
-import pkg_resources
 
 import homeassistant.util.package as pkg_util
 from homeassistant.core import HomeAssistant
@@ -28,16 +24,12 @@ async def async_process_requirements(hass: HomeAssistant, name: str,
     if pip_lock is None:
         pip_lock = hass.data[DATA_PIP_LOCK] = asyncio.Lock()
 
-    pkg_cache = hass.data.get(DATA_PKG_CACHE)
-    if pkg_cache is None:
-        pkg_cache = hass.data[DATA_PKG_CACHE] = PackageLoadable(hass)
-
     pip_install = partial(pkg_util.install_package,
                           **pip_kwargs(hass.config.config_dir))
 
     async with pip_lock:
         for req in requirements:
-            if await pkg_cache.loadable(req):
+            if pkg_util.is_installed(req):
                 continue
 
             ret = await hass.async_add_executor_job(pip_install, req)
@@ -58,50 +50,3 @@ def pip_kwargs(config_dir: Optional[str]) -> Dict[str, Any]:
     if not (config_dir is None or pkg_util.is_virtual_env()):
         kwargs['target'] = os.path.join(config_dir, 'deps')
     return kwargs
-
-
-class PackageLoadable:
-    """Class to check if a package is loadable, with built-in cache."""
-
-    def __init__(self, hass: HomeAssistant) -> None:
-        """Initialize the PackageLoadable class."""
-        self.dist_cache = {}  # type: Dict[str, pkg_resources.Distribution]
-        self.hass = hass
-
-    async def loadable(self, package: str) -> bool:
-        """Check if a package is what will be loaded when we import it.
-
-        Returns True when the requirement is met.
-        Returns False when the package is not installed or doesn't meet req.
-        """
-        dist_cache = self.dist_cache
-
-        try:
-            req = pkg_resources.Requirement.parse(package)
-        except ValueError:
-            # This is a zip file. We no longer use this in Home Assistant,
-            # leaving it in for custom components.
-            req = pkg_resources.Requirement.parse(urlparse(package).fragment)
-
-        req_proj_name = req.project_name.lower()
-        dist = dist_cache.get(req_proj_name)
-
-        if dist is not None:
-            return dist in req
-
-        for path in sys.path:
-            # We read the whole mount point as we're already here
-            # Caching it on first call makes subsequent calls a lot faster.
-            await self.hass.async_add_executor_job(self._fill_cache, path)
-
-            dist = dist_cache.get(req_proj_name)
-            if dist is not None:
-                return dist in req
-
-        return False
-
-    def _fill_cache(self, path: str) -> None:
-        """Add packages from a path to the cache."""
-        dist_cache = self.dist_cache
-        for dist in pkg_resources.find_distributions(path):
-            dist_cache.setdefault(dist.project_name.lower(), dist)

homeassistant/scripts/__init__.py

@@ -9,9 +9,9 @@ from typing import List
 
 from homeassistant.bootstrap import async_mount_local_lib_path
 from homeassistant.config import get_default_config_dir
-from homeassistant.core import HomeAssistant
+from homeassistant.requirements import pip_kwargs
-from homeassistant.requirements import pip_kwargs, PackageLoadable
+from homeassistant.util.package import (
-from homeassistant.util.package import install_package, is_virtual_env
+    install_package, is_virtual_env, is_installed)
 
 
 def run(args: List) -> int:
@@ -49,10 +49,8 @@ def run(args: List) -> int:
 
     logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
-    hass = HomeAssistant(loop)
-    pkgload = PackageLoadable(hass)
     for req in getattr(script, 'REQUIREMENTS', []):
-        if loop.run_until_complete(pkgload.loadable(req)):
+        if is_installed(req):
             continue
 
         if not install_package(req, **_pip_kwargs):

homeassistant/util/package.py

@@ -5,6 +5,11 @@ import os
 from subprocess import PIPE, Popen
 import sys
 from typing import Optional
+from urllib.parse import urlparse
+
+import pkg_resources
+from importlib_metadata import version, PackageNotFoundError
+
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -16,6 +21,25 @@ def is_virtual_env() -> bool:
             hasattr(sys, 'real_prefix'))
 
 
+def is_installed(package: str) -> bool:
+    """Check if a package is installed and will be loaded when we import it.
+
+    Returns True when the requirement is met.
+    Returns False when the package is not installed or doesn't meet req.
+    """
+    try:
+        req = pkg_resources.Requirement.parse(package)
+    except ValueError:
+        # This is a zip file. We no longer use this in Home Assistant,
+        # leaving it in for custom components.
+        req = pkg_resources.Requirement.parse(urlparse(package).fragment)
+
+    try:
+        return version(req.project_name) in req
+    except PackageNotFoundError:
+        return False
+
+
 def install_package(package: str, upgrade: bool = True,
                     target: Optional[str] = None,
                     constraints: Optional[str] = None) -> bool:

requirements_all.txt

@@ -5,6 +5,7 @@ async_timeout==3.0.1
 attrs==19.1.0
 bcrypt==3.1.6
 certifi>=2018.04.16
+importlib-metadata==0.15
 jinja2>=2.10
 PyJWT==1.7.1
 cryptography==2.6.1

setup.py

@@ -38,6 +38,7 @@ REQUIRES = [
     'attrs==19.1.0',
     'bcrypt==3.1.6',
     'certifi>=2018.04.16',
+    'importlib-metadata==0.15',
     'jinja2>=2.10',
     'PyJWT==1.7.1',
     # PyJWT has loose dependency. We want the latest one.

tests/test_requirements.py

@@ -4,21 +4,11 @@ from unittest.mock import patch, call
 
 from homeassistant import setup
 from homeassistant.requirements import (
-    CONSTRAINT_FILE, PackageLoadable, async_process_requirements)
+    CONSTRAINT_FILE, async_process_requirements)
-
-import pkg_resources
 
 from tests.common import (
     get_test_home_assistant, MockModule, mock_coro, mock_integration)
 
-RESOURCE_DIR = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), '..', 'resources'))
-
-TEST_NEW_REQ = 'pyhelloworld3==1.0.0'
-
-TEST_ZIP_REQ = 'file://{}#{}' \
-    .format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ)
-
 
 class TestRequirements:
     """Test the requirements module."""
@@ -80,47 +70,10 @@ async def test_install_existing_package(hass):
 
     assert len(mock_inst.mock_calls) == 1
 
-    with patch('homeassistant.requirements.PackageLoadable.loadable',
+    with patch('homeassistant.util.package.is_installed', return_value=True), \
-               return_value=mock_coro(True)), \
             patch(
                 'homeassistant.util.package.install_package') as mock_inst:
         assert await async_process_requirements(
             hass, 'test_component', ['hello==1.0.0'])
 
     assert len(mock_inst.mock_calls) == 0
-
-
-async def test_check_package_global(hass):
-    """Test for an installed package."""
-    installed_package = list(pkg_resources.working_set)[0].project_name
-    assert await PackageLoadable(hass).loadable(installed_package)
-
-
-async def test_check_package_zip(hass):
-    """Test for an installed zip package."""
-    assert not await PackageLoadable(hass).loadable(TEST_ZIP_REQ)
-
-
-async def test_package_loadable_installed_twice(hass):
-    """Test that a package is loadable when installed twice.
-
-    If a package is installed twice, only the first version will be imported.
-    Test that package_loadable will only compare with the first package.
-    """
-    v1 = pkg_resources.Distribution(project_name='hello', version='1.0.0')
-    v2 = pkg_resources.Distribution(project_name='hello', version='2.0.0')
-
-    with patch('pkg_resources.find_distributions', side_effect=[[v1]]):
-        assert not await PackageLoadable(hass).loadable('hello==2.0.0')
-
-    with patch('pkg_resources.find_distributions', side_effect=[[v1], [v2]]):
-        assert not await PackageLoadable(hass).loadable('hello==2.0.0')
-
-    with patch('pkg_resources.find_distributions', side_effect=[[v2], [v1]]):
-        assert await PackageLoadable(hass).loadable('hello==2.0.0')
-
-    with patch('pkg_resources.find_distributions', side_effect=[[v2]]):
-        assert await PackageLoadable(hass).loadable('hello==2.0.0')
-
-    with patch('pkg_resources.find_distributions', side_effect=[[v2]]):
-        assert await PackageLoadable(hass).loadable('Hello==2.0.0')

tests/util/test_package.py

@@ -6,13 +6,20 @@ import sys
 from subprocess import PIPE
 from unittest.mock import MagicMock, call, patch
 
+import pkg_resources
 import pytest
 
 import homeassistant.util.package as package
 
 
+RESOURCE_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'resources'))
+
 TEST_NEW_REQ = 'pyhelloworld3==1.0.0'
 
+TEST_ZIP_REQ = 'file://{}#{}' \
+    .format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ)
+
 
 @pytest.fixture
 def mock_sys():
@@ -176,3 +183,14 @@ def test_async_get_user_site(mock_env_copy):
         stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
         env=env)
     assert ret == os.path.join(deps_dir, 'lib_dir')
+
+
+def test_check_package_global():
+    """Test for an installed package."""
+    installed_package = list(pkg_resources.working_set)[0].project_name
+    assert package.is_installed(installed_package)
+
+
+def test_check_package_zip():
+    """Test for an installed zip package."""
+    assert not package.is_installed(TEST_ZIP_REQ)