Use importlib metadata to check installed packages (#24114)
* Use importlib metadata * Fix script * Remove unused import * Update requirements"
This commit is contained in:
parent
96b7bb625d
commit
179fb0f3b5
8 changed files with 52 additions and 111 deletions
|
@ -4,6 +4,7 @@ async_timeout==3.0.1
|
|||
attrs==19.1.0
|
||||
bcrypt==3.1.6
|
||||
certifi>=2018.04.16
|
||||
importlib-metadata==0.15
|
||||
jinja2>=2.10
|
||||
PyJWT==1.7.1
|
||||
cryptography==2.6.1
|
||||
|
|
|
@ -3,11 +3,7 @@ import asyncio
|
|||
from functools import partial
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pkg_resources
|
||||
|
||||
import homeassistant.util.package as pkg_util
|
||||
from homeassistant.core import HomeAssistant
|
||||
|
@ -28,16 +24,12 @@ async def async_process_requirements(hass: HomeAssistant, name: str,
|
|||
if pip_lock is None:
|
||||
pip_lock = hass.data[DATA_PIP_LOCK] = asyncio.Lock()
|
||||
|
||||
pkg_cache = hass.data.get(DATA_PKG_CACHE)
|
||||
if pkg_cache is None:
|
||||
pkg_cache = hass.data[DATA_PKG_CACHE] = PackageLoadable(hass)
|
||||
|
||||
pip_install = partial(pkg_util.install_package,
|
||||
**pip_kwargs(hass.config.config_dir))
|
||||
|
||||
async with pip_lock:
|
||||
for req in requirements:
|
||||
if await pkg_cache.loadable(req):
|
||||
if pkg_util.is_installed(req):
|
||||
continue
|
||||
|
||||
ret = await hass.async_add_executor_job(pip_install, req)
|
||||
|
@ -58,50 +50,3 @@ def pip_kwargs(config_dir: Optional[str]) -> Dict[str, Any]:
|
|||
if not (config_dir is None or pkg_util.is_virtual_env()):
|
||||
kwargs['target'] = os.path.join(config_dir, 'deps')
|
||||
return kwargs
|
||||
|
||||
|
||||
class PackageLoadable:
|
||||
"""Class to check if a package is loadable, with built-in cache."""
|
||||
|
||||
def __init__(self, hass: HomeAssistant) -> None:
|
||||
"""Initialize the PackageLoadable class."""
|
||||
self.dist_cache = {} # type: Dict[str, pkg_resources.Distribution]
|
||||
self.hass = hass
|
||||
|
||||
async def loadable(self, package: str) -> bool:
|
||||
"""Check if a package is what will be loaded when we import it.
|
||||
|
||||
Returns True when the requirement is met.
|
||||
Returns False when the package is not installed or doesn't meet req.
|
||||
"""
|
||||
dist_cache = self.dist_cache
|
||||
|
||||
try:
|
||||
req = pkg_resources.Requirement.parse(package)
|
||||
except ValueError:
|
||||
# This is a zip file. We no longer use this in Home Assistant,
|
||||
# leaving it in for custom components.
|
||||
req = pkg_resources.Requirement.parse(urlparse(package).fragment)
|
||||
|
||||
req_proj_name = req.project_name.lower()
|
||||
dist = dist_cache.get(req_proj_name)
|
||||
|
||||
if dist is not None:
|
||||
return dist in req
|
||||
|
||||
for path in sys.path:
|
||||
# We read the whole mount point as we're already here
|
||||
# Caching it on first call makes subsequent calls a lot faster.
|
||||
await self.hass.async_add_executor_job(self._fill_cache, path)
|
||||
|
||||
dist = dist_cache.get(req_proj_name)
|
||||
if dist is not None:
|
||||
return dist in req
|
||||
|
||||
return False
|
||||
|
||||
def _fill_cache(self, path: str) -> None:
|
||||
"""Add packages from a path to the cache."""
|
||||
dist_cache = self.dist_cache
|
||||
for dist in pkg_resources.find_distributions(path):
|
||||
dist_cache.setdefault(dist.project_name.lower(), dist)
|
||||
|
|
|
@ -9,9 +9,9 @@ from typing import List
|
|||
|
||||
from homeassistant.bootstrap import async_mount_local_lib_path
|
||||
from homeassistant.config import get_default_config_dir
|
||||
from homeassistant.core import HomeAssistant
|
||||
from homeassistant.requirements import pip_kwargs, PackageLoadable
|
||||
from homeassistant.util.package import install_package, is_virtual_env
|
||||
from homeassistant.requirements import pip_kwargs
|
||||
from homeassistant.util.package import (
|
||||
install_package, is_virtual_env, is_installed)
|
||||
|
||||
|
||||
def run(args: List) -> int:
|
||||
|
@ -49,10 +49,8 @@ def run(args: List) -> int:
|
|||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
|
||||
|
||||
hass = HomeAssistant(loop)
|
||||
pkgload = PackageLoadable(hass)
|
||||
for req in getattr(script, 'REQUIREMENTS', []):
|
||||
if loop.run_until_complete(pkgload.loadable(req)):
|
||||
if is_installed(req):
|
||||
continue
|
||||
|
||||
if not install_package(req, **_pip_kwargs):
|
||||
|
|
|
@ -5,6 +5,11 @@ import os
|
|||
from subprocess import PIPE, Popen
|
||||
import sys
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pkg_resources
|
||||
from importlib_metadata import version, PackageNotFoundError
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
@ -16,6 +21,25 @@ def is_virtual_env() -> bool:
|
|||
hasattr(sys, 'real_prefix'))
|
||||
|
||||
|
||||
def is_installed(package: str) -> bool:
|
||||
"""Check if a package is installed and will be loaded when we import it.
|
||||
|
||||
Returns True when the requirement is met.
|
||||
Returns False when the package is not installed or doesn't meet req.
|
||||
"""
|
||||
try:
|
||||
req = pkg_resources.Requirement.parse(package)
|
||||
except ValueError:
|
||||
# This is a zip file. We no longer use this in Home Assistant,
|
||||
# leaving it in for custom components.
|
||||
req = pkg_resources.Requirement.parse(urlparse(package).fragment)
|
||||
|
||||
try:
|
||||
return version(req.project_name) in req
|
||||
except PackageNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def install_package(package: str, upgrade: bool = True,
|
||||
target: Optional[str] = None,
|
||||
constraints: Optional[str] = None) -> bool:
|
||||
|
|
|
@ -5,6 +5,7 @@ async_timeout==3.0.1
|
|||
attrs==19.1.0
|
||||
bcrypt==3.1.6
|
||||
certifi>=2018.04.16
|
||||
importlib-metadata==0.15
|
||||
jinja2>=2.10
|
||||
PyJWT==1.7.1
|
||||
cryptography==2.6.1
|
||||
|
|
1
setup.py
1
setup.py
|
@ -38,6 +38,7 @@ REQUIRES = [
|
|||
'attrs==19.1.0',
|
||||
'bcrypt==3.1.6',
|
||||
'certifi>=2018.04.16',
|
||||
'importlib-metadata==0.15',
|
||||
'jinja2>=2.10',
|
||||
'PyJWT==1.7.1',
|
||||
# PyJWT has loose dependency. We want the latest one.
|
||||
|
|
|
@ -4,21 +4,11 @@ from unittest.mock import patch, call
|
|||
|
||||
from homeassistant import setup
|
||||
from homeassistant.requirements import (
|
||||
CONSTRAINT_FILE, PackageLoadable, async_process_requirements)
|
||||
|
||||
import pkg_resources
|
||||
CONSTRAINT_FILE, async_process_requirements)
|
||||
|
||||
from tests.common import (
|
||||
get_test_home_assistant, MockModule, mock_coro, mock_integration)
|
||||
|
||||
RESOURCE_DIR = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), '..', 'resources'))
|
||||
|
||||
TEST_NEW_REQ = 'pyhelloworld3==1.0.0'
|
||||
|
||||
TEST_ZIP_REQ = 'file://{}#{}' \
|
||||
.format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ)
|
||||
|
||||
|
||||
class TestRequirements:
|
||||
"""Test the requirements module."""
|
||||
|
@ -80,47 +70,10 @@ async def test_install_existing_package(hass):
|
|||
|
||||
assert len(mock_inst.mock_calls) == 1
|
||||
|
||||
with patch('homeassistant.requirements.PackageLoadable.loadable',
|
||||
return_value=mock_coro(True)), \
|
||||
with patch('homeassistant.util.package.is_installed', return_value=True), \
|
||||
patch(
|
||||
'homeassistant.util.package.install_package') as mock_inst:
|
||||
assert await async_process_requirements(
|
||||
hass, 'test_component', ['hello==1.0.0'])
|
||||
|
||||
assert len(mock_inst.mock_calls) == 0
|
||||
|
||||
|
||||
async def test_check_package_global(hass):
|
||||
"""Test for an installed package."""
|
||||
installed_package = list(pkg_resources.working_set)[0].project_name
|
||||
assert await PackageLoadable(hass).loadable(installed_package)
|
||||
|
||||
|
||||
async def test_check_package_zip(hass):
|
||||
"""Test for an installed zip package."""
|
||||
assert not await PackageLoadable(hass).loadable(TEST_ZIP_REQ)
|
||||
|
||||
|
||||
async def test_package_loadable_installed_twice(hass):
|
||||
"""Test that a package is loadable when installed twice.
|
||||
|
||||
If a package is installed twice, only the first version will be imported.
|
||||
Test that package_loadable will only compare with the first package.
|
||||
"""
|
||||
v1 = pkg_resources.Distribution(project_name='hello', version='1.0.0')
|
||||
v2 = pkg_resources.Distribution(project_name='hello', version='2.0.0')
|
||||
|
||||
with patch('pkg_resources.find_distributions', side_effect=[[v1]]):
|
||||
assert not await PackageLoadable(hass).loadable('hello==2.0.0')
|
||||
|
||||
with patch('pkg_resources.find_distributions', side_effect=[[v1], [v2]]):
|
||||
assert not await PackageLoadable(hass).loadable('hello==2.0.0')
|
||||
|
||||
with patch('pkg_resources.find_distributions', side_effect=[[v2], [v1]]):
|
||||
assert await PackageLoadable(hass).loadable('hello==2.0.0')
|
||||
|
||||
with patch('pkg_resources.find_distributions', side_effect=[[v2]]):
|
||||
assert await PackageLoadable(hass).loadable('hello==2.0.0')
|
||||
|
||||
with patch('pkg_resources.find_distributions', side_effect=[[v2]]):
|
||||
assert await PackageLoadable(hass).loadable('Hello==2.0.0')
|
||||
|
|
|
@ -6,13 +6,20 @@ import sys
|
|||
from subprocess import PIPE
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pkg_resources
|
||||
import pytest
|
||||
|
||||
import homeassistant.util.package as package
|
||||
|
||||
|
||||
RESOURCE_DIR = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), '..', 'resources'))
|
||||
|
||||
TEST_NEW_REQ = 'pyhelloworld3==1.0.0'
|
||||
|
||||
TEST_ZIP_REQ = 'file://{}#{}' \
|
||||
.format(os.path.join(RESOURCE_DIR, 'pyhelloworld3.zip'), TEST_NEW_REQ)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sys():
|
||||
|
@ -176,3 +183,14 @@ def test_async_get_user_site(mock_env_copy):
|
|||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
|
||||
env=env)
|
||||
assert ret == os.path.join(deps_dir, 'lib_dir')
|
||||
|
||||
|
||||
def test_check_package_global():
|
||||
"""Test for an installed package."""
|
||||
installed_package = list(pkg_resources.working_set)[0].project_name
|
||||
assert package.is_installed(installed_package)
|
||||
|
||||
|
||||
def test_check_package_zip():
|
||||
"""Test for an installed zip package."""
|
||||
assert not package.is_installed(TEST_ZIP_REQ)
|
||||
|
|
Loading…
Add table
Reference in a new issue