requirements.py

Go to the documentation of this file.

"""Module to handle installing requirements."""
 
from __future__ import annotations
 
import asyncio
from collections.abc import Iterable
import contextlib
import logging
import os
from typing import Any
 
from packaging.requirements import Requirement
 
from .core import HomeAssistant, callback
from .exceptions import HomeAssistantError
from .helpers import singleton
from .loader import Integration, IntegrationNotFound, async_get_integration
from .util import package as pkg_util
 
# The default is too low when the internet connection is satellite or high latency
PIP_TIMEOUT = 60
MAX_INSTALL_FAILURES = 3
DATA_REQUIREMENTS_MANAGER = "requirements_manager"
CONSTRAINT_FILE = "package_constraints.txt"
DISCOVERY_INTEGRATIONS: dict[str, Iterable[str]] = {
    "dhcp": ("dhcp",),
    "mqtt": ("mqtt",),
    "ssdp": ("ssdp",),
    "zeroconf": ("zeroconf", "homekit"),
}
_LOGGER = logging.getLogger(__name__)
 
 
class RequirementsNotFound(HomeAssistantError):
    """Raised when a component is not found."""
 
    def __init__(self, domain: str, requirements: list[str]) -> None:
        """Initialize a component not found error."""
        super().__init__(f"Requirements for {domain} not found: {requirements}.")
        self.domaindomain = domain
        self.requirementsrequirements = requirements
 
 
async def async_get_integration_with_requirements(
    hass: HomeAssistant, domain: str
) -> Integration:
    """Get an integration with all requirements installed, including the dependencies.
 
    This can raise IntegrationNotFound if manifest or integration
    is invalid, RequirementNotFound if there was some type of
    failure to install requirements.
    """
    manager = _async_get_manager(hass)
    return await manager.async_get_integration_with_requirements(domain)
 
 
async def async_process_requirements(
    hass: HomeAssistant, name: str, requirements: list[str]
) -> None:
    """Install the requirements for a component or platform.
 
    This method is a coroutine. It will raise RequirementsNotFound
    if an requirement can't be satisfied.
    """
    await _async_get_manager(hass).async_process_requirements(name, requirements)
 
 
async def async_load_installed_versions(
    hass: HomeAssistant, requirements: set[str]
) -> None:
    """Load the installed version of requirements."""
    await _async_get_manager(hass).async_load_installed_versions(requirements)
 
 
@callback
@singleton.singleton(DATA_REQUIREMENTS_MANAGER)
def _async_get_manager(hass: HomeAssistant) -> RequirementsManager:
    """Get the requirements manager."""
    return RequirementsManager(hass)
 
 
@callback
def async_clear_install_history(hass: HomeAssistant) -> None:
    """Forget the install history."""
    _async_get_manager(hass).install_failure_history.clear()
 
 
def pip_kwargs(config_dir: str | None) -> dict[str, Any]:
    """Return keyword arguments for PIP install."""
    is_docker = pkg_util.is_docker_env()
    kwargs = {
        "constraints": os.path.join(os.path.dirname(__file__), CONSTRAINT_FILE),
        "timeout": PIP_TIMEOUT,
    }
    if not (config_dir is None or pkg_util.is_virtual_env()) and not is_docker:
        kwargs["target"] = os.path.join(config_dir, "deps")
    return kwargs
 
 
def _install_with_retry(requirement: str, kwargs: dict[str, Any]) -> bool:
    """Try to install a package up to MAX_INSTALL_FAILURES times."""
    for _ in range(MAX_INSTALL_FAILURES):
        if pkg_util.install_package(requirement, **kwargs):
            return True
    return False
 
 
def _install_requirements_if_missing(
    requirements: list[str], kwargs: dict[str, Any]
) -> tuple[set[str], set[str]]:
    """Install requirements if missing."""
    installed: set[str] = set()
    failures: set[str] = set()
    for req in requirements:
        if pkg_util.is_installed(req) or _install_with_retry(req, kwargs):
            installed.add(req)
            continue
        failures.add(req)
    return installed, failures
 
 
class RequirementsManager:
    """Manage requirements."""
 
    def __init__(self, hass: HomeAssistant) -> None:
        """Init the requirements manager."""
        self.hasshass = hass
        self.pip_lockpip_lock = asyncio.Lock()
        self.integrations_with_reqs: dict[
            str, Integration | asyncio.Future[Integration]
        ] = {}
        self.install_failure_history: set[str] = set()
        self.is_installed_cache: set[str] = set()
 
    async def async_get_integration_with_requirements(
        self, domain: str, done: set[str] | None = None
    ) -> Integration:
        """Get an integration with all requirements installed, including dependencies.
 
        This can raise IntegrationNotFound if manifest or integration
        is invalid, RequirementNotFound if there was some type of
        failure to install requirements.
        """
        if done is None:
            done = {domain}
        else:
            done.add(domain)
 
        cache = self.integrations_with_reqs
        if int_or_fut := cache.get(domain):
            if isinstance(int_or_fut, Integration):
                return int_or_fut
            return await int_or_fut
 
        future = cache[domain] = self.hasshass.loop.create_future()
        try:
            integration = await async_get_integration(self.hasshass, domain)
            if not self.hasshass.config.skip_pip:
                await self._async_process_integration_async_process_integration(integration, done)
        except BaseException as ex:
            # We do not cache failures as we want to retry, or
            # else people can't fix it and then restart, because
            # their config will never be valid.
            del cache[domain]
            future.set_exception(ex)
            with contextlib.suppress(BaseException):
                # Clear the flag as its normal that nothing
                # will wait for this future to be resolved
                # if there are no concurrent requirements fetches.
                await future
            raise
 
        cache[domain] = integration
        future.set_result(integration)
        return integration
 
    async def _async_process_integration(
        self, integration: Integration, done: set[str]
    ) -> None:
        """Process an integration and requirements."""
        if integration.requirements:
            await self.async_process_requirementsasync_process_requirements(
                integration.domain, integration.requirements
            )
 
        cache = self.integrations_with_reqs
 
        deps_to_check = {
            dep
            for dep in integration.dependencies + integration.after_dependencies
            if dep not in done
            # If the dep is in the cache and it's an Integration
            # it's already been checked for the requirements and we should
            # not check it again.
            and (
                not (cached_integration := cache.get(dep))
                or type(cached_integration) is not Integration
            )
        }
 
        for check_domain, to_check in DISCOVERY_INTEGRATIONS.items():
            if (
                check_domain not in done
                and check_domain not in deps_to_check
                # If the integration is in the cache and it's an Integration
                # it's already been checked for the requirements and we should
                # not check it again.
                and (
                    not (cached_integration := cache.get(check_domain))
                    or type(cached_integration) is not Integration
                )
                and any(check in integration.manifest for check in to_check)
            ):
                deps_to_check.add(check_domain)
 
        if not deps_to_check:
            return
 
        exceptions: list[Exception] = []
        # We don't create tasks here since everything waits for the pip lock
        # anyways and we want to make sure we don't start a bunch of tasks
        # that will just wait for the lock.
        for dep in deps_to_check:
            # We want all the async_get_integration_with_requirements calls to
            # happen even if one fails. So we catch the exception and store it
            # to raise the first one after all are done to behave like asyncio
            # gather.
            try:
                await self.async_get_integration_with_requirementsasync_get_integration_with_requirements(dep, done)
            except IntegrationNotFound as ex:
                if (
                    integration.is_built_in
                    or ex.domain not in integration.after_dependencies
                ):
                    exceptions.append(ex)
            except Exception as ex:  # noqa: BLE001
                exceptions.insert(0, ex)
 
        if exceptions:
            raise exceptions[0]
 
    async def async_process_requirements(
        self, name: str, requirements: list[str]
    ) -> None:
        """Install the requirements for a component or platform.
 
        This method is a coroutine. It will raise RequirementsNotFound
        if an requirement can't be satisfied.
        """
        if self.hasshass.config.skip_pip_packages:
            skipped_requirements = {
                req
                for req in requirements
                if Requirement(req).name in self.hasshass.config.skip_pip_packages
            }
 
            for req in skipped_requirements:
                _LOGGER.warning("Skipping requirement %s. This may cause issues", req)
 
            requirements = [r for r in requirements if r not in skipped_requirements]
 
        if not (missing := self._find_missing_requirements_find_missing_requirements(requirements)):
            return
        self._raise_for_failed_requirements_raise_for_failed_requirements(name, missing)
 
        async with self.pip_lockpip_lock:
            # Recalculate missing again now that we have the lock
            if missing := self._find_missing_requirements_find_missing_requirements(requirements):
                await self._async_process_requirements_async_process_requirements(name, missing)
 
    def _find_missing_requirements(self, requirements: list[str]) -> list[str]:
        """Find requirements that are missing in the cache."""
        return [req for req in requirements if req not in self.is_installed_cache]
 
    def _raise_for_failed_requirements(
        self, integration: str, missing: list[str]
    ) -> None:
        """Raise for failed installing integration requirements.
 
        Raise RequirementsNotFound so we do not keep trying requirements
        that have already failed.
        """
        for req in missing:
            if req in self.install_failure_history:
                _LOGGER.info(
                    (
                        "Multiple attempts to install %s failed, install will be"
                        " retried after next configuration check or restart"
                    ),
                    req,
                )
                raise RequirementsNotFound(integration, [req])
 
    async def _async_process_requirements(
        self,
        name: str,
        requirements: list[str],
    ) -> None:
        """Install a requirement and save failures."""
        kwargs = pip_kwargs(self.hasshass.config.config_dir)
        installed, failures = await self.hasshass.async_add_executor_job(
            _install_requirements_if_missing, requirements, kwargs
        )
        self.is_installed_cache |= installed
        self.install_failure_history |= failures
        if failures:
            raise RequirementsNotFound(name, list(failures))
 
    async def async_load_installed_versions(
        self,
        requirements: set[str],
    ) -> None:
        """Load the installed version of requirements."""
        if not (requirements_to_check := requirements - self.is_installed_cache):
            return
 
        self.is_installed_cache |= await self.hasshass.async_add_executor_job(
            pkg_util.get_installed_versions, requirements_to_check
        )