conda.py

#-----------------------------------------------------------------------------
# Copyright (c) 2005-2023, PyInstaller Development Team.
#
# Distributed under the terms of the GNU General Public License (version 2
# or later) with exception for distributing the bootloader.
#
# The full license is in the file COPYING.txt, distributed with this software.
#
# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
#-----------------------------------------------------------------------------

# language=rst
"""
Additional helper methods for working specifically with Anaconda distributions are found at
:mod:`PyInstaller.utils.hooks.conda_support<PyInstaller.utils.hooks.conda>`
which is designed to mimic (albeit loosely) the `importlib.metadata`_ package. These functions find and parse the
distribution metadata from json files located in the ``conda-meta`` directory.

.. versionadded:: 4.2.0

This module is available only if run inside a Conda environment. Usage of this module should therefore be wrapped in
a conditional clause::

    from PyInstaller.compat import is_pure_conda

    if is_pure_conda:
        from PyInstaller.utils.hooks import conda_support

        # Code goes here. e.g.
        binaries = conda_support.collect_dynamic_libs("numpy")
        ...

Packages are all referenced by the *distribution name* you use to install it, rather than the *package name* you import
it with. I.e., use ``distribution("pillow")`` instead of ``distribution("PIL")`` or use ``package_distribution("PIL")``.
"""
from __future__ import annotations

import fnmatch
import json
import sys
from pathlib import Path
from typing import Iterable, List
from importlib.metadata import PackagePath as _PackagePath

from PyInstaller import compat
from PyInstaller.log import logger

# Conda virtual environments each get their own copy of `conda-meta` so the use of `sys.prefix` instead of
# `sys.base_prefix`, `sys.real_prefix` or anything from our `compat` module is intentional.
CONDA_ROOT = Path(sys.prefix)
CONDA_META_DIR = CONDA_ROOT / "conda-meta"

# Find all paths in `sys.path` that are inside Conda root.
PYTHONPATH_PREFIXES = []
for _path in sys.path:
    _path = Path(_path)
    try:
        PYTHONPATH_PREFIXES.append(_path.relative_to(sys.prefix))
    except ValueError:
        pass

PYTHONPATH_PREFIXES.sort(key=lambda p: len(p.parts), reverse=True)


class Distribution:
    """
    A bucket class representation of a Conda distribution.

    This bucket exports the following attributes:

    :ivar name: The distribution's name.
    :ivar version: Its version.
    :ivar files: All filenames as :meth:`PackagePath`\\ s included with this distribution.
    :ivar dependencies: Names of other distributions that this distribution depends on (with version constraints
                        removed).
    :ivar packages: Names of importable packages included in this distribution.

    This class is not intended to be constructed directly by users. Rather use :meth:`distribution` or
    :meth:`package_distribution` to provide one for you.
    """
    def __init__(self, json_path: str):
        try:
            self._json_path = Path(json_path)
            assert self._json_path.exists()
        except (TypeError, AssertionError):
            raise TypeError(
                "Distribution requires a path to a conda-meta json. Perhaps you want "
                "`distribution({})` instead?".format(repr(json_path))
            )

        # Everything we need (including this distribution's name) is kept in the metadata json.
        self.raw: dict = json.loads(self._json_path.read_text())

        # Unpack the more useful contents of the json.
        self.name: str = self.raw["name"]
        self.version: str = self.raw["version"]
        self.files = [PackagePath(i) for i in self.raw["files"]]
        self.dependencies = self._init_dependencies()
        self.packages = self._init_package_names()

    def __repr__(self):
        return "{}(name=\"{}\", packages={})".format(type(self).__name__, self.name, self.packages)

    def _init_dependencies(self):
        """
        Read dependencies from ``self.raw["depends"]``.

        :return: Dependent distribution names.
        :rtype: list

        The names in ``self.raw["depends"]`` come with extra version constraint information which must be stripped.
        """
        dependencies = []
        # For each dependency:
        for dependency in self.raw["depends"]:
            # ``dependency`` is a string of the form: "[name] [version constraints]"
            name, *version_constraints = dependency.split(maxsplit=1)
            dependencies.append(name)
        return dependencies

    def _init_package_names(self):
        """
        Search ``self.files`` for package names shipped by this distribution.

        :return: Package names.
        :rtype: list

        These are names you would ``import`` rather than names you would install.
        """
        packages = []
        for file in self.files:
            package = _get_package_name(file)
            if package is not None:
                packages.append(package)
        return packages

    @classmethod
    def from_name(cls, name: str):
        """
        Get distribution information for a given distribution **name** (i.e., something you would ``conda install``).

        :rtype: :class:`Distribution`
        """
        if name in distributions:
            return distributions[name]
        raise ModuleNotFoundError(
            "Distribution {} is either not installed or was not installed using Conda.".format(name)
        )

    @classmethod
    def from_package_name(cls, name: str):
        """
        Get distribution information for a **package** (i.e., something you would import).

        :rtype: :class:`Distribution`

        For example, the package ``pkg_resources`` belongs to the distribution ``setuptools``, which contains three
        packages.

        >>> package_distribution("pkg_resources")
        Distribution(name="setuptools",
                     packages=['easy_install', 'pkg_resources', 'setuptools'])
        """
        if name in distributions_by_package:
            return distributions_by_package[name]
        raise ModuleNotFoundError("Package {} is either not installed or was not installed using Conda.".format(name))


distribution = Distribution.from_name
package_distribution = Distribution.from_package_name


class PackagePath(_PackagePath):
    """
    A filename relative to Conda's root (``sys.prefix``).

    This class inherits from :class:`pathlib.PurePosixPath` even on non-Posix OSs. To convert to a :class:`pathlib.Path`
    pointing to the real file, use the :meth:`locate` method.
    """
    def locate(self):
        """
        Return a path-like object for this path pointing to the file's true location.
        """
        return Path(sys.prefix) / self


def walk_dependency_tree(initial: str, excludes: Iterable[str] | None = None):
    """
    Collect a :class:`Distribution` and all direct and indirect dependencies of that distribution.

    Arguments:
        initial:
            Distribution name to collect from.
        excludes:
            Distributions to exclude.
    Returns:
        A ``{name: distribution}`` mapping where ``distribution`` is the output of
        :func:`conda_support.distribution(name) <distribution>`.
    """
    if excludes is not None:
        excludes = set(excludes)

    # Rather than use true recursion, mimic it with a to-do queue.
    from collections import deque
    done = {}
    names_to_do = deque([initial])

    while names_to_do:
        # Grab a distribution name from the to-do list.
        name = names_to_do.pop()
        try:
            # Collect and save it's metadata.
            done[name] = distribution = Distribution.from_name(name)
            logger.debug("Collected Conda distribution '%s', a dependency of '%s'.", name, initial)
        except ModuleNotFoundError:
            logger.warning(
                "Conda distribution '%s', dependency of '%s', was not found. "
                "If you installed this distribution with pip then you may ignore this warning.", name, initial
            )
            continue
        # For each dependency:
        for _name in distribution.dependencies:
            if _name in done:
                # Skip anything already done.
                continue
            if _name == name:
                # Avoid infinite recursion if a distribution depends on itself. This will probably never happen but I
                # certainly would not chance it.
                continue
            if excludes is not None and _name in excludes:
                # Do not recurse to excluded dependencies.
                continue
            names_to_do.append(_name)
    return done


def _iter_distributions(name, dependencies, excludes):
    if dependencies:
        return walk_dependency_tree(name, excludes).values()
    else:
        return [Distribution.from_name(name)]


def requires(name: str, strip_versions: bool = False) -> List[str]:
    """
    List requirements of a distribution.

    Arguments:
        name:
            The name of the distribution.
        strip_versions:
            List only their names, not their version constraints.
    Returns:
        A list of distribution names.
    """
    if strip_versions:
        return distribution(name).dependencies
    return distribution(name).raw["depends"]


def files(name: str, dependencies: bool = False, excludes: list | None = None) -> List[PackagePath]:
    """
    List all files belonging to a distribution.

    Arguments:
        name:
            The name of the distribution.
        dependencies:
            Recursively collect files of dependencies too.
        excludes:
            Distributions to ignore if **dependencies** is true.
    Returns:
        All filenames belonging to the given distribution.

    With ``dependencies=False``, this is just a shortcut for::

        conda_support.distribution(name).files
    """
    return [file for dist in _iter_distributions(name, dependencies, excludes) for file in dist.files]


if compat.is_win:
    lib_dir = PackagePath("Library", "bin")
else:
    lib_dir = PackagePath("lib")


def collect_dynamic_libs(name: str, dest: str = ".", dependencies: bool = True, excludes: Iterable[str] | None = None):
    """
    Collect DLLs for distribution **name**.

    Arguments:
        name:
            The distribution's project-name.
        dest:
            Target destination, defaults to ``'.'``.
        dependencies:
            Recursively collect libs for dependent distributions (recommended).
        excludes:
            Dependent distributions to skip, defaults to ``None``.
    Returns:
        List of DLLs in PyInstaller's ``(source, dest)`` format.

    This collects libraries only from Conda's shared ``lib`` (Unix) or ``Library/bin`` (Windows) folders. To collect
    from inside a distribution's installation use the regular :func:`PyInstaller.utils.hooks.collect_dynamic_libs`.
    """
    DLL_SUFFIXES = ("*.dll", "*.dylib", "*.so", "*.so.*")
    _files = []
    for file in files(name, dependencies, excludes):
        # A file is classified as a dynamic library if:
        #  1) it lives inside the dedicated ``lib_dir`` DLL folder
        if file.parent != lib_dir:
            continue
        #  2) it is a file (and not a directory or a symbolic link pointing to a directory)
        resolved_file = file.locate()
        if not resolved_file.is_file():
            continue
        #  3) has a correct suffix
        if not any([resolved_file.match(suffix) for suffix in DLL_SUFFIXES]):
            continue

        _files.append((str(resolved_file), dest))
    return _files


# --- Map packages to distributions and vice-versa ---


def _get_package_name(file: PackagePath):
    """
    Determine the package name of a Python file in :data:`sys.path`.

    Arguments:
        file:
            A Python filename relative to Conda root (sys.prefix).
    Returns:
        Package name or None.

    This function only considers single file packages e.g. ``foo.py`` or top level ``foo/__init__.py``\\ s.
    Anything else is ignored (returning ``None``).
    """
    file = Path(file)
    # TODO: Handle PEP 420 namespace packages (which are missing `__init__` module). No such Conda PEP 420 namespace
    # packages are known.

    # Get top-level folders by finding parents of `__init__.xyz`s
    if file.stem == "__init__" and file.suffix in compat.ALL_SUFFIXES:
        file = file.parent
    elif file.suffix not in compat.ALL_SUFFIXES:
        # Keep single-file packages but skip DLLs, data and junk files.
        return

    # Check if this file/folder's parent is in ``sys.path`` i.e. it's directly importable. This intentionally excludes
    # submodules which would cause confusion because ``sys.prefix`` is in ``sys.path``, meaning that every file in an
    # Conda installation is a submodule.
    for prefix in PYTHONPATH_PREFIXES:
        if len(file.parts) != len(prefix.parts) + 1:
            # This check is redundant but speeds it up quite a bit.
            continue
        # There are no wildcards involved here. The use of ``fnmatch`` is simply to handle the `if case-insensitive
        # file system: use case-insensitive string matching.`
        if fnmatch.fnmatch(str(file.parent), str(prefix)):
            return file.stem


# All the information we want is organised the wrong way.

# We want to look up distribution based on package names, but we can only search for packages using distribution names.
# And we would like to search for a distribution's json file, but, due to the noisy filenames of the jsons, we can only
# find a json's distribution rather than a distribution's json.

# So we have to read everything, then regroup distributions in the ways we want them grouped. This will likely be a
# spectacular bottleneck on full-blown Conda (non miniconda) with 250+ packages by default at several GiBs. I suppose we
# could cache this on a per-json basis if it gets too much.


def _init_distributions():
    distributions = {}
    for path in CONDA_META_DIR.glob("*.json"):
        dist = Distribution(path)
        distributions[dist.name] = dist
    return distributions


distributions = _init_distributions()


def _init_packages():
    distributions_by_package = {}
    for distribution in distributions.values():
        for package in distribution.packages:
            distributions_by_package[package] = distribution
    return distributions_by_package


distributions_by_package = _init_packages()