diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py index 84e7d3f086..9c8b40fca6 100644 --- a/mkdocs/__main__.py +++ b/mkdocs/__main__.py @@ -136,6 +136,9 @@ def __del__(self): ) shell_help = "Use the shell when invoking Git." watch_help = "A directory or file to watch for live reloading. Can be supplied multiple times." +projects_file_help = ( + "URL or local path of the registry file that declares all known MkDocs-related projects." +) def add_options(*opts): @@ -201,7 +204,7 @@ def callback(ctx, param, value): PKG_DIR = os.path.dirname(os.path.abspath(__file__)) -@click.group(context_settings={'help_option_names': ['-h', '--help']}) +@click.group(context_settings=dict(help_option_names=['-h', '--help'], max_content_width=120)) @click.version_option( __version__, '-V', @@ -287,6 +290,30 @@ def gh_deploy_command( ) +@cli.command(name="get-deps") +@verbose_option +@click.option('-f', '--config-file', type=click.File('rb'), help=config_help) +@click.option( + '-p', + '--projects-file', + default='https://raw.githubusercontent.com/mkdocs/best-of-mkdocs/main/projects.yaml', + help=projects_file_help, + show_default=True, +) +def get_deps_command(config_file, projects_file): + """Show required PyPI packages inferred from plugins in mkdocs.yml""" + from mkdocs.commands import get_deps + + warning_counter = utils.CountHandler() + warning_counter.setLevel(logging.WARNING) + logging.getLogger('mkdocs').addHandler(warning_counter) + + get_deps.get_deps(projects_file_url=projects_file, config_file_path=config_file) + + if warning_counter.get_counts(): + sys.exit(1) + + @cli.command(name="new") @click.argument("project_directory") @common_options diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py new file mode 100644 index 0000000000..d303ebdaa6 --- /dev/null +++ b/mkdocs/commands/get_deps.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import dataclasses +import datetime +import functools +import logging +from typing import Iterator, Mapping, Optional + +import yaml + +from mkdocs import utils +from mkdocs.config.base import _open_config_file +from mkdocs.plugins import EntryPoint, entry_points +from mkdocs.utils.cache import download_and_cache_url + +log = logging.getLogger(__name__) + + +def _extract_names(cfg, key: str) -> Iterator[str]: + """Get names of plugins/extensions from the config - in either a list of dicts or a dict.""" + try: + items = iter(cfg.get(key, ())) + except TypeError: + log.error(f"Invalid config entry '{key}'") + for item in items: + try: + if not isinstance(item, str): + [item] = item + yield item + except (ValueError, TypeError): + log.error(f"Invalid config entry '{key}': {item}") + + +@functools.lru_cache() +def _entry_points(group: str) -> Mapping[str, EntryPoint]: + eps = {ep.name: ep for ep in entry_points(group=group)} + log.debug(f"Available '{group}' entry points: {sorted(eps)}") + return eps + + +@dataclasses.dataclass(frozen=True) +class PluginKind: + projects_key: str + entry_points_key: str + + def __str__(self) -> str: + return self.projects_key.rpartition('_')[-1] + + +def get_deps(projects_file_url: str, config_file_path: Optional[str] = None) -> None: + """ + Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects. + + Parameters: + projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects. + The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}] + config_file_path: Non-default path to mkdocs.yml. + """ + with _open_config_file(config_file_path) as f: + cfg = utils.yaml_load(f) + + if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')): + log.warning("The passed config file doesn't seem to be a mkdocs.yml config file") + + try: + theme = cfg['theme']['name'] + except (KeyError, TypeError): + theme = cfg.get('theme') + themes = {theme} if theme else set() + + plugins = set(_extract_names(cfg, 'plugins')) + extensions = set(_extract_names(cfg, 'markdown_extensions')) + + wanted_plugins = ( + (PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}), + (PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}), + (PluginKind('markdown_extension', 'markdown.extensions'), extensions), + ) + for kind, wanted in wanted_plugins: + log.debug(f'Wanted {kind}s: {sorted(wanted)}') + + content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7)) + projects = yaml.safe_load(content)['projects'] + + packages_to_install = set() + for project in projects: + for kind, wanted in wanted_plugins: + available = project.get(kind.projects_key, ()) + if isinstance(available, str): + available = (available,) + for entry_name in available: + if entry_name in wanted or ( + # Also check theme-namespaced plugin names against the current theme. + '/' in entry_name + and theme is not None + and kind.projects_key == 'mkdocs_plugin' + and entry_name.startswith(f'{theme}/') + and entry_name[len(theme) + 1 :] in wanted + ): + if 'pypi_id' in project: + install_name = project['pypi_id'] + elif 'github_id' in project: + install_name = 'git+https://github.com/{github_id}'.format_map(project) + else: + log.error( + f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}" + ) + continue + packages_to_install.add(install_name) + wanted.remove(entry_name) + + for kind, wanted in wanted_plugins: + for entry_name in sorted(wanted): + dist_name = None + ep = _entry_points(kind.entry_points_key).get(entry_name) + if ep is not None and ep.dist is not None: + dist_name = ep.dist.name + if dist_name not in ('mkdocs', 'Markdown'): + warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project" + if ep is not None: + warning += " but is installed locally" + if dist_name: + warning += f" from '{dist_name}'" + log.info(warning) + else: + log.warning(warning) + + for pkg in sorted(packages_to_install): + print(pkg) diff --git a/mkdocs/utils/cache.py b/mkdocs/utils/cache.py new file mode 100644 index 0000000000..fce05b7c9a --- /dev/null +++ b/mkdocs/utils/cache.py @@ -0,0 +1,64 @@ +import datetime +import hashlib +import logging +import os +import urllib.parse +import urllib.request + +import click +import platformdirs + +log = logging.getLogger(__name__) + + +def download_and_cache_url( + url: str, + cache_duration: datetime.timedelta, + comment: bytes = b'# ', +) -> bytes: + """ + Downloads a file from the URL, stores it under ~/.cache/, and returns its content. + + If the URL is a local path, it is simply read and returned instead. + + For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime. + + Parameters: + url: URL or local path of the file to use. + cache_duration: how long to consider the URL content cached. + comment: The appropriate comment prefix for this file format. + """ + + if urllib.parse.urlsplit(url).scheme not in ('http', 'https'): + with open(url, 'rb') as f: + return f.read() + + directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache') + name_hash = hashlib.sha256(url.encode()).hexdigest()[:32] + path = os.path.join(directory, name_hash + os.path.splitext(url)[1]) + + now = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) + prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode()) + # Check for cached file and try to return it + if os.path.isfile(path): + try: + with open(path, 'rb') as f: + line = f.readline() + if line.startswith(prefix): + line = line[len(prefix) :] + timestamp = int(line) + if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration: + log.debug(f"Using cached '{path}' for '{url}'") + return f.read() + except (IOError, ValueError) as e: + log.debug(f'{type(e).__name__}: {e}') + + # Download and cache the file + log.debug(f"Downloading '{url}' to '{path}'") + with urllib.request.urlopen(url) as resp: + content = resp.read() + os.makedirs(directory, exist_ok=True) + with click.open_file(path, 'wb', atomic=True) as f: + f.write(b'%s%d\n' % (prefix, now)) + f.write(content) + return content diff --git a/pyproject.toml b/pyproject.toml index 9c32c949d5..3ebcc192f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "typing_extensions >=3.10; python_version < '3.8'", "packaging >=20.5", "mergedeep >=1.3.4", + "platformdirs >=2.2.0", "colorama >=0.4; platform_system == 'Windows'", ] [project.optional-dependencies] @@ -63,6 +64,7 @@ min-versions = [ "typing_extensions ==3.10; python_version < '3.8'", "packaging ==20.5", "mergedeep ==1.3.4", + "platformdirs ==2.2.0", "colorama ==0.4; platform_system == 'Windows'", "babel ==2.9.0", ]