From bfaf49fd03f53bc480168da3325f60b5b8747902 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Wed, 11 Oct 2023 17:52:04 -0500 Subject: [PATCH] Integrate `docs/bin/*.py` files into the nox config (#529) This PR adds a clone-core nox session and applies the Python linters to the files in `docs/bin`. (cherry picked from commit 881b3260c3defb7f18c461787d60ecb336154876) --- .mypy.ini | 5 +++ docs/bin/clone-core.py | 46 +++++++++++++-------- docs/bin/find-plugin-refs.py | 79 +++++++++++++++++++++--------------- noxfile.py | 20 ++++++++- 4 files changed, 99 insertions(+), 51 deletions(-) create mode 100644 .mypy.ini diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 00000000000..e176756d259 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,5 @@ +[mypy] +check_untyped_defs = True + +[mypy-ansible.*] +ignore_missing_imports = True diff --git a/docs/bin/clone-core.py b/docs/bin/clone-core.py index dee5dfe8512..7c675ad3818 100755 --- a/docs/bin/clone-core.py +++ b/docs/bin/clone-core.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -"""Clone relevant portions of ansible-core from ansible/ansible into the current source tree to facilitate building docs.""" +""" +Clone relevant portions of ansible-core from ansible/ansible into the current +source tree to facilitate building docs. +""" from __future__ import annotations @@ -14,46 +17,57 @@ def main() -> None: keep_dirs = [ - 'bin', - 'lib', - 'packaging', - 'test/lib', + "bin", + "lib", + "packaging", + "test/lib", ] keep_files = [ - 'MANIFEST.in', - 'pyproject.toml', - 'requirements.txt', - 'setup.cfg', - 'setup.py', + "MANIFEST.in", + "pyproject.toml", + "requirements.txt", + "setup.cfg", + "setup.py", ] - branch = (ROOT / 'docs' / 'ansible-core-branch.txt').read_text().strip() + branch = (ROOT / "docs" / "ansible-core-branch.txt").read_text().strip() with tempfile.TemporaryDirectory() as temp_dir: - subprocess.run(['git', 'clone', 'https://github.com/ansible/ansible', '--depth=1', '-b', branch, temp_dir], check=True) + subprocess.run( + [ + "git", + "clone", + "https://github.com/ansible/ansible", + "--depth=1", + "-b", + branch, + temp_dir, + ], + check=True, + ) for keep_dir in keep_dirs: src = pathlib.Path(temp_dir, keep_dir) dst = pathlib.Path.cwd() / keep_dir - print(f'Updating {keep_dir!r} ...', file=sys.stderr, flush=True) + print(f"Updating {keep_dir!r} ...", file=sys.stderr, flush=True) if dst.exists(): shutil.rmtree(dst) shutil.copytree(src, dst, symlinks=True) - (dst / '.gitignore').write_text('*') + (dst / ".gitignore").write_text("*") for keep_file in keep_files: src = pathlib.Path(temp_dir, keep_file) dst = pathlib.Path.cwd() / keep_file - print(f'Updating {keep_file!r} ...', file=sys.stderr, flush=True) + print(f"Updating {keep_file!r} ...", file=sys.stderr, flush=True) shutil.copyfile(src, dst) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/docs/bin/find-plugin-refs.py b/docs/bin/find-plugin-refs.py index d603409d688..8c68114c3e8 100755 --- a/docs/bin/find-plugin-refs.py +++ b/docs/bin/find-plugin-refs.py @@ -1,68 +1,74 @@ #!/usr/bin/env python -# To run this script, first make webdocs in the toplevel of the checkout. This will generate all -# rst files from their sources. Then run this script ./docs/bin/find-plugin-refs.py +# To run this script, first make webdocs in the toplevel of the checkout. +# This will generate all rst files from their sources. +# Then run this script ./docs/bin/find-plugin-refs.py # -# No output means that there are no longer any bare module and plugin names referenced via :ref: +# No output means that there are no longer any bare module and plugin names +# referenced via :ref: # -# For my listing of what needs to be changed after running this script, see the comment at the end -# of the file +# For my listing of what needs to be changed after running this script, +# see the comment at the end of the file -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import glob import os import re +from collections.abc import Collection -from ansible.module_utils._text import to_text +TOPDIR = os.path.join(os.path.dirname(__file__), "..", "docsite", "rst") -TOPDIR = os.path.join(os.path.dirname(__file__), '..', 'docsite', 'rst') - - -def plugin_names(topdir): +def plugin_names(topdir: str | os.PathLike[str]): plugins = set() # Modules are in a separate directory - for module_filename in glob.glob(os.path.join(topdir, 'modules', '*_module.rst')): + for module_filename in glob.glob(os.path.join(topdir, "modules", "*_module.rst")): module_filename = os.path.basename(module_filename) - module_name = module_filename[:module_filename.index('_module.rst')] + module_name = module_filename[: module_filename.index("_module.rst")] plugins.add(module_name) - for plugin_filename in glob.glob(os.path.join(topdir, 'plugins', '*', '*.rst')): + for plugin_filename in glob.glob(os.path.join(topdir, "plugins", "*", "*.rst")): plugin_filename = os.path.basename(plugin_filename) - plugin_name = plugin_filename[:plugin_filename.index('.rst')] + plugin_name = plugin_filename[: plugin_filename.index(".rst")] plugins.add(plugin_name) return plugins -def process_refs(topdir, plugin_names): - REF_RE = re.compile(':ref:`([^`]*)`') - LABEL_RE = re.compile('<([^>]*)>$') +def process_refs(topdir: str | os.PathLike[str], plugin_names: Collection[str]): + REF_RE = re.compile(":ref:`([^`]*)`") + LABEL_RE = re.compile("<([^>]*)>$") - # Walk the whole docs tree looking for :ref:. Anywhere those are found, search for `([^`]*)` + # Walk the whole docs tree looking for :ref:. + # Anywhere those are found, search for `([^`]*)` for dirpath, dirnames, filenames in os.walk(topdir): for filename in filenames: - with open(os.path.join(dirpath, filename), 'rb') as f: + with open(os.path.join(dirpath, filename), "r") as f: data = f.read() - data = to_text(data) for ref_match in re.finditer(REF_RE, data): label = ref_match.group(1) - # If the ref label includes "<", then search for the label inside of the "<>" + # If the ref label includes "<", then search for the label + # inside of the "<>" label_match = re.search(LABEL_RE, label) if label_match: label = label_match.group(1) - # If the ref label is listed in plugins, then print that the file contains an unported ref + # If the ref label is listed in plugins, then print that + # the file contains an unported ref if label in plugin_names: - print(':ref:`{0}` matching plugin {1} was found in {2}'.format(ref_match.group(1), label, os.path.join(dirpath, filename))) - + print( + ":ref:`{0}` matching plugin {1} was found in {2}".format( + ref_match.group(1), + label, + os.path.join(dirpath, filename), + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": plugins = plugin_names(TOPDIR) process_refs(TOPDIR, plugins) @@ -71,15 +77,22 @@ def process_refs(topdir, plugin_names): # - t = _MODULE.sub(r":ref:`\1 <\1>`", t) # + t = _MODULE.sub(r":ref:`\1 `", t) # - # These have @{module}@ in the template and need to have something like module_@{module}@ - # If any of these list plugins as well as modules, they will need to have a conditional or extra - # data passed in to handle that in a generic fashion: + # These have @{module}@ in the template and need to have something like + # module_@{module}@ + # If any of these list plugins as well as modules, + # they will need to have a conditional or extra data passed in to handle + # that in a generic fashion: # # docs/templates/list_of_CATEGORY_modules.rst.j2 # docs/templates/list_of_CATEGORY_plugins.rst.j2 # docs/templates/modules_by_support.rst.j2 # # These are just a simple manual fix: - # :ref:`command` matching plugin command was found in ./../docsite/rst/user_guide/intro_adhoc.rst - # :ref:`shell` matching plugin shell was found in ./../docsite/rst/user_guide/intro_adhoc.rst - # :ref:`config` matching plugin config was found in ./../docsite/rst/installation_guide/intro_configuration.rst + # :ref:`command` matching plugin command was found in + # ./../docsite/rst/user_guide/intro_adhoc.rst + # + # :ref:`shell` matching plugin shell was found in + # ./../docsite/rst/user_guide/intro_adhoc.rst + # + # :ref:`config` matching plugin config was found in + # ./../docsite/rst/installation_guide/intro_configuration.rst diff --git a/noxfile.py b/noxfile.py index 1bd12b88f9a..cc7e39bc507 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,11 +1,18 @@ +from __future__ import annotations + import os +from glob import iglob from pathlib import Path import nox -LINT_FILES = ("hacking/pr_labeler/label.py", "noxfile.py") +LINT_FILES: tuple[str, ...] = ( + "hacking/pr_labeler/label.py", + "noxfile.py", + *iglob("docs/bin/*.py"), +) PINNED = os.environ.get("PINNED", "true").lower() in {"1", "true"} -nox.options.sessions = ("lint",) +nox.options.sessions = ("clone-core", "lint") def install(session: nox.Session, *args, req: str, **kwargs): @@ -83,3 +90,12 @@ def pip_compile(session: nox.Session, req: str): f"tests/{req}.in", ) # fmt: on + + +@nox.session(name="clone-core", venv_backend="none") +def clone_core(session: nox.Session): + """ + Clone relevant portions of ansible-core from ansible/ansible into the current + source tree to facilitate building docs. + """ + session.run_always("python", "docs/bin/clone-core.py")