test_install_all_packages.py

"""
This module uses the pytest infrastructure to produce reports on a large list
of packages.  It verifies installation with and without an intact system PATH.
It also generates report summaries and error reports files.

Test pytest outcomes:
    PASS - if no pip errors, and no pipx issues, and package apps verified
            all installed correctly
    XFAIL - if there is a pip error, i.e. an installation problem out of pipx's
            control
    FAIL - if there is no pip error, but there is a problem due to pipx,
            including a pipx error or warning, incorrect list of
            installed apps, etc.
"""

import io
import os
import re
import subprocess
import sys
import textwrap
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Optional, Tuple

import pytest  # type: ignore[import-not-found]

from helpers import run_pipx_cli
from package_info import PKG

REPORTS_DIR = "./reports"
REPORT_FILENAME_ROOT = "all_packages"
PACKAGE_NAME_LIST = [
    "ansible",
    "awscli",
    "b2",
    "beancount",
    "beets",
    "black",
    "cactus",
    "chert",
    "cloudtoken",
    "coala",
    "cookiecutter",
    "cython",
    "datasette",
    "diffoscope",
    "doc2dash",
    "doitlive",
    "gdbgui",
    "gns3-gui",
    "grow",
    "guake",
    "gunicorn",
    "howdoi",
    "httpie",
    "hyde",
    "ipython",
    "isort",
    "zest-releaser",
    "kaggle",
    "kibitzr",
    "klaus",
    "kolibri",
    "lektor",
    "localstack",
    "mackup",
    "magic-wormhole",
    "mayan-edms",
    "mkdocs",
    "mycli",
    "nikola",
    "nox",
    "pelican",
    "platformio",
    "ppci",
    "prosopopee",
    "ptpython",
    "pycowsay",
    "pylint",
    "retext",
    "robotframework",
    "shell-functools",
    "speedtest-cli",
    "sphinx",
    "sqlmap",
    "streamlink",
    "taguette",
    "term2048",
    "tox-ini-fmt",
    "visidata",
    "vulture",
    "weblate",
    "youtube-dl",
    "zeo",
]


class PackageData:
    def __init__(self):
        self.package_name: str = ""
        self.package_spec: str = ""
        self.clear_elapsed_time: Optional[float] = None
        self.clear_pip_pass: Optional[bool] = None
        self.clear_pipx_pass: Optional[bool] = None
        self.sys_elapsed_time: Optional[float] = None
        self.sys_pip_pass: Optional[bool] = None
        self.sys_pipx_pass: Optional[bool] = None
        self.overall_pass: Optional[bool] = None

    @property
    def clear_pip_pf_str(self) -> str:
        return self._get_pass_fail_str("clear_pip_pass")

    @property
    def clear_pipx_pf_str(self) -> str:
        return self._get_pass_fail_str("clear_pipx_pass")

    @property
    def sys_pip_pf_str(self) -> str:
        return self._get_pass_fail_str("sys_pip_pass")

    @property
    def sys_pipx_pf_str(self) -> str:
        return self._get_pass_fail_str("sys_pipx_pass")

    @property
    def overall_pf_str(self) -> str:
        return self._get_pass_fail_str("overall_pass")

    def _get_pass_fail_str(self, test_attr: str) -> str:
        if getattr(self, test_attr) is not None:
            return "PASS" if getattr(self, test_attr) else "FAIL"
        else:
            return ""


class ModuleGlobalsData:
    def __init__(self):
        self.errors_path = Path(".")
        self.install_data: List[PackageData] = []
        self.py_version_display = "Python {0.major}.{0.minor}.{0.micro}".format(sys.version_info)
        self.py_version_short = "{0.major}.{0.minor}".format(sys.version_info)
        self.report_path = Path(".")
        self.sys_platform = sys.platform
        self.test_class = ""
        self.test_start = datetime.now()
        self.test_end = datetime.now()  # default, must be set later

    def reset(self, test_class: str = "") -> None:
        self.errors_path = Path(".")
        self.install_data = []
        self.report_path = Path(".")
        self.test_class = test_class
        self.test_start = datetime.now()


@pytest.fixture(scope="module")
def module_globals() -> ModuleGlobalsData:
    return ModuleGlobalsData()


def pip_cache_purge() -> None:
    subprocess.run([sys.executable, "-m", "pip", "cache", "purge"], check=True)


def write_report_legend(report_legend_path: Path) -> None:
    with report_legend_path.open("w", encoding="utf-8") as report_legend_fh:
        print(
            textwrap.dedent(
                """
                LEGEND
                ===========
                cleared_PATH = PATH used for pipx tests with only pipx bin dir and nothing else
                sys_PATH = Normal system PATH with all default directories included

                overall = PASS or FAIL for complete end-to-end pipx install, PASS if no errors
                        or warnings and all the proper apps were installed and linked
                pip = PASS or FAIL sub-category based only if pip inside of pipx installs
                        package with/without error
                pipx = PASS or FAIL sub-category based on the non-pip parts of pipx, including
                        whether any errors or warnings are present, and if all the proper apps
                        were installed and linked
                """
            ).strip(),
            file=report_legend_fh,
        )


def format_report_table_header(module_globals: ModuleGlobalsData) -> str:
    header_string = "\n\n"
    header_string += "=" * 79 + "\n"

    header_string += f"{module_globals.sys_platform:16}"
    header_string += f"{module_globals.py_version_display:16}"
    header_string += f"{module_globals.test_start.strftime('%Y-%m-%d %H:%M:%S')}\n\n"

    header_string += f"{'package_spec':24}{'overall':12}{'cleared_PATH':24}"
    header_string += f"{'system_PATH':24}\n"

    header_string += f"{'':24}{'':12}{'pip':8}{'pipx':8}{'time':8}"
    header_string += f"{'pip':8}{'pipx':8}{'time':8}\n"

    header_string += "-" * 79

    return header_string


def format_report_table_row(package_data: PackageData) -> str:
    clear_install_time = f"{package_data.clear_elapsed_time:>3.0f}s"
    if package_data.sys_elapsed_time is not None:
        sys_install_time = f"{package_data.sys_elapsed_time:>3.0f}s"
    else:
        sys_install_time = ""

    return (
        f"{package_data.package_spec:24}{package_data.overall_pf_str:12}"
        f"{package_data.clear_pip_pf_str:8}{package_data.clear_pipx_pf_str:8}"
        f"{clear_install_time:8}"
        f"{package_data.sys_pip_pf_str:8}{package_data.sys_pipx_pf_str:8}"
        f"{sys_install_time:8}"
    )


def format_report_table_footer(module_globals: ModuleGlobalsData) -> str:
    fail_list = []
    prebuild_list = []

    footer_string = "\nSummary\n"
    footer_string += "-" * 79 + "\n"
    for package_data in module_globals.install_data:
        clear_pip_pass = package_data.clear_pip_pass
        clear_pipx_pass = package_data.clear_pipx_pass
        sys_pip_pass = package_data.sys_pip_pass
        sys_pipx_pass = package_data.sys_pipx_pass

        if clear_pip_pass and clear_pipx_pass:
            continue
        elif not clear_pip_pass and sys_pip_pass and sys_pipx_pass:
            prebuild_list.append(package_data.package_spec)
        else:
            fail_list.append(package_data.package_spec)
    if fail_list:
        footer_string += "FAILS:\n"
        for failed_package_spec in sorted(fail_list, key=str.lower):
            footer_string += f"    {failed_package_spec}\n"
    if prebuild_list:
        footer_string += "Needs prebuilt wheel:\n"
        for prebuild_package_spec in sorted(prebuild_list, key=str.lower):
            footer_string += f"    {prebuild_package_spec}\n"

    dt_string = module_globals.test_end.strftime("%Y-%m-%d %H:%M:%S")
    el_datetime = module_globals.test_end - module_globals.test_start
    el_datetime = el_datetime - timedelta(microseconds=el_datetime.microseconds)
    footer_string += f"\nFinished {dt_string}\n"
    footer_string += f"Elapsed: {el_datetime}"

    return footer_string


def verify_installed_resources(
    resource_type: str,
    captured_outerr,
    package_name: str,
    test_error_fh: io.StringIO,
    deps: bool = False,
) -> bool:
    resource_name = {"app": "apps", "man": "man_pages"}[resource_type]
    resource_name_long = {"app": "apps", "man": "manual pages"}[resource_type]
    package_resources = PKG[package_name][resource_name].copy()
    if deps:
        package_resources += PKG[package_name][f"{resource_name}_of_dependencies"]
    if len(package_resources) == 0:
        return True

    reported_resources_re = re.search(
        r"These " + resource_name_long + r" are now globally available\n((?:    - [^\n]+\n)*)",
        captured_outerr.out,
        re.DOTALL,
    )
    if reported_resources_re:
        reported_resources = [x.strip()[2:] for x in reported_resources_re.group(1).strip().split("\n")]
        if set(reported_resources) != set(package_resources):
            resource_success = False
            print("verify_install: REPORTED APPS DO NOT MATCH PACKAGE", file=test_error_fh)
            print(
                f"pipx reported %s: {reported_resources}" % resource_name,
                file=test_error_fh,
            )
            print(
                f" true package %s: {package_resources}" % resource_name,
                file=test_error_fh,
            )
        else:
            resource_success = True
    else:
        resource_success = False
        print("verify_install: APPS TESTING ERROR", file=test_error_fh)

    return resource_success


def verify_post_install(
    pipx_exit_code: int,
    captured_outerr,
    caplog,
    package_name: str,
    test_error_fh: io.StringIO,
    using_clear_path: bool,
    deps: bool = False,
) -> Tuple[bool, Optional[bool], Optional[Path]]:
    pip_error_file = None
    caplog_problem = False
    install_success = f"installed package {package_name}" in captured_outerr.out
    for record in caplog.records:
        if "⚠️" in record.message or "WARNING" in record.message:
            if using_clear_path or "was already on your PATH" not in record.message:
                caplog_problem = True
            print("verify_install: WARNING IN CAPLOG:", file=test_error_fh)
            print(record.message, file=test_error_fh)
        if "Fatal error from pip prevented installation" in record.message:
            pip_error_file_re = re.search(r"pip output in file:\s+(\S.+)$", record.message)
            if pip_error_file_re:
                pip_error_file = Path(pip_error_file_re.group(1))

    if install_success and PKG[package_name].get("apps", None) is not None:
        app_success = verify_installed_resources("app", captured_outerr, package_name, test_error_fh, deps=deps)
    else:
        app_success = True
    if install_success and (
        PKG[package_name].get("man_pages", None) is not None
        or PKG[package_name].get("man_pages_of_dependencies", None) is not None
    ):
        man_success = verify_installed_resources("man", captured_outerr, package_name, test_error_fh, deps=deps)
    else:
        man_success = True

    pip_pass = not ((pipx_exit_code != 0) and f"Error installing {package_name}" in captured_outerr.err)
    pipx_pass: Optional[bool]
    if pip_pass:
        pipx_pass = install_success and not caplog_problem and app_success and man_success
    else:
        pipx_pass = None

    return pip_pass, pipx_pass, pip_error_file


def print_error_report(
    module_globals: ModuleGlobalsData,
    command_captured,
    test_error_fh: io.StringIO,
    package_spec: str,
    test_type: str,
    pip_error_file: Optional[Path],
) -> None:
    with module_globals.errors_path.open("a", encoding="utf-8") as errors_fh:
        print("\n\n", file=errors_fh)
        print("=" * 79, file=errors_fh)
        print(
            f"{package_spec:24}{test_type:16}{module_globals.sys_platform:16}{module_globals.py_version_display}",
            file=errors_fh,
        )
        print("\nSTDOUT:", file=errors_fh)
        print("-" * 76, file=errors_fh)
        print(command_captured.out, end="", file=errors_fh)
        print("\nSTDERR:", file=errors_fh)
        print("-" * 76, file=errors_fh)
        print(command_captured.err, end="", file=errors_fh)
        if pip_error_file is not None:
            print("\nPIP ERROR LOG FILE:", file=errors_fh)
            print("-" * 76, file=errors_fh)
            with pip_error_file.open("r") as pip_error_fh:
                print(pip_error_fh.read(), end="", file=errors_fh)
        print("\n\nTEST WARNINGS / ERRORS:", file=errors_fh)
        print("-" * 76, file=errors_fh)
        print(test_error_fh.getvalue(), end="", file=errors_fh)


def install_and_verify(
    capsys: pytest.CaptureFixture,
    caplog,
    monkeypatch,
    module_globals: ModuleGlobalsData,
    using_clear_path: bool,
    package_data: PackageData,
    deps: bool,
) -> Tuple[bool, Optional[bool], float]:
    _ = capsys.readouterr()
    caplog.clear()

    test_error_fh = io.StringIO()

    monkeypatch.setenv("PATH", os.getenv("PATH_TEST" if using_clear_path else "PATH_ORIG"))

    start_time = time.time()
    pipx_exit_code = run_pipx_cli(
        ["install", package_data.package_spec, "--verbose"] + (["--include-deps"] if deps else [])
    )
    elapsed_time = time.time() - start_time
    captured = capsys.readouterr()

    pip_pass, pipx_pass, pip_error_file = verify_post_install(
        pipx_exit_code,
        captured,
        caplog,
        package_data.package_name,
        test_error_fh,
        using_clear_path=using_clear_path,
        deps=deps,
    )

    if not pip_pass or not pipx_pass:
        print_error_report(
            module_globals,
            captured,
            test_error_fh,
            package_data.package_spec,
            "clear PATH" if using_clear_path else "sys PATH",
            pip_error_file,
        )

    return pip_pass, pipx_pass, elapsed_time


def install_package_both_paths(
    monkeypatch,
    capsys: pytest.CaptureFixture,
    caplog,
    module_globals: ModuleGlobalsData,
    pipx_temp_env,
    package_name: str,
    deps: bool = False,
) -> bool:
    package_data = PackageData()
    module_globals.install_data.append(package_data)
    package_data.package_name = package_name
    package_data.package_spec = PKG[package_name]["spec"]

    (
        package_data.clear_pip_pass,
        package_data.clear_pipx_pass,
        package_data.clear_elapsed_time,
    ) = install_and_verify(
        capsys,
        caplog,
        monkeypatch,
        module_globals,
        using_clear_path=True,
        package_data=package_data,
        deps=deps,
    )
    if not package_data.clear_pip_pass:
        # if we fail to install due to pip install error, try again with
        #   full system path
        (
            package_data.sys_pip_pass,
            package_data.sys_pipx_pass,
            package_data.sys_elapsed_time,
        ) = install_and_verify(
            capsys,
            caplog,
            monkeypatch,
            module_globals,
            using_clear_path=False,
            package_data=package_data,
            deps=deps,
        )

    package_data.overall_pass = bool(
        (package_data.clear_pip_pass and package_data.clear_pipx_pass)
        or (package_data.sys_pip_pass and package_data.sys_pipx_pass)
    )

    with module_globals.report_path.open("a", encoding="utf-8") as report_fh:
        print(format_report_table_row(package_data), file=report_fh, flush=True)

    if not package_data.clear_pip_pass and not package_data.sys_pip_pass:
        # Use xfail to specify error that is from a pip installation problem
        pytest.xfail("pip installation error")

    return package_data.overall_pass


# use class scope to start and finish at end of all parametrized tests
@pytest.fixture(scope="class")
def start_end_test_class(module_globals: ModuleGlobalsData, request):
    reports_path = Path(REPORTS_DIR)
    reports_path.mkdir(exist_ok=True, parents=True)

    module_globals.reset()
    module_globals.test_class = getattr(request.cls, "test_class", "unknown")
    report_filename = (
        f"{REPORT_FILENAME_ROOT}_"
        f"{module_globals.test_class}_"
        f"report_"
        f"{module_globals.sys_platform}_"
        f"{module_globals.py_version_short}_"
        f"{module_globals.test_start.strftime('%Y%m%d')}.txt"
    )
    errors_filename = (
        f"{REPORT_FILENAME_ROOT}_"
        f"{module_globals.test_class}_"
        f"errors_"
        f"{module_globals.sys_platform}_"
        f"{module_globals.py_version_short}_"
        f"{module_globals.test_start.strftime('%Y%m%d')}.txt"
    )
    module_globals.report_path = reports_path / report_filename
    module_globals.errors_path = reports_path / errors_filename

    write_report_legend(reports_path / f"{REPORT_FILENAME_ROOT}_report_legend.txt")

    with module_globals.report_path.open("a", encoding="utf-8") as report_fh:
        print(format_report_table_header(module_globals), file=report_fh)

    yield

    module_globals.test_end = datetime.now()
    with module_globals.report_path.open("a", encoding="utf-8") as report_fh:
        print(format_report_table_footer(module_globals), file=report_fh)


class TestAllPackagesNoDeps:
    test_class = "nodeps"

    @pytest.mark.parametrize("package_name", PACKAGE_NAME_LIST)
    @pytest.mark.all_packages
    def test_all_packages(
        self,
        monkeypatch,
        capsys: pytest.CaptureFixture,
        caplog,
        module_globals: ModuleGlobalsData,
        start_end_test_class,
        pipx_temp_env,
        package_name: str,
    ):
        pip_cache_purge()
        assert install_package_both_paths(
            monkeypatch,
            capsys,
            caplog,
            module_globals,
            pipx_temp_env,
            package_name,
            deps=False,
        )


class TestAllPackagesDeps:
    test_class = "deps"

    @pytest.mark.parametrize("package_name", PACKAGE_NAME_LIST)
    @pytest.mark.all_packages
    def test_deps_all_packages(
        self,
        monkeypatch,
        capsys: pytest.CaptureFixture,
        caplog,
        module_globals: ModuleGlobalsData,
        start_end_test_class,
        pipx_temp_env,
        package_name: str,
    ):
        pip_cache_purge()
        assert install_package_both_paths(
            monkeypatch,
            capsys,
            caplog,
            module_globals,
            pipx_temp_env,
            package_name,
            deps=True,
        )