From 27bfbf130023536ac6838d6ae7093917fdbf182e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 8 Dec 2023 22:25:21 +0100 Subject: [PATCH 01/20] Extract "package manager" functionality into plugins More and more we see the need for primitives like "install a package". Plugins start to require packages to be present on guests, we already do have `prepare/install` plugin, and the upcoming all mighty install plugin. To support all these functions, `tmt.package_managers` would provide necessary primitives, allowing implementations to be shipped as plugins. The patch starts building the primitives, converting `tmt.steps.prepare.install` to use them in the process. --- .pre-commit-config.yaml | 11 + Makefile | 2 +- docs/releases.rst | 7 + plans/features/basic.fmf | 1 + plans/features/extended-unit-tests.fmf | 29 + plans/main.fmf | 3 +- pyproject.toml | 18 +- tests/unit/__init__.py | 26 +- tests/unit/main.fmf | 64 +- tests/unit/test.sh | 57 ++ tests/unit/test_package_managers.py | 954 +++++++++++++++++++++++++ tests/unit/test_steps_prepare.py | 58 +- tmt/package_managers/__init__.py | 184 +++++ tmt/package_managers/apt.py | 195 +++++ tmt/package_managers/dnf.py | 218 ++++++ tmt/package_managers/rpm_ostree.py | 134 ++++ tmt/plugins/__init__.py | 4 + tmt/steps/prepare/install.py | 573 +++++++-------- tmt/steps/provision/__init__.py | 126 +--- tmt/steps/provision/podman.py | 17 +- tmt/utils.py | 14 +- 21 files changed, 2263 insertions(+), 432 deletions(-) create mode 100644 plans/features/extended-unit-tests.fmf create mode 100755 tests/unit/test.sh create mode 100644 tests/unit/test_package_managers.py create mode 100644 tmt/package_managers/__init__.py create mode 100644 tmt/package_managers/apt.py create mode 100644 tmt/package_managers/dnf.py create mode 100644 tmt/package_managers/rpm_ostree.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02a4f1f2a8..c6eb3b9ace 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,10 @@ repos: - "requests>=2.25.1" # 2.28.2 / 2.31.0 - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 + # Help installation by reducing the set of inspected botocore release. + # There is *a lot* of them, and hatch might fetch many of them. + - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 + # report-junit - "junit_xml>=1.9" @@ -75,6 +79,10 @@ repos: - "requests>=2.25.1" # 2.28.2 / 2.31.0 - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 + # Help installation by reducing the set of inspected botocore release. + # There is *a lot* of them, and hatch might fetch many of them. + - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 + # report-junit - "junit_xml>=1.9" @@ -122,3 +130,6 @@ repos: - "docutils>=0.16" # 0.16 is the current one available for RHEL9 - "pint<0.20" - "pygments>=2.7.4" # 2.7.4 is the current one available for RHEL9 + # Help installation by reducing the set of inspected botocore release. + # There is *a lot* of them, and hatch might fetch many of them. + - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 diff --git a/Makefile b/Makefile index b297bb350e..c422a865da 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ images: ## Build tmt images for podman/docker ## Development ## develop: _deps ## Install development requirements - sudo dnf install -y gcc git python3-nitrate {libvirt,krb5,libpq}-devel jq podman + sudo dnf install -y gcc git python3-nitrate {libvirt,krb5,libpq,python3}-devel jq podman buildah # Git vim tags and cleanup tags: diff --git a/docs/releases.rst b/docs/releases.rst index 4b992d0c67..dd833c14d0 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -24,6 +24,13 @@ connections, and may force reboot of the guest when it becomes unresponsive. This is the first step towards helping tests handle kernel panics and similar situations. +Internal implementation of basic package manager actions has been +refactored. tmt now supports package implementations to be shipped as +plugins, therefore allowing for tmt to work natively with distributions +beyond the ecosystem of rpm-based distributions. As a preview, ``apt``, +the package manager used by Debian and Ubuntu, has been included in this +release. + __ https://pagure.io/testcloud/ diff --git a/plans/features/basic.fmf b/plans/features/basic.fmf index 21e0a088c7..feb8150a06 100644 --- a/plans/features/basic.fmf +++ b/plans/features/basic.fmf @@ -6,3 +6,4 @@ description: discover: how: fmf filter: 'tier: 2 & tag:-provision-only' + exclude: '/tests/unit/.*?/extended' diff --git a/plans/features/extended-unit-tests.fmf b/plans/features/extended-unit-tests.fmf new file mode 100644 index 0000000000..7b5550af40 --- /dev/null +++ b/plans/features/extended-unit-tests.fmf @@ -0,0 +1,29 @@ +summary: Unit tests working with containers +description: | + A subset of unit tests that spawns containers. + +discover: + how: fmf + test: '/tests/unit/.*?/extended' + +# Disable systemd-resolved to prevent dns failures +# See: https://github.com/teemtee/tmt/issues/2063 +adjust+: + - when: initiator == packit and distro == fedora + prepare+: + - name: disable systemd resolved + how: shell + script: | + systemctl unmask systemd-resolved + systemctl disable systemd-resolved + systemctl mask systemd-resolved + rm -f /etc/resolv.conf + systemctl restart NetworkManager + sleep 5 + cat /etc/resolv.conf + + - when: trigger == commit + provision: + hardware: + cpu: + processors: ">= 8" diff --git a/plans/main.fmf b/plans/main.fmf index 6b2b03ae5f..64a42c9bda 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -17,7 +17,8 @@ prepare+: # have to run as root to do that, and who's running tmt test suite # as root? # - - pip3 install --user yq || pip3 install yq + - pip3 install --user hatch yq || pip3 install hatch yq + - hatch --help - yq --help # Use the internal executor diff --git a/pyproject.toml b/pyproject.toml index c24c78f951..2cb45a05d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,9 @@ dependencies = [ # F39 / PyPI "requests>=2.25.1", # 2.28.2 / 2.31.0 "ruamel.yaml>=0.16.6", # 0.17.32 / 0.17.32 "urllib3>=1.26.5, <3.0", # 1.26.16 / 2.0.4 + # Help installation by reducing the set of inspected botocore release. + # There is *a lot* of them, and hatch might fetch many of them. + "botocore>=1.25.10", # 1.25.10 is the current one available for RHEL9 ] [project.optional-dependencies] @@ -119,6 +122,8 @@ dependencies = [ "mypy", "pytest", "python-coveralls", + "pytest-container", + "pytest-xdist", "requre", "yq==3.1.1", "pre-commit", @@ -141,16 +146,16 @@ lint = ["autopep8 {args:.}", "ruff --fix {args:.}"] type = ["mypy {args:tmt}"] check = ["lint", "type"] -unit = "pytest -vvv -ra --showlocals tests/unit" -smoke = "pytest -vvv -ra --showlocals tests/unit/test_cli.py" +unit = "pytest -vvv -ra --showlocals -n 0 tests/unit" +smoke = "pytest -vvv -ra --showlocals -n 0 tests/unit/test_cli.py" cov = [ - "coverage run --source=tmt -m pytest -vvv -ra --showlocals tests", + "coverage run --source=tmt -m pytest -vvv -ra --showlocals -n 0 tests", "coverage report", "coverage annotate", ] requre = [ "cd {root}/tests/integration", - "pytest -vvv -ra --showlocals", + "pytest -vvv -ra --showlocals -n 0", "requre-patch purge --replaces :milestone_url:str:SomeText --replaces :latency:float:0 tests/integration/test_data/test_nitrate/*", ] @@ -314,4 +319,7 @@ extend-immutable-calls = ["tmt.utils.field"] known-first-party = ["tmt"] [tool.pytest.ini_options] -markers = ["web: tests which need to access the web"] +markers = [ + "containers: tests which need to spawn containers", + "web: tests which need to access the web" + ] diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 493eea371b..3ca0758d8c 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -7,7 +7,7 @@ import _pytest.logging import pytest -import tmt.utils +from tmt.utils import remove_color class PatternMatching: @@ -56,6 +56,7 @@ def __init__(self, pattern: str) -> None: def _assert_log( caplog: _pytest.logging.LogCaptureFixture, evaluator: Callable[[Iterable[Any]], bool] = any, + remove_colors: bool = False, not_present: bool = False, **tests: Any ) -> None: @@ -86,12 +87,15 @@ def _assert_log( operators: list[tuple[Callable[[Any], Any], str, Callable[[Any, Any], bool], Any]] = [] for field_name, expected_value in tests.items(): - if field_name.startswith('details_'): + if field_name == 'message' and remove_colors: + def field_getter(record, name): return remove_color(getattr(record, name)) + + elif field_name.startswith('details_'): field_name = field_name.replace('details_', '') def field_getter(record, name): return getattr(record.details, name, None) elif field_name == 'message': - def field_getter(record, name): return tmt.utils.remove_color(getattr(record, name)) + def field_getter(record, name): return remove_color(getattr(record, name)) else: def field_getter(record, name): return getattr(record, name) @@ -164,14 +168,26 @@ def _report(message: str) -> None: def assert_log( caplog: _pytest.logging.LogCaptureFixture, evaluator: Callable[[Iterable[Any]], bool] = any, + remove_colors: bool = False, **tests: Any ) -> None: - _assert_log(caplog, evaluator=evaluator, not_present=False, **tests) + _assert_log( + caplog, + evaluator=evaluator, + remove_colors=remove_colors, + not_present=False, + **tests) def assert_not_log( caplog: _pytest.logging.LogCaptureFixture, evaluator: Callable[[Iterable[Any]], bool] = any, + remove_colors: bool = False, **tests: Any ) -> None: - _assert_log(caplog, evaluator=evaluator, not_present=True, **tests) + _assert_log( + caplog, + evaluator=evaluator, + remove_colors=remove_colors, + not_present=True, + **tests) diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 11d7eaad51..5241d19679 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -1,10 +1,64 @@ summary: Python unit tests description: Run all available python unit tests using pytest. -test: python3 -m pytest -vvv -ra --showlocals -framework: shell -require: - - python3-pytest +duration: 30m environment: LANG: en_US.UTF-8 -tier: 0 + + ENABLE_PARALLELIZATION: "no" + ENABLE_CONTAINERS: "no" + WITH_SYSTEM_PACKAGES: "no" + +require+: + - gcc + - git + - python3-nitrate + - libvirt-devel + - krb5-devel + - libpq-devel + - python3-devel + - jq + - podman + - buildah + +# Run against development packages via `hatch`. +/with-development-packages: + enabled: true + + /basic: + summary: Basic unit tests (development packages) + tier: 0 + + /extended: + summary: Extended unit tests (development packages) + tier: 2 + duration: 1h + + environment+: &extended_environment + ENABLE_PARALLELIZATION: "yes" + ENABLE_CONTAINERS: "yes" + +# Run against system, distro-packaged ones via `venv`. +/with-system-packages: + enabled: false + + environment+: + WITH_SYSTEM_PACKAGES: "yes" + + require+: + - python3-pytest + + adjust+: + - when: initiator == packit + enabled: true + + /basic: + summary: Basic unit tests (system packages) + tier: 0 + + /extended: + summary: Extended unit tests (system packages) + tier: 2 + duration: 1h + + environment+: *extended_environment diff --git a/tests/unit/test.sh b/tests/unit/test.sh new file mode 100755 index 0000000000..037c5a7373 --- /dev/null +++ b/tests/unit/test.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +rlJournalStart + rlPhaseStartSetup + ENABLE_PARALLELIZATION="${ENABLE_PARALLELIZATION:-no}" + ENABLE_CONTAINERS="${ENABLE_CONTAINERS:-no}" + # TODO: `test` seems more natural, but creates 3 environments, + # one per available Python installation. I need to check whether + # to disable or take advantage of it. + HATCH_ENVIRONMENT="${HATCH_ENVIRONMENT:-dev}" + + rlLogInfo "ENABLE_PARALLELIZATION=$ENABLE_PARALLELIZATION" + rlLogInfo "ENABLE_CONTAINERS=$ENABLE_CONTAINERS" + rlLogInfo "WITH_SYSTEM_PACKAGES=$WITH_SYSTEM_PACKAGES" + rlLogInfo "HATCH_ENVIRONMENT=$HATCH_ENVIRONMENT" + + if [ "$ENABLE_PARALLELIZATION" = "yes" ]; then + PYTEST_PARALLELIZE="-n auto" + else + PYTEST_PARALLELIZE="-n 0" + fi + + if [ "$ENABLE_CONTAINERS" = "yes" ]; then + PYTEST_MARK="-m containers" + else + PYTEST_MARK="-m 'not containers'" + fi + + rlLogInfo "PYTEST_PARALLELIZE=$PYTEST_PARALLELIZE" + rlLogInfo "PYTEST_MARK=$PYTEST_MARK" + + rlRun "PYTEST_COMMAND='pytest -vvv -ra --showlocals'" + rlPhaseEnd + + if [ "$WITH_SYSTEM_PACKAGES" = "yes" ]; then + rlPhaseStartTest "Unit tests against system Python packages" + rlRun "TEST_VENV=$(mktemp -d)" + + rlRun "python3 -m venv $TEST_VENV --system-site-packages" + rlRun "$TEST_VENV/bin/pip install pytest-container pytest-xdist" + + # Note: we're not in the root directory! + rlRun "$TEST_VENV/bin/python3 -m $PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK ." + + rlRun "rm -rf $TEST_VENV" + rlPhaseEnd + else + rlPhaseStartTest "Unit tests" + rlRun "hatch -v run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" + rlPhaseEnd + fi + + rlPhaseStartCleanup + rlPhaseEnd +rlJournalEnd diff --git a/tests/unit/test_package_managers.py b/tests/unit/test_package_managers.py new file mode 100644 index 0000000000..3290549712 --- /dev/null +++ b/tests/unit/test_package_managers.py @@ -0,0 +1,954 @@ +from collections.abc import Iterator +from inspect import isclass +from typing import Optional + +import _pytest.logging +import pytest +from pytest_container import Container +from pytest_container.container import ContainerData + +import tmt.log +import tmt.package_managers +import tmt.package_managers.apt +import tmt.package_managers.dnf +import tmt.package_managers.rpm_ostree +import tmt.plugins +import tmt.steps.provision.podman +import tmt.utils +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + Package, + PackageManager, + PackageManagerClass, + ) +from tmt.steps.provision.podman import GuestContainer, PodmanGuestData +from tmt.utils import ShellScript + +from . import MATCH, assert_log + +# We will need a logger... +logger = tmt.Logger.create() +logger.add_console_handler() + +# Explore available plugins +tmt.plugins.explore(logger) + + +CONTAINER_FEDORA_RAWHIDE = Container(url='registry.fedoraproject.org/fedora:rawhide') +CONTAINER_FEDORA_39 = Container(url='registry.fedoraproject.org/fedora:39') +CONTAINER_CENTOS_STREAM_8 = Container(url='quay.io/centos/centos:stream8') +CONTAINER_CENTOS_7 = Container(url='quay.io/centos/centos:7') +CONTAINER_UBUNTU_2204 = Container(url='docker.io/library/ubuntu:22.04') +CONTAINER_FEDORA_COREOS = Container(url='quay.io/fedora/fedora-coreos:stable') + +PACKAGE_MANAGER_DNF5 = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('dnf5') +PACKAGE_MANAGER_DNF = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('dnf') +PACKAGE_MANAGER_YUM = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('yum') +PACKAGE_MANAGER_APT = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('apt') +PACKAGE_MANAGER_RPMOSTREE = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY \ + .get_plugin('rpm-ostree') + + +CONTAINER_BASE_MATRIX = [ + # Fedora + (CONTAINER_FEDORA_RAWHIDE, PACKAGE_MANAGER_DNF5), + (CONTAINER_FEDORA_RAWHIDE, PACKAGE_MANAGER_DNF), + (CONTAINER_FEDORA_RAWHIDE, PACKAGE_MANAGER_YUM), + + (CONTAINER_FEDORA_39, PACKAGE_MANAGER_DNF5), + (CONTAINER_FEDORA_39, PACKAGE_MANAGER_DNF), + (CONTAINER_FEDORA_39, PACKAGE_MANAGER_YUM), + + # CentOS Stream + (CONTAINER_CENTOS_STREAM_8, PACKAGE_MANAGER_DNF), + (CONTAINER_CENTOS_STREAM_8, PACKAGE_MANAGER_YUM), + + # CentOS + (CONTAINER_CENTOS_7, PACKAGE_MANAGER_YUM), + + # Ubuntu + (CONTAINER_UBUNTU_2204, PACKAGE_MANAGER_APT), + + # Fedora CoreOS + (CONTAINER_FEDORA_COREOS, PACKAGE_MANAGER_RPMOSTREE), + ] + +CONTAINER_MATRIX_IDS = [ + f'{container.url} / {package_manager_class.__name__.lower()}' + for container, package_manager_class in CONTAINER_BASE_MATRIX + ] + + +@pytest.fixture(name='guest') +def fixture_guest(container: ContainerData, root_logger: tmt.log.Logger) -> GuestContainer: + guest_data = PodmanGuestData( + image=container.image_url_or_id, + container=container.container_id + ) + + guest = GuestContainer( + logger=root_logger, + data=guest_data, + name='dummy-container') + + guest.start() + + return guest + + +@pytest.fixture(name='guest_per_test') +def fixture_guest_per_test( + container_per_test: ContainerData, + root_logger: tmt.log.Logger) -> GuestContainer: + guest_data = PodmanGuestData( + image=container_per_test.image_url_or_id, + container=container_per_test.container_id + ) + + guest = GuestContainer( + logger=root_logger, + data=guest_data, + name='dummy-container') + + guest.start() + + return guest + + +def create_package_manager( + container: ContainerData, + guest: GuestContainer, + package_manager_class: PackageManagerClass, + logger: tmt.log.Logger) -> PackageManager: + guest_data = tmt.steps.provision.podman.PodmanGuestData( + image=container.image_url_or_id, + container=container.container_id + ) + + guest = tmt.steps.provision.podman.GuestContainer( + logger=logger, + data=guest_data, + name='dummy-container') + guest.start() + + if package_manager_class is tmt.package_managers.dnf.Dnf5: + guest.execute(ShellScript('dnf install --nogpgcheck -y dnf5')) + + elif package_manager_class is tmt.package_managers.apt.Apt: + guest.execute(ShellScript('apt update')) + + return package_manager_class(guest=guest, logger=logger) + + +def _parametrize_test_install() -> \ + Iterator[tuple[Container, PackageManagerClass, str, Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Yum: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree \|\| yum install -y tree && rpm -q --whatprovides tree", \ + 'Installed:\n tree', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree \|\| dnf install -y tree", \ + 'Installed:\n tree', \ + None + + elif package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree \|\| dnf5 install -y tree", \ + None, \ + None + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tree \|\| apt install -y tree", \ + 'Setting up tree', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree \|\| rpm-ostree install --apply-live --idempotent --allow-inactive tree", \ + 'Installing: tree', \ + None # noqa: E501 + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container_per_test', + 'package_manager_class', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_parametrize_test_install()), + indirect=["container_per_test"], + ids=CONTAINER_MATRIX_IDS) +def test_install( + container_per_test: ContainerData, + guest_per_test: GuestContainer, + package_manager_class: PackageManagerClass, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager( + container_per_test, + guest_per_test, + package_manager_class, + root_logger) + + output = package_manager.install(Package('tree')) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + if expected_stdout: + assert output.stdout is not None + assert expected_stdout in output.stdout + + if expected_stderr: + assert output.stderr is not None + assert expected_stderr in output.stderr + + +def _parametrize_test_install_nonexistent() -> \ + Iterator[tuple[Container, PackageManagerClass, str, Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| dnf5 install -y tree-but-spelled-wrong", \ + None, \ + 'No match for argument: tree-but-spelled-wrong' # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| dnf install -y tree-but-spelled-wrong", \ + None, \ + 'Error: Unable to find a match: tree-but-spelled-wrong' # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Yum: + if 'fedora' in container.url: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + None, \ + 'Error: Unable to find a match: tree-but-spelled-wrong' # noqa: E501 + + elif 'centos' in container.url and 'centos:7' not in container.url: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + 'No match for argument: tree-but-spelled-wrong', \ + 'Error: Unable to find a match: tree-but-spelled-wrong' # noqa: E501 + + else: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + 'No package tree-but-spelled-wrong available.', \ + 'Error: Nothing to do' # noqa: E501 + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tree-but-spelled-wrong \|\| apt install -y tree-but-spelled-wrong", \ + None, \ + 'E: Unable to locate package tree-but-spelled-wrong' # noqa: E501 + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| rpm-ostree install --apply-live --idempotent --allow-inactive tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + 'error: Packages not found: tree-but-spelled-wrong' # noqa: E501 + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container', + 'package_manager_class', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_parametrize_test_install_nonexistent()), + indirect=["container"], + ids=CONTAINER_MATRIX_IDS) +def test_install_nonexistent( + container: ContainerData, + guest: GuestContainer, + package_manager_class: PackageManagerClass, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager(container, guest, package_manager_class, root_logger) + + with pytest.raises(tmt.utils.RunError) as excinfo: + package_manager.install(Package('tree-but-spelled-wrong')) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + assert excinfo.type is tmt.utils.RunError + assert excinfo.value.returncode != 0 + + if expected_stdout: + assert excinfo.value.stdout is not None + assert expected_stdout in excinfo.value.stdout + + if expected_stderr: + assert excinfo.value.stderr is not None + assert expected_stderr in excinfo.value.stderr + + +def _parametrize_test_install_nonexistent_skip() -> \ + Iterator[tuple[Container, PackageManagerClass, str, Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| dnf5 install -y --skip-unavailable tree-but-spelled-wrong", \ + None, \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| dnf install -y --skip-broken tree-but-spelled-wrong", \ + 'No match for argument: tree-but-spelled-wrong', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Yum: + if 'fedora' in container.url: # noqa: SIM114 + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y --skip-broken tree-but-spelled-wrong \|\| /bin/true", \ + 'No match for argument: tree-but-spelled-wrong', \ + None # noqa: E501 + + elif 'centos' in container.url and 'centos:7' not in container.url: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y --skip-broken tree-but-spelled-wrong \|\| /bin/true", \ + 'No match for argument: tree-but-spelled-wrong', \ + None # noqa: E501 + + else: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| yum install -y --skip-broken tree-but-spelled-wrong \|\| /bin/true", \ + 'No package tree-but-spelled-wrong available.', \ + 'Error: Nothing to do' # noqa: E501 + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tree-but-spelled-wrong \|\| apt install -y --ignore-missing tree-but-spelled-wrong \|\| /bin/true", \ + None, \ + 'E: Unable to locate package tree-but-spelled-wrong' # noqa: E501 + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + r"rpm -q --whatprovides tree-but-spelled-wrong \|\| rpm-ostree install --apply-live --idempotent --allow-inactive tree-but-spelled-wrong \|\| /bin/true", \ + 'no package provides tree-but-spelled-wrong', \ + 'error: Packages not found: tree-but-spelled-wrong' # noqa: E501 + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container', + 'package_manager_class', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_parametrize_test_install_nonexistent_skip()), + indirect=["container"], + ids=CONTAINER_MATRIX_IDS) +def test_install_nonexistent_skip( + container: ContainerData, + guest: GuestContainer, + package_manager_class: PackageManagerClass, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager(container, guest, package_manager_class, root_logger) + + output = package_manager.install( + Package('tree-but-spelled-wrong'), + options=Options(skip_missing=True) + ) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + if expected_stdout: + assert output.stdout is not None + assert expected_stdout in output.stdout + + if expected_stderr: + assert output.stderr is not None + assert expected_stderr in output.stderr + + +def _parametrize_test_install_dont_check_first() -> \ + Iterator[tuple[Container, PackageManagerClass, str, Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + r"dnf5 install -y tree", \ + None, \ + None + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + r"dnf install -y tree", \ + 'Installed:\n tree', \ + None + + elif package_manager_class is tmt.package_managers.dnf.Yum: + yield container, \ + package_manager_class, \ + r"yum install -y tree && rpm -q --whatprovides tree", \ + 'Installed:\n tree', \ + None + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + r"export DEBIAN_FRONTEND=noninteractive; apt install -y tree", \ + 'Setting up tree', \ + None + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + r"rpm-ostree install --apply-live --idempotent --allow-inactive tree", \ + 'Installing: tree', \ + None + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container_per_test', + 'package_manager_class', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_parametrize_test_install_dont_check_first()), + indirect=["container_per_test"], + ids=CONTAINER_MATRIX_IDS) +def test_install_dont_check_first( + container_per_test: ContainerData, + guest_per_test: GuestContainer, + package_manager_class: PackageManagerClass, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager( + container_per_test, + guest_per_test, + package_manager_class, + root_logger) + + output = package_manager.install( + Package('tree'), + options=Options(check_first=False) + ) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + if expected_stdout: + assert output.stdout is not None + assert expected_stdout in output.stdout + + if expected_stderr: + assert output.stderr is not None + assert expected_stderr in output.stderr + + +def _parametrize_test_reinstall() -> Iterator[tuple[ + Container, PackageManagerClass, Optional[str], Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Yum: + if 'centos:7' in container.url: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tar && yum reinstall -y tar && rpm -q --whatprovides tar", \ + 'Reinstalling:\n tar', \ + None # noqa: E501 + + else: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tar && yum reinstall -y tar && rpm -q --whatprovides tar", \ + 'Reinstalled:\n tar', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tar && dnf reinstall -y tar", \ + 'Reinstalled:\n tar', \ + None + + elif package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tar && dnf5 reinstall -y tar", \ + 'Reinstalling tar', \ + None + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + True, \ + r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tar && apt reinstall -y tar", \ + 'Setting up tar', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + False, \ + None, \ + None, \ + None + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container_per_test', + 'package_manager_class', + 'supported', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_parametrize_test_reinstall()), + indirect=["container_per_test"], + ids=CONTAINER_MATRIX_IDS) +def test_reinstall( + container_per_test: ContainerData, + guest_per_test: GuestContainer, + package_manager_class: PackageManagerClass, + supported: bool, + expected_command: Optional[str], + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager( + container_per_test, + guest_per_test, + package_manager_class, + root_logger) + + if supported: + assert expected_command is not None + + output = package_manager.reinstall(Package('tar')) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + else: + with pytest.raises(tmt.utils.GeneralError) as excinfo: + package_manager.reinstall(Package('tar')) + + assert excinfo.value.message \ + == "rpm-ostree does not support reinstall operation." + + if expected_stdout: + assert output.stdout is not None + assert expected_stdout in output.stdout + + if expected_stderr: + assert output.stderr is not None + assert expected_stderr in output.stderr + + +def _generate_test_reinstall_nonexistent_matrix() -> Iterator[tuple[ + Container, PackageManagerClass, Optional[str], Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tree-but-spelled-wrong && dnf5 reinstall -y tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tree-but-spelled-wrong && dnf reinstall -y tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.dnf.Yum: + if 'fedora' in container.url: # noqa: SIM114 + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tree-but-spelled-wrong && yum reinstall -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + None # noqa: E501 + + elif 'centos' in container.url and 'centos:7' not in container.url: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tree-but-spelled-wrong && yum reinstall -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + None # noqa: E501 + + else: + yield container, \ + package_manager_class, \ + True, \ + r"rpm -q --whatprovides tree-but-spelled-wrong && yum reinstall -y tree-but-spelled-wrong && rpm -q --whatprovides tree-but-spelled-wrong", \ + 'no package provides tree-but-spelled-wrong', \ + None # noqa: E501 + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + True, \ + r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tree-but-spelled-wrong && apt reinstall -y tree-but-spelled-wrong", \ + None, \ + 'dpkg-query: no packages found matching tree-but-spelled-wrong' # noqa: E501 + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + False, \ + None, \ + None, \ + None + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container', + 'package_manager_class', + 'supported', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_generate_test_reinstall_nonexistent_matrix()), + indirect=["container"], + ids=CONTAINER_MATRIX_IDS) +def test_reinstall_nonexistent( + container: ContainerData, + guest: GuestContainer, + package_manager_class: PackageManagerClass, + supported: bool, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager(container, guest, package_manager_class, root_logger) + + if supported: + assert expected_command is not None + + with pytest.raises(tmt.utils.RunError) as excinfo: + package_manager.reinstall(Package('tree-but-spelled-wrong')) + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + assert excinfo.type is tmt.utils.RunError + assert excinfo.value.returncode != 0 + + else: + with pytest.raises(tmt.utils.GeneralError) as excinfo: + package_manager.reinstall(Package('tree-but-spelled-wrong')) + + assert excinfo.value.message \ + == "rpm-ostree does not support reinstall operation." + + if expected_stdout: + assert excinfo.value.stdout is not None + assert expected_stdout in excinfo.value.stdout + + if expected_stderr: + assert excinfo.value.stderr is not None + assert expected_stderr in excinfo.value.stderr + + +def _generate_test_check_presence() -> Iterator[ + tuple[Container, PackageManagerClass, Installable, str, Optional[str], Optional[str]]]: + + for container, package_manager_class in CONTAINER_BASE_MATRIX: + if package_manager_class is tmt.package_managers.dnf.Dnf5: + yield container, \ + package_manager_class, \ + Package('util-linux-core'), \ + True, \ + r"rpm -q --whatprovides util-linux-core", \ + r'\s+out:\s+util-linux-core-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/flock", \ + r'\s+out:\s+util-linux-core-', \ + None + + elif package_manager_class is tmt.package_managers.dnf.Dnf: + if 'centos:stream8' in container.url: + yield container, \ + package_manager_class, \ + Package('util-linux'), \ + True, \ + r"rpm -q --whatprovides util-linux", \ + r'\s+out:\s+util-linux-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/flock", \ + r'\s+out:\s+util-linux-', \ + None + + else: + yield container, \ + package_manager_class, \ + Package('util-linux-core'), \ + True, \ + r"rpm -q --whatprovides util-linux-core", \ + r'\s+out:\s+util-linux-core-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/flock", \ + r'\s+out:\s+util-linux-core-', \ + None + + elif package_manager_class is tmt.package_managers.dnf.Yum: + if 'centos:stream8' in container.url or 'centos:7' in container.url: + yield container, \ + package_manager_class, \ + Package('util-linux'), \ + True, \ + r"rpm -q --whatprovides util-linux", \ + r'\s+out:\s+util-linux-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/flock", \ + r'\s+out:\s+util-linux-', \ + None + + else: + yield container, \ + package_manager_class, \ + Package('util-linux-core'), \ + True, \ + r"rpm -q --whatprovides util-linux-core", \ + r'\s+out:\s+util-linux-core-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/flock", \ + r'\s+out:\s+util-linux-core-', \ + None + + elif package_manager_class is tmt.package_managers.apt.Apt: + yield container, \ + package_manager_class, \ + Package('util-linux'), \ + True, \ + r"dpkg-query --show util-linux", \ + r'\s+out:\s+util-linux', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"dpkg-query --show tree-but-spelled-wrong", \ + None, \ + r'\s+err:\s+dpkg-query: no packages found matching tree-but-spelled-wrong' + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"dpkg-query --show util-linux", \ + r'\s+out:\s+util-linux', \ + None + + elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: + yield container, \ + package_manager_class, \ + Package('util-linux'), \ + True, \ + r"rpm -q --whatprovides util-linux", \ + r'\s+out:\s+util-linux', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/flock'), \ + True, \ + r"rpm -qf /usr/bin/flock", \ + r'\s+out:\s+util-linux-core', \ + None + + else: + pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") + + +def _generate_test_check_presence_ids(value) -> str: + if isinstance(value, Container): + return value.url + + if isclass(value) and issubclass(value, tmt.package_managers.PackageManager): + return value.__name__.lower() + + if isinstance(value, (Package, FileSystemPath)): + return str(value) + + return '' + + +@pytest.mark.containers() +@pytest.mark.parametrize(('container', + 'package_manager_class', + 'installable', + 'expected_result', + 'expected_command', + 'expected_stdout', + 'expected_stderr'), + list(_generate_test_check_presence()), + indirect=["container"], + ids=_generate_test_check_presence_ids) +def test_check_presence( + container: ContainerData, + guest: GuestContainer, + package_manager_class: PackageManagerClass, + installable: Installable, + expected_result: bool, + expected_command: str, + expected_stdout: Optional[str], + expected_stderr: Optional[str], + root_logger: tmt.log.Logger, + caplog: _pytest.logging.LogCaptureFixture) -> None: + package_manager = create_package_manager(container, guest, package_manager_class, root_logger) + + assert package_manager.check_presence(installable) == {installable: expected_result} + + assert_log(caplog, message=MATCH( + rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) + + if expected_stdout: + assert_log(caplog, remove_colors=True, message=MATCH(expected_stdout)) + + if expected_stderr: + assert_log(caplog, remove_colors=True, message=MATCH(expected_stderr)) diff --git a/tests/unit/test_steps_prepare.py b/tests/unit/test_steps_prepare.py index f4558b74a1..ffc3e58f5d 100644 --- a/tests/unit/test_steps_prepare.py +++ b/tests/unit/test_steps_prepare.py @@ -1,50 +1,32 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock -import tmt -from tmt.log import Logger +from tmt.base import DependencySimple from tmt.steps.prepare.install import InstallBase -def prepare_command(self): - """ Fake prepare_command() for InstallBase """ - return ("command", "options") - - -def get(what, default=None): - """ Fake get() for parent PrepareInstall """ - - if what == "directory": - return [] - - if what == "missing": - return "skip" - - if what == "package": - return [ - # Regular packages - "wget", - "debuginfo-something", - "elfutils-debuginfod", - # Debuginfo packages - "grep-debuginfo", - "elfutils-debuginfod-debuginfo", - ] - - return None - - -@patch.object(tmt.steps.prepare.install.InstallBase, 'prepare_command', prepare_command) -def test_debuginfo(): +def test_debuginfo(root_logger): """ Check debuginfo package parsing """ - logger = Logger.create() parent = MagicMock() - parent.get = get guest = MagicMock() - install = InstallBase(parent=parent, logger=logger, guest=guest) - - assert install.repository_packages == [ + install = InstallBase( + parent=parent, + dependencies=[ + # Regular packages + DependencySimple("wget"), + DependencySimple("debuginfo-something"), + DependencySimple("elfutils-debuginfod"), + # Debuginfo packages + DependencySimple("grep-debuginfo"), + DependencySimple("elfutils-debuginfod-debuginfo"), + ], + directories=[], + exclude=[], + logger=root_logger, + guest=guest) + + assert install.packages == [ "wget", "debuginfo-something", "elfutils-debuginfod", diff --git a/tmt/package_managers/__init__.py b/tmt/package_managers/__init__.py new file mode 100644 index 0000000000..1a696127ee --- /dev/null +++ b/tmt/package_managers/__init__.py @@ -0,0 +1,184 @@ +import dataclasses +import shlex +import sys +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any, Callable, Optional, Union + +import tmt +import tmt.log +import tmt.plugins +import tmt.utils +from tmt.utils import Command, CommandOutput, Path + +if TYPE_CHECKING: + from tmt.steps.provision import Guest + + # Using TypeAlias and typing-extensions under the guard of TYPE_CHECKING, + # to avoid the necessity of requiring the package in runtime. This way, + # we can deal with it in build time and when running tests. + if sys.version_info >= (3, 10): + from typing import TypeAlias + else: + from typing_extensions import TypeAlias + + #: A type of package manager names. + GuestPackageManager: TypeAlias = str + + +# +# Installable objects +# +class Package(str): + """ A package name """ + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, (Package, PackageUrl, FileSystemPath, PackagePath)): + raise NotImplementedError + + return str(self) < str(other) + + +class PackageUrl(str): + """ A URL of a package file """ + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, (Package, PackageUrl, FileSystemPath, PackagePath)): + raise NotImplementedError + + return str(self) < str(other) + + +class FileSystemPath(Path): + """ A filesystem path provided by a package """ + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, (Package, PackageUrl, FileSystemPath, PackagePath)): + raise NotImplementedError + + return str(self) < str(other) + + +class PackagePath(Path): + """ A path to a package file """ + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, (Package, PackageUrl, FileSystemPath, PackagePath)): + raise NotImplementedError + + return str(self) < str(other) + + +#: All installable objects. +Installable = Union[Package, FileSystemPath, PackagePath, PackageUrl] + + +PackageManagerClass = type['PackageManager'] + + +_PACKAGE_MANAGER_PLUGIN_REGISTRY: tmt.plugins.PluginRegistry[PackageManagerClass] = \ + tmt.plugins.PluginRegistry() + + +def provides_package_manager( + package_manager: str) -> Callable[[PackageManagerClass], PackageManagerClass]: + """ + A decorator for registering package managers. + + Decorate a package manager plugin class to register a package manager. + """ + + def _provides_package_manager(package_manager_cls: PackageManagerClass) -> PackageManagerClass: + _PACKAGE_MANAGER_PLUGIN_REGISTRY.register_plugin( + plugin_id=package_manager, + plugin=package_manager_cls, + logger=tmt.log.Logger.get_bootstrap_logger()) + + return package_manager_cls + + return _provides_package_manager + + +def find_package_manager(name: 'GuestPackageManager') -> 'PackageManagerClass': + """ + Find a package manager by its name. + + :raises GeneralError: when the plugin does not exist. + """ + + plugin = _PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin(name) + + if plugin is None: + raise tmt.utils.GeneralError( + f"Package manager '{name}' was not found in package manager registry.") + + return plugin + + +def escape_installables(*installables: Installable) -> Iterator[str]: + for installable in installables: + yield shlex.quote(str(installable)) + + +# TODO: find a better name, "options" is soooo overloaded... +@dataclasses.dataclass(frozen=True) +class Options: + #: A list of packages to exclude from installation. + excluded_packages: list[Package] = dataclasses.field(default_factory=list) + + #: If set, a failure to install a given package would not cause an error. + skip_missing: bool = False + + #: If set, check whether the package is already installed, and do not + #: attempt to install it if it is already present. + check_first: bool = True + + #: If set, install packages under this path instead of the usual system + #: root. + install_root: Optional[Path] = None + + #: If set, instruct package manager to behave as if the distribution release + #: was ``release_version``. + release_version: Optional[str] = None + + +class PackageManager(tmt.utils.Common): + """ A base class for package manager plugins """ + + #: A command to run to check whether the package manager is available on + #: a guest. + probe_command: Command + + command: Command + options: Command + + def __init__(self, *, guest: 'Guest', logger: tmt.log.Logger) -> None: + super().__init__(logger=logger) + + self.guest = guest + self.command, self.options = self.prepare_command() + + def prepare_command(self) -> tuple[Command, Command]: + """ Prepare installation command and subcommand options """ + raise NotImplementedError + + def check_presence(self, *installables: Installable) -> dict[Installable, bool]: + """ Return a presence status for each given installable """ + raise NotImplementedError + + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise NotImplementedError + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise NotImplementedError + + def install_debuginfo( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise NotImplementedError diff --git a/tmt/package_managers/apt.py b/tmt/package_managers/apt.py new file mode 100644 index 0000000000..b821fc9289 --- /dev/null +++ b/tmt/package_managers/apt.py @@ -0,0 +1,195 @@ +import re +from typing import Optional, Union + +import tmt.package_managers +import tmt.utils +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + Package, + PackagePath, + escape_installables, + provides_package_manager, + ) +from tmt.utils import ( + Command, + CommandOutput, + Environment, + EnvVarValue, + GeneralError, + RunError, + ShellScript, + ) + +ReducedPackages = list[Union[Package, PackagePath]] + + +@provides_package_manager('apt') +class Apt(tmt.package_managers.PackageManager): + probe_command = Command('apt', '--version') + + install_command = Command('install') + + _sudo_prefix: Command + + def prepare_command(self) -> tuple[Command, Command]: + """ Prepare installation command for apt """ + + if self.guest.facts.is_superuser is False: + self._sudo_prefix = Command('sudo') + + else: + self._sudo_prefix = Command() + + command = Command() + options = Command('-y') + + command += self._sudo_prefix + command += Command('apt') + + return (command, options) + + def _enable_apt_file(self) -> None: + self.install(Package('apt-file')) + self.guest.execute(ShellScript(f'{self._sudo_prefix} apt-file update')) + + def path_to_package(self, path: FileSystemPath) -> Package: + """ + Find a package providing given filesystem path. + + This is not trivial as some are used to from ``yum`` or ``dnf``, + it requires installation of ``apt-file`` utility and building + an index of packages and filesystem paths. + """ + + self._enable_apt_file() + + output = self.guest.execute(ShellScript(f'apt-file search {path} || exit 0')) + + assert output.stdout is not None + + package_names = output.stdout.strip().splitlines() + + if not package_names: + raise GeneralError(f"No package provides {path}.") + + return Package(package_names[0].split(':')[0]) + + def _reduce_to_packages(self, *installables: Installable) -> ReducedPackages: + packages: ReducedPackages = [] + + for installable in installables: + if isinstance(installable, (Package, PackagePath)): + packages.append(installable) + + elif isinstance(installable, FileSystemPath): + packages.append(self.path_to_package(installable)) + + else: + raise GeneralError(f"Package specification '{installable}' is not supported.") + + return packages + + def _construct_presence_script( + self, *installables: Installable) -> tuple[ReducedPackages, ShellScript]: + reduced_packages = self._reduce_to_packages(*installables) + + return reduced_packages, ShellScript( + f'dpkg-query --show {" ".join(escape_installables(*reduced_packages))}') + + def check_presence(self, *installables: Installable) -> dict[Installable, bool]: + reduced_packages, presence_script = self._construct_presence_script(*installables) + + try: + output = self.guest.execute(presence_script) + stdout, stderr = output.stdout, output.stderr + + except RunError as exc: + stdout, stderr = exc.stdout, exc.stderr + + if stdout is None or stderr is None: + raise GeneralError("apt presence check provided no output") + + results: dict[Installable, bool] = {} + + for installable, package in zip(installables, reduced_packages): + match = re.search( + rf'dpkg-query: no packages found matching {re.escape(str(package))}', stderr) + + if match is not None: + results[installable] = False + continue + + match = re.search(rf'^{re.escape(str(package))}\s', stdout) + + if match is not None: + results[installable] = True + continue + + return results + + def _extra_options(self, options: Options) -> Command: + extra_options = Command() + + if options.skip_missing: + extra_options += Command('--ignore-missing') + + return extra_options + + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + options = options or Options() + + extra_options = self._extra_options(options) + packages = self._reduce_to_packages(*installables) + + script = ShellScript( + f'{self.command.to_script()} install ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*packages))}') + + if options.check_first: + script = self._construct_presence_script(*packages)[1] | script + + # TODO: find a better way to handle `skip_missing`, this may hide other + # kinds of errors. But `--ignore-missing` does not turn exit code into + # zero :/ + if options.skip_missing: + script = script | ShellScript('/bin/true') + + return self.guest.execute(script, env=Environment({ + 'DEBIAN_FRONTEND': EnvVarValue('noninteractive') + })) + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + options = options or Options() + + extra_options = self._extra_options(options) + packages = self._reduce_to_packages(*installables) + + script = ShellScript( + f'{self.command.to_script()} reinstall ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*packages))}') + + if options.check_first: + script = self._construct_presence_script(*packages)[1] & script + + if options.skip_missing: + script = script | ShellScript('/bin/true') + + return self.guest.execute(script, env=Environment({ + 'DEBIAN_FRONTEND': EnvVarValue('noninteractive') + })) + + def install_debuginfo( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise tmt.utils.GeneralError("There is no support for debuginfo packages in apt.") diff --git a/tmt/package_managers/dnf.py b/tmt/package_managers/dnf.py new file mode 100644 index 0000000000..dbc5e65258 --- /dev/null +++ b/tmt/package_managers/dnf.py @@ -0,0 +1,218 @@ +import re +from typing import Optional, cast + +import tmt.package_managers +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + escape_installables, + provides_package_manager, + ) +from tmt.utils import Command, CommandOutput, GeneralError, RunError, ShellScript + + +@provides_package_manager('dnf') +class Dnf(tmt.package_managers.PackageManager): + probe_command = Command('dnf', '--version') + + _base_command = Command('dnf') + + skip_missing_option = '--skip-broken' + + def prepare_command(self) -> tuple[Command, Command]: + options = Command('-y') + command = Command() + + if self.guest.facts.is_superuser is False: + command += Command('sudo') + + command += self._base_command + + return (command, options) + + def _extra_dnf_options(self, options: Options) -> Command: + """ Collect additional options for ``yum``/``dnf`` based on given options """ + + extra_options = Command() + + for package in options.excluded_packages: + extra_options += Command('--exclude', package) + + if options.skip_missing: + extra_options += Command(self.skip_missing_option) + + return extra_options + + def _construct_presence_script(self, *installables: Installable) -> ShellScript: + return ShellScript( + f'rpm -q --whatprovides {" ".join(escape_installables(*installables))}' + ) + + def check_presence(self, *installables: Installable) -> dict[Installable, bool]: + try: + output = self.guest.execute(self._construct_presence_script(*installables)) + stdout = output.stdout + + except RunError as exc: + stdout = exc.stdout + + if stdout is None: + raise GeneralError("rpm presence check provided no output") + + results: dict[Installable, bool] = {} + + for line, installable in zip(stdout.strip().splitlines(), installables): + match = re.match(rf'package {re.escape(str(installable))} is not installed', line) + if match is not None: + results[installable] = False + continue + + match = re.match(rf'no package provides {re.escape(str(installable))}', line) + if match is not None: + results[installable] = False + continue + + results[installable] = True + + return results + + def _construct_install_script( + self, + *installables: Installable, + options: Optional[Options] = None) -> ShellScript: + options = options or Options() + + extra_options = self._extra_dnf_options(options) + + script = ShellScript( + f'{self.command.to_script()} install ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*installables))}') + + if options.check_first: + script = self._construct_presence_script(*installables) | script + + return script + + def _construct_reinstall_script( + self, + *installables: Installable, + options: Optional[Options] = None) -> ShellScript: + options = options or Options() + + extra_options = self._extra_dnf_options(options) + + script = ShellScript( + f'{self.command.to_script()} reinstall ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*installables))}') + + if options.check_first: + script = self._construct_presence_script(*installables) & script + + return script + + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + return self.guest.execute(self._construct_install_script( + *installables, + options=options)) + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + return self.guest.execute(self._construct_reinstall_script( + *installables, + options=options + )) + + def install_debuginfo( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + # Make sure debuginfo-install is present on the target system + self.install(FileSystemPath('/usr/bin/debuginfo-install')) + + options = options or Options() + + extra_options = self._extra_dnf_options(options) + + return self.guest.execute(ShellScript( + f'debuginfo-install -y ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*installables))}')) + + +@provides_package_manager('dnf5') +class Dnf5(Dnf): + probe_command = Command('dnf5', '--version') + + _base_command = Command('dnf5') + skip_missing_option = '--skip-unavailable' + + +@provides_package_manager('yum') +class Yum(Dnf): + probe_command = Command('yum', '--version') + + _base_command = Command('yum') + + # TODO: get rid of those `type: ignore` below. I think it's caused by the + # decorator, it might be messing with the class inheritance as seen by pyright, + # but mypy sees no issue, pytest sees no issue, everything works. Silencing + # for now. + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + + options = options or Options() + + script = cast( # type: ignore[redundant-cast] + ShellScript, + self._construct_install_script( # type: ignore[reportGeneralIssues,unused-ignore] + *installables, + options=options + )) + + # Extra ignore/check for yum to workaround BZ#1920176 + if options.skip_missing: + script |= ShellScript('/bin/true') + + else: + script &= cast( # type: ignore[redundant-cast] + ShellScript, + self._construct_presence_script( # type: ignore[reportGeneralIssues,unused-ignore] + *installables)) + + return self.guest.execute(script) + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + + options = options or Options() + + script = cast( # type: ignore[redundant-cast] + ShellScript, + self._construct_reinstall_script( # type: ignore[reportGeneralIssues,unused-ignore] + *installables, + options=options + )) + + # Extra ignore/check for yum to workaround BZ#1920176 + if options.skip_missing: + script |= ShellScript('/bin/true') + + else: + script &= cast( # type: ignore[redundant-cast] + ShellScript, + self._construct_presence_script( # type: ignore[reportGeneralIssues,unused-ignore] + *installables)) + + return self.guest.execute(script) diff --git a/tmt/package_managers/rpm_ostree.py b/tmt/package_managers/rpm_ostree.py new file mode 100644 index 0000000000..ac5d49d4f2 --- /dev/null +++ b/tmt/package_managers/rpm_ostree.py @@ -0,0 +1,134 @@ +import re +from typing import Optional + +import tmt.package_managers +import tmt.utils +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + escape_installables, + provides_package_manager, + ) +from tmt.utils import Command, CommandOutput, GeneralError, RunError, ShellScript + + +@provides_package_manager('rpm-ostree') +class RpmOstree(tmt.package_managers.PackageManager): + probe_command = Command('stat', '/run/ostree-booted') + + def prepare_command(self) -> tuple[Command, Command]: + """ Prepare installation command for rpm-ostree""" + + command = Command() + + if self.guest.facts.is_superuser is False: + command += Command('sudo') + + command += Command('rpm-ostree') + + options = Command('--apply-live', '--idempotent', '--allow-inactive') + + return (command, options) + + def _construct_presence_script(self, *installables: Installable) -> ShellScript: + if len(installables) == 1 and isinstance(installables[0], FileSystemPath): + return ShellScript(f'rpm -qf {installables[0]}') + + return ShellScript( + f'rpm -q --whatprovides {" ".join(escape_installables(*installables))}') + + def check_presence(self, *installables: Installable) -> dict[Installable, bool]: + if len(installables) == 1 and isinstance(installables[0], FileSystemPath): + try: + self.guest.execute(ShellScript(f'rpm -qf {installables[0]}')) + + except RunError as exc: + if exc.returncode == 1: + return {installables[0]: False} + + raise exc + + return {installables[0]: True} + + try: + output = self.guest.execute(ShellScript( + f'rpm -q --whatprovides {" ".join(escape_installables(*installables))}')) + stdout = output.stdout + + except RunError as exc: + stdout = exc.stdout + + if stdout is None: + raise GeneralError("rpm presence check provided no output") + + results: dict[Installable, bool] = {} + + for line, installable in zip(stdout.strip().splitlines(), installables): + match = re.match(rf'package {re.escape(str(installable))} is not installed', line) + if match is not None: + results[installable] = False + continue + + match = re.match(rf'no package provides {re.escape(str(installable))}', line) + if match is not None: + results[installable] = False + continue + + results[installable] = True + + return results + + def _extra_options( + self, + options: Options) -> Command: + extra_options = Command() + + for package in options.excluded_packages: + self.warn("There is no support for rpm-ostree exclude," + f" package '{package}' may still be installed.") + + if options.install_root is not None: + extra_options += Command(f'--installroot={options.install_root}') + + if options.release_version is not None: + extra_options += Command(f'--releasever={options.release_version}') + + return extra_options + + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + if len(installables) != 1: + raise GeneralError( + "rpm-ostree package manager does not support installation of multiple packages.") + + options = options or Options() + + extra_options = self._extra_options(options) + + script = ShellScript( + f'{self.command.to_script()} install ' + f'{self.options.to_script()} {extra_options} ' + f'{" ".join(escape_installables(*installables))}') + + if options.check_first: + script = self._construct_presence_script(*installables) | script + + if options.skip_missing: + script = script | ShellScript('/bin/true') + + return self.guest.execute(script) + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise GeneralError("rpm-ostree does not support reinstall operation.") + + def install_debuginfo( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise GeneralError("rpm-ostree does not support debuginfo packages.") diff --git a/tmt/plugins/__init__.py b/tmt/plugins/__init__.py index c081aa74c9..450a1845ff 100644 --- a/tmt/plugins/__init__.py +++ b/tmt/plugins/__init__.py @@ -68,6 +68,7 @@ def _discover_packages() -> list[tuple[str, Path]]: ('tmt.export', Path('export')), ('tmt.frameworks', Path('frameworks')), ('tmt.checks', Path('checks')), + ('tmt.package_managers', Path('package_managers')), ] @@ -298,3 +299,6 @@ def iter_plugin_ids(self) -> Iterator[str]: def iter_plugins(self) -> Iterator[RegisterableT]: yield from self._plugins.values() + + def items(self) -> Iterator[tuple[str, RegisterableT]]: + yield from self._plugins.items() diff --git a/tmt/steps/prepare/install.py b/tmt/steps/prepare/install.py index 3acd260d07..73624251e9 100644 --- a/tmt/steps/prepare/install.py +++ b/tmt/steps/prepare/install.py @@ -1,7 +1,8 @@ import dataclasses import re import shutil -from typing import Literal, Optional, cast +from collections.abc import Iterator +from typing import Any, Literal, Optional, TypeVar, Union, cast import fmf import fmf.utils @@ -10,206 +11,127 @@ import tmt.base import tmt.log import tmt.options +import tmt.package_managers import tmt.steps import tmt.steps.prepare import tmt.utils -from tmt.steps.provision import Guest, GuestPackageManager +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + Package, + PackagePath, + PackageUrl, + ) +from tmt.steps.provision import Guest from tmt.utils import Command, Path, ShellScript, field COPR_URL = 'https://copr.fedorainfracloud.org/coprs' +T = TypeVar('T') + + class InstallBase(tmt.utils.Common): """ Base class for installation implementations """ - # Each installer knows its package manager and copr plugin - package_manager: str - copr_plugin: str - - # Save a prepared command and options for package operations - command: Command - options: Command + guest: Guest skip_missing: bool = False + exclude: list[Package] - packages: list[str] - directories: list[Path] - exclude: list[str] - - local_packages: list[Path] - remote_packages: list[str] - debuginfo_packages: list[str] - repository_packages: list[str] + packages: list[Union[Package, FileSystemPath]] + local_packages: list[PackagePath] + remote_packages: list[PackageUrl] + debuginfo_packages: list[Package] - rpms_directory: Path + package_directory: Path def __init__( self, *, parent: 'PrepareInstall', guest: Guest, - logger: tmt.log.Logger) -> None: + dependencies: list[tmt.base.DependencySimple], + directories: list[Path], + exclude: list[str], + logger: tmt.log.Logger, + **kwargs: Any) -> None: """ Initialize installation data """ - super().__init__(logger=logger, parent=parent, relative_indent=0) - self.guest = guest - - # Get package related data from the plugin - self.packages = parent.get("package", []) - self.directories = cast(list[Path], parent.get("directory", [])) - self.exclude = parent.get("exclude", []) + super().__init__(logger=logger, parent=parent, relative_indent=0, guest=guest, **kwargs) - if not self.packages and not self.directories: + if not dependencies and not directories: self.debug("No packages for installation found.", level=3) + self.guest = guest + self.exclude = [Package(package) for package in exclude] + self.skip_missing = bool(parent.get('missing') == 'skip') # Prepare package lists and installation command - self.prepare_packages() - - self.command, self.options = self.prepare_command() - - self.debug(f"Using '{self.command}' for all package operations.") - self.debug(f"Options for package operations are '{self.options}'.") + self.prepare_installables(dependencies, directories) - def prepare_packages(self) -> None: + def prepare_installables( + self, + dependencies: list[tmt.base.DependencySimple], + directories: list[Path]) -> None: """ Process package names and directories """ + self.packages = [] self.local_packages = [] self.remote_packages = [] self.debuginfo_packages = [] - self.repository_packages = [] # Detect local, debuginfo and repository packages - for package in self.packages: - if re.match(r"^http(s)?://", package): - self.remote_packages.append(package) - elif package.endswith(".rpm"): - self.local_packages.append(Path(package)) - elif re.search(r"-debug(info|source)(\.|$)", package): + for dependency in dependencies: + if re.match(r"^http(s)?://", dependency): + self.remote_packages.append(PackageUrl(dependency)) + elif dependency.endswith(".rpm"): + self.local_packages.append(PackagePath(dependency)) + elif re.search(r"-debug(info|source)(\.|$)", dependency): # Strip the '-debuginfo' string from package name # (installing with it doesn't work on RHEL7) - package = re.sub(r"-debuginfo((?=\.)|$)", "", package) - self.debuginfo_packages.append(package) + self.debuginfo_packages.append( + Package( + re.sub( + r"-debuginfo((?=\.)|$)", + "", + str(dependency)))) + + elif dependency.startswith('/'): + self.packages.append(FileSystemPath(dependency)) + else: - self.repository_packages.append(package) + self.packages.append(Package(dependency)) # Check rpm packages in local directories - for directory in self.directories: + for directory in directories: self.info('directory', directory, 'green') if not directory.is_dir(): raise tmt.utils.PrepareError(f"Packages directory '{directory}' not found.") for filepath in directory.iterdir(): if filepath.suffix == '.rpm': self.debug(f"Found rpm '{filepath}'.", level=3) - self.local_packages.append(filepath) - - def prepare_command(self) -> tuple[Command, Command]: - """ Prepare installation command and subcommand options """ - raise NotImplementedError + self.local_packages.append(PackagePath(filepath)) def prepare_repository(self) -> None: """ Configure additional repository """ raise NotImplementedError - def operation_script(self, subcommand: Command, args: Command) -> ShellScript: - """ - Render a shell script to perform the requested package operation. - - .. warning:: - - Each and every argument from ``args`` **will be** sanitized by - escaping. This is not compatible with operations that wish to use - shell wildcards. Such operations need to be constructed manually. - - :param subcommand: package manager subcommand, e.g. ``install`` or ``erase``. - :param args: arguments for the subcommand, e.g. package names. - """ - - return ShellScript( - f"{self.command.to_script()} {subcommand.to_script()} " - f"{self.options.to_script()} {args.to_script()}") - - def perform_operation(self, subcommand: Command, args: Command) -> tmt.utils.CommandOutput: - """ - Perform the requested package operation. - - .. warning:: - - Each and every argument from ``args`` **will be** sanitized by - escaping. This is not compatible with operations that wish to use - shell wildcards. Such operations need to be constructed manually. - - :param subcommand: package manager subcommand, e.g. ``install`` or ``erase``. - :param args: arguments for the subcommand, e.g. package names. - :returns: command output. - """ - - return self.guest.execute(self.operation_script(subcommand, args)) - - def list_packages(self, packages: list[str], title: str) -> Command: + def list_installables(self, title: str, *installables: Installable) -> Iterator[Installable]: """ Show package info and return package names """ # Show a brief summary by default if not self.verbosity_level: - summary = fmf.utils.listed(packages, max=3) + summary = fmf.utils.listed(installables, max=3) self.info(title, summary, 'green') # Provide a full list of packages in verbose mode else: - summary = fmf.utils.listed(packages, 'package') + summary = fmf.utils.listed(installables, 'package') self.info(title, summary + ' requested', 'green') - for package in sorted(packages): - self.verbose(package, shift=1) - - return Command(*packages) - - def enable_copr_epel6(self, copr: str) -> None: - """ Manually enable copr repositories for epel6 """ - # Parse the copr repo name - matched = re.match("^(@)?([^/]+)/([^/]+)$", copr) - if not matched: - raise tmt.utils.PrepareError(f"Invalid copr repository '{copr}'.") - group, name, project = matched.groups() - group = 'group_' if group else '' - # Prepare the repo file url - parts = [COPR_URL] + (['g'] if group else []) - parts += [name, project, 'repo', 'epel-6'] - parts += [f"{group}{name}-{project}-epel-6.repo"] - url = '/'.join(parts) - # Download the repo file on guest - try: - self.guest.execute(Command('curl', '-LOf', url), - cwd=Path('/etc/yum.repos.d'), silent=True) - except tmt.utils.RunError as error: - if error.stderr and 'not found' in error.stderr.lower(): - raise tmt.utils.PrepareError( - f"Copr repository '{copr}' not found.") - raise + for package in sorted(installables): + self.verbose(str(package), shift=1) - def enable_copr(self) -> None: - """ Enable requested copr repositories """ - # FIXME: cast() - https://github.com/teemtee/tmt/issues/1372 - coprs = cast(PrepareInstall, self.parent).get('copr') - if not coprs: - return - # Try to install copr plugin - self.debug('Make sure the copr plugin is available.') - try: - self.guest.execute( - ShellScript(f'rpm -q {self.copr_plugin}') - | self.operation_script(Command('install'), Command('-y', self.copr_plugin)), - silent=True) - # Enable repositories manually for epel6 - except tmt.utils.RunError: - for copr in coprs: - self.info('copr', copr, 'green') - self.enable_copr_epel6(copr) - # Enable repositories using copr plugin - else: - for copr in coprs: - self.info('copr', copr, 'green') - self.perform_operation( - Command('copr'), - Command('enable', '-y', copr) - ) + yield from installables def prepare_install_local(self) -> None: """ Copy packages to the test system """ @@ -218,14 +140,14 @@ def prepare_install_local(self) -> None: workdir = cast(PrepareInstall, self.parent).step.workdir if not workdir: raise tmt.utils.GeneralError('workdir should not be empty') - self.rpms_directory = workdir / 'rpms' - self.rpms_directory.mkdir(parents=True) + self.package_directory = workdir / 'packages' + self.package_directory.mkdir(parents=True) # Copy local packages into workdir, push to guests for package in self.local_packages: self.verbose(package.name, shift=1) - self.debug(f"Copy '{package}' to '{self.rpms_directory}'.", level=3) - shutil.copy(package, self.rpms_directory) + self.debug(f"Copy '{package}' to '{self.package_directory}'.", level=3) + shutil.copy(package, self.package_directory) self.guest.push() def install_from_repository(self) -> None: @@ -251,7 +173,7 @@ def install(self) -> None: self.install_local() if self.remote_packages: self.install_from_url() - if self.repository_packages: + if self.packages: self.install_from_repository() if self.debuginfo_packages: self.install_debuginfo() @@ -266,182 +188,179 @@ def rpm_check(self, package: str, mode: str = '-q') -> None: self.debug(f"Package '{output.stdout.strip()}' already installed.") -class InstallDnf(InstallBase): - """ Install packages using dnf """ +class Copr(tmt.utils.Common): + copr_plugin: str - package_manager = "dnf" - copr_plugin = "dnf-plugins-core" - skip_missing_option = "--skip-broken" + guest: Guest - def prepare_command(self) -> tuple[Command, Command]: - """ Prepare installation command """ + # Keep this method around, to correctly support Python's method resolution order. + def __init__(self, *args: Any, guest: Guest, **kwargs: Any) -> None: + super().__init__(*args, guest=guest, **kwargs) - options = Command('-y') + self.guest = guest - for package in self.exclude: - options += Command('--exclude', package) + def enable_copr_epel6(self, copr: str) -> None: + """ Manually enable copr repositories for epel6 """ + # Parse the copr repo name + matched = re.match("^(@)?([^/]+)/([^/]+)$", copr) + if not matched: + raise tmt.utils.PrepareError(f"Invalid copr repository '{copr}'.") + group, name, project = matched.groups() + group = 'group_' if group else '' + # Prepare the repo file url + parts = [COPR_URL] + (['g'] if group else []) + parts += [name, project, 'repo', 'epel-6'] + parts += [f"{group}{name}-{project}-epel-6.repo"] + url = '/'.join(parts) + # Download the repo file on guest + try: + self.guest.execute(Command('curl', '-LOf', url), + cwd=Path('/etc/yum.repos.d'), silent=True) + except tmt.utils.RunError as error: + if error.stderr and 'not found' in error.stderr.lower(): + raise tmt.utils.PrepareError( + f"Copr repository '{copr}' not found.") + raise + + def enable_copr(self, repositories: list[str]) -> None: + """ Enable requested copr repositories """ - command = Command() + if not repositories: + return - if self.guest.facts.is_superuser is False: - command += Command('sudo') + # Try to install copr plugin + self.debug('Make sure the copr plugin is available.') + try: + self.guest.package_manager.install(Package(self.copr_plugin)) - command += Command(self.package_manager) + # Enable repositories manually for epel6 + except tmt.utils.RunError: + for repository in repositories: + self.info('copr', repository, 'green') + self.enable_copr_epel6(repository) - if self.skip_missing: - options += Command(self.skip_missing_option) + # Enable repositories using copr plugin + else: + for repository in repositories: + self.info('copr', repository, 'green') - return (command, options) + self.guest.execute(ShellScript( + f"{self.guest.package_manager.command.to_script()} copr " + f"{self.guest.package_manager.options.to_script()} enable -y {repository}" + )) + + +class InstallDnf(InstallBase, Copr): + """ Install packages using dnf """ + + copr_plugin = "dnf-plugins-core" def install_local(self) -> None: - """ Install copied local packages """ + """ Install packages stored in a local directory """ + # Use both dnf install/reinstall to get all packages refreshed # FIXME Simplify this once BZ#1831022 is fixed/implemeted. - self.guest.execute(ShellScript( - f""" - {self.command.to_script()} install {self.options.to_script()} {self.rpms_directory}/* - """ - )) - self.guest.execute(ShellScript( - f""" - {self.command.to_script()} reinstall {self.options.to_script()} {self.rpms_directory}/* - """ - )) + filelist = [PackagePath(self.package_directory / filename) + for filename in self.local_packages] + + self.guest.package_manager.install( + *filelist, + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) + ) + + self.guest.package_manager.reinstall( + *filelist, + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) + ) summary = fmf.utils.listed([str(path) for path in self.local_packages], 'local package') self.info('total', f"{summary} installed", 'green') def install_from_url(self) -> None: - """ Install packages directly from URL """ - self.perform_operation( - Command('install'), - self.list_packages(self.remote_packages, title="remote package") + """ Install packages stored on a remote URL """ + + self.guest.package_manager.install( + *self.list_installables("remote package", *self.remote_packages), + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) ) def install_from_repository(self) -> None: - """ Install packages from the repository """ - packages = self.list_packages(self.repository_packages, title="package") + """ Install packages from a repository """ - # Check and install - self.guest.execute( - ShellScript(f'rpm -q --whatprovides {packages.to_script()}') - | self.operation_script(Command('install'), packages) + self.guest.package_manager.install( + *self.list_installables("package", *self.packages), + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) ) def install_debuginfo(self) -> None: """ Install debuginfo packages """ - packages = self.list_packages(self.debuginfo_packages, title="debuginfo") + packages = self.list_installables("debuginfo", *self.debuginfo_packages) - # Make sure debuginfo-install is present on the target system - self.perform_operation( - Command('install'), - Command('-y', '/usr/bin/debuginfo-install') + self.guest.package_manager.install_debuginfo( + *packages, + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) ) - command = Command('debuginfo-install', '-y') - - if self.skip_missing: - command += Command('--skip-broken') - - command += packages - - self.guest.execute(command) - # Check the packages are installed on the guest because 'debuginfo-install' # returns 0 even though it didn't manage to install the required packages if not self.skip_missing: - packages_debuginfo = [f'{package}-debuginfo' for package in self.debuginfo_packages] - command = Command('rpm', '-q', *packages_debuginfo) - self.guest.execute(command) - - -class InstallDnf5(InstallDnf): - """ Install packages using dnf5 """ - - package_manager = "dnf5" - copr_plugin = "dnf5-plugins" - skip_missing_option = "--skip-unavailable" + self.guest.package_manager.check_presence( + *[Package(f'{package}-debuginfo') for package in self.debuginfo_packages]) class InstallYum(InstallDnf): """ Install packages using yum """ - package_manager = "yum" copr_plugin = "yum-plugin-copr" - def install_from_repository(self) -> None: - """ Install packages from the repository """ - packages = self.list_packages(self.repository_packages, title="package") - - # Extra ignore/check for yum to workaround BZ#1920176 - check = ShellScript(f'rpm -q --whatprovides {packages.to_script()}') - script = check | self.operation_script(Command('install'), packages) - if self.skip_missing: - script |= ShellScript('true') - else: - script &= check - - # Check and install - self.guest.execute(script) - - -class InstallRpmOstree(InstallBase): +class InstallRpmOstree(InstallBase, Copr): """ Install packages using rpm-ostree """ - package_manager = "rpm-ostree" copr_plugin = "dnf-plugins-core" + recommended_packages: list[Union[Package, FileSystemPath]] + required_packages: list[Union[Package, FileSystemPath]] + def sort_packages(self) -> None: """ Identify required and recommended packages """ self.recommended_packages = [] self.required_packages = [] - for package in self.repository_packages: - try: - if package.startswith('/'): - # Dependencies can also be files, let's check those as well - # TODO: PR 2557 needs to check for files as well - self.rpm_check(package=package, mode='-qf') - else: - self.rpm_check(package=package) - except tmt.utils.RunError: + for package in self.packages: + presence = self.guest.package_manager.check_presence(package) + + if not all(presence.values()): if self.skip_missing: self.recommended_packages.append(package) else: self.required_packages.append(package) - def prepare_command(self) -> tuple[Command, Command]: - """ Prepare installation command for rpm-ostree""" - - command = Command() - - if self.guest.facts.is_superuser is False: - command += Command('sudo') - - command += Command('rpm-ostree') - - options = Command('--apply-live', '--idempotent', '--allow-inactive') - - for package in self.exclude: - # exclude not supported in rpm-ostree - self.warn(f"there is no support for rpm-ostree exclude. " - f"Package '{package}' may still be installed.") - - return (command, options) - def install_debuginfo(self) -> None: """ Install debuginfo packages """ self.warn("Installation of debuginfo packages not supported yet.") def install_local(self) -> None: """ Install copied local packages """ - local_packages_installed = [] + local_packages_installed: list[PackagePath] = [] for package in self.local_packages: try: - self.perform_operation( - Command('install'), - Command(f'{self.rpms_directory / package.name}') - ) + self.guest.package_manager.install( + PackagePath(self.package_directory / package.name)) local_packages_installed.append(package) except tmt.utils.RunError as error: self.warn(f"Local package '{package}' not installed: {error.stderr}") @@ -454,13 +373,10 @@ def install_from_repository(self) -> None: # Install recommended packages if self.recommended_packages: - self.list_packages(self.recommended_packages, title="package") + self.list_installables("package", *self.recommended_packages) for package in self.recommended_packages: try: - self.perform_operation( - Command('install'), - Command(package) - ) + self.guest.package_manager.install(package) except tmt.utils.RunError as error: self.debug(f"Package installation failed: {error}") self.warn(f"Unable to install recommended package '{package}'.") @@ -468,10 +384,69 @@ def install_from_repository(self) -> None: # Install required packages if self.required_packages: - self.perform_operation( - Command('install'), - self.list_packages(self.required_packages, title="package") + self.guest.package_manager.install( + *self.list_installables("package", *self.required_packages)) + + +class InstallApt(InstallBase): + """ Install packages using apt """ + + def install_local(self) -> None: + """ Install packages stored in a local directory """ + + filelist = [PackagePath(self.package_directory / filename) + for filename in self.local_packages] + + self.guest.package_manager.install( + *self.list_installables('local packages', *filelist), + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) + ) + + summary = fmf.utils.listed([str(path) for path in self.local_packages], 'local package') + self.info('total', f"{summary} installed", 'green') + + def install_from_url(self) -> None: + """ Install packages stored on a remote URL """ + + self.guest.package_manager.install( + *self.list_installables("remote package", *self.remote_packages), + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) + ) + + def install_from_repository(self) -> None: + """ Install packages from a repository """ + + self.guest.package_manager.install( + *self.list_installables("package", *self.packages), + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing + ) + ) + + def install_debuginfo(self) -> None: + """ Install debuginfo packages """ + packages = self.list_installables("debuginfo", *self.debuginfo_packages) + + self.guest.package_manager.install_debuginfo( + *packages, + options=Options( + excluded_packages=self.exclude, + skip_missing=self.skip_missing ) + ) + + # Check the packages are installed on the guest because 'debuginfo-install' + # returns 0 even though it didn't manage to install the required packages + if not self.skip_missing: + self.guest.package_manager.check_presence( + *[Package(f'{package}-debuginfo') for package in self.debuginfo_packages]) @dataclasses.dataclass @@ -598,24 +573,54 @@ def go( return # Pick the right implementation - # TODO: it'd be nice to use a "plugin registry" and make the implementations - # discovered as any other plugins. - if guest.facts.package_manager == GuestPackageManager.RPM_OSTREE: - installer: InstallBase = InstallRpmOstree(logger=logger, parent=self, guest=guest) - - elif guest.facts.package_manager == GuestPackageManager.DNF: - installer = InstallDnf(logger=logger, parent=self, guest=guest) - - elif guest.facts.package_manager == GuestPackageManager.DNF5: - installer = InstallDnf5(logger=logger, parent=self, guest=guest) - - elif guest.facts.package_manager == GuestPackageManager.YUM: - installer = InstallYum(logger=logger, parent=self, guest=guest) + # TODO: it'd be nice to use a "plugin registry" and make the + # implementations discovered as any other plugins. Package managers are + # shipped as plugins, but we still need a matching *installation* class. + # But do we really need a class per package manager family? Maybe the + # code could be integrated into package manager plugins directly. + if guest.facts.package_manager == 'rpm-ostree': + installer: InstallBase = InstallRpmOstree( + logger=logger, + parent=self, + dependencies=self.data.package, + directories=self.data.directory, + exclude=self.data.exclude, + guest=guest) + + elif guest.facts.package_manager in ('dnf', 'dnf5'): + installer = InstallDnf( + logger=logger, + parent=self, + dependencies=self.data.package, + directories=self.data.directory, + exclude=self.data.exclude, + guest=guest) + + elif guest.facts.package_manager == 'yum': + installer = InstallYum( + logger=logger, + parent=self, + dependencies=self.data.package, + directories=self.data.directory, + exclude=self.data.exclude, + guest=guest) + + elif guest.facts.package_manager == 'apt': + installer = InstallApt( + logger=logger, + parent=self, + dependencies=self.data.package, + directories=self.data.directory, + exclude=self.data.exclude, + guest=guest) else: raise tmt.utils.PrepareError( f'Package manager "{guest.facts.package_manager}" is not supported.') - # Enable copr repositories and install packages - installer.enable_copr() + # Enable copr repositories... + if isinstance(installer, Copr): + installer.enable_copr(self.data.copr) + + # ... and install packages. installer.install() diff --git a/tmt/steps/provision/__init__.py b/tmt/steps/provision/__init__.py index 097bf6f864..267a6c963b 100644 --- a/tmt/steps/provision/__init__.py +++ b/tmt/steps/provision/__init__.py @@ -34,12 +34,14 @@ import tmt import tmt.hardware import tmt.log +import tmt.package_managers import tmt.plugins import tmt.queue import tmt.steps import tmt.utils from tmt.log import Logger from tmt.options import option +from tmt.package_managers import FileSystemPath, Package from tmt.plugins import PluginRegistry from tmt.steps import Action, ActionTask, PhaseQueue from tmt.utils import ( @@ -102,13 +104,6 @@ class CheckRsyncOutcome(enum.Enum): INSTALLED = 'installed' -class GuestPackageManager(enum.Enum): - DNF = 'dnf' - DNF5 = 'dnf5' - YUM = 'yum' - RPM_OSTREE = 'rpm-ostree' - - T = TypeVar('T') @@ -138,12 +133,10 @@ class GuestFacts(SerializableContainer): arch: Optional[str] = None distro: Optional[str] = None kernel_release: Optional[str] = None - package_manager: Optional[GuestPackageManager] = field( + package_manager: Optional['tmt.package_managers.GuestPackageManager'] = field( # cast: since the default is None, mypy cannot infere the full type, # and reports `package_manager` parameter to be `object`. - default=cast(Optional[GuestPackageManager], None), - serialize=lambda package_manager: package_manager.value if package_manager else None, - unserialize=lambda raw_value: GuestPackageManager(raw_value) if raw_value else None) + default=cast(Optional['tmt.package_managers.GuestPackageManager'], None)) has_selinux: Optional[bool] = None is_superuser: Optional[bool] = None @@ -340,16 +333,15 @@ def _query_kernel_release(self, guest: 'Guest') -> Optional[str]: (Command('uname', '-r'), r'(.+)') ]) - def _query_package_manager(self, guest: 'Guest') -> Optional[GuestPackageManager]: + def _query_package_manager( + self, + guest: 'Guest') -> Optional['tmt.package_managers.GuestPackageManager']: return self._probe( guest, [ - (Command('stat', '/run/ostree-booted'), GuestPackageManager.RPM_OSTREE), - (Command('dnf5', '--version'), GuestPackageManager.DNF5), - (Command('dnf', '--version'), GuestPackageManager.DNF), - (Command('yum', '--version'), GuestPackageManager.YUM), - # And, one day, we'd follow up on this with... - # (Command('dpkg', '-l', 'apt'), 'apt') + (package_manager_class.probe_command, plugin_id) + for plugin_id, package_manager_class + in tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.items() ]) def _query_has_selinux(self, guest: 'Guest') -> Optional[bool]: @@ -409,7 +401,7 @@ def format(self) -> Iterator[tuple[str, str, str]]: yield 'kernel_release', 'kernel', self.kernel_release or 'unknown' yield 'package_manager', \ 'package manager', \ - self.package_manager.value if self.package_manager else 'unknown' + self.package_manager if self.package_manager else 'unknown' yield 'has_selinux', 'selinux', 'yes' if self.has_selinux else 'no' yield 'is_superuser', 'is superuser', 'yes' if self.is_superuser else 'no' @@ -686,6 +678,15 @@ def is_ready(self) -> bool: raise NotImplementedError + @cached_property + def package_manager(self) -> 'tmt.package_managers.PackageManager': + if not self.facts.package_manager: + raise tmt.utils.GeneralError( + f"Package manager was not detected on guest '{self.name}'.") + + return tmt.package_managers.find_package_manager( + self.facts.package_manager)(guest=self, logger=self._logger) + @classmethod def options(cls, how: Optional[str] = None) -> list[tmt.options.ClickOptionDecoratorType]: """ Prepare command line options related to guests """ @@ -849,8 +850,9 @@ def _prepare_environment( # Plan environment and variables provided on the command line # override environment provided to execute(). # FIXME: cast() - https://github.com/teemtee/tmt/issues/1372 - parent = cast(Provision, self.parent) - environment.update(parent.plan.environment) + if self.parent: + parent = cast(Provision, self.parent) + environment.update(parent.plan.environment) return environment @staticmethod @@ -1141,28 +1143,23 @@ def _check_rsync(self) -> CheckRsyncOutcome: except tmt.utils.RunError: pass - # Check the package manager - self.debug("Check the package manager.") - try: - self.execute(Command('dnf', '--version')) - package_manager = "dnf" - except tmt.utils.RunError: - package_manager = "yum" - # Install under '/root/pkg' for read-only distros # (for now the check is based on 'rpm-ostree' presence) # FIXME: Find a better way how to detect read-only distros - self.debug("Check for a read-only distro.") - try: - self.execute(Command('rpm-ostree', '--version')) - readonly = ( - " --installroot=/root/pkg --releasever / " - "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync") - except tmt.utils.RunError: - readonly = "" + # self.debug("Check for a read-only distro.") + if self.facts.package_manager == 'rpm-ostree': + self.package_manager.install( + Package('rsync'), + options=tmt.package_managers.Options( + install_root=Path('/root/pkg'), + release_version='/' + ) + ) - # Install the rsync - self.execute(ShellScript(f"{package_manager} install -y rsync" + readonly)) + self.execute(Command('ln', '-sf', '/root/pkg/bin/rsync', '/usr/local/bin/rsync')) + + else: + self.package_manager.install(Package('rsync')) return CheckRsyncOutcome.INSTALLED @@ -1457,15 +1454,7 @@ def setup(self) -> None: if self.is_dry_run: return if not self.facts.is_superuser and self.become: - assert self.facts.package_manager is not None - # TODO: refactor this after PR #2557 is completed - self.execute( - Command( - 'sudo', - f'{self.facts.package_manager.value}', - 'install', - '-y', - 'acl')) + self.package_manager.install(FileSystemPath('/usr/bin/setfacl')) workdir_root = effective_workdir_root() self.execute(ShellScript( f""" @@ -1836,47 +1825,6 @@ def remove(self) -> None: """ self.debug(f"Doing nothing to remove guest '{self.primary_address}'.") - def _check_rsync(self) -> CheckRsyncOutcome: - """ - Make sure that rsync is installed on the guest - - On read-only distros install it under the '/root/pkg' directory. - Returns 'already installed' when rsync is already present. - """ - - # Check for rsync (nothing to do if already installed) - self.debug("Ensure that rsync is installed on the guest.") - try: - self.execute(Command('rsync', '--version')) - return CheckRsyncOutcome.ALREADY_INSTALLED - except tmt.utils.RunError: - pass - - # Check the package manager - self.debug("Check the package manager.") - try: - self.execute(Command('dnf', '--version')) - package_manager = "dnf" - except tmt.utils.RunError: - package_manager = "yum" - - # Install under '/root/pkg' for read-only distros - # (for now the check is based on 'rpm-ostree' presence) - # FIXME: Find a better way how to detect read-only distros - self.debug("Check for a read-only distro.") - try: - self.execute(Command('rpm-ostree', '--version')) - readonly = ( - " --installroot=/root/pkg --releasever / " - "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync") - except tmt.utils.RunError: - readonly = "" - - # Install the rsync - self.execute(ShellScript(f"{package_manager} install -y rsync" + readonly)) - - return CheckRsyncOutcome.INSTALLED - @dataclasses.dataclass class ProvisionStepData(tmt.steps.StepData): diff --git a/tmt/steps/provision/podman.py b/tmt/steps/provision/podman.py index acc9a1b15f..e0c3102641 100644 --- a/tmt/steps/provision/podman.py +++ b/tmt/steps/provision/podman.py @@ -158,6 +158,19 @@ def start(self) -> None: """ Start provisioned guest """ if self.is_dry_run: return + + if self.container: + self.primary_address = self.topology_address = self.container + + self.verbose('primary address', self.primary_address, 'green') + self.verbose('topology address', self.topology_address, 'green') + + return + + self.container = self.primary_address = self.topology_address = self._tmt_name() + self.verbose('primary address', self.primary_address, 'green') + self.verbose('topology address', self.topology_address, 'green') + # Check if the image is available assert self.image is not None @@ -184,9 +197,7 @@ def start(self) -> None: # Mount the whole plan directory in the container workdir = self.parent.plan.workdir - self.container = self.primary_address = self.topology_address = self._tmt_name() - self.verbose('primary address', self.primary_address, 'green') - self.verbose('topology address', self.topology_address, 'green') + self.verbose('name', self.container, 'green') additional_args = [] diff --git a/tmt/utils.py b/tmt/utils.py index a6ce811c4c..cada92a04c 100644 --- a/tmt/utils.py +++ b/tmt/utils.py @@ -1064,14 +1064,26 @@ def __str__(self) -> str: return self._script def __add__(self, other: 'ShellScript') -> 'ShellScript': + if not other: + return self + return ShellScript.from_scripts([self, other]) def __and__(self, other: 'ShellScript') -> 'ShellScript': + if not other: + return self + return ShellScript(f'{self} && {other}') def __or__(self, other: 'ShellScript') -> 'ShellScript': + if not other: + return self + return ShellScript(f'{self} || {other}') + def __bool__(self) -> bool: + return bool(self._script) + @classmethod def from_scripts(cls, scripts: list['ShellScript']) -> 'ShellScript': """ @@ -1083,7 +1095,7 @@ def from_scripts(cls, scripts: list['ShellScript']) -> 'ShellScript': :param scripts: scripts to merge into one. """ - return ShellScript('; '.join(script._script for script in scripts)) + return ShellScript('; '.join(script._script for script in scripts if bool(script))) def to_element(self) -> _CommandElement: """ Convert a shell script to a command element """ From 851afd65ae4834da36edc1e68d0d4ded3fb8d6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 13:36:26 +0100 Subject: [PATCH 02/20] squash: try use packaged hatch when possible --- plans/features/core.fmf | 12 ++++++++++++ plans/features/extended-unit-tests.fmf | 11 +++++++++++ plans/main.fmf | 3 +-- tests/unit/main.fmf | 10 ++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/plans/features/core.fmf b/plans/features/core.fmf index 5ecd021e54..7322a1e688 100644 --- a/plans/features/core.fmf +++ b/plans/features/core.fmf @@ -6,3 +6,15 @@ description: discover: how: fmf filter: 'tier: 0, 1 & tag:-provision-only' + +adjust+: + # /tests/unit/with-development-packages/* needs `hatch` but there seem + # to be none for CentOS Stream 9. Test will request it, but on CentOS, + # we need a `prepare` phase to install it from PyPI. If we find + # `hatch` packaged, we can drop this workaround. + - when: distro == centos-stream + prepare+: + - how: shell + script: + - pip3 install --user hatch || pip3 install hatch + - hatch --help diff --git a/plans/features/extended-unit-tests.fmf b/plans/features/extended-unit-tests.fmf index 7b5550af40..89a0a13ff7 100644 --- a/plans/features/extended-unit-tests.fmf +++ b/plans/features/extended-unit-tests.fmf @@ -27,3 +27,14 @@ adjust+: hardware: cpu: processors: ">= 8" + + # /tests/unit/with-development-packages/* needs `hatch` but there seem + # to be none for CentOS Stream 9. Test will request it, but on CentOS, + # we need a `prepare` phase to install it from PyPI. If we find + # `hatch` packaged, we can drop this workaround. + - when: distro == centos-stream + prepare+: + - how: shell + script: + - pip3 install --user hatch || pip3 install hatch + - hatch --help diff --git a/plans/main.fmf b/plans/main.fmf index 64a42c9bda..6b2b03ae5f 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -17,8 +17,7 @@ prepare+: # have to run as root to do that, and who's running tmt test suite # as root? # - - pip3 install --user hatch yq || pip3 install hatch yq - - hatch --help + - pip3 install --user yq || pip3 install yq - yq --help # Use the internal executor diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 5241d19679..0b6cdf620f 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -25,6 +25,16 @@ require+: /with-development-packages: enabled: true + # /tests/unit/with-development-packages/* needs `hatch` but there seem + # to be none for CentOS Stream 9. Test will request it, but on CentOS, + # we need a `prepare` phase to install it from PyPI. If we find + # `hatch` packaged, we can drop this workaround and make `hatch` a + # regular required package. + adjust+: + - when: distro == centos-stream + require+: + - hatch + /basic: summary: Basic unit tests (development packages) tier: 0 From 79d9bbc2d452ff45f97cd2d239d609f88ab0f379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 13:48:26 +0100 Subject: [PATCH 03/20] squash: drop adjusted hatch --- plans/features/core.fmf | 12 ------------ plans/features/extended-unit-tests.fmf | 11 ----------- tests/unit/main.fmf | 11 +---------- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/plans/features/core.fmf b/plans/features/core.fmf index 7322a1e688..5ecd021e54 100644 --- a/plans/features/core.fmf +++ b/plans/features/core.fmf @@ -6,15 +6,3 @@ description: discover: how: fmf filter: 'tier: 0, 1 & tag:-provision-only' - -adjust+: - # /tests/unit/with-development-packages/* needs `hatch` but there seem - # to be none for CentOS Stream 9. Test will request it, but on CentOS, - # we need a `prepare` phase to install it from PyPI. If we find - # `hatch` packaged, we can drop this workaround. - - when: distro == centos-stream - prepare+: - - how: shell - script: - - pip3 install --user hatch || pip3 install hatch - - hatch --help diff --git a/plans/features/extended-unit-tests.fmf b/plans/features/extended-unit-tests.fmf index 89a0a13ff7..7b5550af40 100644 --- a/plans/features/extended-unit-tests.fmf +++ b/plans/features/extended-unit-tests.fmf @@ -27,14 +27,3 @@ adjust+: hardware: cpu: processors: ">= 8" - - # /tests/unit/with-development-packages/* needs `hatch` but there seem - # to be none for CentOS Stream 9. Test will request it, but on CentOS, - # we need a `prepare` phase to install it from PyPI. If we find - # `hatch` packaged, we can drop this workaround. - - when: distro == centos-stream - prepare+: - - how: shell - script: - - pip3 install --user hatch || pip3 install hatch - - hatch --help diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 0b6cdf620f..3f485f02ac 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -20,21 +20,12 @@ require+: - jq - podman - buildah + - hatch # Run against development packages via `hatch`. /with-development-packages: enabled: true - # /tests/unit/with-development-packages/* needs `hatch` but there seem - # to be none for CentOS Stream 9. Test will request it, but on CentOS, - # we need a `prepare` phase to install it from PyPI. If we find - # `hatch` packaged, we can drop this workaround and make `hatch` a - # regular required package. - adjust+: - - when: distro == centos-stream - require+: - - hatch - /basic: summary: Basic unit tests (development packages) tier: 0 From 06883ff7ba690f55514484a7d246061ab16756c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 20:50:45 +0100 Subject: [PATCH 04/20] squash: try updating pip on centos stream --- plans/main.fmf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plans/main.fmf b/plans/main.fmf index 6b2b03ae5f..84541a6ab3 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -31,3 +31,10 @@ adjust: how: install directory: tmp/RPMS/noarch when: how == full + + - when: distro == centos-stream-9 + prepare+: + - name: Install recent pip + how: shell + script: + - pip3 install --user pip From 8c5bb84f85ede9b4ac8068497d8bbef5fd0016eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 21:44:25 +0100 Subject: [PATCH 05/20] squash: more logging --- tests/unit/test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test.sh b/tests/unit/test.sh index 037c5a7373..d10f764887 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -32,6 +32,10 @@ rlJournalStart rlLogInfo "PYTEST_MARK=$PYTEST_MARK" rlRun "PYTEST_COMMAND='pytest -vvv -ra --showlocals'" + + rlLogInfo "pip is $(which pip), $(pip --version)" + rlLogInfo "hatch is $(which hatch), $(hatch --version)" + rlLogInfo "hatch is $(which hatch), $(hatch --version)" rlPhaseEnd if [ "$WITH_SYSTEM_PACKAGES" = "yes" ]; then @@ -48,7 +52,7 @@ rlJournalStart rlPhaseEnd else rlPhaseStartTest "Unit tests" - rlRun "hatch -v run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" + rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd fi From 5a2255282f2a2cf8b734d99af244191d38364877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 22:22:06 +0100 Subject: [PATCH 06/20] squash: upgrade pip --- tests/unit/test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test.sh b/tests/unit/test.sh index d10f764887..602b87fc65 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -35,7 +35,6 @@ rlJournalStart rlLogInfo "pip is $(which pip), $(pip --version)" rlLogInfo "hatch is $(which hatch), $(hatch --version)" - rlLogInfo "hatch is $(which hatch), $(hatch --version)" rlPhaseEnd if [ "$WITH_SYSTEM_PACKAGES" = "yes" ]; then @@ -52,6 +51,10 @@ rlJournalStart rlPhaseEnd else rlPhaseStartTest "Unit tests" + # Only pip 21.3+ will be able to install project in editable mode. + # See https://github.com/pypa/hatch/discussions/806#discussioncomment-5503233 + rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:pip install -U '>=21.3'" + rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd fi From 562d2b7f13e06278e044dc06b28654405e8298e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 13 Mar 2024 22:28:32 +0100 Subject: [PATCH 07/20] squash: upgrade pip --- plans/main.fmf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/plans/main.fmf b/plans/main.fmf index 84541a6ab3..6b2b03ae5f 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -31,10 +31,3 @@ adjust: how: install directory: tmp/RPMS/noarch when: how == full - - - when: distro == centos-stream-9 - prepare+: - - name: Install recent pip - how: shell - script: - - pip3 install --user pip From 717591afdb72217ed4665095fe7c3036823c5306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Thu, 14 Mar 2024 08:41:16 +0100 Subject: [PATCH 08/20] squash: fix typo --- tests/unit/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test.sh b/tests/unit/test.sh index 602b87fc65..2829a8ce04 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -53,7 +53,7 @@ rlJournalStart rlPhaseStartTest "Unit tests" # Only pip 21.3+ will be able to install project in editable mode. # See https://github.com/pypa/hatch/discussions/806#discussioncomment-5503233 - rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:pip install -U '>=21.3'" + rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:pip install -U 'pip>=21.3'" rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd From adcaadd853b88dc2891ed0a278253635cc2f926e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Thu, 14 Mar 2024 09:44:18 +0100 Subject: [PATCH 09/20] squash: fix typo --- tests/unit/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test.sh b/tests/unit/test.sh index 2829a8ce04..8f0437ab3e 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -51,9 +51,9 @@ rlJournalStart rlPhaseEnd else rlPhaseStartTest "Unit tests" - # Only pip 21.3+ will be able to install project in editable mode. - # See https://github.com/pypa/hatch/discussions/806#discussioncomment-5503233 - rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:pip install -U 'pip>=21.3'" + if rlIsCentOS; then + rlRun "hatch -vv run $HATCH_ENVIRONMENT:ls" 1 + fi rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd From 5fd02462305eaf2e3549eb25db2b9ca0323e50c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Thu, 14 Mar 2024 13:03:01 +0100 Subject: [PATCH 10/20] squash: reduce verbosity --- tests/unit/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test.sh b/tests/unit/test.sh index 8f0437ab3e..ac9ee6081e 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -52,10 +52,10 @@ rlJournalStart else rlPhaseStartTest "Unit tests" if rlIsCentOS; then - rlRun "hatch -vv run $HATCH_ENVIRONMENT:ls" 1 + rlRun "hatch -v run $HATCH_ENVIRONMENT:ls" 1 fi - rlRun "hatch -vvvv run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" + rlRun "hatch -v run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd fi From 957b32e55c06283e508c0d8a7003f8b01134c8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 14:14:52 +0100 Subject: [PATCH 11/20] squash: use special environment on centos --- pyproject.toml | 5 +++++ tests/unit/main.fmf | 5 +++++ tests/unit/test.sh | 4 ---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2cb45a05d8..34dade07d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,11 @@ requre = [ "requre-patch purge --replaces :milestone_url:str:SomeText --replaces :latency:float:0 tests/integration/test_data/test_nitrate/*", ] +[tool.hatch.envs.dev-not-editable] +template = "dev" +description = "Same as 'dev', but not using editable install" +dev-mode = false + [tool.hatch.envs.test] template = "dev" description = "Run scripts with multiple Python versions" diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 3f485f02ac..292bba1e9e 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -9,6 +9,11 @@ environment: ENABLE_CONTAINERS: "no" WITH_SYSTEM_PACKAGES: "no" +adjust+: + - when: distro == centos-stream-9 + environment+: + HATCH_ENVIRONMENT: dev-not-editable + require+: - gcc - git diff --git a/tests/unit/test.sh b/tests/unit/test.sh index ac9ee6081e..7c2094bb6a 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -51,10 +51,6 @@ rlJournalStart rlPhaseEnd else rlPhaseStartTest "Unit tests" - if rlIsCentOS; then - rlRun "hatch -v run $HATCH_ENVIRONMENT:ls" 1 - fi - rlRun "hatch -v run $HATCH_ENVIRONMENT:$PYTEST_COMMAND $PYTEST_PARALLELIZE $PYTEST_MARK tests/unit" rlPhaseEnd fi From 08c60731dd3c78e3db87ee7483f8db56df11742b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 15:22:00 +0100 Subject: [PATCH 12/20] squash: help pip a bit more... --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 34dade07d1..f2bb8202fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ # F39 / PyPI # Help installation by reducing the set of inspected botocore release. # There is *a lot* of them, and hatch might fetch many of them. "botocore>=1.25.10", # 1.25.10 is the current one available for RHEL9 + "boto3>=1.22.10", # 1.22.10 is the current one available for RHEL9 ] [project.optional-dependencies] From 38c85355844347f719d9ad41cee0b9e13351e70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 15:23:00 +0100 Subject: [PATCH 13/20] squash: help pip a bit more... --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6eb3b9ace..e891c67241 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: # Help installation by reducing the set of inspected botocore release. # There is *a lot* of them, and hatch might fetch many of them. - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 + - "boto3>=1.22.10" # 1.22.10 is the current one available for RHEL9 # report-junit - "junit_xml>=1.9" @@ -82,6 +83,7 @@ repos: # Help installation by reducing the set of inspected botocore release. # There is *a lot* of them, and hatch might fetch many of them. - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 + - "boto3>=1.22.10" # 1.22.10 is the current one available for RHEL9 # report-junit - "junit_xml>=1.9" From becebef401a6595ebbc816321e71c5c94d27a12d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 16:27:09 +0100 Subject: [PATCH 14/20] squash: longer duration on centos --- tests/unit/main.fmf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 292bba1e9e..72cb485ab0 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -13,6 +13,7 @@ adjust+: - when: distro == centos-stream-9 environment+: HATCH_ENVIRONMENT: dev-not-editable + duration: 1h require+: - gcc @@ -44,6 +45,10 @@ require+: ENABLE_PARALLELIZATION: "yes" ENABLE_CONTAINERS: "yes" + adjust+: + - when: distro == centos-stream-9 + duration: 2h + # Run against system, distro-packaged ones via `venv`. /with-system-packages: enabled: false @@ -68,3 +73,7 @@ require+: duration: 1h environment+: *extended_environment + + adjust+: + - when: distro == centos-stream-9 + duration: 2h From c6aa0763411707422b2a0acebea8950b612d7b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 19:52:10 +0100 Subject: [PATCH 15/20] squash: try preventing conflict --- .pre-commit-config.yaml | 8 ++++++-- pyproject.toml | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e891c67241..5ceb7d0bab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,9 @@ repos: # which is the version a developer encounters given the requirements are not # frozen. - "types-Markdown" - - "types-requests" + # prevent conflict between types-requests and urllib3 + - "types-requests<2.31.0.7; python_version < '3.10'" + - "types-requests; python_version >= '3.10'" - "types-setuptools" - "types-jsonschema" - "types-urllib3" @@ -95,7 +97,9 @@ repos: # which is the version a developer encounters given the requirements are not # frozen. - "types-Markdown" - - "types-requests" + # prevent conflict between types-requests and urllib3 + - "types-requests<2.31.0.7; python_version < '3.10'" + - "types-requests; python_version >= '3.10'" - "types-setuptools" - "types-jsonschema" - "types-urllib3" diff --git a/pyproject.toml b/pyproject.toml index f2bb8202fa..4fa5adec78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,7 +132,9 @@ dependencies = [ # which is the version a developer encounters given the requirements are not # frozen. "types-Markdown", - "types-requests", + # prevent conflict between types-requests and urllib3 + "types-requests<2.31.0.7; python_version < '3.10'", + "types-requests; python_version >= '3.10'", "types-setuptools", "types-jsonschema", "types-urllib3", From 354435a8e16d1f70f2525ad4f8d6108980c786b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 21:01:52 +0100 Subject: [PATCH 16/20] squash: more tests... --- .pre-commit-config.yaml | 8 -------- plans/features/extended-unit-tests.fmf | 2 ++ pyproject.toml | 4 ---- tests/unit/main.fmf | 9 --------- 4 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ceb7d0bab..15b73709f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,10 +34,6 @@ repos: - "requests>=2.25.1" # 2.28.2 / 2.31.0 - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 - # Help installation by reducing the set of inspected botocore release. - # There is *a lot* of them, and hatch might fetch many of them. - - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 - - "boto3>=1.22.10" # 1.22.10 is the current one available for RHEL9 # report-junit - "junit_xml>=1.9" @@ -82,10 +78,6 @@ repos: - "requests>=2.25.1" # 2.28.2 / 2.31.0 - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 - # Help installation by reducing the set of inspected botocore release. - # There is *a lot* of them, and hatch might fetch many of them. - - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 - - "boto3>=1.22.10" # 1.22.10 is the current one available for RHEL9 # report-junit - "junit_xml>=1.9" diff --git a/plans/features/extended-unit-tests.fmf b/plans/features/extended-unit-tests.fmf index 7b5550af40..7fbf94cf5e 100644 --- a/plans/features/extended-unit-tests.fmf +++ b/plans/features/extended-unit-tests.fmf @@ -14,6 +14,7 @@ adjust+: - name: disable systemd resolved how: shell script: | + set -x systemctl unmask systemd-resolved systemctl disable systemd-resolved systemctl mask systemd-resolved @@ -21,6 +22,7 @@ adjust+: systemctl restart NetworkManager sleep 5 cat /etc/resolv.conf + ps xa | grep resolv - when: trigger == commit provision: diff --git a/pyproject.toml b/pyproject.toml index 4fa5adec78..5493d5c805 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,10 +39,6 @@ dependencies = [ # F39 / PyPI "requests>=2.25.1", # 2.28.2 / 2.31.0 "ruamel.yaml>=0.16.6", # 0.17.32 / 0.17.32 "urllib3>=1.26.5, <3.0", # 1.26.16 / 2.0.4 - # Help installation by reducing the set of inspected botocore release. - # There is *a lot* of them, and hatch might fetch many of them. - "botocore>=1.25.10", # 1.25.10 is the current one available for RHEL9 - "boto3>=1.22.10", # 1.22.10 is the current one available for RHEL9 ] [project.optional-dependencies] diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 72cb485ab0..292bba1e9e 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -13,7 +13,6 @@ adjust+: - when: distro == centos-stream-9 environment+: HATCH_ENVIRONMENT: dev-not-editable - duration: 1h require+: - gcc @@ -45,10 +44,6 @@ require+: ENABLE_PARALLELIZATION: "yes" ENABLE_CONTAINERS: "yes" - adjust+: - - when: distro == centos-stream-9 - duration: 2h - # Run against system, distro-packaged ones via `venv`. /with-system-packages: enabled: false @@ -73,7 +68,3 @@ require+: duration: 1h environment+: *extended_environment - - adjust+: - - when: distro == centos-stream-9 - duration: 2h From 9fd2a20c0c92b9efd4cee72eb08cf329d34adb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Fri, 15 Mar 2024 21:59:04 +0100 Subject: [PATCH 17/20] squash: more tests... --- plans/features/extended-unit-tests.fmf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plans/features/extended-unit-tests.fmf b/plans/features/extended-unit-tests.fmf index 7fbf94cf5e..080238b265 100644 --- a/plans/features/extended-unit-tests.fmf +++ b/plans/features/extended-unit-tests.fmf @@ -17,12 +17,14 @@ adjust+: set -x systemctl unmask systemd-resolved systemctl disable systemd-resolved + systemctl stop systemd-resolved systemctl mask systemd-resolved rm -f /etc/resolv.conf systemctl restart NetworkManager sleep 5 cat /etc/resolv.conf ps xa | grep resolv + netstat -pnl - when: trigger == commit provision: From 5248903b1e80781fe9c74f969eb01d6baa2e959e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Mon, 18 Mar 2024 21:24:42 +0100 Subject: [PATCH 18/20] squash: move extended test suite around --- .packit.yaml | 22 ++++++++++++++++++++++ tests/unit/main.fmf | 10 +++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.packit.yaml b/.packit.yaml index 21e5896c26..dcc0c09c4e 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -49,6 +49,8 @@ jobs: name: /plans/features/(core|basic) # Test pull requests (full) + # Do not run extended unit tests, that plan gets its own job because + # of podman vs systemd-resolved flakiness. - job: tests identifier: full trigger: pull_request @@ -61,6 +63,26 @@ jobs: - full test absent: - discuss + tf_extra_params: + test: + tmt: + name: '^(?!/plans/features/extended-unit-tests).*$' + + - job: tests + identifier: extended-unit-tests + trigger: pull_request + targets: + - fedora-39 + require: + label: + present: + - full test + absent: + - discuss + tf_extra_params: + test: + tmt: + name: '/plans/features/extended-unit-tests$' # Test pull requests (provision) - job: tests diff --git a/tests/unit/main.fmf b/tests/unit/main.fmf index 292bba1e9e..1430360ae8 100644 --- a/tests/unit/main.fmf +++ b/tests/unit/main.fmf @@ -29,7 +29,13 @@ require+: # Run against development packages via `hatch`. /with-development-packages: - enabled: true + enabled: false + + adjust+: + - when: initiator is not defined or distro == fedora-39 + because: Enable locally or in CI on Fedora 39 + + enabled: true /basic: summary: Basic unit tests (development packages) @@ -56,6 +62,8 @@ require+: adjust+: - when: initiator == packit + because: Enable in CI only + enabled: true /basic: From 342350fabab4970469f122c91ba0dadf6e2ede55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Mon, 18 Mar 2024 22:23:25 +0100 Subject: [PATCH 19/20] squash: fix test for rawhide --- tests/unit/test_package_managers.py | 62 ++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_package_managers.py b/tests/unit/test_package_managers.py index 3290549712..0ebc0275e9 100644 --- a/tests/unit/test_package_managers.py +++ b/tests/unit/test_package_managers.py @@ -727,10 +727,10 @@ def _generate_test_check_presence() -> Iterator[ if package_manager_class is tmt.package_managers.dnf.Dnf5: yield container, \ package_manager_class, \ - Package('util-linux-core'), \ + Package('coreutils'), \ True, \ - r"rpm -q --whatprovides util-linux-core", \ - r'\s+out:\s+util-linux-core-', \ + r"rpm -q --whatprovides coreutils", \ + r'\s+out:\s+coreutils-', \ None yield container, \ @@ -743,10 +743,10 @@ def _generate_test_check_presence() -> Iterator[ yield container, \ package_manager_class, \ - FileSystemPath('/usr/bin/flock'), \ + FileSystemPath('/usr/bin/arch'), \ True, \ - r"rpm -q --whatprovides /usr/bin/flock", \ - r'\s+out:\s+util-linux-core-', \ + r"rpm -q --whatprovides /usr/bin/arch", \ + r'\s+out:\s+coreutils-', \ None elif package_manager_class is tmt.package_managers.dnf.Dnf: @@ -775,6 +775,31 @@ def _generate_test_check_presence() -> Iterator[ r'\s+out:\s+util-linux-', \ None + elif 'fedora:rawhide' in container.url: + yield container, \ + package_manager_class, \ + Package('coreutils'), \ + True, \ + r"rpm -q --whatprovides coreutils", \ + r'\s+out:\s+coreutils-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/arch'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/arch", \ + r'\s+out:\s+coreutils-', \ + None + else: yield container, \ package_manager_class, \ @@ -826,6 +851,31 @@ def _generate_test_check_presence() -> Iterator[ r'\s+out:\s+util-linux-', \ None + elif 'fedora:rawhide' in container.url: + yield container, \ + package_manager_class, \ + Package('coreutils'), \ + True, \ + r"rpm -q --whatprovides coreutils", \ + r'\s+out:\s+coreutils-', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"rpm -q --whatprovides tree-but-spelled-wrong", \ + r'\s+out:\s+no package provides tree-but-spelled-wrong', \ + None + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/arch'), \ + True, \ + r"rpm -q --whatprovides /usr/bin/arch", \ + r'\s+out:\s+coreutils-', \ + None + else: yield container, \ package_manager_class, \ From 190a0c2f7b3f22f2dea8cdbf4fe8892f79746e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Tue, 19 Mar 2024 11:15:18 +0100 Subject: [PATCH 20/20] squash: use fedora-latest-stable --- .packit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.packit.yaml b/.packit.yaml index dcc0c09c4e..dcb11a4b22 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -72,7 +72,7 @@ jobs: identifier: extended-unit-tests trigger: pull_request targets: - - fedora-39 + - fedora-latest-stable require: label: present: