Extract "package manager" functionality into plugins

.packit.yaml

@@ -49,6 +49,8 @@ jobs:
                 name: /plans/features/(core|basic)
 
   # Test pull requests (full)
+  # Do not run extended unit tests, that plan gets its own job because
+  # of podman vs systemd-resolved flakiness.
   - job: tests
     identifier: full
     trigger: pull_request
@@ -61,6 +63,26 @@ jobs:
             - full test
           absent:
             - discuss
+    tf_extra_params:
+        test:
+            tmt:
+                name: '^(?!/plans/features/extended-unit-tests).*$'
+
+  - job: tests
+    identifier: extended-unit-tests
+    trigger: pull_request
+    targets:
+      - fedora-latest-stable
+    require:
+        label:
+          present:
+            - full test
+          absent:
+            - discuss
+    tf_extra_params:
+        test:
+            tmt:
+                name: '/plans/features/extended-unit-tests$'
 
   # Test pull requests (provision)
   - job: tests

.pre-commit-config.yaml

@@ -34,6 +34,7 @@ repos:
           - "requests>=2.25.1"       # 2.28.2 / 2.31.0
           - "ruamel.yaml>=0.16.6"    # 0.17.32 / 0.17.32
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
+
           # report-junit
           - "junit_xml>=1.9"
 
@@ -44,7 +45,9 @@ repos:
           # which is the version a developer encounters given the requirements are not
           # frozen.
           - "types-Markdown"
-          - "types-requests"
+          # prevent conflict between types-requests and urllib3
+          - "types-requests<2.31.0.7; python_version < '3.10'"
+          - "types-requests; python_version >= '3.10'"
           - "types-setuptools"
           - "types-jsonschema"
           - "types-urllib3"
@@ -75,6 +78,7 @@ repos:
           - "requests>=2.25.1"       # 2.28.2 / 2.31.0
           - "ruamel.yaml>=0.16.6"    # 0.17.32 / 0.17.32
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
+
           # report-junit
           - "junit_xml>=1.9"
 
@@ -85,7 +89,9 @@ repos:
           # which is the version a developer encounters given the requirements are not
           # frozen.
           - "types-Markdown"
-          - "types-requests"
+          # prevent conflict between types-requests and urllib3
+          - "types-requests<2.31.0.7; python_version < '3.10'"
+          - "types-requests; python_version >= '3.10'"
           - "types-setuptools"
           - "types-jsonschema"
           - "types-urllib3"
@@ -122,3 +128,6 @@ repos:
         - "docutils>=0.16"         # 0.16 is the current one available for RHEL9
         - "pint<0.20"
         - "pygments>=2.7.4"        # 2.7.4 is the current one available for RHEL9
+        # Help installation by reducing the set of inspected botocore release.
+        # There is *a lot* of them, and hatch might fetch many of them.
+        - "botocore>=1.25.10"      # 1.25.10 is the current one available for RHEL9

Makefile

@@ -86,7 +86,7 @@ images:  ## Build tmt images for podman/docker
 ## Development
 ##
 develop: _deps  ## Install development requirements
-	sudo dnf install -y gcc git python3-nitrate {libvirt,krb5,libpq}-devel jq podman
+	sudo dnf install -y gcc git python3-nitrate {libvirt,krb5,libpq,python3}-devel jq podman buildah
 
 # Git vim tags and cleanup
 tags:

docs/releases.rst

@@ -24,6 +24,13 @@ connections, and may force reboot of the guest when it becomes
 unresponsive. This is the first step towards helping tests handle kernel
 panics and similar situations.
 
+Internal implementation of basic package manager actions has been
+refactored. tmt now supports package implementations to be shipped as
+plugins, therefore allowing for tmt to work natively with distributions
+beyond the ecosystem of rpm-based distributions. As a preview, ``apt``,
+the package manager used by Debian and Ubuntu, has been included in this
+release.
+
 __ https://pagure.io/testcloud/
 
 

plans/features/basic.fmf

@@ -6,3 +6,4 @@ description:
 discover:
     how: fmf
     filter: 'tier: 2 & tag:-provision-only'
+    exclude: '/tests/unit/.*?/extended'

plans/features/extended-unit-tests.fmf

@@ -0,0 +1,33 @@
+summary: Unit tests working with containers
+description: |
+    A subset of unit tests that spawns containers.
+
+discover:
+    how: fmf
+    test: '/tests/unit/.*?/extended'
+
+# Disable systemd-resolved to prevent dns failures
+# See: https://github.com/teemtee/tmt/issues/2063
+adjust+:
+  - when: initiator == packit and distro == fedora
+    prepare+:
+      - name: disable systemd resolved
+        how: shell
+        script: |
+            set -x
+            systemctl unmask systemd-resolved
+            systemctl disable systemd-resolved
+            systemctl stop systemd-resolved
+            systemctl mask systemd-resolved
+            rm -f /etc/resolv.conf
+            systemctl restart NetworkManager
+            sleep 5
+            cat /etc/resolv.conf
+            ps xa | grep resolv
+            netstat -pnl
+
+  - when: trigger == commit
+    provision:
+        hardware:
+          cpu:
+            processors: ">= 8"

pyproject.toml

@@ -119,14 +119,18 @@ dependencies = [
     "mypy",
     "pytest",
     "python-coveralls",
+    "pytest-container",
+    "pytest-xdist",
     "requre",
     "yq==3.1.1",
     "pre-commit",
     # Do not install *types-click* - it's not recommended with Click 8 & newer,
     # which is the version a developer encounters given the requirements are not
     # frozen.
     "types-Markdown",
-    "types-requests",
+    # prevent conflict between types-requests and urllib3
+    "types-requests<2.31.0.7; python_version < '3.10'",
+    "types-requests; python_version >= '3.10'",
     "types-setuptools",
     "types-jsonschema",
     "types-urllib3",
@@ -141,19 +145,24 @@ lint = ["autopep8 {args:.}", "ruff --fix {args:.}"]
 type = ["mypy {args:tmt}"]
 check = ["lint", "type"]
 
-unit = "pytest -vvv -ra --showlocals tests/unit"
-smoke = "pytest -vvv -ra --showlocals tests/unit/test_cli.py"
+unit = "pytest -vvv -ra --showlocals -n 0 tests/unit"
+smoke = "pytest -vvv -ra --showlocals -n 0 tests/unit/test_cli.py"
 cov = [
-    "coverage run --source=tmt -m pytest -vvv -ra --showlocals tests",
+    "coverage run --source=tmt -m pytest -vvv -ra --showlocals -n 0 tests",
     "coverage report",
     "coverage annotate",
     ]
 requre = [
     "cd {root}/tests/integration",
-    "pytest -vvv -ra --showlocals",
+    "pytest -vvv -ra --showlocals -n 0",
     "requre-patch purge --replaces :milestone_url:str:SomeText --replaces :latency:float:0 tests/integration/test_data/test_nitrate/*",
     ]
 
+[tool.hatch.envs.dev-not-editable]
+template = "dev"
+description = "Same as 'dev', but not using editable install"
+dev-mode = false
+
 [tool.hatch.envs.test]
 template = "dev"
 description = "Run scripts with multiple Python versions"
@@ -314,4 +323,7 @@ extend-immutable-calls = ["tmt.utils.field"]
 known-first-party = ["tmt"]
 
 [tool.pytest.ini_options]
-markers = ["web: tests which need to access the web"]
+markers = [
+    "containers: tests which need to spawn containers",
+    "web: tests which need to access the web"
+    ]

tests/unit/__init__.py

@@ -7,7 +7,7 @@
 import _pytest.logging
 import pytest
 
-import tmt.utils
+from tmt.utils import remove_color
 
 
 class PatternMatching:
@@ -56,6 +56,7 @@ def __init__(self, pattern: str) -> None:
 def _assert_log(
         caplog: _pytest.logging.LogCaptureFixture,
         evaluator: Callable[[Iterable[Any]], bool] = any,
+        remove_colors: bool = False,
         not_present: bool = False,
         **tests: Any
         ) -> None:
@@ -86,12 +87,15 @@ def _assert_log(
     operators: list[tuple[Callable[[Any], Any], str, Callable[[Any, Any], bool], Any]] = []
 
     for field_name, expected_value in tests.items():
-        if field_name.startswith('details_'):
+        if field_name == 'message' and remove_colors:
+            def field_getter(record, name): return remove_color(getattr(record, name))
+
+        elif field_name.startswith('details_'):
             field_name = field_name.replace('details_', '')
             def field_getter(record, name): return getattr(record.details, name, None)
 
         elif field_name == 'message':
-            def field_getter(record, name): return tmt.utils.remove_color(getattr(record, name))
+            def field_getter(record, name): return remove_color(getattr(record, name))
 
         else:
             def field_getter(record, name): return getattr(record, name)
@@ -164,14 +168,26 @@ def _report(message: str) -> None:
 def assert_log(
         caplog: _pytest.logging.LogCaptureFixture,
         evaluator: Callable[[Iterable[Any]], bool] = any,
+        remove_colors: bool = False,
         **tests: Any
         ) -> None:
-    _assert_log(caplog, evaluator=evaluator, not_present=False, **tests)
+    _assert_log(
+        caplog,
+        evaluator=evaluator,
+        remove_colors=remove_colors,
+        not_present=False,
+        **tests)
 
 
 def assert_not_log(
         caplog: _pytest.logging.LogCaptureFixture,
         evaluator: Callable[[Iterable[Any]], bool] = any,
+        remove_colors: bool = False,
         **tests: Any
         ) -> None:
-    _assert_log(caplog, evaluator=evaluator, not_present=True, **tests)
+    _assert_log(
+        caplog,
+        evaluator=evaluator,
+        remove_colors=remove_colors,
+        not_present=True,
+        **tests)

tests/unit/main.fmf

@@ -1,10 +1,78 @@
 summary: Python unit tests
 description:
     Run all available python unit tests using pytest.
-test: python3 -m pytest -vvv -ra --showlocals
-framework: shell
-require:
-  - python3-pytest
+duration: 30m
 environment:
     LANG: en_US.UTF-8
-tier: 0
+
+    ENABLE_PARALLELIZATION: "no"
+    ENABLE_CONTAINERS: "no"
+    WITH_SYSTEM_PACKAGES: "no"
+
+adjust+:
+  - when: distro == centos-stream-9
+    environment+:
+      HATCH_ENVIRONMENT: dev-not-editable
+
+require+:
+  - gcc
+  - git
+  - python3-nitrate
+  - libvirt-devel
+  - krb5-devel
+  - libpq-devel
+  - python3-devel
+  - jq
+  - podman
+  - buildah
+  - hatch
+
+# Run against development packages via `hatch`.
+/with-development-packages:
+    enabled: false
+
+    adjust+:
+      - when: initiator is not defined or distro == fedora-39
+        because: Enable locally or in CI on Fedora 39
+
+        enabled: true
+
+    /basic:
+        summary: Basic unit tests (development packages)
+        tier: 0
+
+    /extended:
+        summary: Extended unit tests (development packages)
+        tier: 2
+        duration: 1h
+
+        environment+: &extended_environment
+            ENABLE_PARALLELIZATION: "yes"
+            ENABLE_CONTAINERS: "yes"
+
+# Run against system, distro-packaged ones via `venv`.
+/with-system-packages:
+    enabled: false
+
+    environment+:
+        WITH_SYSTEM_PACKAGES: "yes"
+
+    require+:
+      - python3-pytest
+
+    adjust+:
+      - when: initiator == packit
+        because: Enable in CI only
+
+        enabled: true
+
+    /basic:
+        summary: Basic unit tests (system packages)
+        tier: 0
+
+    /extended:
+        summary: Extended unit tests (system packages)
+        tier: 2
+        duration: 1h
+
+        environment+: *extended_environment