tox: add tox

`tox` is a standard testing tool for Python projects, this allows you to
test locally with all your installed Python version with the following
command:

`tox -m test -p all`

To run the tests in parallel for all supported Python versions.

To run linters or type analysis:

```
tox -m lint -p all
tox -m type -p all

tox -e py36,py37,py38 -p all
```

This commit *also* disables the `import-error` warning from `pylint`,
not all Python versions have the system-installed Python libraries
available and they can't be fetched from PyPI.

Some linters have been added and the general order linters run in has
been changed. This allows for quicker test failure when running
`tox -m lint`. As a consequence the `test_pylint` test has been removed
as it's role can now be fulfilled by `tox`.

Other assorted linter fixes due to newer versions:
- use a str.join method (`consider-using-join`)
- fix various (newer) mypy and pylint issues
- comments starting with `#` and no space due to `autopep8`

And some compatibility issues that were found for 3.6/3.7:
- Fixes the subscript of OrderedDict which is only available after 3.7.
- SimpleHTTPServer has no `directory` argument pre-3.7, use
  path_translate.

This also changes our CI to use the new `tox` setup and on top of that
pins the versions of linters used. This might move into separate
requirements.txt files later on to allow for easier updating of those
dependencies.

.github/workflows/check.yml

@@ -0,0 +1,72 @@
+name: Check
+
+on: [pull_request, push]
+
+permissions:
+  contents: read
+
+jobs:
+  spelling_checker:
+    name: "Spelling"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: codespell-project/actions-codespell@master
+        with:
+          ignore_words_list: msdos, pullrequest
+          skip: ./.git,coverity,rpmbuild,samples
+
+  python_code_linters:
+      name: "Python Linters"
+      runs-on: ubuntu-latest
+      strategy:
+        fail-fast: false
+        matrix:
+          linter:
+          - "ruff"
+          - "pylint"
+          - "autopep8"
+          - "isort"
+      steps:
+      - name: "Clone Repository"
+        uses: actions/checkout@v3
+      - name: "Run Linters"
+        uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
+        with:
+          image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
+          run: |
+            tox -e "${{ matrix.linter }}"
+
+  python_code_types:
+    name: "Python Typing"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        typer:
+        - "mypy"
+    steps:
+    - name: "Clone Repository"
+      uses: actions/checkout@v3
+    - name: "Run Linters"
+      uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
+      with:
+        image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
+        run: |
+          tox -e "${{ matrix.typer }}"
+
+  shell_linters:
+    name: "Shell Linters"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: "Clone Repository"
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: "Differential ShellCheck"
+      uses: redhat-plumbers-in-action/differential-shellcheck@v3
+      with:
+        severity: warning
+        token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/checks.yml → .github/workflows/generate.yml

@@ -1,10 +1,10 @@
-name: Checks
+name: Generate
 
 on: [pull_request, push]
 
 jobs:
-  documentation:
-    name: "📚 Documentation"
+  generate_documentation:
+    name: "Documentation"
     runs-on: ubuntu-latest
     container:
       image: docker.io/library/python:3.7
@@ -33,26 +33,16 @@ jobs:
           test -d docs
           test -f docs/osbuild.1
 
-  test_data:
-    name: "Regenerate Test Data"
+  generate_test_data:
+    name: "Test Data"
     runs-on: ubuntu-latest
     steps:
     - name: "Clone Repository"
       uses: actions/checkout@v3
     - name: "Regenerate Test Data"
       uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
       with:
-        image: ghcr.io/osbuild/osbuild-ci:latest-202304110753
+        image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
         run: |
           make test-data
           git diff --exit-code -- ./test/data
-
-  codespell:
-    name: "Spell check"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: codespell-project/actions-codespell@master
-        with:
-          ignore_words_list: msdos, pullrequest
-          skip: ./.git,coverity,rpmbuild,samples

.github/workflows/tests.yml → .github/workflows/test.yml

@@ -1,10 +1,10 @@
-name: Tests
+name: Test
 
 on: [pull_request, push]
 
 jobs:
   test_suite:
-    name: "Test Suite"
+    name: "Test"
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -19,17 +19,21 @@ jobs:
         - "test.run.test_noop"
         - "test.run.test_sources"
         - "test.run.test_stages"
-        - "test.src"
+        environment:
+        - "py36"
+        - "py37"
+        - "py38"
+        - "py39"
+        - "py310"
+        - "py311"
+        - "py312"
     steps:
     - name: "Clone Repository"
       uses: actions/checkout@v3
-    - name: "Run Tests"
+    - name: "Run"
       uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
       with:
-        image: ghcr.io/osbuild/osbuild-ci:latest-202304110753
+        image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
         run: |
-          python3 -m pytest \
-            --pyargs "${{ matrix.test }}" \
-            --rootdir=. \
-            --cov-report=xml --cov=osbuild \
-            -v
+          TEST_CATEGORY="${{ matrix.test }}" \
+          tox -e "${{ matrix.environment }}"

.gitignore

@@ -20,4 +20,6 @@ cov-int/
 /docs/osbuild-manifest.5
 
 venv
-.venv
+.venv
+
+/.tox

.ruff.toml

@@ -0,0 +1,6 @@
+line-length = 120
+
+ignore = [
+    "E741",  # ambiguous variable names
+    "E501",  # line too long
+]

assemblers/org.osbuild.oci-archive

@@ -228,7 +228,7 @@ def create_oci_dir(tree, output_dir, options):
     blobs = os.path.join(output_dir, "blobs", "sha256")
     os.makedirs(blobs)
 
-    ## layers / rootfs
+    # layers / rootfs
 
     digest, info = blobs_add_layer(blobs, tree)
 

assemblers/org.osbuild.qemu

@@ -502,7 +502,7 @@ def grub2_partition_id(pt: PartitionTable):
     return label2grub[pt.label]
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def install_grub2(image: str, pt: PartitionTable, options):
     """Install grub2 to image"""
     platform = options.get("platform", "i386-pc")
@@ -637,7 +637,7 @@ def install_zipl(root: str, device: str, pt: PartitionTable):
                    check=True)
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def main(tree, output_dir, options, loop_client):
     fmt = options["format"]
     filename = options["filename"]

osbuild/buildroot.py

@@ -15,7 +15,7 @@
 import subprocess
 import tempfile
 import time
-from typing import Optional, Set
+from typing import Set
 
 from osbuild.api import BaseAPI
 from osbuild.util import linux
@@ -101,7 +101,7 @@ def __init__(self, root, runner, libdir, var, *, rundir="/run/osbuild"):
         self.proc = None
         self.tmp = None
         self.mount_boot = True
-        self.caps: Optional[set] = None
+        self.caps = None
 
     @staticmethod
     def _mknod(path, name, mode, major, minor):

osbuild/formats/v2.py

@@ -2,7 +2,7 @@
 
 Second, and current, version of the manifest description
 """
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from osbuild.meta import Index, ModuleInfo, ValidationResult
 
@@ -197,7 +197,7 @@ def sort_devices(devices: Dict) -> Dict:
             desc = devices[name]
 
             parent = desc.get("parent")
-            if parent and not parent in result:
+            if parent and parent not in result:
                 # if the parent is not in the `result` list, it must
                 # be in `todo`; otherwise it is missing
                 if parent not in todo:
@@ -391,8 +391,8 @@ def load(description: Dict, index: Index) -> Manifest:
     return manifest
 
 
-#pylint: disable=too-many-branches
-def output(manifest: Manifest, res: Dict, store: ObjectStore = None) -> Dict:
+# pylint: disable=too-many-branches
+def output(manifest: Manifest, res: Dict, store: Optional[ObjectStore] = None) -> Dict:
     """Convert a result into the v2 format"""
 
     def collect_metadata(p: Pipeline) -> Dict[str, Any]:

osbuild/host.py

@@ -272,7 +272,7 @@ def serve(self):
                 # an exception in `sock.send` later.
                 self._check_fds(reply_fds)
 
-            except:  # pylint: disable=bare-except
+            except Exception:  # pylint: disable=broad-exception-caught
                 reply_fds = self._close_all(reply_fds)
                 _, val, tb = sys.exc_info()
                 reply = self.protocol.encode_exception(val, tb)
@@ -351,7 +351,7 @@ def call(self, method: str, args: Optional[Any] = None) -> Any:
     def call_with_fds(self, method: str,
                       args: Optional[Union[List[str], Dict[str, Any]]] = None,
                       fds: Optional[List[int]] = None,
-                      on_signal: Callable[[Any, Optional[Iterable[int]]], None] = None
+                      on_signal: Optional[Callable[[Any, Optional[Iterable[int]]], None]] = None
                       ) -> Tuple[Any, Optional[Iterable[int]]]:
         """
         Remotely call a method and return the result, including file

osbuild/loop.py

@@ -119,7 +119,7 @@ def __init__(self, minor, dir_fd=None):
 
         self.devname = f"loop{minor}"
         self.minor = minor
-        self.on_close: Optional[Callable[["Loop"], None]] = None
+        self.on_close = None
 
         with contextlib.ExitStack() as stack:
             if not dir_fd:

osbuild/objectstore.py

@@ -338,7 +338,7 @@ def maximum_size(self, size: Union[int, str]):
 
     @property
     def active(self) -> bool:
-        #pylint: disable=protected-access
+        # pylint: disable=protected-access
         return self.cache._is_active()
 
     @property

osbuild/pipeline.py

@@ -383,7 +383,7 @@ class Manifest:
 
     def __init__(self):
         self.pipelines = collections.OrderedDict()
-        self.sources: List[Source] = []
+        self.sources = []
 
     def add_pipeline(
         self,

osbuild/util/ostree.py

@@ -6,7 +6,8 @@
 import sys
 import tempfile
 import typing
-from typing import Any, List
+# pylint doesn't understand the string-annotation below
+from typing import Any, List  # pylint: disable=unused-import
 
 from .types import PathLike
 
@@ -222,7 +223,7 @@ class SubIdsDB:
     """
 
     def __init__(self) -> None:
-        self.db: collections.OrderedDict[str, Any] = collections.OrderedDict()
+        self.db: 'collections.OrderedDict[str, Any]' = collections.OrderedDict()
 
     def read(self, fp) -> int:
         idx = 0

setup.cfg

@@ -9,7 +9,8 @@ disable=missing-docstring,
         consider-using-with,
         consider-using-from-import,
         line-too-long,
-        useless-option-value
+        useless-option-value,
+        import-error
 
 [pylint.TYPECHECK]
 ignored-classes=osbuild.loop.LoopInfo

stages/org.osbuild.dracut

@@ -129,7 +129,7 @@ def yesno(name: str, value: bool) -> str:
     return f"--{prefix}{name}"
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def main(tree, options):
     kernels = options["kernel"]
     compress = options.get("compress")

stages/org.osbuild.first-boot

@@ -50,7 +50,7 @@ After=network-online.target"""
 
     execs = "\n"
     for command in commands:
-        execs += f"ExecStart={command}\n"
+        execs += f"ExecStart={command}\n"  # pylint: disable=consider-using-join
 
     service = f"""[Unit]
 Description=OSBuild First Boot Service

stages/org.osbuild.grub2

@@ -484,7 +484,7 @@ class GrubConfig:
         return data
 
 
-#pylint: disable=too-many-statements,too-many-branches
+# pylint: disable=too-many-statements,too-many-branches
 def main(tree, options):
     root_fs = options.get("rootfs")
     boot_fs = options.get("bootfs")

stages/org.osbuild.oci-archive

@@ -309,7 +309,7 @@ def create_oci_dir(inputs, output_dir, options, create_time):
     blobs = os.path.join(output_dir, "blobs", "sha256")
     os.makedirs(blobs)
 
-    ## layers / rootfs
+    # layers / rootfs
     for ip in sorted(inputs.keys()):
         tree = inputs[ip]["path"]
         digest, info = blobs_add_layer(blobs, tree)