tox: add tox

`tox` is a standard testing tool for Python projects, this allows you to
test locally with all your installed Python version with the following
command:

`tox -e py36,py37,py38,py39,py310,py311,py312 -p all`

To run the tests in parallel.

This commit *also* disables the `import-error` warning from `pylint`,
not all Python versions have the system-installed Python libraries
available and they can't be fetched from PyPI.

Some linters have been added and the general order linters run in has
been changed. This allows for quicker test failure when running
`tox -e lint`. As a consequence the `test_pylint` test has been removed
as it's role can now be fulfilled by `tox`.

Other assorted linter fixes due to newer versions:
- use a str.join method (`consider-using-join`)
- fix various (newer) mypy and pylint issues
- comments starting with `#` and no space due to `autopep8`

And some compatibility issues that were found for 3.6/3.7:
- Fixes the subscript of OrderedDict which is only available after 3.7.
- SimpleHTTPServer has no `directory` argument pre-3.7, use chdir.

.github/workflows/lint.yml

@@ -0,0 +1,19 @@
+name: Linters
+
+on: [pull_request, push]
+
+jobs:
+  test_suite:
+    name: "Lint Suite"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - name: "Clone Repository"
+      uses: actions/checkout@v3
+    - name: "Run Linters"
+      uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
+      with:
+        image: ghcr.io/osbuild/osbuild-ci:latest-202304110753
+        run: |
+          tox -m lint -p all

.github/workflows/tests.yml → .github/workflows/test.yml

@@ -19,7 +19,6 @@ jobs:
         - "test.run.test_noop"
         - "test.run.test_sources"
         - "test.run.test_stages"
-        - "test.src"
     steps:
     - name: "Clone Repository"
       uses: actions/checkout@v3
@@ -28,8 +27,5 @@ jobs:
       with:
         image: ghcr.io/osbuild/osbuild-ci:latest-202304110753
         run: |
-          python3 -m pytest \
-            --pyargs "${{ matrix.test }}" \
-            --rootdir=. \
-            --cov-report=xml --cov=osbuild \
-            -v
+          TEST_CATEGORY="${{ matrix.test }}" \
+          tox -m test -p all

.github/workflows/type.yml

@@ -0,0 +1,19 @@
+name: Static Type Analysis
+
+on: [pull_request, push]
+
+jobs:
+  test_suite:
+    name: "Static Type Analysis"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - name: "Clone Repository"
+      uses: actions/checkout@v3
+    - name: "Run Linters"
+      uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
+      with:
+        image: ghcr.io/osbuild/osbuild-ci:latest-202304110753
+        run: |
+          tox -m type -p all

.gitignore

@@ -20,4 +20,6 @@ cov-int/
 /docs/osbuild-manifest.5
 
 venv
-.venv
+.venv
+
+/.tox

.ruff.toml

@@ -0,0 +1,6 @@
+line-length = 120
+
+ignore = [
+    "E741",  # ambiguous variable names
+    "E501",  # line too long
+]

assemblers/org.osbuild.oci-archive

@@ -228,7 +228,7 @@ def create_oci_dir(tree, output_dir, options):
     blobs = os.path.join(output_dir, "blobs", "sha256")
     os.makedirs(blobs)
 
-    ## layers / rootfs
+    # layers / rootfs
 
     digest, info = blobs_add_layer(blobs, tree)
 

assemblers/org.osbuild.qemu

@@ -502,7 +502,7 @@ def grub2_partition_id(pt: PartitionTable):
     return label2grub[pt.label]
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def install_grub2(image: str, pt: PartitionTable, options):
     """Install grub2 to image"""
     platform = options.get("platform", "i386-pc")
@@ -637,7 +637,7 @@ def install_zipl(root: str, device: str, pt: PartitionTable):
                    check=True)
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def main(tree, output_dir, options, loop_client):
     fmt = options["format"]
     filename = options["filename"]

osbuild/buildroot.py

@@ -15,7 +15,7 @@
 import subprocess
 import tempfile
 import time
-from typing import Optional, Set
+from typing import Set
 
 from osbuild.api import BaseAPI
 from osbuild.util import linux
@@ -101,7 +101,7 @@ def __init__(self, root, runner, libdir, var, *, rundir="/run/osbuild"):
         self.proc = None
         self.tmp = None
         self.mount_boot = True
-        self.caps: Optional[set] = None
+        self.caps = None
 
     @staticmethod
     def _mknod(path, name, mode, major, minor):

osbuild/formats/v2.py

@@ -2,7 +2,7 @@
 
 Second, and current, version of the manifest description
 """
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from osbuild.meta import Index, ModuleInfo, ValidationResult
 
@@ -197,7 +197,7 @@ def sort_devices(devices: Dict) -> Dict:
             desc = devices[name]
 
             parent = desc.get("parent")
-            if parent and not parent in result:
+            if parent and parent not in result:
                 # if the parent is not in the `result` list, it must
                 # be in `todo`; otherwise it is missing
                 if parent not in todo:
@@ -391,8 +391,8 @@ def load(description: Dict, index: Index) -> Manifest:
     return manifest
 
 
-#pylint: disable=too-many-branches
-def output(manifest: Manifest, res: Dict, store: ObjectStore = None) -> Dict:
+# pylint: disable=too-many-branches
+def output(manifest: Manifest, res: Dict, store: Optional[ObjectStore] = None) -> Dict:
     """Convert a result into the v2 format"""
 
     def collect_metadata(p: Pipeline) -> Dict[str, Any]:

osbuild/host.py

@@ -272,7 +272,7 @@ def serve(self):
                 # an exception in `sock.send` later.
                 self._check_fds(reply_fds)
 
-            except:  # pylint: disable=bare-except
+            except Exception:  # pylint: disable=broad-exception-caught
                 reply_fds = self._close_all(reply_fds)
                 _, val, tb = sys.exc_info()
                 reply = self.protocol.encode_exception(val, tb)
@@ -351,7 +351,7 @@ def call(self, method: str, args: Optional[Any] = None) -> Any:
     def call_with_fds(self, method: str,
                       args: Optional[Union[List[str], Dict[str, Any]]] = None,
                       fds: Optional[List[int]] = None,
-                      on_signal: Callable[[Any, Optional[Iterable[int]]], None] = None
+                      on_signal: Optional[Callable[[Any, Optional[Iterable[int]]], None]] = None
                       ) -> Tuple[Any, Optional[Iterable[int]]]:
         """
         Remotely call a method and return the result, including file

osbuild/loop.py

@@ -109,7 +109,7 @@ def __init__(self, minor, dir_fd=None):
 
         self.devname = f"loop{minor}"
         self.minor = minor
-        self.on_close: Optional[Callable[["Loop"], None]] = None
+        self.on_close = None
 
         with contextlib.ExitStack() as stack:
             if not dir_fd:

osbuild/objectstore.py

@@ -338,7 +338,7 @@ def maximum_size(self, size: Union[int, str]):
 
     @property
     def active(self) -> bool:
-        #pylint: disable=protected-access
+        # pylint: disable=protected-access
         return self.cache._is_active()
 
     @property

osbuild/pipeline.py

@@ -383,7 +383,7 @@ class Manifest:
 
     def __init__(self):
         self.pipelines = collections.OrderedDict()
-        self.sources: List[Source] = []
+        self.sources = []
 
     def add_pipeline(
         self,

osbuild/util/ostree.py

@@ -6,7 +6,8 @@
 import sys
 import tempfile
 import typing
-from typing import Any, List
+# pylint doesn't understand the string-annotation below
+from typing import Any, List  # pylint: disable=unused-import
 
 from .types import PathLike
 
@@ -222,7 +223,7 @@ class SubIdsDB:
     """
 
     def __init__(self) -> None:
-        self.db: collections.OrderedDict[str, Any] = collections.OrderedDict()
+        self.db: 'collections.OrderedDict[str, Any]' = collections.OrderedDict()
 
     def read(self, fp) -> int:
         idx = 0

setup.cfg

@@ -9,7 +9,8 @@ disable=missing-docstring,
         consider-using-with,
         consider-using-from-import,
         line-too-long,
-        useless-option-value
+        useless-option-value,
+        import-error
 
 [pylint.TYPECHECK]
 ignored-classes=osbuild.loop.LoopInfo

stages/org.osbuild.dracut

@@ -129,7 +129,7 @@ def yesno(name: str, value: bool) -> str:
     return f"--{prefix}{name}"
 
 
-#pylint: disable=too-many-branches
+# pylint: disable=too-many-branches
 def main(tree, options):
     kernels = options["kernel"]
     compress = options.get("compress")

stages/org.osbuild.first-boot

@@ -50,7 +50,7 @@ After=network-online.target"""
 
     execs = "\n"
     for command in commands:
-        execs += f"ExecStart={command}\n"
+        execs += f"ExecStart={command}\n"  # pylint: disable=consider-using-join
 
     service = f"""[Unit]
 Description=OSBuild First Boot Service

stages/org.osbuild.grub2

@@ -484,7 +484,7 @@ class GrubConfig:
         return data
 
 
-#pylint: disable=too-many-statements,too-many-branches
+# pylint: disable=too-many-statements,too-many-branches
 def main(tree, options):
     root_fs = options.get("rootfs")
     boot_fs = options.get("bootfs")

stages/org.osbuild.oci-archive

@@ -309,7 +309,7 @@ def create_oci_dir(inputs, output_dir, options, create_time):
     blobs = os.path.join(output_dir, "blobs", "sha256")
     os.makedirs(blobs)
 
-    ## layers / rootfs
+    # layers / rootfs
     for ip in sorted(inputs.keys()):
         tree = inputs[ip]["path"]
         digest, info = blobs_add_layer(blobs, tree)

stages/org.osbuild.ovf

@@ -36,7 +36,7 @@ SCHEMA_2 = """
 OVF_TEMPLATE = """<?xml version="1.0"?>
 <Envelope xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <References>
-	  <File ovf:href="{image_name}" ovf:id="file1" ovf:size="{vmdk_size}"/>
+    <File ovf:href="{image_name}" ovf:id="file1" ovf:size="{vmdk_size}"/>
   </References>
   <DiskSection>
     <Info>Virtual disk information</Info>
@@ -57,7 +57,7 @@ OVF_TEMPLATE = """<?xml version="1.0"?>
         <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
         <vssd:InstanceID>0</vssd:InstanceID>
         <vssd:VirtualSystemIdentifier>image</vssd:VirtualSystemIdentifier>
-	<vssd:VirtualSystemType>vmx-15</vssd:VirtualSystemType>
+        <vssd:VirtualSystemType>vmx-15</vssd:VirtualSystemType>
       </System>
       <Item>
         <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
@@ -88,7 +88,7 @@ OVF_TEMPLATE = """<?xml version="1.0"?>
         <rasd:Description>SCSI Controller</rasd:Description>
         <rasd:ElementName>SCSI Controller 0</rasd:ElementName>
         <rasd:InstanceID>3</rasd:InstanceID>
-	<rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
+        <rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
         <rasd:ResourceType>6</rasd:ResourceType>
       </Item>
       <Item>

test/run/test_sources.py

@@ -73,7 +73,8 @@ def can_setup_netns() -> bool:
 def runFileServer(barrier, directory):
     class Handler(http.server.SimpleHTTPRequestHandler):
         def __init__(self, request, client_address, server):
-            super().__init__(request, client_address, server, directory=directory)
+            os.chdir(directory)
+            super().__init__(request, client_address, server)
 
         def guess_type(self, path):
             try: