Merge branch 'release/1.17.0'

.github/workflows/package-build.yml

@@ -11,9 +11,6 @@ jobs:
     strategy:
       matrix:
        include:
-         - distribution: debian
-           distribution_release: buster
-           repository_type: packagecloud
          - distribution: debian
            distribution_release: bullseye
            repository_type: packagecloud

debian/changelog

@@ -1,3 +1,11 @@
+edi (1.17.0) mantic; urgency=medium
+
+  [ Matthias Luescher ]
+  * Removed Debian buster build.
+  * Added handling for podman images within command runner.
+
+ -- Matthias Lüscher (Launchpad) <m.luescher@datacomm.ch>  Fri, 22 Mar 2024 14:41:55 +0100
+
 edi (1.16.1) mantic; urgency=medium
 
   [ Matthias Luescher ]

debian/compat

@@ -1 +1 @@
-9
+10

debian/control

@@ -3,7 +3,7 @@ Section: devel
 Priority: optional
 Build-Depends: ansible (>= 2.9),
                binfmt-support,
-               debhelper (>= 9),
+               debhelper (>= 10),
                debootstrap,
                dh-python,
                flake8,
@@ -30,7 +30,7 @@ Build-Depends: ansible (>= 2.9),
                zstd,
 Maintainer: Matthias Luescher <lueschem@gmail.com>
 Standards-Version: 3.9.7
-X-Python3-Version: >= 3.5
+X-Python3-Version: >= 3.8
 
 Package: edi
 Architecture: all

debian/rules

@@ -3,7 +3,7 @@
 #DH_VERBOSE=1
 
 %:
-	dh $@ --with python3,sphinxdoc --buildsystem=pybuild --fail-missing
+	dh $@ --with python3 --buildsystem=pybuild --fail-missing
 
 override_dh_installudev:
 	dh_installudev --priority=81

docs/conf.py

@@ -55,8 +55,8 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.16.1'
-release = '1.16.1'
+version = '1.17.0'
+release = '1.17.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

edi/lib/buildahhelpers.py

@@ -79,7 +79,6 @@ def is_container_existing(name):
 
 @require('buildah', buildah_install_hint, BuildahVersion.check)
 def delete_container(name):
-    # needs to be stopped first!
     cmd = [buildah_exec(), "rm", name]
 
     run(cmd, log_threshold=logging.INFO)

edi/lib/commandrunner.py

@@ -33,11 +33,14 @@
 from edi.lib.shellhelpers import run, safely_remove_artifacts_folder
 from edi.lib.configurationparser import remove_passwords
 from edi.lib.yamlhelpers import LiteralString
+from edi.lib.podmanhelpers import is_image_existing, try_delete_image, untag_image
 
 
 class ArtifactType(Enum):
     PATH = 'path'
     BUILDAH_CONTAINER = 'buildah-container'
+    PODMAN_IMAGE = 'podman-image'  # Owned by non-root user.
+    PODMAN_IMAGE_ROOT = 'podman-image-root'  # Owned by root.
 
 
 Artifact = namedtuple("Artifact", "name, location, type")
@@ -114,8 +117,7 @@ def require_real_root(self):
         commands = self._get_commands()
 
         for command in commands:
-            if (not self._are_all_artifacts_available(command.output_artifacts)
-                    and self._require_real_root(command.config_node.get('require_root', False))):
+            if self._require_real_root(command.config_node.get('require_root', False)):
                 return True
 
         return False
@@ -132,7 +134,7 @@ def _require_real_root(config_value):
     def require_real_root_for_clean(self):
         for command in self._get_commands():
             if (self._require_real_root(command.config_node.get('require_root', False))
-                    and self._is_an_artifact_a_directory(command.output_artifacts)):
+                    and self._is_root_required_for_removal(command.output_artifacts)):
                 return True
 
         return False
@@ -158,19 +160,32 @@ def clean(self):
         commands = self._get_commands()
         for command in commands:
             for _, artifact in command.output_artifacts.items():
-                if not str(get_workdir()) in str(artifact.location):
-                    raise FatalError(('Output artifact {} is not within the current working directory!'
-                                      ).format(artifact.location))
-
-                if os.path.isfile(artifact.location):
-                    logging.info("Removing '{}'.".format(artifact.location))
-                    os.remove(artifact.location)
-                    print_success("Removed image file artifact {}.".format(artifact.location))
-                elif os.path.isdir(artifact.location):
-                    safely_remove_artifacts_folder(artifact.location,
-                                                   sudo=self._require_real_root(command.config_node.get('require_root',
-                                                                                                        False)))
-                    print_success("Removed image directory artifact {}.".format(artifact.location))
+                if artifact.type is ArtifactType.PATH:
+                    if not str(get_workdir()) in str(artifact.location):
+                        raise FatalError(('Output artifact {} is not within the current working directory!'
+                                          ).format(artifact.location))
+
+                    if os.path.isfile(artifact.location):
+                        logging.info("Removing '{}'.".format(artifact.location))
+                        os.remove(artifact.location)
+                        print_success("Removed image file artifact {}.".format(artifact.location))
+                    elif os.path.isdir(artifact.location):
+                        safely_remove_artifacts_folder(artifact.location,
+                                                       sudo=self._require_real_root(
+                                                           command.config_node.get('require_root', False)))
+                        print_success("Removed image directory artifact {}.".format(artifact.location))
+                elif artifact.type in [ArtifactType.PODMAN_IMAGE, ArtifactType.PODMAN_IMAGE_ROOT]:
+                    image_name = artifact.location
+                    require_sudo = artifact.type is ArtifactType.PODMAN_IMAGE_ROOT
+                    if is_image_existing(image_name, sudo=require_sudo):
+                        if try_delete_image(image_name, sudo=require_sudo):
+                            print_success(f"Removed podman image {artifact.location}.")
+                        else:
+                            logging.info(f"Podman image '{artifact.location}' is still in use, going to untag it.")
+                            untag_image(image_name, sudo=require_sudo)
+                            print_success(f"Untagged podman image {artifact.location}.")
+                else:
+                    raise FatalError(f"Unhandled removal of artifact type '{artifact.type}'.")
 
     def result(self):
         commands = self._get_commands()
@@ -185,7 +200,6 @@ def _run_command(command_file, require_root):
         run(cmd, log_threshold=logging.INFO, sudo=CommandRunner._require_real_root(require_root))
 
     def _get_commands(self):
-        artifact_directory = get_artifact_dir()
         commands = self.config.get_ordered_path_items(self.config_section)
         augmented_commands = []
         artifacts = dict()
@@ -200,15 +214,7 @@ def _get_commands(self):
 
             new_artifacts = dict()
             for artifact_key, artifact_item in output.items():
-                if str(artifact_item) != os.path.basename(artifact_item):
-                    raise FatalError((('''The specified output artifact '{}' within the '''
-                                       '''command node '{}' is invalid.\n'''
-                                       '''The output shall be a file or a folder (no '/' in string).''')
-                                      ).format(artifact_key, name))
-                artifact_path = os.path.join(artifact_directory, artifact_item)
-                new_artifacts[artifact_key] = Artifact(name=artifact_key,
-                                                       location=str(artifact_path),
-                                                       type=ArtifactType.PATH)
+                new_artifacts[artifact_key] = self._get_artifact(name, artifact_key, artifact_item)
 
             artifacts.update(new_artifacts)
             dictionary.update({key: val.location for key, val in artifacts.items()})
@@ -220,30 +226,78 @@ def _get_commands(self):
 
         return augmented_commands
 
+    @staticmethod
+    def _get_artifact(node_name, artifact_key, artifact_item):
+        if type(artifact_item) is dict:
+            artifact_location = artifact_item.get('location', '')
+            artifact_type_string = artifact_item.get('type', 'path')
+            try:
+                artifact_type = ArtifactType(artifact_type_string)
+            except ValueError:
+                raise FatalError((f"The specified output artifact '{artifact_key}' within the "
+                                  f"command node '{node_name}' has an invalid artifact type '{artifact_type_string}'."))
+        else:
+            artifact_location = artifact_item
+            artifact_type = ArtifactType.PATH
+
+        if not artifact_location:
+            raise FatalError((f"The specified output artifact '{artifact_key}' within the "
+                              f"command node '{node_name}' must not be empty."))
+
+        if artifact_type is ArtifactType.PATH:
+            if str(artifact_location) != str(os.path.basename(artifact_location)):
+                raise FatalError((('''The specified output artifact '{}' within the '''
+                                   '''command node '{}' is invalid. '''
+                                   '''The output shall be a file or a folder (no '/' in string).''')
+                                  ).format(artifact_key, node_name))
+            artifact_location = os.path.join(get_artifact_dir(), artifact_location)
+
+        return Artifact(name=artifact_key, location=str(artifact_location), type=artifact_type)
+
     @staticmethod
     def _are_all_artifacts_available(artifacts):
         for _, artifact in artifacts.items():
-            if not os.path.isfile(artifact.location) and not os.path.isdir(artifact.location):
-                return False
+            if artifact.type is ArtifactType.PATH:
+                if not os.path.isfile(artifact.location) and not os.path.isdir(artifact.location):
+                    return False
+            elif artifact.type in [ArtifactType.PODMAN_IMAGE, ArtifactType.PODMAN_IMAGE_ROOT]:
+                require_sudo = artifact.type is ArtifactType.PODMAN_IMAGE_ROOT
+                if not is_image_existing(artifact.location, sudo=require_sudo):
+                    return False
+            else:
+                raise FatalError(f"Unhandled presence checking for artifact type '{artifact.type}'.")
 
         return True
 
     @staticmethod
-    def _is_an_artifact_a_directory(artifacts):
+    def _is_root_required_for_removal(artifacts):
         for _, artifact in artifacts.items():
-            if os.path.isdir(artifact.location):
+            if artifact.type is ArtifactType.PATH and os.path.isdir(artifact.location):
+                return True
+            if artifact.type is ArtifactType.PODMAN_IMAGE_ROOT:
+                # Unable to check presence as we might not be running as root.
                 return True
 
         return False
 
     @staticmethod
     def _post_process_artifacts(command_name, expected_artifacts):
         for _, artifact in expected_artifacts.items():
-            if not os.path.isfile(artifact.location) and not os.path.isdir(artifact.location):
-                raise FatalError(('''The command '{}' did not generate '''
-                                  '''the specified output artifact '{}'.'''.format(command_name, artifact.location)))
-            elif os.path.isfile(artifact.location):
-                chown_to_user(artifact.location)
+            if artifact.type is ArtifactType.PATH:
+                if not os.path.isfile(artifact.location) and not os.path.isdir(artifact.location):
+                    raise FatalError(('''The command '{}' did not generate '''
+                                      '''the specified output artifact '{}'.'''.format(command_name,
+                                                                                       artifact.location)))
+                elif os.path.isfile(artifact.location):
+                    chown_to_user(artifact.location)
+            elif artifact.type in [ArtifactType.PODMAN_IMAGE, ArtifactType.PODMAN_IMAGE_ROOT]:
+                require_sudo = artifact.type is ArtifactType.PODMAN_IMAGE_ROOT
+                if not is_image_existing(artifact.location, sudo=require_sudo):
+                    raise FatalError(('''The command '{}' did not generate '''
+                                      '''the specified podman image '{}'.'''.format(command_name,
+                                                                                    artifact.location)))
+            else:
+                raise FatalError(f"Missing postprocessing rule for artifact type '{artifact.type}'.")
 
     @staticmethod
     def _render_command_file(input_file, dictionary):

edi/lib/podmanhelpers.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2024 Matthias Luescher
+#
+# Authors:
+#  Matthias Luescher
+#
+# This file is part of edi.
+#
+# edi is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# edi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with edi.  If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+import yaml
+import logging
+from packaging.version import Version
+from edi.lib.helpers import FatalError
+from edi.lib.versionhelpers import get_stripped_version
+from edi.lib.shellhelpers import run, Executables, require
+
+
+podman_install_hint = "'sudo apt install podman'"
+
+
+def podman_exec():
+    return Executables.get('podman')
+
+
+def get_podman_version():
+    if not Executables.has('podman'):
+        return '0.0.0'
+
+    cmd = [Executables.get("podman"), "version", "--format=json"]
+    result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    parsed_result = yaml.safe_load(result.stdout)
+    return parsed_result.get('Client').get('Version')
+
+
+class PodmanVersion:
+    """
+    Make sure that the podman version is >= 4.3.1.
+    """
+    _check_done = False
+    _required_minimal_version = '4.3.1'
+
+    def __init__(self, clear_cache=False):
+        if clear_cache:
+            PodmanVersion._check_done = False
+
+    @staticmethod
+    def check():
+        if PodmanVersion._check_done:
+            return
+
+        if Version(get_stripped_version(get_podman_version())) < Version(PodmanVersion._required_minimal_version):
+            raise FatalError(('The current podman installation ({}) does not meet the minimal requirements (>={}).\n'
+                              'Please update your podman installation!'
+                              ).format(get_podman_version(), PodmanVersion._required_minimal_version))
+        else:
+            PodmanVersion._check_done = True
+
+
+@require('podman', podman_install_hint, PodmanVersion.check)
+def is_image_existing(name, sudo=False):
+    cmd = [podman_exec(), "image", "exists", name]
+    result = run(cmd, check=False, stderr=subprocess.PIPE, sudo=sudo)
+    return result.returncode == 0
+
+
+@require('podman', podman_install_hint, PodmanVersion.check)
+def try_delete_image(name, sudo=False):
+    cmd = [podman_exec(), "image", "rm", name]
+    result = run(cmd, check=False, stderr=subprocess.PIPE, sudo=sudo)
+    return result.returncode == 0
+
+
+@require('podman', podman_install_hint, PodmanVersion.check)
+def untag_image(name, sudo=False):
+    cmd = [podman_exec(), "image", "untag", name]
+    run(cmd, log_threshold=logging.INFO, sudo=sudo)

edi/lib/versionhelpers.py

@@ -20,13 +20,13 @@
 # along with edi.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import pkg_resources
+import importlib.metadata
 import re
 from edi.lib.helpers import FatalError
 
 # The do_release script will update this version!
 # During launchpad debuild neither the git version nor the package version is available.
-edi_fallback_version = '1.16.1'
+edi_fallback_version = '1.17.0'
 
 
 def get_edi_version():
@@ -43,8 +43,8 @@ def get_edi_version():
         return get_version(root=project_root)
     else:
         try:
-            return pkg_resources.get_distribution('edi').version
-        except pkg_resources.DistributionNotFound:
+            return importlib.metadata.version('edi')
+        except importlib.metadata.PackageNotFoundError:
             return edi_fallback_version
 
 

setup.py

@@ -38,7 +38,7 @@
 setup(
     name='edi',
 
-    version='1.16.1',
+    version='1.17.0',
 
     description='Embedded Development Infrastructure - edi',
     long_description=long_description,