Merge branch 'release/1.16.0'

.github/workflows/package-build.yml

@@ -26,9 +26,6 @@ jobs:
          - distribution: ubuntu
            distribution_release: "22.04"
            repository_type: launchpad
-         - distribution: ubuntu
-           distribution_release: "23.04"
-           repository_type: launchpad
          - distribution: ubuntu
            distribution_release: "23.10"
            repository_type: launchpad

debian/changelog

@@ -1,3 +1,16 @@
+edi (1.16.0) jammy; urgency=medium
+
+  [ Matthias Luescher ]
+  * Added project commands.
+  * CommandRunner: allow multiple input artifacts.
+  * Added buildah related stuff.
+  * Added parsing of preprocessing_commands section.
+  * Implemented project make command.
+  * Added possibility to run command in fakeroot environment.
+  * Removed Ubuntu 23.04.
+
+ -- Matthias Lüscher (Launchpad) <m.luescher@datacomm.ch>  Fri, 02 Feb 2024 22:56:50 +0100
+
 edi (1.15.9) jammy; urgency=medium
 
   * Removed obsolete QEMU handling. Closes #87.

debian/control

@@ -36,7 +36,6 @@ Package: edi
 Architecture: all
 Depends: ${misc:Depends},
          ${python3:Depends},
-         ${ansible:Depends},
          ansible (>= 2.9),
          binfmt-support,
          debootstrap,
@@ -46,6 +45,15 @@ Depends: ${misc:Depends},
          qemu-user-static (>= 1:4.2),
          sudo,
          zstd,
+Recommends: debian-archive-keyring,
+            mmdebstrap,
+Suggests: buildah,
+          distrobox,
+          dosfstools,
+          e2fsprogs,
+          genimage,
+          mtools,
+          podman,
 Description: Embedded Development Infrastructure - edi
  Driven by the DevOps mindset edi helps you to streamline your embedded
  development infrastructure. To achieve this goal, edi leverages top-notch

debian/rules

@@ -2,22 +2,9 @@
 
 #DH_VERBOSE=1
 
-VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID)
-
-ifeq ($(VERSION_ID),16.04)
-	SUBSTVARS = -Vansible:Depends=""
-else ifeq ($(VERSION_ID),9)
-	SUBSTVARS = -Vansible:Depends=""
-else
-	SUBSTVARS = -Vansible:Depends="python3-distutils"
-endif
-
 %:
 	dh $@ --with python3,sphinxdoc --buildsystem=pybuild --fail-missing
 
-override_dh_gencontrol:
-	dh_gencontrol -- $(SUBSTVARS)
-
 override_dh_installudev:
 	dh_installudev --priority=81
 

docs/conf.py

@@ -55,8 +55,8 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.15.9'
-release = '1.15.9'
+version = '1.16.0'
+release = '1.16.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

edi/commands/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016 Matthias Luescher
+# Copyright (C) 2023 Matthias Luescher
 #
 # Authors:
 #  Matthias Luescher
@@ -25,5 +25,7 @@
 from edi.commands.configcommands import configclean, configinit  # noqa: ignore=F401
 from edi.commands.targetcommands import targetconfigure  # noqa: ignore=F401
 from edi.commands.documentationcommands import render  # noqa: ignore=F401
+from edi.commands.projectcommands import prepare, configure, make, clean  # noqa: ignore=F401
 
-__all__ = ["config", "image", "lxc", "version", "clean", "target", "documentation"]
+
+__all__ = ["config", "image", "lxc", "version", "clean", "target", "documentation", "project"]

edi/commands/imagecommands/create.py

@@ -54,7 +54,7 @@ def _dry_run(self):
         if self._input_artifact() is not None:
             plugins.update(Export().dry_run(self.config.get_base_config_file()))
         command_runner = CommandRunner(self.config, self.section, Artifact(name='edi_input_artifact',
-                                                                           url=self._input_artifact(),
+                                                                           location=self._input_artifact(),
                                                                            type=ArtifactType.PATH))
         plugins.update(command_runner.get_plugin_report())
         return plugins
@@ -64,10 +64,10 @@ def run(self, config_file):
 
     def _run(self):
         command_runner = CommandRunner(self.config, self.section, Artifact(name='edi_input_artifact',
-                                                                           url=self._input_artifact(),
+                                                                           location=self._input_artifact(),
                                                                            type=ArtifactType.PATH))
 
-        if command_runner.require_root():
+        if command_runner.require_real_root():
             self._require_sudo()
 
         if self._input_artifact() is not None:
@@ -80,7 +80,7 @@ def _run(self):
         result = command_runner.run()
 
         if result:
-            formatted_results = [f"{a.name}: {a.url}" for a in result]
+            formatted_results = [f"{a.name}: {a.location}" for a in result]
             print_success(("Completed the image creation post processing commands.\n"
                            "The following artifacts are now available:\n- {}".format('\n- '.join(formatted_results))))
         return result
@@ -94,9 +94,9 @@ def clean(self, config_file):
 
     def _clean(self):
         command_runner = CommandRunner(self.config, self.section, Artifact(name='edi_input_artifact',
-                                                                           url=self._input_artifact(),
+                                                                           location=self._input_artifact(),
                                                                            type=ArtifactType.PATH))
-        if command_runner.require_root_for_clean():
+        if command_runner.require_real_root_for_clean():
             self._require_sudo()
         command_runner.clean()
 

edi/commands/project.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Matthias Luescher
+#
+# Authors:
+#  Matthias Luescher
+#
+# This file is part of edi.
+#
+# edi is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# edi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with edi.  If not, see <http://www.gnu.org/licenses/>.
+
+from edi.lib.edicommand import EdiCommand
+
+
+class Project(EdiCommand):
+
+    @classmethod
+    def advertise(cls, subparsers):
+        help_text = "EXPERIMENTAL: handle edi projects"
+        description_text = "EXPERIMENTAL: Do processing of edi projects."
+        parser = subparsers.add_parser(cls._get_short_command_name(),
+                                       help=help_text,
+                                       description=description_text)
+
+        cls._add_sub_commands(parser)
+
+    def run_cli(self, cli_args):
+        self._run_sub_command_cli(cli_args)

edi/commands/projectcommands/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2016 Matthias Luescher
+#
+# Authors:
+#  Matthias Luescher
+#
+# This file is part of edi.
+#
+# edi is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# edi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with edi.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ["prepare", "configure", "make", "clean"]

edi/commands/projectcommands/clean.py

@@ -0,0 +1,35 @@
+#
+# edi is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# edi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with edi.  If not, see <http://www.gnu.org/licenses/>.
+
+from edi.commands.project import Project
+
+
+class Clean(Project):
+    @classmethod
+    def advertise(cls, subparsers):
+        help_text = "EXPERIMENTAL: clean all intermediate project artifacts"
+        description_text = "EXPERIMENTAL: Clean all intermediate project artifacts."
+        parser = subparsers.add_parser(cls._get_short_command_name(),
+                                       help=help_text,
+                                       description=description_text)
+        cls._require_config_file(parser)
+
+    def run_cli(self, cli_args):
+        self.run(cli_args.config_file)
+
+    def run(self, config_file):
+        self.clean(config_file)
+
+    def clean(self, config_file):
+        self._clean_siblings_and_sub_commands(config_file)

edi/commands/projectcommands/configure.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Matthias Luescher
+#
+# Authors:
+#  Matthias Luescher
+#
+# This file is part of edi.
+#
+# edi is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# edi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with edi.  If not, see <http://www.gnu.org/licenses/>.
+
+import hashlib
+import logging
+import os
+from edi.commands.project import Project
+from edi.commands.projectcommands.prepare import Prepare
+from edi.lib.playbookrunner import PlaybookRunner
+from edi.lib.helpers import print_success, get_artifact_dir
+from edi.lib.shellhelpers import run
+from edi.lib.buildahhelpers import create_container, delete_container, is_container_existing
+from edi.lib.configurationparser import command_context
+from edi.lib.commandrunner import ArtifactType, Artifact, find_artifact
+
+
+class Configure(Project):
+
+    def __init__(self):
+        super().__init__()
+        self.ansible_connection = 'buildah'
+        self._prepare_results = None
+
+    @classmethod
+    def advertise(cls, subparsers):
+        help_text = "EXPERIMENTAL: configure a project container using Ansible playbook(s)"
+        description_text = "EXPERIMENTAL: Configure a project container."
+        parser = subparsers.add_parser(cls._get_short_command_name(),
+                                       help=help_text,
+                                       description=description_text)
+        cls._offer_options(parser, introspection=True, clean=True)
+        cls._require_config_file(parser)
+
+    @staticmethod
+    def _unpack_cli_args(cli_args):
+        return [cli_args.config_file]
+
+    def run_cli(self, cli_args):
+        self._dispatch(*self._unpack_cli_args(cli_args), run_method=self._get_run_method(cli_args))
+
+    def dry_run(self, config_file):
+        return self._dispatch(config_file, run_method=self._dry_run)
+
+    def _dry_run(self):
+        plugins = {}
+        plugins.update(Prepare().dry_run(self.config.get_base_config_file()))
+        playbook_runner = PlaybookRunner(self.config, self._result(), self.ansible_connection)
+        plugins.update(playbook_runner.get_plugin_report())
+        return plugins
+
+    def run(self, config_file):
+        return self._dispatch(config_file, run_method=self._run)
+
+    def _run(self):
+        container_name = self._get_container_artifact().location
+
+        if is_container_existing(container_name):
+            logging.info(f"Project container {container_name} is already existing. "
+                         f"Delete the container to get it re-created from scratch.")
+        else:
+            self._prepare_results = Prepare().run(self.config.get_base_config_file())
+
+            bootstrapped_rootfs = find_artifact(self._prepare_results, "edi_bootstrapped_rootfs",
+                                                "configure", "prepare")
+
+            print(f"Going to create project container '{container_name}'\n"
+                  f"based on content of '{bootstrapped_rootfs}'.")
+            create_container(container_name, bootstrapped_rootfs.location)
+
+        seal_file = self._get_seal_artifact().location
+
+        if os.path.exists(seal_file):
+            logging.info(f"Project container {container_name} is already fully configured. "
+                         f"Delete {seal_file} to get it re-configured.")
+        else:
+            print(f"Going to configure project container '{container_name}' - be patient.")
+
+            playbook_runner = PlaybookRunner(self.config, container_name, self.ansible_connection)
+            playbook_runner.run_all()
+            run(["touch", seal_file])
+
+        print_success(f"Configured project container '{container_name}'.")
+
+        collected_results = self._result()
+        if collected_results:
+            formatted_results = [f"{a.name}: {a.location}" for a in collected_results]
+            print_success(("Completed the project configure command.\n"
+                           "The following artifacts are now available:\n- {}".format('\n- '.join(formatted_results))))
+
+        return collected_results
+
+    def clean_recursive(self, config_file, depth):
+        self.clean_depth = depth
+        self._dispatch(config_file, run_method=self._clean)
+
+    def _clean(self):
+        seal_file = self._get_seal_artifact().location
+        if os.path.exists(seal_file):
+            os.remove(seal_file)
+            print_success(f"Deleted seal file '{seal_file}'.")
+
+        if self.clean_depth > 0:
+            container_name = self._get_container_artifact().location
+            if is_container_existing(container_name):
+                delete_container(container_name)
+                print_success(f"Deleted project container '{container_name}'.")
+
+        if self.clean_depth > 1:
+            Prepare().clean_recursive(self.config.get_base_config_file(), self.clean_depth - 2)
+
+    def _dispatch(self, config_file, run_method):
+        with command_context({'edi_create_distributable_image': True}):
+            self._setup_parser(config_file)
+            return run_method()
+
+    def _result(self):
+        if not self._prepare_results:
+            self._prepare_results = Prepare().result(self.config.get_base_config_file())
+
+        all_results = self._prepare_results.copy()
+        all_results.append(self._get_container_artifact())
+        all_results.append(self._get_seal_artifact())
+        return all_results
+
+    def result(self, config_file):
+        return self._dispatch(config_file, run_method=self._result)
+
+    def _get_container_artifact(self):
+        container_name = "edi-{}-{}".format(
+            hashlib.sha256(self.config.get_configuration_name().encode()).hexdigest()[:8],
+            self.config.get_project_directory_hash())
+        return Artifact(name="edi_project_container", location=container_name, type=ArtifactType.BUILDAH_CONTAINER)
+
+    def _get_seal_artifact(self):
+        seal_artifact = f"{self.config.get_configuration_name()}.seal"
+        return Artifact(name="edi_project_container_seal",
+                        location=str(os.path.join(get_artifact_dir(), seal_artifact)),
+                        type=ArtifactType.PATH)