Merge branch 'release/1.6.0'

debian/changelog

@@ -1,3 +1,12 @@
+edi (1.6.0) bionic; urgency=medium
+
+  [ Matthias Lüscher ]
+  * Carefully handle mounts that might be a leftover from interrupted debootstrap.
+  * Install a udev rule that switches off tx offloading on edibr0.
+  * Added possibility to use custom bridge for LXC containers (e.g. edibr0).
+
+ -- Matthias Luescher <matthias.luescher@schindler.com>  Wed, 29 Apr 2020 19:21:24 +0200
+
 edi (1.5.1) bionic; urgency=medium
 
   [ Matthias Lüscher ]

debian/control

@@ -38,6 +38,7 @@ Depends: ${misc:Depends},
          ansible (>=2.1),
          binfmt-support,
          debootstrap,
+         ethtool,
          liblzma5,
          openssh-client,
          python3-apt (>= 1),

debian/edi.udev

@@ -0,0 +1,3 @@
+# Switch off tx offloading on edibr* to avoid dhcp issues on old or emulated containers.
+# This rule gets added by the edi package.
+ACTION=="add", SUBSYSTEM=="net", KERNEL=="edibr*", RUN+="/sbin/ethtool -K $env{INTERFACE} tx off"

debian/rules

@@ -18,6 +18,9 @@ endif
 override_dh_gencontrol:
 	dh_gencontrol -- $(SUBSTVARS)
 
+override_dh_installudev:
+	dh_installudev --priority=81
+
 override_dh_auto_build:
 	dh_auto_build
 	PYTHONPATH=. http_proxy='127.0.0.1:9' sphinx-build -N -bman docs/ build/man # Manpage generator

docs/conf.py

@@ -55,8 +55,8 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.5.1'
-release = '1.5.1'
+version = '1.6.0'
+release = '1.6.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/config_management/plugins.rst

@@ -53,21 +53,34 @@ The following profiles have proven to be useful for various projects:
 
 .. _LXD Profile Documentation: https://lxd.readthedocs.io/en/latest/profiles/
 
+.. _default_network_interface:
+
 Default Network Interface
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This profile adds a default network interface to the container named according to the value of
-:code:`edi_lxc_network_interface_name`. The interface is of type :code:`bridged` and its parent is
-:code:`lxdbr0`.
+:code:`edi_lxc_network_interface_name`. The interface is of type :code:`bridged` and its default parent is
+:code:`lxdbr0`. To properly deal with legacy or emulated containers please use :code:`edibr0` instead of
+:code:`lxdbr0` by setting the general parameter :code:`edi_lxc_bridge_interface_name`:
 
 .. code-block:: yaml
   :caption: Configuration Example
 
+  general:
+    ...
+    edi_lxc_bridge_interface_name: edibr0
+    ...
+
+  ...
+
   lxc_profiles:
     ...
     100_lxc_networking:
         path: lxc_profiles/general/lxc_networking/default_interface.yml
-    ...
+     ...
+
+On the bridges matching :code:`edibr*` checksum offloading gets disabled (e.g. :code:`ethtool -K edibr0 tx off`).
+This is to make sure that emulated and legacy containers can get an IPv4 address assigned.
 
 Default Root Device
 ^^^^^^^^^^^^^^^^^^^
@@ -487,8 +500,8 @@ The changelog template can be used to document the changes of each package:
         edi_doc_replacements:
         - pattern: '(CVE-[0-9]{4}-[0-9]{4,6})'
           replacement: '`\1 <https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1>`_'
-        - pattern: '[#]*((?i)Closes:\s[#])([0-9]{6,10})'
+        - pattern: '(?i)[#]*(Closes:\s[#])([0-9]{6,10})'
           replacement: '`\1\2 <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=\2>`_'
-        - pattern: '[#]*((?i)LP:\s[#])([0-9]{6,10})'
+        - pattern: '(?i)[#]*(LP:\s[#])([0-9]{6,10})'
           replacement: '`\1\2 <https://bugs.launchpad.net/ubuntu/+source/nano/+bug/\2>`_'
   ...

docs/config_management/yaml.rst

@@ -57,6 +57,11 @@ edi supports the following settings:
    *edi_lxc_network_interface_name:*
       The default network interface that will be used for the lxc container.
       If unspecified edi will name the container interface :code:`lxcif0`.
+   *edi_lxc_bridge_interface_name:*
+      The bridge that the container will get attached to.
+      If unspecified edi will take the bridge :code:`lxdbr0`. If the specified bridge does not exist, :code:`edi`
+      will automatically create it.
+      Please check the chapter :ref:`default network interface <default_network_interface>` for more information.
    *edi_config_management_user_name:*
       The target system user that will be used for configuration management tasks.
       Please note that direct lxc container management uses the root user.
@@ -307,9 +312,9 @@ A documentation step can look like this:
          edi_doc_include_changelog: True
          edi_doc_changelog_baseline: 2019-12-01 00:00:00 GMT
          edi_doc_replacements:
-         - pattern: '[#]*((?i)Closes:\s[#])([0-9]{6,10})'
+         - pattern: '(?i)[#]*(Closes:\s[#])([0-9]{6,10})'
            replacement: '`\1\2 <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=\2>`_'
-         - pattern: '[#]*((?i)LP:\s[#])([0-9]{6,10})'
+         - pattern: '(?i)[#]*(LP:\s[#])([0-9]{6,10})'
            replacement: '`\1\2 <https://bugs.launchpad.net/ubuntu/+source/nano/+bug/\2>`_'
      ...
 
@@ -337,4 +342,4 @@ The documentation steps can be fine tuned using the following parameters:
       :code:`re.sub(pattern, replacement, changelog_line)` will be applied to the changelog lines in the given list
       order.
 
-.. _edi-pi: https://www.github.com/lueschem/edi-pi
+.. _edi-pi: https://www.github.com/lueschem/edi-pi

edi/commands/imagecommands/bootstrap.py

@@ -19,7 +19,6 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with edi.  If not, see <http://www.gnu.org/licenses/>.
 
-import tempfile
 import os
 import shutil
 import logging
@@ -30,7 +29,7 @@
 from edi.lib.helpers import (FatalError, chown_to_user, print_success,
                              get_workdir, get_artifact_dir, create_artifact_dir)
 from edi.lib.configurationparser import command_context
-from edi.lib.shellhelpers import run, get_chroot_cmd, require
+from edi.lib.shellhelpers import run, get_chroot_cmd, require, mount_aware_tempdir
 from edi.lib.proxyhelpers import ProxySetup
 from edi.lib.keyhelpers import fetch_repository_key, build_keyring
 
@@ -78,7 +77,7 @@ def _run(self):
 
         workdir = get_workdir()
 
-        with tempfile.TemporaryDirectory(dir=workdir) as tempdir:
+        with mount_aware_tempdir(workdir, log_warning=True) as tempdir:
             chown_to_user(tempdir)
             key_data = fetch_repository_key(self.config.get_bootstrap_repository_key())
             keyring_file = build_keyring(tempdir, "temp_keyring.gpg", key_data)

edi/commands/lxccommands/launch.py

@@ -27,7 +27,7 @@
 from edi.lib.networkhelpers import is_valid_hostname
 from edi.lib.lxchelpers import (is_container_existing, is_container_running, start_container,
                                 launch_container, get_container_profiles, stop_container,
-                                apply_profiles, try_delete_container)
+                                apply_profiles, try_delete_container, is_bridge_available, create_bridge)
 
 
 class Launch(Lxc):
@@ -51,6 +51,12 @@ def advertise(cls, subparsers):
     def _unpack_cli_args(cli_args):
         return [cli_args.container_name, cli_args.config_file]
 
+    def _setup_bridge(self):
+        bridge_name = self.config.get_lxc_bridge_interface_name()
+        if not is_bridge_available(bridge_name):
+            print("Creating new bridge '{}'.".format(bridge_name))
+            create_bridge(bridge_name)
+
     def run_cli(self, cli_args):
         self._dispatch(*self._unpack_cli_args(cli_args), run_method=self._get_run_method(cli_args))
 
@@ -93,11 +99,13 @@ def _run(self):
             if not is_container_running(self._result()):
                 logging.info(("Starting existing container {0}."
                               ).format(self._result()))
+                self._setup_bridge()
                 start_container(self._result())
                 print_success("Started container {}.".format(self._result()))
         else:
             image = Import().run(self.config.get_base_config_file())
             profiles = Profile().run(self.config.get_base_config_file(), include_post_config_profiles=False)
+            self._setup_bridge()
             print("Going to launch container.")
             launch_container(image, self._result(), profiles)
             print_success("Launched container {}.".format(self._result()))

edi/lib/configurationparser.py

@@ -189,6 +189,9 @@ def get_lxc_stop_timeout(self):
             raise FatalError('''The value of 'edi_lxc_stop_timeout' must be an integer.''')
         return timeout
 
+    def get_lxc_bridge_interface_name(self):
+        return self._get_general_item("edi_lxc_bridge_interface_name", "lxdbr0")
+
     def get_general_parameters(self):
         return self._get_general_item("parameters", {})
 
@@ -387,6 +390,7 @@ def _get_node_dictionary(self, node):
 
         node_dict["edi_lxc_network_interface_name"] = self._get_general_item("edi_lxc_network_interface_name",
                                                                              "lxcif0")
+        node_dict["edi_lxc_bridge_interface_name"] = self.get_lxc_bridge_interface_name()
         node_dict["edi_config_management_user_name"] = self._get_general_item("edi_config_management_user_name",
                                                                               "edicfgmgmt")
 

edi/lib/lxchelpers.py

@@ -130,23 +130,37 @@ def is_container_running(name):
         raise FatalError("Unable to parse lxc output ({}).".format(exc))
 
 
+@require('lxc', lxd_install_hint, LxdVersion.check)
+def is_bridge_available(bridge_name):
+    cmd = [lxc_exec(), "network", "list", "--format=json"]
+    result = run(cmd, stdout=subprocess.PIPE)
+
+    try:
+        parsed_result = yaml.safe_load(result.stdout)
+        bridge_names = [item.get("name") for item in parsed_result]
+        if bridge_name in bridge_names:
+            return True
+        else:
+            return False
+    except yaml.YAMLError as exc:
+        raise FatalError("Unable to parse lxc output ({}).".format(exc))
+
+
+@require('lxc', lxd_install_hint, LxdVersion.check)
+def create_bridge(bridge_name):
+    cmd = [lxc_exec(), "network", "create", bridge_name]
+    run(cmd)
+
+
 @require('lxc', lxd_install_hint, LxdVersion.check)
 def launch_container(image, name, profiles):
     cmd = [lxc_exec(), "launch", "local:{}".format(image), name]
     for profile in profiles:
         cmd.extend(["-p", profile])
     result = run(cmd, check=False, stderr=subprocess.PIPE, log_threshold=logging.INFO)
     if result.returncode != 0:
-        if 'Missing parent' in result.stderr and 'lxdbr0' in result.stderr:
-            raise FatalError(('''Launching image '{}' failed with the following message:\n{}'''
-                              'Please make sure that lxdbr0 is available. Use one of the following commands to '
-                              'create lxdbr0:\n'
-                              'lxd init\n'
-                              'or (for lxd >= 2.3)\n'
-                              'lxc network create lxdbr0').format(image, result.stderr))
-        else:
-            raise FatalError(('''Launching image '{}' failed with the following message:\n{}'''
-                              ).format(image, result.stderr))
+        raise FatalError(('''Launching image '{}' failed with the following message:\n{}'''
+                          ).format(image, result.stderr))
 
 
 @require('lxc', lxd_install_hint, LxdVersion.check)

edi/lib/shellhelpers.py

@@ -22,6 +22,8 @@
 import logging
 import subprocess
 import os
+from shutil import rmtree
+from tempfile import mkdtemp
 from contextlib import contextmanager
 from edi.lib.helpers import get_user, get_artifact_dir, FatalError, which
 from edi.lib import mockablerun
@@ -120,6 +122,22 @@ def safely_remove_artifacts_folder(abs_folder_path, sudo=False):
     run(['rm', '-rf', abs_folder_path], sudo=sudo)
 
 
+@contextmanager
+def mount_aware_tempdir(parent_dir, log_warning=False):
+    directory = mkdtemp(dir=parent_dir)
+    assert str(parent_dir) in str(directory)
+    try:
+        yield directory
+    finally:
+        mount_points = run(['findmnt', '--noheadings', '--raw', '--output=target'], stdout=subprocess.PIPE)
+        for mount_point in mount_points.stdout.splitlines():
+            if str(parent_dir) in mount_point:
+                if log_warning:
+                    logging.warning("Going to unmount '{}'.".format(mount_point))
+                run(['umount', mount_point], sudo=True)
+        rmtree(directory)
+
+
 @contextmanager
 def gpg_agent(directory):
     """

edi/lib/versionhelpers.py

@@ -26,7 +26,7 @@
 
 # The do_release script will update this version!
 # During launchpad debuild neither the git version nor the package version is available.
-edi_fallback_version = '1.5.1'
+edi_fallback_version = '1.6.0'
 
 
 def get_edi_version():

edi/plugins/config_templates/project_tree/configuration/base/PROJECTNAME.yml

@@ -1,4 +1,5 @@
 general:
+    edi_lxc_bridge_interface_name: edibr0
 {% if edi_edi_version %}
     edi_required_minimal_edi_version: {{ edi_edi_version }}
 {% endif %}

edi/plugins/lxc_profiles/general/lxc_networking/default_interface.yml

@@ -5,5 +5,5 @@ devices:
   {{ edi_lxc_network_interface_name }}:
     name: "{{ edi_lxc_network_interface_name }}"
     nictype: bridged
-    parent: lxdbr0
+    parent: {{ edi_lxc_bridge_interface_name or 'lxdbr0' }}
     type: nic

setup.py

@@ -38,7 +38,7 @@
 setup(
     name='edi',
 
-    version='1.5.1',
+    version='1.6.0',
 
     description='Embedded Development Infrastructure - edi',
     long_description=long_description,

tests/data/test_documentation/configuration/base/documentation.yml

@@ -29,9 +29,9 @@ documentation_steps:
             edi_doc_replacements:
             - pattern: '(CVE-[0-9]{4}-[0-9]{4,6})'
               replacement: '`\1 <https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1>`_'
-            - pattern: '[#]*((?i)Closes:\s[#])([0-9]{6,10})'
+            - pattern: '(?i)[#]*(Closes:\s[#])([0-9]{6,10})'
               replacement: '`\1\2 <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=\2>`_'
-            - pattern: '[#]*((?i)LP:\s[#])([0-9]{6,10})'
+            - pattern: '(?i)[#]*(LP:\s[#])([0-9]{6,10})'
               replacement: '`\1\2 <https://bugs.launchpad.net/ubuntu/+source/nano/+bug/\2>`_'
 
 

tests/lib/test_lxchelpers.py

@@ -22,12 +22,13 @@
 
 import subprocess
 import pytest
+from subprocess import CalledProcessError
 from contextlib import contextmanager
 from edi.lib.helpers import FatalError
 from edi.lib.lxchelpers import (get_server_image_compression_algorithm,
-                                get_file_extension_from_image_compression_algorithm,
-                                get_lxd_version, LxdVersion)
-from edi.lib.shellhelpers import mockablerun
+                                get_file_extension_from_image_compression_algorithm, lxc_exec,
+                                get_lxd_version, LxdVersion, is_bridge_available, create_bridge)
+from edi.lib.shellhelpers import mockablerun, run
 from tests.libtesting.helpers import get_command, get_sub_command
 from tests.libtesting.contextmanagers.mocked_executable import mocked_executable, mocked_lxd_version_check
 
@@ -122,3 +123,17 @@ def test_valid_version(monkeypatch):
     with mocked_executable('lxd', '/here/is/no/lxd'):
         with clear_lxd_version_check_cache():
             LxdVersion.check()
+
+
+@pytest.mark.requires_lxc
+def test_is_bridge_available():
+    bridge_name = "edibrtesttest42"
+    assert not is_bridge_available(bridge_name)
+    create_bridge(bridge_name)
+    assert is_bridge_available(bridge_name)
+    with pytest.raises(CalledProcessError) as e:
+        create_bridge(bridge_name)
+    assert 'non-zero exit status' in str(e)
+    cmd = [lxc_exec(), "network", "delete", bridge_name]
+    run(cmd)
+    assert not is_bridge_available(bridge_name)