Add Ubuntu 26.04 support for OSMorphing minions

coriolis/osmorphing/base.py

@@ -5,6 +5,7 @@
 import itertools
 import os
 import re
+import shlex
 import uuid
 
 from oslo_log import log as logging
@@ -250,7 +251,18 @@ def post_packages_uninstall(self, package_names):
         pass
 
     def set_environment(self, environment):
-        self._environment = environment
+        """Merges the given environment into the tools' own environment.
+
+        The variables declared by the tools' constructor (such as
+        DEBIAN_FRONTEND for the Debian-based tools) are preserved, unless
+        the given environment explicitly overrides them.
+
+        The variables are copied over instead of the mapping being stored
+        as-is, as the environment provided by the OSMount tools is shared
+        between the OSDetect and the OSMorphing tools of both the export
+        and the import providers.
+        """
+        self._environment.update(environment or {})
 
 
 class BaseLinuxOSMorphingTools(BaseOSMorphingTools):
@@ -383,12 +395,12 @@ def run_user_script(self, user_script):
             utils.exec_ssh_cmd(
                 self._conn,
                 "sudo chmod +x %s" % script_path,
-                get_pty=True)
+                get_pty=False)
 
             utils.exec_ssh_cmd(
                 self._conn,
                 'sudo "%s" "%s"' % (script_path, self._os_root_dir),
-                get_pty=True)
+                get_pty=False)
         except Exception as err:
             raise exception.CoriolisException(
                 "Failed to run user script.") from err
@@ -434,7 +446,7 @@ def _exec_cmd(self, cmd, timeout=None):
             timeout = self._osmorphing_operation_timeout
         try:
             return utils.exec_ssh_cmd(
-                self._ssh, cmd, environment=self._environment, get_pty=True,
+                self._ssh, cmd, environment=self._environment, get_pty=False,
                 timeout=timeout)
         except exception.MinionMachineCommandTimeout as ex:
             raise exception.OSMorphingSSHOperationTimeout(
@@ -446,7 +458,7 @@ def _exec_cmd_chroot(self, cmd, timeout=None):
         try:
             return utils.exec_ssh_cmd_chroot(
                 self._ssh, self._os_root_dir, cmd,
-                environment=self._environment, get_pty=True, timeout=timeout)
+                environment=self._environment, get_pty=False, timeout=timeout)
         except exception.MinionMachineCommandTimeout as ex:
             raise exception.OSMorphingSSHOperationTimeout(
                 cmd=cmd, timeout=timeout) from ex
@@ -465,7 +477,7 @@ def _write_file_sudo(self, chroot_path, content):
         self._exec_cmd_chroot("cp /%s /%s" % (tmp_file, chroot_path))
         self._exec_cmd_chroot("rm /%s" % tmp_file)
         utils.exec_ssh_cmd(
-            self._ssh, "sudo sync", self._environment, get_pty=True)
+            self._ssh, "sudo sync", self._environment, get_pty=False)
 
     def _enable_systemd_service(self, service_name):
         self._exec_cmd_chroot("systemctl enable %s.service" % service_name)
@@ -827,20 +839,19 @@ def _validate_grub_config_obj(self, config_obj):
     def set_grub_value(self, option, value, config_obj, replace=True):
         self._validate_grub_config_obj(config_obj)
 
+        # the sed script and the config path are shell-quoted so that
+        # values holding spaces, quotes or leading dashes reach sed
+        # as a single argument instead of being split into separate options.
         def append_to_cfg(opt, val):
-            cmd = "sed -ie '$a%(o)s=\"%(v)s\"' %(cfg)s" % {
-                "o": opt,
-                "v": val,
-                "cfg": config_obj["location"]
-            }
+            cmd = "sed -ie %s %s" % (
+                shlex.quote('$a%s="%s"' % (opt, val)),
+                shlex.quote(config_obj["location"]))
             self._exec_cmd_chroot(cmd)
 
         def replace_in_cfg(opt, val):
-            cmd = "sed -i 's|^%(o)s=.*|%(o)s=\"%(v)s\"|g' %(cfg)s" % {
-                "o": opt,
-                "v": val,
-                "cfg": config_obj["location"]
-            }
+            cmd = "sed -i %s %s" % (
+                shlex.quote('s|^%s=.*|%s="%s"|g' % (opt, opt, val)),
+                shlex.quote(config_obj["location"]))
             self._exec_cmd_chroot(cmd)
 
         if config_obj["contents"].get(option, False):

coriolis/osmorphing/debian.py

@@ -33,6 +33,21 @@ class BaseDebianMorphingTools(base.BaseLinuxOSMorphingTools):
 
     netplan_base = "etc/netplan"
 
+    def __init__(self, conn, os_root_dir, os_root_dev, hypervisor,
+                 event_manager, detected_os_info, osmorphing_parameters,
+                 operation_timeout=None):
+        super(BaseDebianMorphingTools, self).__init__(
+            conn, os_root_dir, os_root_dev, hypervisor, event_manager,
+            detected_os_info, osmorphing_parameters, operation_timeout)
+
+        # NOTE: every dpkg invocation may run maintainer scripts which prompt
+        # through debconf (e.g. asking for a keyboard layout, or acknowledging
+        # a pending kernel upgrade), which would hang the OSMorphing operation
+        # indefinitely. Both the install and the uninstall paths are affected,
+        # so the frontend is declared non-interactive for all the commands run
+        # by these tools instead of being set on individual operations.
+        self.set_environment({'DEBIAN_FRONTEND': 'noninteractive'})
+
     @classmethod
     def check_os_supported(cls, detected_os_info):
         if detected_os_info['distribution_name'] != (
@@ -242,9 +257,8 @@ def install_packages(self, package_names):
             self._exec_cmd_chroot(deb_reconfigure_cmd)
 
             apt_get_cmd = (
-                '/bin/bash -c "DEBIAN_FRONTEND=noninteractive '
                 'apt-get install %s -y '
-                '-o Dpkg::Options::=\'--force-confdef\'"' % (
+                '-o Dpkg::Options::=--force-confdef' % (
                     " ".join(package_names)))
             self._exec_cmd_chroot(apt_get_cmd)
         except Exception as err:

coriolis/osmorphing/osdetect/base.py

@@ -86,7 +86,7 @@ def _exec_cmd(self, cmd, timeout=None):
             timeout = self._osdetect_operation_timeout
         try:
             return utils.exec_ssh_cmd(
-                self._conn, cmd, environment=self._environment, get_pty=True,
+                self._conn, cmd, environment=self._environment, get_pty=False,
                 timeout=timeout)
         except exception.MinionMachineCommandTimeout as ex:
             raise exception.OSMorphingSSHOperationTimeout(
@@ -98,7 +98,7 @@ def _exec_cmd_chroot(self, cmd, timeout=None):
         try:
             return utils.exec_ssh_cmd_chroot(
                 self._conn, self._os_root_dir, cmd,
-                environment=self._environment, get_pty=True, timeout=timeout)
+                environment=self._environment, get_pty=False, timeout=timeout)
         except exception.MinionMachineCommandTimeout as ex:
             raise exception.OSMorphingSSHOperationTimeout(
                 cmd=cmd, timeout=timeout) from ex

coriolis/osmorphing/osmount/base.py

@@ -102,6 +102,7 @@ def _connect(self):
         self._ssh = ssh
 
     def setup(self):
+        utils.check_env_command(self._ssh)
         if self._allow_ssh_env_vars():
             self._ssh.close()
             self._connect()
@@ -123,11 +124,36 @@ def _exec_cmd(self, cmd, timeout=None):
             timeout = self._osmount_operation_timeout
         try:
             return utils.exec_ssh_cmd(self._ssh, cmd, self._environment,
-                                      get_pty=True, timeout=timeout)
+                                      get_pty=False, timeout=timeout)
         except exception.MinionMachineCommandTimeout as ex:
             raise exception.OSMorphingSSHOperationTimeout(
                 cmd=cmd, timeout=timeout) from ex
 
+    def _exec_sudo_env_cmd(self, cmd, timeout=None):
+        """
+        Runs a sudo command that also passes all the environment variables to
+        the underlying command. Replaces sudo's -E flag, which is not currently
+        supported in all shipped sudo variants (like sudo-rs).
+        """
+
+        if not timeout:
+            timeout = self._osmount_operation_timeout
+        env_cmd = "sudo %s%s" % (
+            utils.get_env_command_prefix(self._environment), cmd)
+        try:
+            return utils.exec_ssh_cmd(
+                self._ssh,
+                env_cmd,
+                environment=self._environment,
+                get_pty=False,
+                timeout=timeout,
+            )
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd,
+                timeout=timeout,
+            ) from ex
+
     def get_connection(self):
         return self._ssh
 
@@ -589,6 +615,11 @@ def _find_and_mount_root(self, devices):
             self._exec_cmd(
                 'sudo mount -o bind /%(dir)s/ %(mount_dir)s' %
                 {'dir': directory, 'mount_dir': mount_dir})
+            # NOTE: the bind above joins the minion's peer group, so a unit
+            # restarting on the minion during OSMorphing has its per-unit
+            # '/run/credentials/<unit>' mount show up under the bind as well,
+            # which would later fail the 'umount -R'.
+            self._exec_cmd('sudo mount --make-private %s' % mount_dir)
 
         self._mask_minion_efi_firmware(os_root_dir)
 
@@ -690,6 +721,15 @@ def mount_os(self):
 
     def dismount_os(self, root_dir):
         self._exec_cmd('sudo fuser --kill --mount %s || true' % root_dir)
+        # NOTE: the binds are made private as they are made, but a mount can
+        # still propagate in before that happens, and chroots mounted by an
+        # older worker were never severed at all. Nothing may be left shared:
+        # unmounting below must not propagate back out to the minion's own
+        # mounts and fail with `target is busy` on one which is still in
+        # use there.
+        self._exec_cmd(
+            'mountpoint -q %s && sudo mount --make-rprivate %s || true' % (
+                root_dir, root_dir))
         self._exec_cmd(
             'mountpoint -q %s && sudo umount -R %s' % (root_dir, root_dir))
 
@@ -734,12 +774,12 @@ def run_user_script(self, user_script):
             utils.exec_ssh_cmd(
                 self._ssh,
                 "sudo chmod +x %s" % script_path,
-                get_pty=True)
+                get_pty=False)
 
             utils.exec_ssh_cmd(
                 self._ssh,
                 f'sudo "{script_path}"',
-                get_pty=True)
+                get_pty=False)
         except Exception as err:
             raise exception.CoriolisException(
                 "Failed to run user script.") from err

coriolis/osmorphing/osmount/redhat.py

@@ -20,7 +20,7 @@ def check_os(self):
 
     def setup(self):
         super(RedHatOSMountTools, self).setup()
-        self._exec_cmd("sudo -E yum install -y lvm2 psmisc cryptsetup")
+        self._exec_sudo_env_cmd("yum install -y lvm2 psmisc cryptsetup")
         self._exec_cmd("sudo modprobe dm-mod")
         self._exec_cmd("sudo modprobe dm-crypt")
         self._exec_cmd("sudo rm -f /etc/lvm/devices/system.devices")

coriolis/osmorphing/osmount/suse.py

@@ -39,9 +39,9 @@ def _allow_ssh_env_vars(self):
     def setup(self):
         super(SUSEOSMountTools, self).setup()
         retry_ssh_cmd = utils.retry_on_error(
-            max_attempts=10, sleep_seconds=30)(self._exec_cmd)
+            max_attempts=10, sleep_seconds=30)(self._exec_sudo_env_cmd)
         retry_ssh_cmd(
-            "sudo -E zypper --non-interactive install lvm2 psmisc cryptsetup")
+            "zypper --non-interactive install lvm2 psmisc cryptsetup")
         self._exec_cmd("sudo modprobe dm-mod")
         self._exec_cmd("sudo modprobe dm-crypt")
         self._exec_cmd("sudo rm -f /etc/lvm/devices/system.devices")

coriolis/osmorphing/osmount/ubuntu.py

@@ -25,8 +25,8 @@ def setup(self):
         # Apart from relying on possibly not-yet-installed tools like `fuser`,
         # or checking every /proc/*/fd ourselves, we simply retry it:
         retry_ssh_cmd = utils.retry_on_error(
-            max_attempts=10, sleep_seconds=30)(self._exec_cmd)
-        retry_ssh_cmd("sudo -E apt-get update -y")
+            max_attempts=10, sleep_seconds=30)(self._exec_sudo_env_cmd)
+        retry_ssh_cmd("apt-get update -y")
 
         # NOTE(aznashwan): in case an unattended upgrade is already happening
         # and is at the package installation stage (in which case the
@@ -35,9 +35,10 @@ def setup(self):
         # prompts interactively for a keyboard layout unless
         # DEBIAN_FRONTEND=noninteractive is set, which would otherwise hang
         # the install indefinitely.
-        self._exec_cmd(
-            "sudo -E DEBIAN_FRONTEND=noninteractive apt-get "
-            "-o DPkg::Lock::Timeout=600 install lvm2 psmisc cryptsetup -y")
+        self._environment['DEBIAN_FRONTEND'] = 'noninteractive'
+        self._exec_sudo_env_cmd(
+            "apt-get -o DPkg::Lock::Timeout=600 "
+            "install lvm2 psmisc cryptsetup -y")
 
         self._exec_cmd("sudo modprobe dm-mod")
         self._exec_cmd("sudo modprobe dm-crypt")

coriolis/providers/backup_writers.py

@@ -100,20 +100,20 @@ def _disable_lvm2_lvmetad(ssh):
         utils.exec_ssh_cmd(
             ssh,
             'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" '
-            '%s' % cfg, get_pty=True)
+            '%s' % cfg, get_pty=False)
         # NOTE: lvm2-lvmetad is the name of the lvmetad service
         # on both debian and RHEL based systems. It needs to be stopped
         # before we begin disk replication. We disable it in the config
         # just in case some other process starts the daemon later on, as
         # a dependency. As the service may not actually exist, even though
         # the config is present, we ignore errors when stopping it.
         utils.ignore_exceptions(utils.exec_ssh_cmd)(
-            ssh, "sudo service lvm2-lvmetad stop", get_pty=True)
+            ssh, "sudo service lvm2-lvmetad stop", get_pty=False)
         # disable volume groups. Any volume groups that have volumes in use
         # will remain online. However, volume groups belonging to disks
         # that have been synced at least once, will be deactivated.
         utils.ignore_exceptions(utils.exec_ssh_cmd)(
-            ssh, "sudo vgchange -an", get_pty=True)
+            ssh, "sudo vgchange -an", get_pty=False)
 
 
 def _disable_lvm_metad_udev_rule(ssh):
@@ -131,7 +131,7 @@ def _disable_lvm_metad_udev_rule(ssh):
     ]
     for path in rule_paths:
         if utils.test_ssh_path(ssh, path):
-            utils.exec_ssh_cmd(ssh, "sudo rm %s" % path, get_pty=True)
+            utils.exec_ssh_cmd(ssh, "sudo rm %s" % path, get_pty=False)
 
 
 def _check_deserialize_key(key):
@@ -1021,7 +1021,7 @@ def _inject_dport_allow_rule(self, ssh):
             "sudo iptables -I INPUT -p tcp --dport %(port)s -j ACCEPT" % {
                 "port": self._writer_port})
         try:
-            utils.exec_ssh_cmd(ssh, cmd, get_pty=True)
+            utils.exec_ssh_cmd(ssh, cmd, get_pty=False)
         except exception.CoriolisException:
             LOG.warn(
                 "Could not inject TCP FW rule. Error was: %s",
@@ -1030,15 +1030,15 @@ def _inject_dport_allow_rule(self, ssh):
     def _add_firewalld_port(self, ssh):
         cmd = "sudo firewall-cmd --add-port=%s/tcp" % self._writer_port
         try:
-            utils.exec_ssh_cmd(ssh, cmd, get_pty=True)
+            utils.exec_ssh_cmd(ssh, cmd, get_pty=False)
         except exception.CoriolisException:
             LOG.warn("Could not add TCP port to firewalld. Error was: %s",
                      utils.get_exception_details())
 
     def _change_binary_se_context(self, ssh):
         cmd = "sudo chcon -t bin_t %s" % self._writer_cmd
         try:
-            utils.exec_ssh_cmd(ssh, cmd, get_pty=True)
+            utils.exec_ssh_cmd(ssh, cmd, get_pty=False)
         except exception.CoriolisException:
             LOG.warn("Could not change SELinux context of writer binary. "
                      "Error was:%s", utils.get_exception_details())
@@ -1061,12 +1061,12 @@ def _copy_writer(self, ssh):
                     ssh,
                     "sudo mv %s %s" % (
                         remote_tmp_path, self._writer_cmd),
-                    get_pty=True
+                    get_pty=False
                 )
                 utils.exec_ssh_cmd(
                     ssh,
                     "sudo chmod +x %s" % self._writer_cmd,
-                    get_pty=True
+                    get_pty=False
                 )
             finally:
                 sftp.close()
@@ -1075,7 +1075,7 @@ def _fetch_remote_file(self, ssh, remote_file, local_file):
         with open(local_file, 'wb') as fd:
             utils.exec_ssh_cmd(
                 ssh,
-                "sudo chmod +r %s" % remote_file, get_pty=True)
+                "sudo chmod +r %s" % remote_file, get_pty=False)
             data = utils.retry_on_error()(
                 utils.read_ssh_file)(ssh, remote_file)
             fd.write(data)
@@ -1103,7 +1103,7 @@ def _setup_certificates(self, ssh):
 
         if not all(exist):
             utils.exec_ssh_cmd(
-                ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True)
+                ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=False)
             utils.exec_ssh_cmd(
                 ssh,
                 "sudo %(writer_cmd)s generate-certificates -output-dir "
@@ -1112,7 +1112,7 @@ def _setup_certificates(self, ssh):
                     "cert_dir": remote_base_dir,
                     "extra_hosts": self._ip,
                 },
-                get_pty=True)
+                get_pty=False)
 
         return {
             "srv_crt": remote_srv_crt,
@@ -1124,7 +1124,7 @@ def _setup_certificates(self, ssh):
 
     def _read_remote_file_sudo(self, remote_path):
         contents = utils.exec_ssh_cmd(
-            self._ssh, 'sudo cat "%s"' % remote_path, get_pty=True)
+            self._ssh, 'sudo cat "%s"' % remote_path, get_pty=False)
         return contents
 
     def _init_writer(self, ssh, cert_paths):