[RHELC-576, RHELC-596] Merge yum transactions into a single one

convert2rhel/backup.py

@@ -18,6 +18,7 @@
 import abc
 import logging
 import os
+import re
 import shutil
 
 import six
@@ -46,10 +47,22 @@ def track_installed_pkgs(self, pkgs):
         """Track packages installed before the PONR to be able to remove them later (roll them back) if needed."""
         self.installed_pkgs += pkgs
 
-    def backup_and_track_removed_pkg(self, pkg):
+    def backup_and_track_removed_pkg(
+        self,
+        pkg,
+        reposdir=None,
+        set_releasever=False,
+        custom_releasever=None,
+        varsdir=None,
+    ):
         """Add a removed RPM pkg to the list of removed pkgs."""
         restorable_pkg = RestorablePackage(pkg)
-        restorable_pkg.backup()
+        restorable_pkg.backup(
+            reposdir=reposdir,
+            set_releasever=set_releasever,
+            custom_releasever=custom_releasever,
+            varsdir=varsdir,
+        )
         self.removed_pkgs.append(restorable_pkg)
 
     def _remove_installed_pkgs(self):
@@ -309,58 +322,101 @@ def __init__(self, pkgname):
         self.name = pkgname
         self.path = None
 
-    def backup(self):
-        """Save version of RPM package"""
+    def backup(
+        self,
+        reposdir=None,
+        set_releasever=False,
+        custom_releasever=None,
+        varsdir=None,
+    ):
+        """Save version of RPM package.
+
+        :param reposdir: Custom repositories directory to be used in the backup.
+        :type reposdir: str
+        """
         loggerinst.info("Backing up %s." % self.name)
         if os.path.isdir(BACKUP_DIR):
-            reposdir = get_hardcoded_repofiles_dir()
-
-            # One of the reasons we hardcode repofiles pointing to archived repositories of older system
-            # minor versions is that we need to be able to download an older package version as a backup.
-            # Because for example the default repofiles on CentOS Linux 8.4 point only to 8.latest repositories
-            # that already don't contain 8.4 packages.
+            # If we detect that the current system is an EUS release, then we
+            # proceed to use the hardcoded_repofiles, otherwise, we use the
+            # custom reposdir that comes from the method parameter. This is
+            # mainly because of CentOS Linux which we have hardcoded repofiles.
+            # If we ever put Oracle Linux repofiles to ship with convert2rhel,
+            # them the second part of this condition can be dropped.
+            if system_info.corresponds_to_rhel_eus_release() and system_info.id == "centos":
+                reposdir = get_hardcoded_repofiles_dir()
+
+            # One of the reasons we hardcode repofiles pointing to archived
+            # repositories of older system minor versions is that we need to be
+            # able to download an older package version as a backup. Because for
+            # example the default repofiles on CentOS Linux 8.4 point only to
+            # 8.latest repositories that already don't contain 8.4 packages.
             if not system_info.has_internet_access:
                 if reposdir:
                     loggerinst.debug(
                         "Not using repository files stored in %s due to the absence of internet access." % reposdir
                     )
-                self.path = download_pkg(self.name, dest=BACKUP_DIR, set_releasever=False)
+                self.path = download_pkg(
+                    self.name,
+                    dest=BACKUP_DIR,
+                    set_releasever=set_releasever,
+                    custom_releasever=custom_releasever,
+                    varsdir=varsdir,
+                )
             else:
                 if reposdir:
                     loggerinst.debug("Using repository files stored in %s." % reposdir)
                 self.path = download_pkg(
                     self.name,
                     dest=BACKUP_DIR,
-                    set_releasever=False,
+                    set_releasever=set_releasever,
                     reposdir=reposdir,
+                    custom_releasever=custom_releasever,
+                    varsdir=varsdir,
                 )
         else:
             loggerinst.warning("Can't access %s" % BACKUP_DIR)
 
 
-def remove_pkgs(pkgs_to_remove, backup=True, critical=True):
+def remove_pkgs(
+    pkgs_to_remove,
+    backup=True,
+    critical=True,
+    reposdir=None,
+    set_releasever=False,
+    custom_releasever=None,
+    varsdir=None,
+):
     """Remove packages not heeding to their dependencies."""
-
-    # NOTE(r0x0d): This function is tied to the class ChangedRPMPackagesController and
-    # a couple of other places too, ideally, we should decide if we want to use
-    # this function as an entrypoint or the variable `changed_pkgs_control`, so
-    # we can move this piece of code to the `pkghandler.py` where it should be.
-    # Right now, if we move this code to the `pkghandler.py`, we have a
-    # *circular import dependency error*.
-    # @abadger has an implementation in mind to address some of those issues
-    # and actually place a controller in front of classes like this.
-    if backup:
-        # Some packages, when removed, will also remove repo files, making it
-        # impossible to access the repositories to download a backup. For this
-        # reason we first backup all packages and only after that we remove
-        for nvra in pkgs_to_remove:
-            changed_pkgs_control.backup_and_track_removed_pkg(nvra)
+    # NOTE(r0x0d): This function is tied to the class
+    # ChangedRPMPackagesController and a couple of other places too, ideally, we
+    # should decide if we want to use this function as an entrypoint or the
+    # variable `changed_pkgs_control`, so we can move this piece of code to the
+    # `pkghandler.py` where it should be. Right now, if we move this code to the
+    # `pkghandler.py`, we have a *circular import dependency error*. @abadger
+    # has an implementation in mind to address some of those issues and actually
+    # place a controller in front of classes like this.
 
     if not pkgs_to_remove:
         loggerinst.info("No package to remove")
         return
 
-    for nvra in pkgs_to_remove:
+    if backup:
+        # Some packages, when removed, will also remove repo files, making it
+        # impossible to access the repositories to download a backup. For this
+        # reason we first back up *all* packages and only after that we remove them.
+        for nevra in pkgs_to_remove:
+            changed_pkgs_control.backup_and_track_removed_pkg(
+                pkg=nevra,
+                reposdir=reposdir,
+                set_releasever=set_releasever,
+                custom_releasever=custom_releasever,
+                varsdir=varsdir,
+            )
+    for nevra in pkgs_to_remove:
+        # It's necessary to remove an epoch from the NEVRA string returned by yum because the rpm command does not
+        # handle the epoch well and considers the package we want to remove as not installed. On the other hand, the
+        # epoch in NEVRA returned by dnf is handled by rpm just fine.
+        nvra = remove_epoch_from_yum_nevra_notation(nevra)
         loggerinst.info("Removing package: %s" % nvra)
         _, ret_code = run_subprocess(["rpm", "-e", "--nodeps", nvra])
         if ret_code != 0:
@@ -370,5 +426,22 @@ def remove_pkgs(pkgs_to_remove, backup=True, critical=True):
                 loggerinst.warning("Couldn't remove %s." % nvra)
 
 
+def remove_epoch_from_yum_nevra_notation(package_nevra):
+    """Remove epoch from the NEVRA string returned by yum.
+
+    Yum prints epoch only when it's non-zero. It's printed differently by yum and dnf:
+      yum - epoch before name: "7:oraclelinux-release-7.9-1.0.9.el7.x86_64"
+      dnf - epoch before version: "oraclelinux-release-8:8.2-1.0.8.el8.x86_64"
+
+    This function removes the epoch from the yum notation only.
+    It's safe to pass the dnf notation string with an epoch. This function will return it as is.
+    """
+    epoch_match = re.search(r"^\d+:(.*)", package_nevra)
+    if epoch_match:
+        # Return NVRA without the found epoch
+        return epoch_match.group(1)
+    return package_nevra
+
+
 changed_pkgs_control = ChangedRPMPackagesController()  # pylint: disable=C0103
 backup_control = BackupController()

convert2rhel/checks.py

@@ -72,6 +72,7 @@ def perform_pre_checks():
 def perform_pre_ponr_checks():
     """Late checks before ponr should be added here."""
     ensure_compatibility_of_kmods()
+    validate_package_manager_transaction()
 
 
 def check_convert2rhel_latest():
@@ -295,6 +296,15 @@ def ensure_compatibility_of_kmods():
         logger.info("Kernel modules are compatible.")
 
 
+def validate_package_manager_transaction():
+    """Validate the package manager transaction is passing the tests."""
+    logger.task("Validate the %s transaction", pkgmanager.TYPE)
+    transaction_handler = pkgmanager.create_transaction_handler()
+    transaction_handler.run_transaction(
+        validate_transaction=True,
+    )
+
+
 def get_loaded_kmods():
     """Get a set of kernel modules loaded on host.
 

convert2rhel/data/7/x86_64/configs/oracle-7-x86_64.cfg

@@ -24,6 +24,7 @@ excluded_pkgs =
   yum-rhn-plugin
   python-requests
   gnome-documents-libs
+  shim-x64
 
 # List of packages that either contain repofiles or affect variables in the repofiles (e.g. $releasever).
 # The rhn-client-tools package contains repofiles on OL 7.6 and older, but when it's installed it's not possible to

convert2rhel/initialize.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright(C) 2022 Red Hat, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+import sys
+
+
+def set_locale():
+    """Set the C locale, also known as the POSIX locale, for the main process as well as the child processes.
+
+    The reason is to get predictable output from the executables we call, not
+    influenced by non-default locale. We need to be setting not only LC_ALL but
+    LANG as well because subscription-manager considers LANG to have priority
+    over LC_ALL even though it goes against POSIX which specifies that LC_ALL
+    overrides LANG.
+
+    .. note::
+        Since we introduced a new way to interact with packages that is not
+        through the `yum` cli calls, but with the Python API, we had to move
+        this function to a new module to initialize all the settings in
+        Convert2RHEL before-hand as this was causing problems related to the
+        locales. The main problem that this function solves by being here is by
+        overriding any user set locale in their machine, to actually being the
+        ones we require during the process execution.
+    """
+    os.environ.update({"LC_ALL": "C", "LANG": "C"})
+
+
+def run():
+    """Wrapper around the main function.
+
+    This function is intended to initialize all early code and function calls
+    before any other main imports.
+    """
+    # prepare environment
+    set_locale()
+
+    from convert2rhel import main
+
+    sys.exit(main.main())

convert2rhel/main.py

@@ -17,11 +17,20 @@
 
 import logging
 import os
-import sys
 
 from convert2rhel import backup, breadcrumbs, cert, checks, grub
 from convert2rhel import logger as logger_module
-from convert2rhel import pkghandler, redhatrelease, repo, special_cases, subscription, systeminfo, toolopts, utils
+from convert2rhel import (
+    pkghandler,
+    pkgmanager,
+    redhatrelease,
+    repo,
+    special_cases,
+    subscription,
+    systeminfo,
+    toolopts,
+    utils,
+)
 
 
 loggerinst = logging.getLogger(__name__)
@@ -66,9 +75,6 @@ def main():
     # initialize logging
     initialize_logger("convert2rhel.log", logger_module.LOG_DIR)
 
-    # prepare environment
-    utils.set_locale()
-
     # handle command line arguments
     toolopts.CLI()
 
@@ -98,6 +104,7 @@ def main():
         redhatrelease.system_release_file.backup()
         redhatrelease.os_release_file.backup()
         repo.backup_yum_repos()
+        repo.backup_varsdir()
 
         # begin conversion process
         process_phase = ConversionPhase.PRE_PONR_CHANGES
@@ -138,7 +145,6 @@ def main():
     except (Exception, SystemExit, KeyboardInterrupt) as err:
         # Catching the three exception types separately due to python 2.4
         # (RHEL 5) - 2.7 (RHEL 7) compatibility.
-
         utils.log_traceback(toolopts.tool_opts.debug)
         no_changes_msg = "No changes were made to the system."
         breadcrumbs.breadcrumbs.finish_collection(success=False)
@@ -235,11 +241,11 @@ def pre_ponr_conversion():
 
 def post_ponr_conversion():
     """Perform main steps for system conversion."""
-
+    transaction_handler = pkgmanager.create_transaction_handler()
+    loggerinst.task("Convert: Replace system packages")
+    transaction_handler.run_transaction()
     loggerinst.task("Convert: Prepare kernel")
     pkghandler.preserve_only_rhel_kernel()
-    loggerinst.task("Convert: Replace packages")
-    pkghandler.replace_non_red_hat_packages()
     loggerinst.task("Convert: List remaining non-Red Hat packages")
     pkghandler.list_non_red_hat_pkgs_left()
     loggerinst.task("Convert: Configure the bootloader")
@@ -266,6 +272,7 @@ def rollback_changes():
     loggerinst.warning("Abnormal exit! Performing rollback ...")
     subscription.rollback()
     backup.changed_pkgs_control.restore_pkgs()
+    repo.restore_varsdir()
     repo.restore_yum_repos()
     redhatrelease.system_release_file.restore()
     redhatrelease.os_release_file.restore()
@@ -282,7 +289,3 @@ def rollback_changes():
             raise
 
     return
-
-
-if __name__ == "__main__":
-    sys.exit(main())