DPDK: install path fixes for meson and Ubuntu 24.04

lisa/operating_system.py

@@ -404,13 +404,18 @@ def uninstall_packages(
         package_names = self._get_package_list(packages)
         self._uninstall_packages(package_names, signed, timeout, extra_args)
 
-    def package_exists(self, package: Union[str, Tool, Type[Tool]]) -> bool:
+    def package_exists(
+        self,
+        package: Union[str, Tool, Type[Tool]],
+    ) -> bool:
         """
         Query if a package/tool is installed on the node.
         Return Value - bool
         """
         package_name = self.__resolve_package_name(package)
-        return self._package_exists(package_name)
+        exists = self._package_exists(package_name)
+        self._log.debug(f"package '{package}' exists: {exists}")
+        return exists
 
     def is_package_in_repo(self, package: Union[str, Tool, Type[Tool]]) -> bool:
         """
@@ -2094,6 +2099,30 @@ def _initialize_package_installation(self) -> None:
                 "There are no enabled repositories defined in this image.",
             )
 
+    def _uninstall_packages(
+        self,
+        packages: List[str],
+        signed: bool = True,
+        timeout: int = 600,
+        extra_args: Optional[List[str]] = None,
+    ) -> None:
+        add_args = self._process_extra_package_args(extra_args)
+        command = f"zypper --non-interactive {add_args}"
+        if not signed:
+            command += " --no-gpg-checks "
+        remove_packages = " ".join(packages)
+        command += f" rm {remove_packages}"
+        self.wait_running_process("zypper")
+        uninstall_result = self._node.execute(
+            command, shell=True, sudo=True, timeout=timeout
+        )
+        uninstall_result.assert_exit_code(
+            expected_exit_code=0,
+            message=f"Could not uninstall package(s): {remove_packages}",
+            include_output=True,
+        )
+        self._log.debug(f"{packages} is/are removed successfully.")
+
     def _install_packages(
         self,
         packages: List[str],

lisa/tools/meson.py

@@ -8,20 +8,26 @@
 
 from lisa.executable import Tool
 from lisa.operating_system import Posix
+from lisa.util import parse_version
 
 from .ln import Ln
 from .python import Pip
 from .whoami import Whoami
 
 
 class Meson(Tool):
+    _minimum_version = parse_version("0.52.0")
+
     @property
     def command(self) -> str:
         return "meson"
 
     def _check_exists(self) -> bool:
         result = self.node.execute("meson --version", shell=True)
-        return result.exit_code == 0 and VersionInfo.parse(result.stdout) >= "0.52.0"
+        return (
+            result.exit_code == 0
+            and VersionInfo.parse(result.stdout) >= self._minimum_version
+        )
 
     @property
     def can_install(self) -> bool:
@@ -30,24 +36,78 @@ def can_install(self) -> bool:
     def _install(self) -> bool:
         posix_os: Posix = cast(Posix, self.node.os)
         # use pip to make sure we install a recent version
-        if (not posix_os.package_exists("meson")) or posix_os.get_package_information(
-            "meson", use_cached=False
-        ) < "0.52.0":
-            username = self.node.tools[Whoami].get_username()
-            self.node.tools[Pip].install_packages("meson", install_to_user=True)
-            # environment variables won't expand even when using shell=True :\
-            self.node.tools[Ln].create_link(
-                f"/home/{username}/.local/bin/meson", "/usr/bin/meson", force=True
-            )
-            # ensure sudo has access as well
-            self.node.execute(
-                "pip3 install meson",
-                sudo=True,
-                shell=True,
-                no_debug_log=True,
-                no_info_log=True,
-                no_error_log=True,
-            )
+
+        package_installed = ""
+        package_available = ""
+        # packaged as 'meson' on older systems and 'python3-meson' on newer ones,
+        # since it's actually just a python package.
+        # But now we have a bunch of annoying cases.
+        # 1.  meson is installed and it's the right version
+        # 2. 'meson' is installed but the wrong version
+        # 3. meson is not installed and the right version is in the repo
+        # 4. meson is not installed and the wrong version is in the repo
+
+        for pkg in [
+            "python3-meson",
+            "meson",
+        ]:
+            if posix_os.package_exists(pkg):
+                package_installed = pkg
+            if posix_os.is_package_in_repo(pkg):
+                package_available = pkg
+            if package_installed or package_available:
+                break
+
+        if package_installed:
+            # check the installed version before touching anything
+            if (
+                posix_os.get_package_information(package_installed, use_cached=False)
+                >= self._minimum_version
+            ):
+                # meson is installed and it's the right version
+                return self._check_exists()
+
+        # otherwise, install the available package from the repo
+        if package_available:
+            posix_os.install_packages(package_available)
+            # and update the cached version info
+            posix_os.get_package_information(package_available, use_cached=False)
+            package_installed = package_available
+
+        # check the version, return if it's good, remove if not
+        if package_installed:
+            if (
+                posix_os.get_package_information(package_installed)
+                >= self._minimum_version
+            ):
+                # the right version was in the repo
+                return self._check_exists()
+            else:
+                # the wrong version was in the repo
+                # (or wrong version installed and no update available from repo)
+                posix_os.uninstall_packages(package_installed)
+                package_installed = ""
+
+        # If we get here, we couldn't find a good version from the package manager.
+        # So we will install with pip. This is least desirable since it introduces
+        # unpredictable behavior when running meson or ninja with sudo.
+        # Like sudo ninja install, for example.
+
+        username = self.node.tools[Whoami].get_username()
+        self.node.tools[Pip].install_packages("meson", install_to_user=True)
+        self.node.tools[Ln].create_link(
+            f"/home/{username}/.local/bin/meson", "/usr/bin/meson", force=True
+        )
+        # ensure sudo has access as well
+        self.node.execute(
+            "pip3 install meson",
+            sudo=True,
+            shell=True,
+            no_debug_log=True,
+            no_info_log=True,
+            no_error_log=True,
+        )
+
         return self._check_exists()
 
     def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath:

microsoft/testsuites/dpdk/common.py

@@ -39,10 +39,12 @@ def __init__(
         matcher: Callable[[Posix], bool],
         packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None,
         stop_on_match: bool = False,
+        requires_reboot: bool = False,
     ) -> None:
         self.matcher = matcher
         self.packages = packages
         self.stop_on_match = stop_on_match
+        self.requires_reboot = requires_reboot
 
 
 class DependencyInstaller:
@@ -65,6 +67,10 @@ def install_required_packages(
         for requirement in self.requirements:
             if requirement.matcher(os) and requirement.packages:
                 packages += requirement.packages
+                if requirement.requires_reboot:
+                    os.install_packages(packages=packages, extra_args=extra_args)
+                    packages = []
+                    node.reboot()
                 if requirement.stop_on_match:
                     break
         os.install_packages(packages=packages, extra_args=extra_args)

microsoft/testsuites/dpdk/dpdktestpmd.py

@@ -63,6 +63,7 @@
             and bool(x.get_kernel_information().version >= "5.15.0")
             and x.is_package_in_repo("linux-modules-extra-azure"),
             packages=["linux-modules-extra-azure"],
+            requires_reboot=True,
         ),
         OsPackageDependencies(
             matcher=lambda x: isinstance(x, Debian),
@@ -105,8 +106,6 @@
                 "dpkg-dev",
                 "pkg-config",
                 "python3-pip",
-                "python3-pyelftools",
-                "python-pyelftools",
                 # 18.04 doesn't need linux-modules-extra-azure
                 # since it will never have MANA support
             ],
@@ -120,14 +119,14 @@
             and bool(x.get_kernel_information().version >= "5.15.0")
             and x.is_package_in_repo("linux-modules-extra-azure"),
             packages=["linux-modules-extra-azure"],
+            requires_reboot=True,
         ),
         OsPackageDependencies(
             matcher=lambda x: isinstance(x, Debian),
             packages=[
                 "build-essential",
                 "libnuma-dev",
                 "libmnl-dev",
-                "python3-pyelftools",
                 "libelf-dev",
                 "pkg-config",
             ],
@@ -226,7 +225,17 @@ def _setup_node(self) -> None:
         # which breaks when another tool checks for it's existence before building...
         # like cmake, meson, make, autoconf, etc.
         self._node.tools[Ninja].install()
-        self._node.tools[Pip].install_packages("pyelftools")
+        # try multiple avenues for installing pyelftools, preferring the
+        # more recent convention of prepending apt/dnf/etc python system packages
+        # with python3-...
+        if self._os.is_package_in_repo("python3-pyelftools"):
+            self._os.install_packages("python3-pyelftools")
+        # then try the older package manager name if it's available
+        elif self._os.is_package_in_repo("pyelftools"):
+            self._os.install_packages("pyelftools")
+        else:
+            # otherwise try pip. This can fail to install for the system on ubuntu 24.04
+            self._node.tools[Pip].install_packages("pyelftools")
 
     def _uninstall(self) -> None:
         # undo source installation (thanks ninja)

microsoft/testsuites/dpdk/rdmacore.py

@@ -149,7 +149,8 @@ def _check_if_installed(self) -> bool:
 
     def _setup_node(self) -> None:
         if isinstance(self._os, (Debian, Fedora, Suse)):
-            self._os.uninstall_packages("rdma-core")
+            if self._os.package_exists("rdma-core"):
+                self._os.uninstall_packages("rdma-core")
         if isinstance(self._os, Fedora):
             self._os.group_install_packages("Development Tools")
         super()._setup_node()