test/cvm_boot: optionally add package repository before upgrading packages (#3851)

* tool: add createrepo tool

Signed-off-by: Thien Trung Vuong <tvuong@microsoft.com>

* test/cvm_boot: optionally add package repository before upgrading packages

Signed-off-by: Thien Trung Vuong <tvuong@microsoft.com>

---------

Signed-off-by: Thien Trung Vuong <tvuong@microsoft.com>

Author: trungams

lisa/operating_system.py

@@ -19,6 +19,7 @@
     Type,
     Union,
 )
+from urllib.parse import urlparse
 
 from assertpy import assert_that
 from retry import retry
@@ -378,6 +379,9 @@ def get_kernel_information(self, force_run: bool = False) -> KernelInformation:
 
         return kernel_information
 
+    def add_repository(self, repo: str, **kwargs: Any) -> None:
+        raise NotImplementedError()
+
     def install_packages(
         self,
         packages: Union[
@@ -985,6 +989,7 @@ def add_repository(
         no_gpgcheck: bool = True,
         repo_name: Optional[str] = None,
         keys_location: Optional[List[str]] = None,
+        **kwargs: Any,
     ) -> None:
         self._initialize_package_installation()
         if keys_location:
@@ -1537,6 +1542,7 @@ def add_repository(
         no_gpgcheck: bool = True,
         repo_name: Optional[str] = None,
         keys_location: Optional[List[str]] = None,
+        **kwargs: Any,
     ) -> None:
         self._node.tools[YumConfigManager].add_repository(repo, no_gpgcheck)
 
@@ -1955,6 +1961,35 @@ def add_azure_core_repo(
             sudo=True,
         )
 
+    def add_repository(
+        self,
+        repo: str,
+        no_gpgcheck: bool = True,
+        repo_name: Optional[str] = None,
+        keys_location: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        parsed_url = urlparse(repo)
+        if parsed_url.scheme and parsed_url.netloc:
+            self._node.tools[YumConfigManager].add_repository(repo, no_gpgcheck)
+        else:
+            self._create_local_repo(Path(repo))
+
+    def _create_local_repo(self, source_tarball: Path) -> None:
+        from lisa.tools import CreateRepo, RemoteCopy
+
+        working_path = Path(self._node.get_working_path())
+        tarball_file = source_tarball.name
+        tarball_path = working_path / tarball_file
+
+        # copy tarball to remote node
+        result = self._node.tools[RemoteCopy].copy_to_remote(
+            src=source_tarball, dest=working_path
+        )
+        self._log.debug(f"tarball copied to: {result}")
+
+        self._node.tools[CreateRepo].create_repo_from_tarball(tarball_path)
+
     # Disable KillUserProcesses to avoid test processes being terminated when
     # the SSH session is reset
     def set_kill_user_processes(self) -> None:
@@ -2054,6 +2089,7 @@ def add_repository(
         no_gpgcheck: bool = True,
         repo_name: Optional[str] = None,
         keys_location: Optional[List[str]] = None,
+        **kwargs: Any,
     ) -> None:
         self._initialize_package_installation()
         cmd = "zypper ar"
@@ -2187,6 +2223,7 @@ def add_repository(
         no_gpgcheck: bool = True,
         repo_name: Optional[str] = None,
         keys_location: Optional[List[str]] = None,
+        **kwargs: Any,
     ) -> None:
         raise SkippedException(
             UnsupportedDistroException(

lisa/tools/__init__.py

@@ -25,6 +25,7 @@
 from .chown import Chown
 from .chrony import Chrony
 from .cp import Cp
+from .createrepo import CreateRepo
 from .curl import Curl
 from .date import Date
 from .df import Df
@@ -147,6 +148,7 @@
     "Chown",
     "Chrony",
     "Cp",
+    "CreateRepo",
     "Curl",
     "Date",
     "Df",

lisa/tools/createrepo.py

@@ -0,0 +1,70 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+
+from pathlib import Path
+
+from lisa.executable import Tool
+from lisa.operating_system import CBLMariner
+from lisa.tools.tar import Tar
+from lisa.tools.tee import Tee
+from lisa.util import UnsupportedDistroException
+
+
+class CreateRepo(Tool):
+    @property
+    def command(self) -> str:
+        return "createrepo"
+
+    @property
+    def can_install(self) -> bool:
+        return True
+
+    def _install(self) -> bool:
+        if isinstance(self.node.os, CBLMariner):
+            self.node.os.install_packages("createrepo")
+        else:
+            raise UnsupportedDistroException(
+                self.node.os,
+                f"tool {self.command} can't be installed in {self.node.os.name}",
+            )
+        return self._check_exists()
+
+    def create_repo_from_tarball(self, tarball_path: Path) -> None:
+        workspace = Path(self.node.get_working_path())
+        repo_path = workspace / "rpms"
+
+        # extract tarball
+        self.node.tools[Tar].extract(
+            file=tarball_path.as_posix(),
+            dest_dir=repo_path.as_posix(),
+        )
+
+        # run createrepo
+        self._run_create_repo(path=repo_path.as_posix(), compatibility=True)
+
+        # add repo file
+        repo_file = "[builtpackages]\n"
+        repo_file += "name=Built Packages\n"
+        repo_file += f"baseurl=file://{repo_path.as_posix()}\n"  # noqa: E231
+        repo_file += "enabled=1\n"
+        repo_file += "gpgcheck=0\n"
+        repo_file += "priority=1\n"
+        repo_file += "skip_if_unavailable=True\n"
+
+        # write repo file to /etc/yum/repos.d/
+        repo_file_path = Path("/etc/yum.repos.d/builtpackages.repo")
+        self.node.tools[Tee].write_to_file(repo_file, repo_file_path, sudo=True)
+
+    def _run_create_repo(self, path: str, compatibility: bool = True) -> None:
+        cmd = path
+        if compatibility:
+            cmd = f"--compatibility {cmd}"
+        self.run(
+            cmd,
+            expected_exit_code=0,
+            expected_exit_code_failure_message="Failed to create repository",
+            shell=True,
+            sudo=False,
+            force_run=True,
+        )

microsoft/testsuites/cvm/cvm_boot.py

@@ -21,7 +21,7 @@
     SecurityProfile,
     SecurityProfileSettings,
 )
-from lisa.operating_system import CBLMariner, Posix
+from lisa.operating_system import CBLMariner
 from lisa.sut_orchestrator import AZURE
 from lisa.testsuite import simple_requirement
 from lisa.tools import BootCtl, Lsblk, Tpm2
@@ -96,9 +96,13 @@ def verify_encrypted_root_partition(self, node: RemoteNode) -> None:
         use_new_environment=True,
     )
     def verify_boot_success_after_component_upgrade(
-        self, log: Logger, node: RemoteNode, log_path: Path
+        self,
+        log: Logger,
+        node: RemoteNode,
+        log_path: Path,
+        variables: Dict[str, Any],
     ) -> None:
-        posix_os: Posix = cast(Posix, node.os)
+        os: CBLMariner = cast(CBLMariner, node.os)
         # First boot - no package upgrade has been performed
         # Check PCR values (PCR4, PCR7)
         pcrs_before_reboot = node.tools[Tpm2].pcrread(pcrs=[4, 7])
@@ -107,16 +111,20 @@ def verify_boot_success_after_component_upgrade(
         boot_components = ["shim", "systemd-boot", "kernel-uki"]
         boot_components_versions: Dict[str, str] = dict()
         for pkg in boot_components:
-            pkg_version = posix_os.get_package_information(pkg, use_cached=False)
+            pkg_version = os.get_package_information(pkg, use_cached=False)
             boot_components_versions[pkg] = pkg_version.version_str
 
+        repo_url = variables.get("rpm_repository")
+        if repo_url:
+            os.add_repository(repo_url)
+
         # Upgrade boot components
-        posix_os.update_packages(boot_components)
+        os.update_packages(boot_components)
 
         # Get new boot components versions
         boot_components_new_versions: Dict[str, str] = dict()
         for pkg in boot_components:
-            pkg_version = posix_os.get_package_information(pkg, use_cached=False)
+            pkg_version = os.get_package_information(pkg, use_cached=False)
             boot_components_new_versions[pkg] = pkg_version.version_str
 
         # Reboot