RDK-47063: Add memcr L2 tests

.github/workflows/L2-tests.yml

@@ -123,13 +123,22 @@ jobs:
           sudo chmod 755 /var/volatile/rdk/dobby/bundles
           mkdir -p build
           cd build
-          cmake -DCMAKE_TOOLCHAIN_FILE="${{ env.TOOLCHAIN_FILE }}" -DRDK_PLATFORM=DEV_VM -DCMAKE_INSTALL_PREFIX:PATH=/usr ${{ matrix.extra_flags }}  -DRDK=ON -DUSE_SYSTEMD=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DAI_BUILD_TYPE=AI_DEBUG ..
+          cmake -DCMAKE_TOOLCHAIN_FILE="${{ env.TOOLCHAIN_FILE }}" -DRDK_PLATFORM=DEV_VM -DCMAKE_INSTALL_PREFIX:PATH=/usr ${{ matrix.extra_flags }}  -DRDK=ON -DUSE_SYSTEMD=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DAI_BUILD_TYPE=AI_DEBUG -DDOBBY_HIBERNATE_MEMCR_IMPL=ON ..
           make -j $(nproc)
           sudo make install
           # Link for the DobbyInit tool
           sudo ln -sf /usr/local/libexec/ /usr/libexec
           sudo ln -sf /lib/x86_64-linux-gnu/libreadline.so.* /lib/
 
+      - name: build memcr
+        run: |
+          mkdir -p ~/memcr
+          cd ~/memcr
+          git clone https://github.com/LibertyGlobal/memcr.git .
+          make COMPRESS_LZ4=1 CHECKSUM_MD5=1 ENCRYPT=1
+          mkdir -p scripts
+          cp $GITHUB_WORKSPACE/Dobby/develop/vagrant/memcr/start_memcr.sh ~/memcr/scripts/
+
       - name: Checkout srcThunder
         run: |
          cd $GITHUB_WORKSPACE

develop/vagrant/Vagrantfile

@@ -88,7 +88,7 @@ Vagrant.configure("2") do |config|
     config.vm.provision "shell", inline: <<-SHELL
         figlet DEPENDENCIES  # ASCII banner
 
-        apt install -y build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libjsoncpp1 libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang
+        apt install -y build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libjsoncpp1 libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang liblz4-dev
 
         #
         # To resolve DNS server resolver
@@ -238,6 +238,20 @@ Vagrant.configure("2") do |config|
         echo "****************************************************************************"
     SHELL
 
+    ##################################################
+    #
+    # Clone and build memcr repo
+    # Copy start_memcr script
+    config.vm.provision "shell", privileged: false, inline: <<-SHELL
+        figlet MEMCR  # ASCII banner
+        mkdir -p ~/memcr
+        cd ~/memcr
+        git clone https://github.com/LibertyGlobal/memcr.git .
+        make COMPRESS_LZ4=1 CHECKSUM_MD5=1 ENCRYPT=1
+    SHELL
+
+    config.vm.provision "file", source: "./memcr", destination: "~/memcr/scripts"
+
     config.vm.provision "shell", privileged: false, inline: <<-SHELL
         figlet DONE
         echo "**************************************************************************"

develop/vagrant/memcr/start_memcr.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "starting memcr service"
+
+DUMPSDIR=/media/apps/memcr
+LOCATOR=/tmp/memcrcom
+
+sudo mkdir -p ${DUMPSDIR}
+sudo chmod -R 777 ${DUMPSDIR}
+find ${DUMPSDIR} -mindepth 1 -exec rm {} \;
+sudo ~/memcr/memcr -d ${DUMPSDIR} -N -l ${LOCATOR} -f -z -e

tests/L2_testing/test_runner/container_manipulations.py

@@ -90,7 +90,7 @@ def execute_test():
 
     with test_utils.dobby_daemon():
         for test in tests:
-            process = dobby_tool_command(test.command, test.container_id)
+            process = test_utils.dobby_tool_command(test.command, test.container_id)
             test_utils.print_log("command output = %s" % process.stdout, test_utils.Severity.debug)
             result = test.expected_output in process.stdout
             if test.negation:
@@ -102,32 +102,6 @@ def execute_test():
     return test_utils.count_print_results(output_table)
 
 
-def dobby_tool_command(command, container_id):
-    """Runs DobbyTool command
-
-    Parameters:
-    command (string): command that should be run
-    container_id (string): name of container to run
-
-    Returns:
-    process (process): process that runs selected command
-
-    """
-
-    full_command = [
-            "DobbyTool",
-            command,
-            container_id
-        ]
-    if command == "start":
-        container_path = test_utils.get_container_spec_path(container_id)
-        full_command.append(container_path)
-
-    process = test_utils.run_command_line(full_command)
-
-    return process
-
-
 if __name__ == "__main__":
     test_utils.parse_arguments(__file__)
     execute_test()

tests/L2_testing/test_runner/memcr_tests.py

@@ -0,0 +1,222 @@
+# If not stated otherwise in this file or this component's LICENSE file the
+# following copyright and licenses apply:
+#
+# Copyright 2024 Sky UK
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import test_utils
+import subprocess
+import json
+from time import sleep
+from collections import namedtuple
+from pathlib import Path
+
+# base fields - same as in test_utils.Test
+# test_func - string, function name that performs the test
+Test = namedtuple('Test', ['name',
+                           'container_id',
+                           'expected_output',
+                           'description',
+                           'test_func']
+                  )
+
+tests = (
+    Test("Basic memcr test",
+         "sleepy",
+         True,
+         "Starts container, hibernates it and wakes it up",
+         "basic_memcr_test"),
+)
+
+
+class memcr:
+    """ Starts memcr """
+    def __init__(self):
+        test_utils.print_log("Starting memcr", test_utils.Severity.debug)
+        # as this process is running infinitely we cannot use run_command_line as it waits for execution to end
+        self.subproc = subprocess.Popen(["~/memcr/scripts/start_memcr.sh"],
+                                        universal_newlines=True,
+                                        shell=True,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE
+                                        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, etype, value, traceback):
+        """ Closes all memcr processess including worker processes waiting for restore command.
+        Sends SIGINT (-2) signal. If SIGKILL would be used instead of SIGKILL, memcr would not
+        be able to close its worker processes.
+        """
+        test_utils.print_log("Stopping memcr", test_utils.Severity.debug)
+        subprocess.run(["sudo", "pkill", "-2", "memcr"])
+
+
+def get_container_state(container_id):
+    """ Returns state of container
+
+    Parameters:
+    container_id (string): name of container
+
+    Returns:
+    string: container state or None if getting container info failed
+    """
+    process = test_utils.dobby_tool_command("info", container_id)
+
+    if not process.stdout.startswith("{"):
+        return None
+
+    info_json = json.loads(process.stdout)
+    return info_json.get("state")
+
+
+def get_container_pids(container_id):
+    """ Returns list of pids running in container
+
+    Parameters:
+    container_id (string): name of container
+
+    Returns:
+    list of pids or [] if getting container info failed
+    """
+    process = test_utils.dobby_tool_command("info", container_id)
+
+    if not process.stdout.startswith("{"):
+        return []
+
+    info_json = json.loads(process.stdout)
+    return info_json.get("pids")
+
+
+def get_checkpointed_pids(memcr_dump_dir = "/media/apps/memcr/"):
+    """ Returns pids of processes that are currently checkpointed by memcr.
+    Memcr stores memory of checkpointed processes in files named 'pages-<pid>.img'.
+    By default files are stored in /media/apps/memcr/
+
+    Parameters:
+    memcr_dump_dir (string): default location of memcr pages files
+
+    Returns:
+    list of pids
+    """
+    prefix = "pages-"
+    sufix = ".img"
+    p = Path(memcr_dump_dir)
+
+    checkpointed_pids = [int(x.name[len(prefix):-len(sufix)])
+                            for x in p.iterdir()
+                            if x.is_file() and x.name.startswith("pages-") and x.name.endswith(".img")]
+    test_utils.print_log("checkpointed pids: [" + " ".join(map(str, checkpointed_pids)) + "]", test_utils.Severity.debug)
+    return checkpointed_pids
+
+
+def check_pids_checkpointed(pids):
+    """ Checks if all pids from pids list are currently checkpointed
+
+    Parameters:
+    pids (list(int)): list of pids
+
+    Returns:
+    True if all pids from input parameter are checkpointed, False otherwise
+    """
+    checkpointed_pids = get_checkpointed_pids()
+    for p in pids:
+        if p not in checkpointed_pids:
+            return False
+
+    return True
+
+
+def check_pids_restored(pids):
+    """ Checks if all pids from pids list are currently restored (not checkpointed)
+
+    Parameters:
+    pids (list(int)): list of pids
+
+    Returns:
+    True if non of pids from input parameter is checkpointed, False otherwise
+    """
+    checkpointed_pids = get_checkpointed_pids()
+    for p in pids:
+        if p in checkpointed_pids:
+            return False
+
+    return True
+
+
+def basic_memcr_test(container_id):
+    with test_utils.dobby_daemon(), memcr(), test_utils.untar_bundle(container_id) as bundle_path:
+
+        # start container
+        command = ["DobbyTool", "start", container_id, bundle_path]
+        test_utils.run_command_line(command)
+
+        # give dobby some time to start container
+        sleep(1)
+
+        # check container is in running state
+        if get_container_state(container_id) != "running":
+            return False, "Unable to start container"
+
+        # store container pids
+        pids = get_container_pids(container_id)
+        test_utils.print_log("container pids: [" + " ".join(map(str, pids)) + "]", test_utils.Severity.debug)
+
+        # hibernate container
+        test_utils.dobby_tool_command("hibernate", container_id)
+
+        # give memcr some time to checkpoint everything
+        sleep(1)
+
+        # check container is hibernated
+        if get_container_state(container_id) != "hibernated":
+            return False, "Failed to hibernate container"
+
+        # check if all processes were checkpointed
+        if not check_pids_checkpointed(pids):
+            return False, "Not all pids checkpointed"
+
+        # wakeup/restore the container
+        test_utils.dobby_tool_command("wakeup", container_id)
+
+        # give memcr some time to wakeup container
+        sleep(1)
+
+        # check container is running again
+        if get_container_state(container_id) != "running":
+            return False, "Failed to wakeup container"
+
+        # check if all processes were restored
+        if not check_pids_restored(pids):
+            return False, "Not all pids restored"
+
+        return True, "Test passed"
+
+
+def execute_test():
+    output_table = []
+
+    for test in tests:
+        result = globals()[test.test_func](test.container_id)
+        output = test_utils.create_simple_test_output(test, result[0], result[1])
+        output_table.append(output)
+        test_utils.print_single_result(output)
+
+    return test_utils.count_print_results(output_table)
+
+
+if __name__ == "__main__":
+    test_utils.parse_arguments(__file__)
+    execute_test()

tests/L2_testing/test_runner/pid_limit_tests.py

@@ -22,11 +22,11 @@
     test_utils.Test("Pid limit default",
                     "sleepy",
                     "256",
-                    "Starts contaiern with no pid limit specified, checks if default pid limit is set for container"),
+                    "Starts container with no pid limit specified, checks if default pid limit is set for container"),
     test_utils.Test("Pid limit no override",
                     "sleepy_pid_limit",
                     "1000",
-                    "Starts contaienr with pid limit specified in config.json, checks if that pid limit was not overriden"),
+                    "Starts container with pid limit specified in config.json, checks if that pid limit was not overriden"),
 ]
 
 

tests/L2_testing/test_runner/runner.py

@@ -26,6 +26,7 @@
 import gui_containers
 import network_tests
 import pid_limit_tests
+import memcr_tests
 import sys
 
 from time import sleep
@@ -39,7 +40,8 @@
                    thunder_plugin,
                    network_tests,
                    gui_containers,
-                   pid_limit_tests]
+                   pid_limit_tests,
+                   memcr_tests]
 
 def run_all_tests():
     success_count = 0