Merge pull request #1243 from microsoft/feature-ci-support-redis

[ci] Integrate Redis to CI Infrastructure

.github/workflows/catnap.yml

@@ -123,9 +123,57 @@ jobs:
           **/*.stdout.txt
           **/*.stderr.txt
 
+  redis-pipeline:
+    name: Redis Pipeline
+    needs: release-pipeline
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Setup SSH
+      shell: bash
+      run: |
+        mkdir -p $HOME/.ssh/
+        echo "${{ secrets.SSHKEY }}" > "$HOME/.ssh/id_rsa"
+        chmod 400 $HOME/.ssh/id_rsa
+        echo "Host *" > $HOME/.ssh/config
+        echo -e "\tStrictHostKeyChecking no" >> $HOME/.ssh/config
+        echo -e "\tIdentityFile $HOME/.ssh/id_rsa" >> $HOME/.ssh/config
+        echo -e "\tIdentitiesOnly yes" >> $HOME/.ssh/config
+        echo -e "\tPasswordAuthentication no" >> $HOME/.ssh/config
+        echo -e "\tUser ${{ secrets.USERNAME }}" >> $HOME/.ssh/config
+        echo -e "\tPort ${{ secrets.PORTNUM }}" >> $HOME/.ssh/config
+    - name: Run
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          branch_name="${{ github.head_ref }}"
+        else
+          branch_name="${{ github.ref_name }}"
+        fi
+        pip install --pre azure-data-tables
+        python3 tools/demikernel_ci.py \
+          --server $SERVER \
+          --client $CLIENT \
+          --repository demikernel \
+          --branch origin/$branch_name \
+          --libos $LIBOS \
+          --test-redis --delay 10 \
+          --server-addr $SERVER_ADDR \
+          --client-addr $CLIENT_ADDR
+    - name: Archive Logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: redis-pipeline-logs
+        path: |
+          **/*.stdout.txt
+          **/*.stderr.txt
+
   report-performance:
     name: Report Performance
-    needs: release-pipeline
+    needs: redis-pipeline
     if: ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     permissions:

shim/src/Makefile

@@ -14,7 +14,7 @@ all: all-libs
 
 all-libs:
 	mkdir -p $(LIBDIR)
-	$(CC) $(CFLAGS) $(SRC) -fPIC -shared -o $(LIBDIR)/$(SHIM_LIB) $(LIBDIR)/$(DEMIKERNEL_LIB)
+	$(CC) $(CFLAGS) $(SRC) -fPIC -shared -o $(LIBDIR)/$(SHIM_LIB) -L $(LIBDIR)/ -ldemikernel
 
 clean:
 	rm -f $(LIBDIR)/$(SHIM_LIB)

tools/ci/job/factory.py

@@ -29,6 +29,12 @@ def compile(self) -> BaseJob:
         else:
             return linux.CompileJobOnLinux(self.config)
 
+    def install(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Install is not supported on Windows")
+        else:
+            return linux.InstallJobOnLinux(self.config)
+
     def cleanup(self) -> BaseJob:
         if self.config["platform"] == "windows":
             return windows.CleanupJobOnWindows(self.config)
@@ -90,3 +96,39 @@ def system_test(self, test_name: str, niterations: int = 0, run_mode: str = "",
                     return linux.UdpPushPopTest(self.config)
                 else:
                     raise Exception("Invalid test name")
+
+    def clone_redis(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Clone is not supported on Windows")
+        else:
+            return linux.CloneRedisJobOnLinux(self.config)
+
+    def make_redis(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Make Redis is not supported on Windows")
+        else:
+            return linux.MakeRedisJobOnLinux(self.config)
+
+    def run_redis_server(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Run Redis is not supported on Windows")
+        else:
+            return linux.RunRedisServerJobOnLinux(self.config)
+
+    def run_redis_benchmark(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Run Redis is not supported on Windows")
+        else:
+            return linux.RunRedisBenchmarkJobOnLinux(self.config)
+
+    def stop_redis_server(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Stop Redis is not supported on Windows")
+        else:
+            return linux.StopRedisServerJobOnLinux(self.config)
+
+    def cleanup_redis(self) -> BaseJob:
+        if self.config["platform"] == "windows":
+            raise Exception("Cleanup Redis is not supported on Windows")
+        else:
+            return linux.CleanupRedisJobOnLinux(self.config)

tools/ci/job/generic.py

@@ -15,15 +15,15 @@ def __init__(self, config, name):
         self.name = name
         self.config = config
 
-    def execute(self, serverTask: BaseTask, clientTask: BaseTask = None) -> bool:
+    def execute(self, serverTask: BaseTask, clientTask: BaseTask = None, no_wait: bool = False) -> bool:
         jobs: dict[str, subprocess.Popen[str]] = {}
 
         jobs[self.name + "-server-" + self.server()] = serverTask.execute()
 
         if clientTask is not None:
             jobs[self.name + "-client-" +
                  self.client()] = clientTask.execute()
-        return wait_and_report(self.name, self.log_directory(), jobs)
+        return wait_and_report(self.name, self.log_directory(), jobs, no_wait)
 
     def name(self) -> str:
         return self.name
@@ -66,3 +66,15 @@ def server_addr(self) -> str:
 
     def client_addr(self) -> str:
         return self.config["client_addr"]
+
+    def install_prefix(self) -> str:
+        return self.config["install_prefix"]
+
+    def libshim_path(self) -> str:
+        return self.config["libshim_path"]
+
+    def ld_library_path(self) -> str:
+        return self.config["ld_library_path"]
+
+    def path(self) -> str:
+        return self.config["path"]

tools/ci/job/linux.py

@@ -3,6 +3,7 @@
 
 import subprocess
 import time
+import ci.task.linux as linux
 from ci.task.linux import BaseLinuxTask, CheckoutOnLinux, CompileOnLinux, RunOnLinux, CleanupOnLinux
 from ci.job.utils import wait_and_report
 from ci.job.generic import BaseJob
@@ -16,8 +17,8 @@ class BaseLinuxJob(BaseJob):
     def __init__(self, config, name):
         super().__init__(config, name)
 
-    def execute(self, serverTask: BaseLinuxTask, clientTask: BaseLinuxTask = None) -> bool:
-        return super().execute(serverTask, clientTask)
+    def execute(self, serverTask: BaseLinuxTask, clientTask: BaseLinuxTask = None, no_wait: bool = False) -> bool:
+        return super().execute(serverTask, clientTask, no_wait)
 
 
 class CheckoutJobOnLinux(BaseLinuxJob):
@@ -300,3 +301,95 @@ def job_test_system_rust(
     }
     job: SystemTestJobOnLinux = SystemTestJobOnLinux(config)
     return job.execute()
+
+
+class InstallJobOnLinux(BaseLinuxJob):
+
+    def __init__(self, config: dict):
+        name = "install-{}".format("debug" if config["is_debug"] else "release")
+        super().__init__(config, name)
+
+    def execute(self) -> bool:
+        cmd: str = f"LIBOS={super().libos()} INSTALL_PREFIX={super().install_prefix()} install"
+        serverTask: linux.CompileOnLinux = linux.CompileOnLinux(
+            super().server(), super().repository(), cmd, super().is_debug())
+
+        if not super().enable_nfs():
+            clientTask: linux.CompileOnLinux = linux.CompileOnLinux(
+                super().client(), super().repository(), cmd, super().is_debug())
+            return super().execute(serverTask, clientTask)
+        return super().execute(serverTask)
+
+
+class CloneRedisJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "clone-redis")
+
+    def execute(self) -> bool:
+        serverTask: linux.CloneOnLinux = linux.CloneOnLinux(
+            super().server(), super().path(), super().repository(), super().branch())
+        clientTask: linux.CloneOnLinux = linux.CloneOnLinux(
+            super().client(), super().path(), super().repository(), super().branch())
+        return super().execute(serverTask, clientTask)
+
+
+class MakeRedisJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "make-redis")
+
+    def execute(self) -> bool:
+        serverTask: linux.MakeRedisOnLinux = linux.MakeRedisOnLinux(super().server(), super().path())
+        clientTask: linux.MakeRedisOnLinux = linux.MakeRedisOnLinux(super().client(), super().path())
+        return super().execute(serverTask, clientTask)
+
+
+class RunRedisServerJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "run-redis-server")
+
+    def execute(self) -> bool:
+        serverTask: linux.RunredisServerOnLinux = linux.RunredisServerOnLinux(
+            host=super().server(), redis_path=f"{super().path()}/redis",
+            env=f"CONFIG_PATH={super().config_path()} LD_LIBRARY_PATH={super().ld_library_path()} LD_PRELOAD={super().libshim_path()} LIBOS={super().libos()}",
+            params=f"--bind {super().server_addr()} --protected-mode no --save \\\"\\\" ")
+
+        passed: bool = super().execute(serverTask, no_wait=True)
+
+        # Give some time to Redis server to start.
+        time.sleep(super().delay())
+
+        return passed
+
+
+class RunRedisBenchmarkJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "run-redis-benchmark")
+
+    def execute(self) -> bool:
+        clientTask: linux.RunRedisBenchmarkOnLinux = linux.RunRedisBenchmarkOnLinux(
+            super().client(), f"{super().path()}/redis",
+            f"-h {super().server_addr()} -d 64 -t set,get -c 1 -n 100")
+        return super().execute(clientTask)
+
+
+class StopRedisServerJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "stop-redis-server")
+
+    def execute(self) -> bool:
+        clientTask: linux.StopRedisServerOnLinux = linux.StopRedisServerOnLinux(
+            super().client(), f"{super().path()}/redis",
+            f"-h {super().server_addr()}")
+        return super().execute(clientTask)
+
+
+class CleanupRedisJobOnLinux(BaseLinuxJob):
+    def __init__(self, config: dict):
+        super().__init__(config, "cleanup-redis")
+
+    def execute(self) -> bool:
+        serverTask: linux.CleanupRedisOnLinux = linux.CleanupRedisOnLinux(
+            super().server(), "redis-server", f"{super().path()}/redis")
+        clientTask: linux.CleanupRedisOnLinux = linux.CleanupRedisOnLinux(
+            super().client(), "redis-benchmark", f"{super().path()}/redis")
+        return super().execute(serverTask, clientTask)

tools/ci/job/utils.py

@@ -126,21 +126,22 @@ def extract_performance(job_name, file):
                 table_client.create_entity(entry)
 
 
-def wait_jobs(log_directory: str, jobs: dict):
+def wait_jobs(log_directory: str, jobs: dict, no_wait: bool = False):
     @timing
     def wait_jobs2(log_directory: str, jobs: dict) -> List:
         status: list[int] = []
 
-        for job_name, j in jobs.items():
-            stdout, stderr = j.communicate()
-            status.append((j.pid, j.returncode))
-            with open(log_directory + "/" + job_name + ".stdout.txt", "w") as file:
-                file.write("{}".format(stdout))
-            with open(log_directory + "/" + job_name + ".stdout.txt", "r") as file:
-                extract_performance(job_name, file)
+        if not no_wait:
+            for job_name, j in jobs.items():
+                stdout, stderr = j.communicate()
+                status.append((j.pid, j.returncode))
+                with open(log_directory + "/" + job_name + ".stdout.txt", "w") as file:
+                    file.write("{}".format(stdout))
+                with open(log_directory + "/" + job_name + ".stdout.txt", "r") as file:
+                    extract_performance(job_name, file)
 
-            with open(log_directory + "/" + job_name + ".stderr.txt", "w") as file:
-                file.write("{}".format(stderr))
+                with open(log_directory + "/" + job_name + ".stderr.txt", "w") as file:
+                    file.write("{}".format(stderr))
 
         # Cleanup list of jobs.
         jobs.clear()
@@ -149,8 +150,12 @@ def wait_jobs2(log_directory: str, jobs: dict) -> List:
     return wait_jobs2(log_directory, jobs)
 
 
-def wait_and_report(name: str, log_directory: str, jobs: dict, all_pass=True):
-    ret = wait_jobs(log_directory, jobs)
+def wait_and_report(name: str, log_directory: str, jobs: dict, all_pass=True, no_wait: bool = False):
+    if no_wait:
+        print("[NO WAIT] in {:9.2f} ms {}".format(0, name))
+        return True
+
+    ret = wait_jobs(log_directory, jobs, no_wait)
     passed: bool = False
     status: List = ret[0]
     duration: float = ret[1]

tools/ci/task/linux.py

@@ -40,3 +40,40 @@ def __init__(self, host: str, repository: str, is_sudo: bool, branch: str):
         sudo_cmd: str = "sudo -E" if is_sudo else ""
         cmd: str = f"cd {repository} && {sudo_cmd} make clean && git checkout {branch} && git clean -fdx ; sudo -E rm -rf /dev/shm/demikernel* ; sudo pkill -f demikernel*"
         super().__init__(host, cmd)
+
+
+class CloneOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, path: str, repository: str, branch: str):
+        cmd: str = f"cd {path} && git clone {repository} --branch {branch}"
+        super().__init__(host, cmd)
+
+
+class MakeRedisOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, path: str):
+        cmd: str = f"cd {path}/redis  && make MALLOC=libc all"
+        super().__init__(host, cmd)
+
+
+class RunredisServerOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, redis_path: str, env: str, params: str):
+        redis_cmd: str = f"{env} ./src/redis-server {params} --daemonize yes"
+        cmd: str = f"cd {redis_path} && {redis_cmd}"
+        super().__init__(host, cmd)
+
+
+class RunRedisBenchmarkOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, redis_path: str, params: str, timeout: int = 120):
+        cmd: str = f"cd {redis_path} && timeout {timeout} ./src/redis-benchmark {params}"
+        super().__init__(host, cmd)
+
+
+class StopRedisServerOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, redis_path: str, params: str):
+        cmd: str = f"cd {redis_path} && ./src/redis-cli {params} shutdown"
+        super().__init__(host, cmd)
+
+
+class CleanupRedisOnLinux(BaseLinuxTask):
+    def __init__(self, host: str, process_name: str, redis_path: str):
+        cmd: str = f"sudo pkill -e {process_name} ; rm -rf {redis_path}"
+        super().__init__(host, cmd)