cli: add podman runner for slurm resource manager (#909)

added SlurmRunner for Podman and corresponding tests

src/popper/runner_slurm.py

@@ -8,6 +8,7 @@
 from popper.runner_host import HostRunner
 from popper.runner_host import DockerRunner as HostDockerRunner
 from popper.runner_host import SingularityRunner as HostSingularityRunner
+from popper.runner_host import PodmanRunner as HostPodmanRunner
 
 
 class SlurmRunner(HostRunner):
@@ -162,6 +163,68 @@ def _create_cmd(self, step, img, cid):
         return " ".join(cmd)
 
 
+class PodmanRunner(SlurmRunner, HostPodmanRunner):
+    def __init__(self, **kw):
+        super(PodmanRunner, self).__init__(init_podman_client=False, **kw)
+
+    def __exit__(self, exc_type, exc, traceback):
+        pass
+
+    def run(self, step):
+        """Execute the given step via slurm in the docker engine."""
+        cid = pu.sanitized_name(step.id, self._config.wid)
+        cmd = []
+
+        build, _, img, tag, build_ctx_path = self._get_build_info(step)
+
+        cmd.append(f"podman rm -f {cid} || true")
+
+        if build:
+            cmd.append(f"podman build -t {img}:{tag} {build_ctx_path}")
+        elif not self._config.skip_pull and not step.skip_pull:
+            cmd.append(f"podman pull {img}:{tag}")
+
+        cmd.append(self._create_cmd(step, f"{img}:{tag}", cid))
+        cmd.append(f"podman start --attach {cid}")
+
+        self._spawned_containers.add(cid)
+        ecode = self._submit_batch_job(cmd, step)
+        self._spawned_containers.remove(cid)
+        return ecode
+
+    def _create_cmd(self, step, img, cid):
+        container_args = self._get_container_kwargs(step, img, cid)
+        container_args.pop("detach")
+        cmd = ["podman create"]
+        cmd.append(f"--name {container_args.pop('name')}")
+        cmd.append(f"--workdir {container_args.pop('working_dir')}")
+
+        entrypoint = container_args.pop("entrypoint", None)
+        if entrypoint:
+            cmd.append(f"--entrypoint {' '.join(entrypoint)}")
+
+        # append volume and environment flags
+        for vol in container_args.pop("volumes"):
+            cmd.append(f"-v {vol}")
+        for env_key, env_val in container_args.pop("environment").items():
+            cmd.append(f"-e {env_key}={env_val}")
+
+        command = container_args.pop("command")
+        image = container_args.pop("image")
+
+        # anything else is treated as a flag
+        for k, v in container_args.items():
+            cmd.append(pu.key_value_to_flag(k, v))
+
+        # append the image and the commands
+        cmd.append(image)
+
+        if command:
+            cmd.append(" ".join(command))
+
+        return " ".join(cmd)
+
+
 class SingularityRunner(SlurmRunner, HostSingularityRunner):
     def __init__(self, **kw):
         super(SingularityRunner, self).__init__(init_spython_client=False, **kw)

src/test/test_runner_slurm.py

@@ -9,7 +9,12 @@
 from popper.config import ConfigLoader
 from popper.runner import WorkflowRunner
 from popper.parser import WorkflowParser
-from popper.runner_slurm import SlurmRunner, DockerRunner, SingularityRunner
+from popper.runner_slurm import (
+    SlurmRunner,
+    DockerRunner,
+    SingularityRunner,
+    PodmanRunner,
+)
 from popper.cli import log as log
 
 from .test_common import PopperTest
@@ -312,6 +317,119 @@ def test_run(self, mock_kill):
             self.assertEqual(expected, actual)
 
 
+class TestSlurmPodmanRunner(unittest.TestCase):
+    def setUp(self):
+        log.setLevel("CRITICAL")
+        self.Popen = MockPopen()
+        replacer = Replacer()
+        replacer.replace("popper.runner_host.Popen", self.Popen)
+        self.addCleanup(replacer.restore)
+
+    def tearDown(self):
+        log.setLevel("NOTSET")
+
+    def test_create_cmd(self):
+        self.maxDiff = None
+        config = {"workspace_dir": "/w"}
+        with PodmanRunner(config=ConfigLoader.load(**config)) as prunner:
+            step = Box({"args": ["-two", "-flags"]}, default_box=True)
+            cmd = prunner._create_cmd(step, "foo:1.9", "container_name")
+
+            expected = (
+                "podman create"
+                " --name container_name"
+                " --workdir /workspace"
+                " -v /w:/workspace:Z"
+                "   foo:1.9 -two -flags"
+            )
+
+            self.assertEqual(expected, cmd)
+
+        config_dict = {
+            "engine": {
+                "name": "podman",
+                "options": {
+                    "privileged": True,
+                    "hostname": "popper.local",
+                    "domainname": "www.example.org",
+                    "volumes": ["/path/in/host:/path/in/container"],
+                    "environment": {"FOO": "bar"},
+                },
+            },
+            "resource_manager": {"name": "slurm"},
+        }
+
+        config = {"workspace_dir": "/w", "config_file": config_dict}
+        with PodmanRunner(config=ConfigLoader.load(**config)) as prunner:
+            step = Box({"args": ["-two", "-flags"]}, default_box=True)
+            cmd = prunner._create_cmd(step, "foo:1.9", "container_name")
+
+            expected = (
+                "podman create --name container_name "
+                "--workdir /workspace "
+                "-v /w:/workspace:Z "
+                "-v /path/in/host:/path/in/container "
+                "-e FOO=bar   --privileged --hostname popper.local "
+                "--domainname www.example.org"
+                " foo:1.9 -two -flags"
+            )
+
+            self.assertEqual(expected, cmd)
+
+    @replace("popper.runner_slurm.os.kill", mock_kill)
+    def test_run(self, mock_kill):
+        config_dict = {
+            "engine": {
+                "name": "podman",
+                "options": {
+                    "privileged": True,
+                    "hostname": "popper.local",
+                    "domainname": "www.example.org",
+                    "volumes": ["/path/in/host:/path/in/container"],
+                    "environment": {"FOO": "bar"},
+                },
+            },
+            "resource_manager": {"name": "slurm"},
+        }
+
+        config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
+
+        self.Popen.set_command(
+            f"sbatch --wait --job-name popper_1_{config.wid} "
+            f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
+            f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
+            returncode=0,
+        )
+
+        self.Popen.set_command(
+            f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out", returncode=0
+        )
+
+        with WorkflowRunner(config) as r:
+            wf_data = {
+                "steps": [
+                    {
+                        "uses": "popperized/bin/sh@master",
+                        "runs": ["cat"],
+                        "args": ["README.md"],
+                    }
+                ]
+            }
+            r.run(WorkflowParser.parse(wf_data=wf_data))
+
+        with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
+            # fmt: off
+            expected = f"""#!/bin/bash
+podman rm -f popper_1_{config.wid} || true
+podman build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/{config.wid}/github.com/popperized/bin/sh
+podman create --name popper_1_{config.wid} --workdir /workspace --entrypoint cat -v /w:/workspace:Z -v /path/in/host:/path/in/container -e FOO=bar   --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
+podman start --attach popper_1_{config.wid}"""
+            # fmt: on
+            actual = f.read()
+            self.maxDiff = None
+            self.assertEqual(expected, actual)
+
+
 class TestSlurmSingularityRunner(unittest.TestCase):
     def setUp(self):
         self.Popen = MockPopen()