Skip to content

Commit

Permalink
fix handling of cache directory in slurm runner (#841)
Browse files Browse the repository at this point in the history 
changes handling of cache dir in slurm runner. Updates tests accordingly
  • Loading branch information
@JayjeetAtGithub
JayjeetAtGithub committed May 26, 2020
1 parent b749c64 commit 3fdbf85
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 21 deletions.
2 changes: 1 addition & 1 deletion cli/popper/runner_slurm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def _stop_out_stream(self):

def _submit_batch_job(self, cmd, step):
job_name = pu.sanitized_name(step.id, self._config.wid)
temp_dir = "/tmp/popper/slurm/"
temp_dir = f"{self._config.cache_dir}/slurm/{self._config.wid}"
os.makedirs(temp_dir, exist_ok=True)

job_script = os.path.join(temp_dir, f"{job_name}.sh")
Expand Down
44 changes: 24 additions & 20 deletions cli/test/test_runner_slurm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
from box import Box


config = ConfigLoader.load(workspace_dir="/w")
slurm_cache_dir = f"{os.environ['HOME']}/.cache/popper/slurm/{config.wid}"


def mock_kill(pid, sig):
return 0

Expand Down Expand Up @@ -62,25 +66,25 @@ def test_submit_batch_job(self, mock_kill):
self.Popen.set_command(
"sbatch --wait "
f"--job-name popper_sample_{config.wid} "
f"--output /tmp/popper/slurm/popper_sample_{config.wid}.out "
f"/tmp/popper/slurm/popper_sample_{config.wid}.sh",
f"--output {slurm_cache_dir}/popper_sample_{config.wid}.out "
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
returncode=0,
)
self.Popen.set_command(
f"tail -f /tmp/popper/slurm/popper_sample_{config.wid}.out", returncode=0
f"tail -f {slurm_cache_dir}/popper_sample_{config.wid}.out", returncode=0
)
step = Box({"id": "sample"}, default_box=True)
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open(f"/tmp/popper/slurm/popper_sample_{config.wid}.sh", "r") as f:
with open(f"{slurm_cache_dir}/popper_sample_{config.wid}.sh", "r") as f:
content = f.read()

self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)

call_tail = call.Popen(
["tail", "-f", f"/tmp/popper/slurm/popper_sample_{config.wid}.out"],
["tail", "-f", f"{slurm_cache_dir}/popper_sample_{config.wid}.out"],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
Expand All @@ -96,8 +100,8 @@ def test_submit_batch_job(self, mock_kill):
"--job-name",
f"popper_sample_{config.wid}",
"--output",
f"/tmp/popper/slurm/popper_sample_{config.wid}.out",
f"/tmp/popper/slurm/popper_sample_{config.wid}.sh",
f"{slurm_cache_dir}/popper_sample_{config.wid}.out",
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
],
cwd=os.getcwd(),
env=None,
Expand All @@ -121,13 +125,13 @@ def test_submit_job_failure(self, mock_kill):

self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} "
f"--output /tmp/popper/slurm/popper_1_{config.wid}.out "
f"/tmp/popper/slurm/popper_1_{config.wid}.sh",
f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=12,
)

self.Popen.set_command(
f"tail -f /tmp/popper/slurm/popper_1_{config.wid}.out", returncode=0
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out", returncode=0
)

with WorkflowRunner(config) as r:
Expand All @@ -143,7 +147,7 @@ def test_submit_job_failure(self, mock_kill):
self.assertRaises(SystemExit, r.run, WorkflowParser.parse(wf_data=wf_data))

call_tail = call.Popen(
["tail", "-f", f"/tmp/popper/slurm/popper_1_{config.wid}.out"],
["tail", "-f", f"{slurm_cache_dir}/popper_1_{config.wid}.out"],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
Expand All @@ -159,8 +163,8 @@ def test_submit_job_failure(self, mock_kill):
"--job-name",
f"popper_1_{config.wid}",
"--output",
f"/tmp/popper/slurm/popper_1_{config.wid}.out",
f"/tmp/popper/slurm/popper_1_{config.wid}.sh",
f"{slurm_cache_dir}/popper_1_{config.wid}.out",
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
],
cwd=os.getcwd(),
env=None,
Expand Down Expand Up @@ -273,13 +277,13 @@ def test_run(self, mock_kill):

self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} "
f"--output /tmp/popper/slurm/popper_1_{config.wid}.out "
f"/tmp/popper/slurm/popper_1_{config.wid}.sh",
f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)

self.Popen.set_command(
f"tail -f /tmp/popper/slurm/popper_1_{config.wid}.out", returncode=0
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out", returncode=0
)

with WorkflowRunner(config) as r:
Expand All @@ -294,7 +298,7 @@ def test_run(self, mock_kill):
}
r.run(WorkflowParser.parse(wf_data=wf_data))

with open(f"/tmp/popper/slurm/popper_1_{config.wid}.sh", "r") as f:
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
docker rm -f popper_1_{config.wid} || true
Expand Down Expand Up @@ -378,20 +382,20 @@ def test_slurm_singularity_run(self, mock_kill):

# fmt: off
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} --output /tmp/popper/slurm/popper_1_{config.wid}.out /tmp/popper/slurm/popper_1_{config.wid}.sh",
f"sbatch --wait --job-name popper_1_{config.wid} --output {slurm_cache_dir}/popper_1_{config.wid}.out {slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)
# fmt: on

self.Popen.set_command(
f"tail -f /tmp/popper/slurm/popper_1_{config.wid}.out", returncode=0
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out", returncode=0
)

with WorkflowRunner(config) as r:
wf_data = {"steps": [{"uses": "popperized/bin/sh@master", "args": ["ls"],}]}
r.run(WorkflowParser.parse(wf_data=wf_data))

with open(f"/tmp/popper/slurm/popper_1_{config.wid}.sh", "r") as f:
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/popper_1_{config.wid}.sif ls"""
Expand Down

0 comments on commit 3fdbf85

Please sign in to comment.