From 5668e21fb399975174e692e2904ff9e540341f8b Mon Sep 17 00:00:00 2001 From: Frankie Robertson Date: Wed, 16 Dec 2020 11:35:02 +0200 Subject: [PATCH] Don't specify to SBATCH how much of unspecified resources we want --- yaspi/templates/cpu-proc/template.sh | 10 ++++----- yaspi/templates/gpu-proc/template.sh | 10 ++++----- yaspi/templates/ray/ray-sbatch.sh | 8 +++---- yaspi/yaspi.py | 31 ++++++++++++++++++++-------- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/yaspi/templates/cpu-proc/template.sh b/yaspi/templates/cpu-proc/template.sh index 76a0508..cc0462e 100644 --- a/yaspi/templates/cpu-proc/template.sh +++ b/yaspi/templates/cpu-proc/template.sh @@ -1,11 +1,11 @@ #!/bin/bash #SBATCH --job-name={{job_name}} -#SBATCH --mem={{mem}} +#SBATCH --mem={{mem|ordeleteline}} #SBATCH --array={{array}} -#SBATCH --time={{time_limit}} +#SBATCH --time={{time_limit|ordeleteline}} #SBATCH --output={{log_path}} -#SBATCH --partition={{partition}} -#SBATCH --cpus-per-task={{cpus_per_task}} +#SBATCH --partition={{partition|ordeleteline}} +#SBATCH --cpus-per-task={{cpus_per_task|ordeleteline}} {{sbatch_resources}} {{exclude_nodes}} # ------------------------------- @@ -30,4 +30,4 @@ eval "${prep}" cmd="{{cmd}}" cmd="srun ${cmd} --slurm --worker_id ${worker_id}" echo "running cmd $cmd" -eval "${cmd}" \ No newline at end of file +eval "${cmd}" diff --git a/yaspi/templates/gpu-proc/template.sh b/yaspi/templates/gpu-proc/template.sh index 31b694f..080242f 100644 --- a/yaspi/templates/gpu-proc/template.sh +++ b/yaspi/templates/gpu-proc/template.sh @@ -1,11 +1,11 @@ #!/bin/bash #SBATCH --job-name={{job_name}} -#SBATCH --mem={{mem}} +#SBATCH --mem={{mem|ordeleteline}} #SBATCH --array={{array}} -#SBATCH --time={{time_limit}} +#SBATCH --time={{time_limit|ordeleteline}} #SBATCH --output={{log_path}} -#SBATCH --partition={{partition}} -#SBATCH --cpus-per-task={{cpus_per_task}} +#SBATCH --partition={{partition|ordeleteline}} +#SBATCH --cpus-per-task={{cpus_per_task|ordeleteline}} {{sbatch_resources}} {{exclude_nodes}} # ------------------------------- @@ -31,4 +31,4 @@ eval "${prep}" cmd="{{cmd}}" cmd="srun --unbuffered ${cmd} ${custom_args_queue[${worker_id}]}" echo "running cmd $cmd" -eval "${cmd}" \ No newline at end of file +eval "${cmd}" diff --git a/yaspi/templates/ray/ray-sbatch.sh b/yaspi/templates/ray/ray-sbatch.sh index 264903f..c32c5b6 100644 --- a/yaspi/templates/ray/ray-sbatch.sh +++ b/yaspi/templates/ray/ray-sbatch.sh @@ -1,11 +1,11 @@ #!/bin/bash #SBATCH --job-name={{job_name}} -#SBATCH --mem={{mem}} +#SBATCH --mem={{mem|ordeleteline}} #SBATCH --array={{array}} -#SBATCH --time={{time_limit}} +#SBATCH --time={{time_limit|ordeleteline}} #SBATCH --output={{log_path}} -#SBATCH --partition={{partition}} -#SBATCH --cpus-per-task={{cpus_per_task}} +#SBATCH --partition={{partition|ordeleteline}} +#SBATCH --cpus-per-task={{cpus_per_task|ordeleteline}} {{sbatch_resources}} {{exclude_nodes}} # ------------------------------- diff --git a/yaspi/yaspi.py b/yaspi/yaspi.py index 1c40dda..db5bdd7 100644 --- a/yaspi/yaspi.py +++ b/yaspi/yaspi.py @@ -13,6 +13,9 @@ from watchlogs.watchlogs import Watcher +OR_DELETE_LINE = "|ordeleteline" + + class Yaspi: @beartype @@ -24,15 +27,15 @@ def __init__( recipe: str, gen_script_dir: str, log_dir: str, - partition: str, + partition: NoneTypeOr[str], job_array_size: int, - cpus_per_task: int, - gpus_per_task: int, + cpus_per_task: NoneTypeOr[int], + gpus_per_task: NoneTypeOr[int], refresh_logs: bool, exclude: str, use_custom_ray_tmp_dir: bool, ssh_forward: str, - time_limit: str, + time_limit: NoneTypeOr[str], throttle_array: int, mem: str, constraint_str: str, @@ -274,14 +277,24 @@ def fill_template(self, template_path: Path, rules: dict) -> str: with open(template_path, "r") as f: template = f.read().splitlines() for row in template: + skip_row = False edits = [] regex = r"\{\{(.*?)\}\}" for match in re.finditer(regex, row): groups = match.groups() assert len(groups) == 1, "expected single group" key = groups[0] + ordeleteline = False + if key.endswith(OR_DELETE_LINE): + ordeleteline = True + key = key[:-len(OR_DELETE_LINE)] token = rules[key] + if ordeleteline and token is None: + skip_row = True + break edits.append((match.span(), token)) + if skip_row: + continue if edits: # invert the spans spans = [(None, 0)] + [x[0] for x in edits] + [(len(row), None)] @@ -306,22 +319,22 @@ def main(): help="the SLURM recipe to use to generate scripts") parser.add_argument("--template_dir", help="if given, override directory containing SLURM templates") - parser.add_argument("--partition", default="gpu", + parser.add_argument("--partition", default=None, help="The name of the SLURM partition used to run the job") - parser.add_argument("--time_limit", default="96:00:00", + parser.add_argument("--time_limit", default=None, help="The maximum amount of time allowed to run the job") parser.add_argument("--gen_script_dir", default="data/slurm-gen-scripts", help="directory in which generated slurm scripts will be stored") parser.add_argument("--cmd", default='echo "hello"', help="single command (or comma separated commands) to run") - parser.add_argument("--mem", default='60G', + parser.add_argument("--mem", default=None, help="the memory to be requested for each SLURM worker") parser.add_argument("--prep", default="", help="a command to be run before srun") parser.add_argument("--job_array_size", type=int, default=2, help="The number of SLURM array workers") - parser.add_argument("--cpus_per_task", type=int, default=5, + parser.add_argument("--cpus_per_task", type=int, default=None, help="the number of cpus requested for each SLURM task") - parser.add_argument("--gpus_per_task", type=int, default=1, + parser.add_argument("--gpus_per_task", type=int, default=None, help="the number of gpus requested for each SLURM task") parser.add_argument("--throttle_array", type=int, default=0, help="limit the number of array workers running at once")