From 8c3f962ad8555ff785e2ad73578dd95e4a9f5a0c Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 23 Feb 2021 09:33:33 +0100 Subject: [PATCH 1/7] add preload_cmds options --- reframe/core/pipeline.py | 18 ++++++++++++++++-- reframe/core/schedulers/__init__.py | 5 ++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ef9b66e477..18406aa607 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -322,6 +322,19 @@ def pipeline_hooks(cls): #: :default: :class:`None`. container_platform = ContainerPlatformField(type(None)) + #: .. versionadded:: 3.5.0 + #: + #: List of shell commands to execute before loading the environment for + #: this job. + #: + #: These commands do not execute in the context of ReFrame. + #: Instead, they are emitted in the generated job script just before the + #: actual job launch command. + #: + #: :type: :class:`List[str]` + #: :default: ``[]`` + preload_cmds = fields.TypedField(typ.List[str]) + #: .. versionadded:: 3.0 #: #: List of shell commands to execute before launching this job. @@ -794,6 +807,7 @@ def _rfm_init(self, name=None, prefix=None): self.postbuild_cmds = [] self.executable = os.path.join('.', self.name) self.executable_opts = [] + self.preload_cmds = [] self.prerun_cmds = [] self.postrun_cmds = [] self.keep_files = [] @@ -1253,7 +1267,7 @@ def compile(self): with osext.change_dir(self._stagedir): try: self._build_job.prepare( - build_commands, environs, + build_commands, environs, self.preload_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=True ) @@ -1381,7 +1395,7 @@ def run(self): try: self.logger.debug('Generating the run script') self._job.prepare( - commands, environs, + commands, environs, self.preload_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=rt.runtime().get_option( 'general/0/trap_job_errors' diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index d4872a541e..4391acaff7 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -330,7 +330,7 @@ def nodelist(self): def submit_time(self): return self._submit_time - def prepare(self, commands, environs=None, **gen_opts): + def prepare(self, commands, environs=None, preload_cmds=None, **gen_opts): environs = environs or [] if self.num_tasks <= 0: getlogger().debug(f'[F] Flexible node allocation requested') @@ -357,6 +357,9 @@ def prepare(self, commands, environs=None, **gen_opts): with shell.generate_script(self.script_filename, **gen_opts) as builder: builder.write_prolog(self.scheduler.emit_preamble(self)) + for c in preload_cmds: + builder.write_body(c) + builder.write(runtime.emit_loadenv_commands(*environs)) for c in commands: builder.write_body(c) From 17aaa6a647669dc4e032049a7c83f823eb39f10a Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 23 Feb 2021 09:49:14 +0100 Subject: [PATCH 2/7] fix merge --- reframe/core/pipeline.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index d09956ef8c..48f9478582 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -816,14 +816,6 @@ def _rfm_init(self, name=None, prefix=None): self.descr = self.name self.executable = os.path.join('.', self.name) - self.executable_opts = [] - self.preload_cmds = [] - self.prerun_cmds = [] - self.postrun_cmds = [] - self.keep_files = [] - self.readonly_files = [] - self.tags = set() - self.maintainers = [] self._perfvalues = {} # Static directories of the regression check From 38f52669abb1d8f7abea8b56e6e547d74f528a3a Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 23 Feb 2021 10:18:26 +0100 Subject: [PATCH 3/7] fix merge --- reframe/core/pipeline.py | 4 ++-- reframe/core/schedulers/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 48f9478582..e410eeced1 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -348,7 +348,7 @@ def pipeline_hooks(cls): container_platform = variable(type(None), field=ContainerPlatformField, value=None) - #: .. versionadded:: 3.5.0 + #: .. versionadded:: 3.5 #: #: List of shell commands to execute before loading the environment for #: this job. @@ -359,7 +359,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - preload_cmds = fields.TypedField(typ.List[str]) + preload_cmds = variable(typ.List[str], value=[]) #: .. versionadded:: 3.0 #: diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 4391acaff7..4114711a33 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -330,7 +330,7 @@ def nodelist(self): def submit_time(self): return self._submit_time - def prepare(self, commands, environs=None, preload_cmds=None, **gen_opts): + def prepare(self, commands, environs=None, preload_cmds=[], **gen_opts): environs = environs or [] if self.num_tasks <= 0: getlogger().debug(f'[F] Flexible node allocation requested') From 15e19965a555de5ba06c57231ed0eef4b4d3942d Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 2 Mar 2021 21:52:22 +0100 Subject: [PATCH 4/7] get commands from parition config --- reframe/core/pipeline.py | 19 ++++--------------- reframe/core/systems.py | 14 ++++++++++++-- reframe/schemas/config.json | 7 ++++++- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e410eeced1..a28b193ecd 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -348,19 +348,6 @@ def pipeline_hooks(cls): container_platform = variable(type(None), field=ContainerPlatformField, value=None) - #: .. versionadded:: 3.5 - #: - #: List of shell commands to execute before loading the environment for - #: this job. - #: - #: These commands do not execute in the context of ReFrame. - #: Instead, they are emitted in the generated job script just before the - #: actual job launch command. - #: - #: :type: :class:`List[str]` - #: :default: ``[]`` - preload_cmds = variable(typ.List[str], value=[]) - #: .. versionadded:: 3.0 #: #: List of shell commands to execute before launching this job. @@ -1232,7 +1219,8 @@ def compile(self): with osext.change_dir(self._stagedir): try: self._build_job.prepare( - build_commands, environs, self.preload_cmds, + build_commands, environs, + self._current_partition.preload_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=True ) @@ -1360,7 +1348,8 @@ def run(self): try: self.logger.debug('Generating the run script') self._job.prepare( - commands, environs, self.preload_cmds, + commands, environs, + self._current_partition.preload_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=rt.runtime().get_option( 'general/0/trap_job_errors' diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 4c7c94f981..37f54a8fff 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -22,7 +22,7 @@ class SystemPartition(jsonext.JSONSerializable): def __init__(self, parent, name, sched_type, launcher_type, descr, access, container_environs, resources, - local_env, environs, max_jobs): + local_env, environs, max_jobs, preload_cmds): getlogger().debug(f'Initializing system partition {name!r}') self._parent_system = parent self._name = name @@ -35,6 +35,7 @@ def __init__(self, parent, name, sched_type, launcher_type, self._local_env = local_env self._environs = environs self._max_jobs = max_jobs + self._preload_cmds = preload_cmds self._resources = {r['name']: r['options'] for r in resources} @property @@ -98,6 +99,14 @@ def max_jobs(self): ''' return self._max_jobs + @property + def preload_cmds(self): + '''Commands to be emitted before loading the modules. + + :type: :class:`List[str]` + ''' + return self._preload_cmds + @property def name(self): '''The name of this partition. @@ -317,7 +326,8 @@ def create(cls, site_config): modules=site_config.get(f'{partid}/modules'), variables=site_config.get(f'{partid}/variables') ), - max_jobs=site_config.get(f'{partid}/max_jobs') + max_jobs=site_config.get(f'{partid}/max_jobs'), + preload_cmds=site_config.get(f'{partid}/preload_cmds') ) ) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 519565ad8a..44ca313f06 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -220,6 +220,10 @@ "modules": {"$ref": "#/defs/modules_list"}, "variables": {"$ref": "#/defs/envvar_list"}, "max_jobs": {"type": "number"}, + "preload_cmds": { + "type": "array", + "items": {"type": "string"} + }, "resources": { "type": "array", "items": { @@ -472,6 +476,7 @@ "systems/partitions/modules": [], "systems/partitions/variables": [], "systems/partitions/environs": [], - "systems/partitions/max_jobs": 8 + "systems/partitions/max_jobs": 8, + "systems/partitions/preload_cmds": [] } } From a4b027bcf2659bd3175298b09a68f91c60ba8514 Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 3 Mar 2021 20:59:11 +0100 Subject: [PATCH 5/7] add unittest --- reframe/core/pipeline.py | 4 ++-- reframe/core/schedulers/__init__.py | 4 ++-- reframe/core/systems.py | 10 +++++----- reframe/schemas/config.json | 4 ++-- unittests/test_schedulers.py | 18 ++++++++++++------ 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a28b193ecd..851cf87620 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1220,7 +1220,7 @@ def compile(self): try: self._build_job.prepare( build_commands, environs, - self._current_partition.preload_cmds, + self._current_partition.prepare_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=True ) @@ -1349,7 +1349,7 @@ def run(self): self.logger.debug('Generating the run script') self._job.prepare( commands, environs, - self._current_partition.preload_cmds, + self._current_partition.prepare_cmds, login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=rt.runtime().get_option( 'general/0/trap_job_errors' diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 4114711a33..954f214db8 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -330,7 +330,7 @@ def nodelist(self): def submit_time(self): return self._submit_time - def prepare(self, commands, environs=None, preload_cmds=[], **gen_opts): + def prepare(self, commands, environs=None, prepare_cmds=[], **gen_opts): environs = environs or [] if self.num_tasks <= 0: getlogger().debug(f'[F] Flexible node allocation requested') @@ -357,7 +357,7 @@ def prepare(self, commands, environs=None, preload_cmds=[], **gen_opts): with shell.generate_script(self.script_filename, **gen_opts) as builder: builder.write_prolog(self.scheduler.emit_preamble(self)) - for c in preload_cmds: + for c in prepare_cmds: builder.write_body(c) builder.write(runtime.emit_loadenv_commands(*environs)) diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 37f54a8fff..5cb840a341 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -22,7 +22,7 @@ class SystemPartition(jsonext.JSONSerializable): def __init__(self, parent, name, sched_type, launcher_type, descr, access, container_environs, resources, - local_env, environs, max_jobs, preload_cmds): + local_env, environs, max_jobs, prepare_cmds): getlogger().debug(f'Initializing system partition {name!r}') self._parent_system = parent self._name = name @@ -35,7 +35,7 @@ def __init__(self, parent, name, sched_type, launcher_type, self._local_env = local_env self._environs = environs self._max_jobs = max_jobs - self._preload_cmds = preload_cmds + self._prepare_cmds = prepare_cmds self._resources = {r['name']: r['options'] for r in resources} @property @@ -100,12 +100,12 @@ def max_jobs(self): return self._max_jobs @property - def preload_cmds(self): + def prepare_cmds(self): '''Commands to be emitted before loading the modules. :type: :class:`List[str]` ''' - return self._preload_cmds + return self._prepare_cmds @property def name(self): @@ -327,7 +327,7 @@ def create(cls, site_config): variables=site_config.get(f'{partid}/variables') ), max_jobs=site_config.get(f'{partid}/max_jobs'), - preload_cmds=site_config.get(f'{partid}/preload_cmds') + prepare_cmds=site_config.get(f'{partid}/prepare_cmds') ) ) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 44ca313f06..27dcb87d7a 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -220,7 +220,7 @@ "modules": {"$ref": "#/defs/modules_list"}, "variables": {"$ref": "#/defs/envvar_list"}, "max_jobs": {"type": "number"}, - "preload_cmds": { + "prepare_cmds": { "type": "array", "items": {"type": "string"} }, @@ -477,6 +477,6 @@ "systems/partitions/variables": [], "systems/partitions/environs": [], "systems/partitions/max_jobs": 8, - "systems/partitions/preload_cmds": [] + "systems/partitions/prepare_cmds": [] } } diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 91d7b5221f..e15d3a8366 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -115,10 +115,12 @@ def fake_job(make_job): return ret -def prepare_job(job, command='hostname', pre_run=None, post_run=None): +def prepare_job(job, command='hostname', pre_run=None, post_run=None, + prepare_cmds=None): environs = [Environment(name='foo', modules=['testmod_foo'])] pre_run = pre_run or ['echo prerun'] post_run = post_run or ['echo postrun'] + prepare_cmds = prepare_cmds or ['echo prepare'] with rt.module_use('unittests/modules'): job.prepare( [ @@ -126,16 +128,18 @@ def prepare_job(job, command='hostname', pre_run=None, post_run=None): job.launcher.run_command(job) + ' ' + command, post_run ], - environs + environs, + prepare_cmds ) def assert_job_script_sanity(job): '''Assert the sanity of the produced script file.''' with open(job.script_filename) as fp: - matches = re.findall(r'echo prerun|echo postrun|hostname', + matches = re.findall(r'echo prepare|echo prerun|echo postrun|hostname', fp.read()) - assert ['echo prerun', 'hostname', 'echo postrun'] == matches + assert ['echo prepare', 'echo prerun', 'hostname', + 'echo postrun'] == matches def _expected_slurm_directives(job): @@ -508,7 +512,8 @@ def test_cancel_with_grace(minimal_job, scheduler, local_only): prepare_job(minimal_job, command='sleep 5 &', pre_run=['trap -- "" TERM'], - post_run=['echo $!', 'wait']) + post_run=['echo $!', 'wait'], + prepare_cmds=['']) minimal_job.submit() # Stall a bit here to let the the spawned process start and install its @@ -552,7 +557,8 @@ def test_cancel_term_ignore(minimal_job, scheduler, local_only): command=os.path.join(fixtures.TEST_RESOURCES_CHECKS, 'src', 'sleep_deeply.sh'), pre_run=[''], - post_run=['']) + post_run=[''], + prepare_cmds=['']) minimal_job.submit() # Stall a bit here to let the the spawned process start and install its From d42d10a9a7256859a6fe56d5e22a00734745b27f Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 3 Mar 2021 21:21:26 +0100 Subject: [PATCH 6/7] add documentation --- docs/config_reference.rst | 8 ++++++++ docs/configure.rst | 1 + docs/tutorial_advanced.rst | 3 +++ 3 files changed, 12 insertions(+) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index db3f1f2f4f..10ced097b9 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -308,6 +308,14 @@ System Partition Configuration This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. +.. js:attribute:: .systems[].partitions[].prepare_cmds + + :required: No + :default: ``[]`` + + List of commands to be emitted before the environment modules are loaded. + + .. js:attribute:: .systems[].partitions[].resources :required: No diff --git a/docs/configure.rst b/docs/configure.rst index afe477c98d..d4e3e14eb3 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -112,6 +112,7 @@ The basic properties of a partition are the following: For a complete list of the supported container platforms, see `here `__. * ``max_jobs``: The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. +* ``prepare_cmds``: List of commands to be emitted before the environment modules are loaded. * ``resources``: This is a set of optional additional scheduler resources that the tests can access transparently. For more information, please have a look `here `__. diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 775864447c..508d6a6eae 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -389,12 +389,15 @@ Generally, ReFrame generates the job shell scripts using the following pattern: #!/bin/bash -l {job_scheduler_preamble} + {prepare_cmds} {test_environment} {prerun_cmds} {parallel_launcher} {executable} {executable_opts} {postrun_cmds} The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation. +The ``prepare_cmds`` is set in the partition configuration. +It is used to specify a list of commands that need to be emitted before the environment modules are loaded. The ``test_environment`` are the necessary commands for setting up the environment of the test. These include any modules or environment variables set at the `system partition level `__ or any `modules `__ or `environment variables `__ set at the test level. Then the commands specified in :attr:`prerun_cmds ` follow, while those specified in the :attr:`postrun_cmds ` come after the launch of the parallel job. From 9433b3673129aa6bfbf39cba9858cd8ff6a2a72d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 4 Mar 2021 14:16:37 +0100 Subject: [PATCH 7/7] Minor fixes --- docs/config_reference.rst | 4 +++- docs/configure.rst | 1 - docs/tutorial_advanced.rst | 8 ++++---- reframe/core/schedulers/__init__.py | 3 ++- unittests/test_schedulers.py | 3 ++- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 10ced097b9..8f62c56d8a 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -313,7 +313,9 @@ System Partition Configuration :required: No :default: ``[]`` - List of commands to be emitted before the environment modules are loaded. + List of shell commands to be emitted before any environment loading commands are emitted. + + .. versionadded:: 3.5.0 .. js:attribute:: .systems[].partitions[].resources diff --git a/docs/configure.rst b/docs/configure.rst index d4e3e14eb3..afe477c98d 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -112,7 +112,6 @@ The basic properties of a partition are the following: For a complete list of the supported container platforms, see `here `__. * ``max_jobs``: The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. -* ``prepare_cmds``: List of commands to be emitted before the environment modules are loaded. * ``resources``: This is a set of optional additional scheduler resources that the tests can access transparently. For more information, please have a look `here `__. diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 508d6a6eae..473048df29 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -390,15 +390,15 @@ Generally, ReFrame generates the job shell scripts using the following pattern: #!/bin/bash -l {job_scheduler_preamble} {prepare_cmds} - {test_environment} + {env_load_cmds} {prerun_cmds} {parallel_launcher} {executable} {executable_opts} {postrun_cmds} The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation. -The ``prepare_cmds`` is set in the partition configuration. -It is used to specify a list of commands that need to be emitted before the environment modules are loaded. -The ``test_environment`` are the necessary commands for setting up the environment of the test. +The ``prepare_cmds`` are commands that can be emitted before the test environment commands. +These can be specified with the :js:attr:`prepare_cmds <.systems[].partitions[].prepare_cmds>` partition configuration option. +The ``env_load_cmds`` are the necessary commands for setting up the environment of the test. These include any modules or environment variables set at the `system partition level `__ or any `modules `__ or `environment variables `__ set at the test level. Then the commands specified in :attr:`prerun_cmds ` follow, while those specified in the :attr:`postrun_cmds ` come after the launch of the parallel job. The parallel launch itself consists of three parts: diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 954f214db8..287eb35d3b 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -330,7 +330,7 @@ def nodelist(self): def submit_time(self): return self._submit_time - def prepare(self, commands, environs=None, prepare_cmds=[], **gen_opts): + def prepare(self, commands, environs=None, prepare_cmds=None, **gen_opts): environs = environs or [] if self.num_tasks <= 0: getlogger().debug(f'[F] Flexible node allocation requested') @@ -357,6 +357,7 @@ def prepare(self, commands, environs=None, prepare_cmds=[], **gen_opts): with shell.generate_script(self.script_filename, **gen_opts) as builder: builder.write_prolog(self.scheduler.emit_preamble(self)) + prepare_cmds = prepare_cmds or [] for c in prepare_cmds: builder.write_body(c) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index e15d3a8366..e01003a15c 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -115,7 +115,8 @@ def fake_job(make_job): return ret -def prepare_job(job, command='hostname', pre_run=None, post_run=None, +def prepare_job(job, command='hostname', + pre_run=None, post_run=None, prepare_cmds=None): environs = [Environment(name='foo', modules=['testmod_foo'])] pre_run = pre_run or ['echo prerun']