[feat] Add configuration parameter for automatically capturing command errors and exit code in job scripts

docs/config_reference.rst

@@ -1067,6 +1067,14 @@ General Configuration
    Ignore test name conflicts when loading tests.
 
 
+.. js:attribute:: .general[].trap_job_errors
+
+   :required: No
+   :default: ``false``
+
+   Trap command errors in the generated job scripts and let them exit immediately.
+
+
 .. js:attribute:: .general[].keep_stage_files
 
    :required: No

docs/manpage.rst

@@ -647,6 +647,18 @@ Here is an alphabetical list of the environment variables recognized by ReFrame:
       ================================== ==================
 
 
+.. envvar:: RFM_TRAP_JOB_ERRORS
+
+   Ignore job exit code
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated configuration parameter :js:attr:`trap_job_errors` general configuration parameter
+      ================================== ==================
+
+
 .. envvar:: RFM_IGNORE_REQNODENOTAVAIL
 
    Do not treat specially jobs in pending state with the reason ``ReqNodeNotAvail`` (Slurm only).

reframe/core/pipeline.py

@@ -46,7 +46,7 @@
 #: dependencies will be explicitly specified by the user.
 #:
 #:  This constant is directly available under the :mod:`reframe` module.
-DEPEND_EXACT  = 1
+DEPEND_EXACT = 1
 
 #: Constant to be passed as the ``how`` argument of the
 #: :func:`RegressionTest.depends_on` method. It denotes that the test cases of
@@ -61,7 +61,7 @@
 #: this test depends on all the test cases of the target test.
 #:
 #:  This constant is directly available under the :mod:`reframe` module.
-DEPEND_FULLY  = 3
+DEPEND_FULLY = 3
 
 
 def _run_hooks(name=None):
@@ -1147,7 +1147,7 @@ def compile(self):
 
         # Verify the sourcepath and determine the sourcepath in the stagedir
         if (os.path.isabs(self.sourcepath) or
-            os.path.normpath(self.sourcepath).startswith('..')):
+                os.path.normpath(self.sourcepath).startswith('..')):
             raise PipelineError(
                 'self.sourcepath is an absolute path or does not point to a '
                 'subfolder or a file contained in self.sourcesdir: ' +
@@ -1317,9 +1317,12 @@ def run(self):
                 self._job.prepare(
                     commands, environs,
                     login=rt.runtime().get_option('general/0/use_login_shell'),
+                    trap_errors=rt.runtime().get_option(
+                        'general/0/trap_job_errors'
+                    )
                 )
             except OSError as e:
-                raise PipelineError('failed to prepare job') from e
+                raise PipelineError('failed to prepare run job') from e
 
             self._job.submit()
 
@@ -1404,7 +1407,16 @@ def check_sanity(self):
               more details.
 
         '''
-        if self.sanity_patterns is None:
+        if rt.runtime().get_option('general/0/trap_job_errors'):
+            sanity_patterns = [
+                sn.assert_eq(self.job.exitcode, 0,
+                             msg='job exited with exit code {0}')
+            ]
+            if self.sanity_patterns is not None:
+                sanity_patterns.append(self.sanity_patterns)
+
+            self.sanity_patterns = sn.all(sanity_patterns)
+        elif self.sanity_patterns is None:
             raise SanityError('sanity_patterns not set')
 
         with os_ext.change_dir(self._stagedir):
@@ -1593,7 +1605,7 @@ def depends_on(self, target, how=DEPEND_BY_ENV, subdeps=None):
             raise TypeError("how argument must be of type: `int'")
 
         if (subdeps is not None and
-            not isinstance(subdeps, typ.Dict[str, typ.List[str]])):
+                not isinstance(subdeps, typ.Dict[str, typ.List[str]])):
             raise TypeError("subdeps argument must be of type "
                             "`Dict[str, List[str]]' or `None'")
 

reframe/schemas/config.json

@@ -356,6 +356,7 @@
                     "clean_stagedir": {"type": "boolean"},
                     "colorize": {"type": "boolean"},
                     "ignore_check_conflicts": {"type": "boolean"},
+                    "trap_job_errors": {"type": "boolean"},
                     "keep_stage_files": {"type": "boolean"},
                     "module_map_file": {"type": "string"},
                     "module_mappings": {
@@ -402,6 +403,7 @@
         "general/clean_stagedir": true,
         "general/colorize": true,
         "general/ignore_check_conflicts": false,
+        "general/trap_job_errors": false,
         "general/keep_stage_files": false,
         "general/module_map_file": "",
         "general/module_mappings": [],

unittests/test_pipeline.py

@@ -107,21 +107,6 @@ def remote_exec_ctx(user_system):
     yield partition, environ
 
 
-@pytest.fixture
-def remote_exec_ctx(user_system):
-    partition = fixtures.partition_by_scheduler()
-    if partition is None:
-        pytest.skip('job submission not supported')
-
-    try:
-        environ = partition.environs[0]
-    except IndexError:
-        pytest.skip('no environments configured for partition: %s' %
-                    partition.fullname)
-
-    yield partition, environ
-
-
 @pytest.fixture
 def container_remote_exec_ctx(remote_exec_ctx):
     def _container_exec_ctx(platform):
@@ -789,6 +774,36 @@ def test_registration_of_tests():
             mod.MyBaseTest(10, 20)] == checks
 
 
+def test_trap_job_errors_without_sanity_patterns(local_exec_ctx):
+    rt.runtime().site_config.add_sticky_option('general/trap_job_errors', True)
+
+    @fixtures.custom_prefix('unittests/resources/checks')
+    class MyTest(rfm.RunOnlyRegressionTest):
+        def __init__(self):
+            self.valid_prog_environs = ['*']
+            self.valid_systems = ['*']
+            self.executable = 'exit 10'
+
+    with pytest.raises(SanityError, match='job exited with exit code 10'):
+        _run(MyTest(), *local_exec_ctx)
+
+
+def test_trap_job_errors_with_sanity_patterns(local_exec_ctx):
+    rt.runtime().site_config.add_sticky_option('general/trap_job_errors', True)
+
+    @fixtures.custom_prefix('unittests/resources/checks')
+    class MyTest(rfm.RunOnlyRegressionTest):
+        def __init__(self):
+            self.valid_prog_environs = ['*']
+            self.valid_systems = ['*']
+            self.prerun_cmds = ['echo hello']
+            self.executable = 'true'
+            self.sanity_patterns = sn.assert_not_found(r'hello', self.stdout)
+
+    with pytest.raises(SanityError):
+        _run(MyTest(), *local_exec_ctx)
+
+
 def _run_sanity(test, *exec_ctx, skip_perf=False):
     test.setup(*exec_ctx)
     test.check_sanity()