Merge branch 'master' into develop

README_minimal.md

@@ -9,7 +9,7 @@
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/reframe-hpc)
 [![Downloads](https://pepy.tech/badge/reframe-hpc)](https://pepy.tech/project/reframe-hpc)
 [![Downloads](https://pepy.tech/badge/reframe-hpc/month)](https://pepy.tech/project/reframe-hpc)<br/>
-[![Slack](https://reframe-slack.herokuapp.com/badge.svg)](https://reframe-slack.herokuapp.com/)<br/>
+[![Slack](https://badgen.net/badge/icon/slack?icon=slack&label)](https://join.slack.com/t/reframetalk/shared_invite/zt-1tar8s71w-At0tolJ~~zxT2oG_2Ly9sw)
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![DOI](https://zenodo.org/badge/89384186.svg)](https://zenodo.org/badge/latestdoi/89384186)<br/>
 [![Twitter Follow](https://img.shields.io/twitter/follow/ReFrameHPC?style=social)](https://twitter.com/ReFrameHPC)

bootstrap.sh

@@ -33,6 +33,7 @@ usage()
     echo "Bootstrap ReFrame by pulling all its dependencies"
     echo "  -P EXEC     Use EXEC as Python interpreter"
     echo "  -h          Print this help message and exit"
+    echo "  --ignore-errors Ignore installation errors"
     echo "  --pip-opts  Pass additional options to pip."
     echo "  +docs       Build also the documentation"
     echo "  +pygelf     Install also the pygelf Python package"
@@ -43,17 +44,22 @@ while getopts "hP:-:"  opt; do
     case $opt in
         "P") python=$OPTARG ;;
         "h") usage && exit 0 ;;
-	"-")
-	    case "${OPTARG}" in
+	    "-")
+	        case "${OPTARG}" in
+                "ignore-errors") ignore_errors=1 ;;
                 pip-opts)
-	            PIPOPTS="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) ;;
-		pip-opts=*)
+	                PIPOPTS="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) ;;
+		        pip-opts=*)
                     PIPOPTS=${OPTARG#*=} ;;
             esac;;
         "?") usage && exit 0 ;;
     esac
 done
 
+if [ -z $ignore_errors ]; then
+    set -e
+fi
+
 shift $((OPTIND - 1))
 if [ -z $python ]; then
     python=python3
@@ -82,16 +88,13 @@ fi
 export PIP_USER=0
 
 # Check if ensurepip is installed
-$python -m ensurepip --version &> /dev/null
-epip=$?
-
-export PATH=$(pwd)/external/usr/bin:$PATH
-
-# Install pip for Python 3
-if [ $epip -eq 0 ]; then
+if $python -m ensurepip --version &> /dev/null; then
+    # Install pip for Python 3
     CMD $python -m ensurepip --root $(pwd)/external/ --default-pip
 fi
 
+export PATH=$(pwd)/external/usr/bin:$PATH
+
 # ensurepip installs pip in `external/usr/` whereas the `--root` option installs
 # everything under `external/`. That's why we include both in the PYTHONPATH
 

docs/manpage.rst

@@ -219,6 +219,12 @@ An action must always be specified.
    You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies.
    For more information, have a look in :ref:`generate-ci-pipeline`.
 
+   .. note::
+      The :option:`--ci-generate` option will not work with the test generation options, such as the :option:`--repeat` and :option:`--distribute` options, as the generated child pipeline will use the :option:`-n` option to select the test of the CI job and test filtering happens before any test generation.
+
+      This limitation will be removed in future versions.
+
+
    .. versionadded:: 3.4.1
 
 .. option:: --describe

reframe/__init__.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.3.0-dev.1'
+VERSION = '4.3.0-dev.2'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

reframe/core/config.py

@@ -17,6 +17,7 @@
 import reframe
 import reframe.core.settings as settings
 import reframe.utility as util
+import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.environments import normalize_module_list
 from reframe.core.exceptions import (ConfigError, ReframeFatalError)
@@ -202,7 +203,7 @@ def __repr__(self):
                 f'sources={self._sources!r})')
 
     def __str__(self):
-        return json.dumps(self._pick_config(), indent=2)
+        return jsonext.dumps(self._pick_config(), indent=2)
 
     # Delegate everything to either the original config or to the reduced one
     # if a system is selected

reframe/core/pipeline.py

@@ -198,9 +198,9 @@ def pipeline_hooks(cls):
     #:
     #: .. seealso::
     #:    - `Environment features
-    #:      <config_reference.html#environments-.features>`__
+    #:      <config_reference.html#config.environments.features>`__
     #:    - `Environment extras
-    #:      <config_reference.html#environments-.extras>`__
+    #:      <config_reference.html#config.environments.extras>`__
     #:
     #: .. versionchanged:: 2.12
     #:    Programming environments can now be specified using wildcards.
@@ -268,9 +268,9 @@ def pipeline_hooks(cls):
     #:
     #: .. seealso::
     #:    - `System partition features
-    #:      <config_reference.html#systems-.partitions-.features>`__
+    #:      <config_reference.html#config.systems.partitions.features>`__
     #:    - `System partition extras
-    #:      <config_reference.html#systems-.partitions-.extras>`__
+    #:      <config_reference.html#config.systems.partitions.extras>`__
     #:
     #: .. versionchanged:: 3.3
     #:    Default value changed from ``[]`` to ``None``.
@@ -1645,7 +1645,7 @@ def _setup_paths(self):
         except OSError as e:
             raise PipelineError('failed to set up paths') from e
 
-    def _create_job(self, name, force_local=False, **job_opts):
+    def _create_job(self, job_type, force_local=False, **job_opts):
         '''Setup the job related to this check.'''
 
         if force_local:
@@ -1656,24 +1656,31 @@ def _create_job(self, name, force_local=False, **job_opts):
             launcher = self._current_partition.launcher_type()
 
         self.logger.debug(
-            f'Setting up job {name!r} '
+            f'Setting up {type} job for test {self.name!r} '
             f'(scheduler: {scheduler.registered_name!r}, '
             f'launcher: {launcher.registered_name!r})'
         )
+
+        if job_type == 'build':
+            script_name = 'rfm_build.sh'
+        elif job_type == 'run':
+            script_name = 'rfm_job.sh'
+
         return Job.create(scheduler,
                           launcher,
-                          name=name,
+                          name=f'rfm_{self.short_name}',
+                          script_filename=script_name,
                           workdir=self._stagedir,
                           sched_access=self._current_partition.access,
                           **job_opts)
 
     def _setup_build_job(self, **job_opts):
-        self._build_job = self._create_job(f'rfm_build',
-                                           self.local or self.build_locally,
-                                           **job_opts)
+        self._build_job = self._create_job(
+            'build', self.local or self.build_locally, **job_opts
+        )
 
     def _setup_run_job(self, **job_opts):
-        self._job = self._create_job(f'rfm_job', self.local, **job_opts)
+        self._job = self._create_job(f'run', self.local, **job_opts)
 
     def _setup_container_platform(self):
         try:

reframe/core/schedulers/__init__.py

@@ -8,6 +8,7 @@
 #
 
 import abc
+import os
 import time
 
 import reframe.core.runtime as runtime
@@ -330,9 +331,11 @@ def __init__(self,
         self._cli_options = list(sched_options) if sched_options else []
         self._name = name
         self._workdir = workdir
-        self._script_filename = script_filename or '%s.sh' % name
-        self._stdout = stdout or '%s.out' % name
-        self._stderr = stderr or '%s.err' % name
+        self._script_filename = script_filename or f'{name}.sh'
+
+        basename, _ = os.path.splitext(self._script_filename)
+        self._stdout = stdout or f'{basename}.out'
+        self._stderr = stderr or f'{basename}.err'
 
         # Backend scheduler related information
         self._sched_flex_alloc_nodes = sched_flex_alloc_nodes

reframe/frontend/cli.py

@@ -855,7 +855,7 @@ def restrict_logging():
                     f'no such configuration parameter found: {config_param}'
                 )
             else:
-                printer.info(json.dumps(value, indent=2))
+                printer.info(jsonext.dumps(value, indent=2))
 
         sys.exit(0)
 

reframe/frontend/dependencies.py

@@ -205,6 +205,14 @@ def prune_deps(graph, testcases, max_depth=None):
                 if adj not in pruned_graph:
                     unvisited.append(adj)
 
+    # Re-calculate the in-degree of the pruned graph nodes
+    for u in pruned_graph:
+        u.in_degree = 0
+
+    for u, adjacent in pruned_graph.items():
+        for v in adjacent:
+            v.in_degree += 1
+
     return pruned_graph
 
 

reframe/utility/jsonext.py

@@ -66,6 +66,9 @@ def encode(obj, **kwargs):
     if inspect.istraceback(obj):
         return traceback.format_tb(obj)
 
+    if inspect.isfunction(obj):
+        return f'py::{obj.__qualname__}'
+
     newobj = encode_dict(obj)
     if newobj is not None:
         return newobj

reframe/utility/sanity.py

@@ -790,7 +790,7 @@ def extractall_s(patt, string, tag=0, conv=None):
     ``patt`` in ``string``.
 
     :arg patt: as in :func:`extractall`.
-'   :arg string: The string to examine.
+    :arg string: The string to examine.
     :arg tag: as in :func:`extractall`.
     :arg conv: as in :func:`extractall`.
     :returns: same as :func:`extractall`.

unittests/test_dependencies.py

@@ -697,6 +697,14 @@ def test_prune_deps(default_exec_ctx):
             assert len(pruned_deps[node('t5')]) == 0
             assert len(pruned_deps[node('t0')]) == 0
 
+            # Check the degree of the nodes of the pruned graph
+            assert in_degree(pruned_deps, node('t0')) == 1
+            assert in_degree(pruned_deps, node('t1')) == 2
+            assert in_degree(pruned_deps, node('t2')) == 1
+            assert in_degree(pruned_deps, node('t3')) == 0
+            assert in_degree(pruned_deps, node('t5')) == 1
+            assert in_degree(pruned_deps, node('t7')) == 0
+
 
 def test_toposort(default_exec_ctx):
     #

unittests/test_utility.py

@@ -1585,6 +1585,12 @@ def test_jsonext_dumps():
     )
     assert '{"(1, 2, 3)": 1}' == jsonext.dumps({(1, 2, 3): 1})
 
+    def foo():
+        pass
+
+    assert ('{"fn": "py::test_jsonext_dumps.<locals>.foo"}' ==
+            jsonext.dumps({'fn': foo}))
+
 
 # Classes to test JSON deserialization