Reframe 2.8 public release

.gitignore

@@ -24,6 +24,9 @@ var/
 .installed.cfg
 *.egg
 
+# Ignore Mac DS_Store files
+.DS_Store
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

docs/Makefile

@@ -2,22 +2,21 @@
 #
 
 # You can set these variables from the command line.
+BUILD_CMD   = build
+DOCS        = docs
+DOCS_ALL    = _build/_allversions
+MASTER      = master
 PYTHON      = python
 SPHINXOPTS  =
 SPHINXBUILD =  -msphinx
 SPHINXPROJ  = ReFrame
+SPHINX_VERS = sphinx-versioning
 SOURCEDIR   = .
 BUILDDIR    = $(VERSION)
 PANDOC      = pandoc
 PANDOCOPTS  = --columns 1000
 RM          = /bin/rm -rf
-
-ifeq ($(finstring "darwin", $(OSTYPE)), "darwin")
-SYMLINK_DIR=ln -sfh
-else
-# Assume a GNU/Linux system here
-SYMLINK_DIR=ln -sfn
-endif
+TAG_VERS    = '^v\d+(\.\d+)*[a-z]*'
 
 MDS = about.md \
 	advanced.md \
@@ -58,24 +57,29 @@ TARGET_DOCS := \
 	coverage
 
 all: $(RSTS)
+	@$(SPHINX_VERS) -l conf.py build docs/ html/
+	@rsync -az old/ html/_old/
+	@./link_old_docs.sh
+
+latest: $(RSTS)
 	@make html
 	@touch html/.nojekyll
-	@echo 'Linking to old documentation pages'
-	@cd html && $(SYMLINK_DIR) ../old _old && cd - > /dev/null
+	@rsync -az old/ html/_old/
 
 %.rst: %.md
 	$(PANDOC) $(PANDOCOPTS) --from=markdown --to=rst --output=$(@) $(@:.rst=.md)
 
 clean:
 	@echo 'Removing md files'
-	-$(RM) $(RSTS) doctrees
+	-$(RM) $(RSTS)
 
 distclean: clean
 	@echo 'Removing directories'
-	-$(RM) $(TARGET_DOCS)
+	-$(RM) $(TARGET_DOCS) doctrees
 
 $(TARGET_DOCS): Makefile
 	@$(PYTHON) $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 
-.PHONY: clean Makefile
+
+.PHONY: all distclean clean latest Makefile

docs/_templates/footer.html

@@ -62,7 +62,7 @@
         </div>
         <div  class="cscs_right" style="margin-left: auto; margin-right: auto;">
           <a href="http://www.cscs.ch" target="_blank">
-            <img id="cscs-cscs-logo" src="{{ pathto('../_static/img/logo_cscs.png', 1) }}" style="background-color: transparent; margin-top: 5px;">
+            <img id="cscs-cscs-logo" src="{{ pathto('_static/img/logo_cscs.png', 1) }}" style="background-color: transparent; margin-top: 5px;">
           </a>
         </div>
       </div>

docs/_templates/layout.html

@@ -12,9 +12,9 @@
       <span class="caption-text">Useful Links</span>
     </p>
     <ul>
+      <li class="toctree-l1"><a class="reference internal" href="https://github.com/eth-cscs/reframe" target="_blank">Get ReFrame</a></li>
+      <li class="toctree-l1"><a class="reference internal" href="https://github.com/eth-cscs/production" target="_blank">CSCS Easybuild recipes</a></li>
       <li class="toctree-l1"><a class="reference internal" href="http://www.cscs.ch" target="_blank">CSCS</a></li>
       <li class="toctree-l1"><a class="reference internal" href="http://www.ethz.ch" target="_blank">ETH Zurich</a></li>
     </ul>
 {% endblock %}
-
-

docs/conf.py

@@ -18,6 +18,7 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import re
 import sys
 sys.path.insert(0, os.path.abspath('..'))
 from recommonmark.parser import CommonMarkParser
@@ -74,9 +75,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.7'
+version = '2.8'
 # The full version, including alpha/beta/rc tags.
-release = '2.7'
+release = '2.8'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -88,7 +89,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'old', 'html/_old']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'old', 'html/*']
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
@@ -236,3 +237,8 @@
      author, 'ReFrame', 'ReFrame is a new framework for writing regression tests for HPC systems.',
      'Miscellaneous'),
 ]
+
+
+# Versioning
+scv_whitelist_branches = ('master',)
+scv_whitelist_tags = (re.compile(r'^v\d+\.\d+(\.d+)?[a-z]?'),)

docs/configure.md

@@ -21,6 +21,7 @@ The following example shows a minimal configuration for the [Piz Daint](http://w
             'daint': {
                 'descr': 'Piz Daint',
                 'hostnames': ['daint'],
+                'modules_system': 'tmod',
                 'partitions': {
                     'login': {
                         'scheduler': 'local',
@@ -88,6 +89,8 @@ The valid attributes of a system are the following:
 
 * `descr`: A detailed description of the system (default is the system name).
 * `hostnames`: This is a list of hostname patterns that will be used by ReFrame when it tries to [auto-detect](#system-auto-detection) the current system (default `[]`).
+* `modules_system`: _(New in version 2.8)_ The modules system that should be used for loading environment modules on this system.
+   The only available modules system backend is currently `tmod`, which corresponds to the [TCL implementation](http://modules.sourceforge.net/) of the environment modules (default `None`).
 * `prefix`: Default regression prefix for this system (default `.`).
 * `stagedir`: Default stage directory for this system (default `None`).
 * `outputdir`: Default output directory for this system (default `None`).
@@ -109,16 +112,33 @@ The partitions of a system are defined similarly to systems as a set of key/valu
 The available partition attributes are the following:
 
 * `descr`: A detailed description of the partition (default is the partition name).
-* `scheduler`: The job scheduler to use for launching jobs on this partition.
-   Available values are the following:
-     * `local` (**default**): Jobs on this partition will be launched locally as OS processes.
-     When a job is launched with this scheduler, ReFrame will create a wrapper shell script for running the check on the local machine.
-     * `nativeslurm`: Jobs on this partition will be launched using Slurm and the `srun` command for creating MPI processes.
-     * `slurm+alps`: Jobs on this partition will be launched using Slurm and the `aprun` command for creating MPI processes.
+
+* `scheduler`: _(Changed in version 2.8)_ The job scheduler and parallel program launcher combination that is used on this partition to launch jobs.
+   The syntax of this attribute is `<scheduler>+<launcher>`.
+   The available values for the job scheduler are the following:
+       - `slurm`: Jobs on this partition will be launched using [Slurm](https://www.schedmd.com/).
+       - `local`: Jobs on this partition will be launched locally as OS processes.
+
+     The available values for the parallel program launchers are the following:
+     - `srun`: Programs on this partition will be launched using a bare `srun` command *without* any job allocation options passed to it.
+        This launcher may only be used with the `slurm` scheduler.
+     - `srunalloc`: Programs on this partition will be launched using the `srun` command *with* job allocation options passed automatically to it.
+        This launcher may also be used with the `local` scheduler.
+     - `alps`: Programs on this partition will be launched using the `aprun` command.
+     - `mpirun`: Programs on this partition will be launched using the `mpirun` command.
+     - `mpiexec`: Programs on this partition will be launched using the `mpiexec` command.
+     - `local`: Programs on this partition will be launched as-is without using any parallel program launcher.
+
+     There exist also the following aliases for specific combinations of job schedulers and parallel program launchers:
+     - `nativeslurm`: This is equivalent to `slurm+srun`.
+     - `local`: This is equivalent to `local+local`.
+
 * `access`: A list of scheduler options that will be passed to the generated job script for gaining access to that logical partition (default `[]`).
+
 * `environs`: A list of environments, with which ReFrame will try to run any regression tests written for this partition (default `[]`).
   The environment names must be resolved inside the `environments` section of the `_site_configuration` dictionary (see [Environments Configuration](#environments-configuration) for more information).
 * `modules`: A list of modules to be loaded before running a regression test on that partition (default `[]`).
+
 * `variables`: A set of environment variables to be set before running a regression test on that partition (default `{}`).
   Environment variables can be set as follows (notice that both the variable name and its value are strings):
 
@@ -128,9 +148,11 @@ The available partition attributes are the following:
     'OTHER': 'foo'
 }
 ```
+
 * `max_jobs`: The maximum number of concurrent regression tests that may be active (not completed) on this partition.
    This option is relevant only when Reframe executes with the [asynchronous execution policy](running.html#asynchronous-execution-of-regression-checks).
-* `resources`: A set of custom resource specifications and how these can be requested from the partition's scheduler (default `{}`).
+
+* `resources`: _(Changed in version 2.8)_ A set of custom resource specifications and how these can be requested from the partition's scheduler (default `{}`).
   This variable is a set of key/value pairs with the key being the resource name and the value being a list of options to be passed to the partition's job scheduler.
   The option strings can contain "references" to the resource being required using the syntax `{resource_name}`.
   In such cases, the `{resource_name}` will be replaced by the value of that resource defined in the regression test that is being run.
@@ -144,11 +166,19 @@ The available partition attributes are the following:
 }
 ```
 
-When ReFrame will run a test that defines `self.num_gpus_per_node = 8`, the generated job script will have the following line in its preamble:
+A regression test then may request this resource as follows:
+
+```python
+self.extra_resources = {'num_gpus_per_node': '8'}
+```
+
+and the generated job script will have the following line in its preamble:
 ```bash
 #SBATCH --gres=gpu:8
 ```
 
+Refer to the [reference guide](reference.html#reframe.core.pipeline.RegressionTest.extra_resources) for more information on the use of the `extra_resources` regression test attribute.
+
 ## Environments Configuration
 
 The environments available for testing in different systems are defined under the `environments` key of the top-level `_site_configuration` dictionary.

docs/index.rst

@@ -14,7 +14,7 @@ Reframe also offers a high-level and flexible abstraction for writing sanity and
 Writing system regression tests in a high-level modern programming language, like Python, poses a great advantage in organizing and maintaining the tests.
 Users can create their own test hierarchies or test factories for generating multiple tests at the same time and they can also customize them in a simple and expressive way.
 
-For versions 2.6.1 and older, please refer to `this documentation <_old/index.html>`__.
+For versions 2.6.1 and older, please refer to `this documentation </_old/index.html>`__.
 
 Use Cases
 =========

docs/link_old_docs.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+prefix=${1:-html}
+
+if [[ x$OSTYPE =~ x"darwin" ]]; then
+    symlink="ln -sfh"
+else
+    symlink="ln -sfn"
+fi
+
+link_old_docs()
+{
+    echo "linking old docs for $1 ..."
+    cd $1 && $symlink ../_old _old && cd - > /dev/null
+}
+
+
+link_old_docs $prefix/master
+
+for d in $prefix/v*; do
+    # Verify that $d is actually a version
+    if ! echo $(basename $d) | grep -e 'v[0-9]\+\.[0-9]\+' > /dev/null; then
+        echo Skipping non version directory $d ...
+        continue
+    fi
+
+    link_old_docs $d
+done

docs/pipeline.md

@@ -66,11 +66,9 @@ At this phase, all these directories are created.
 ### Prepare a job for the test
 At this point a _job descriptor_ will be created for the test. A job descriptor in ReFrame is an abstraction of the job scheduler's functionality relevant to the regression framework.
 It is responsible for submitting a job in a job queue and waiting for its completion.
-Currently, the ReFrame framework supports three job scheduler backends:
+ReFrame supports two job scheduler backends that can be combined with several different parallel program launchers.
+For a complete list of the job scheduler/parallel launchers combinations, please refer to ["Partition Configuration"](configure.html#partition-configuration).
 
-* __local__, which is basically a *pseudo-scheduler* that just spawns local OS processes,
-* __nativeslurm__, which is the native [Slurm](https://slurm.schedmd.com) job scheduler and
-* __slurm+alps__, which uses [Slurm](https://slurm.schedmd.com) for job submission, but uses [Cray's ALPS](http://docs.cray.com/books/S-2529-116//S-2529-116.pdf) for launching MPI processes on the compute nodes.
 
 ## 2. The Compilation Phase
 

docs/reference.rst

@@ -20,3 +20,7 @@ Reference Guide
 .. automodule:: reframe.core.launchers
    :members:
    :show-inheritance:
+
+.. automodule:: reframe.core.launchers.registry
+   :members:
+   :show-inheritance:

docs/requirements.txt

@@ -25,6 +25,7 @@ sphinx-bootstrap-theme>=0.5.3
 sphinx-fakeinv>=1.0.0
 sphinx-rtd-theme>=0.2.4
 sphinxcontrib-websupport>=1.0.1
+sphinxcontrib-versioning>=2.2.1
 tornado>=4.5.2
 urllib3>=1.22
 watchdog>=0.8.3

docs/running.md

@@ -34,7 +34,7 @@ An example listing of checks is the following that lists all the tests found und
 The ouput looks like:
 ```
 Command line: ./bin/reframe -c tutorial/ -l
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint103
 Reframe paths
@@ -90,7 +90,7 @@ To run the regression tests you should specify the _run_ action though the `-r`
 The output of the regression run looks like the following:
 ```
 Command line: ./bin/reframe -c tutorial/example1.py -r
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint103
 Reframe paths
@@ -218,7 +218,7 @@ For example, you can select only the `example7_check` from the tutorial as follo
 
 ```
 Command line: ./bin/reframe -c tutorial/ -n example7_check -l
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint103
 Reframe paths
@@ -239,7 +239,7 @@ Similarly, you can exclude this test by passing the `-x example7_check` option:
 
 ```
 Command line: ./bin/reframe -c tutorial/ -x example7_check -l
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint103
 Reframe paths
@@ -357,7 +357,7 @@ This is useful if you want to check the directories that ReFrame will create.
 ```
 ```
 Command line: ./bin/reframe --prefix /foo -t foo -l
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint103
 Reframe paths
@@ -429,7 +429,7 @@ _logging_config = {
         'reframe.log' : {
             'level'     : 'DEBUG',
             'format'    : '[%(asctime)s] %(levelname)s: '
-                          '%(check_name)s: %(message)s',
+                          '%(testcase_name)s: %(message)s',
             'append'    : False,
         },
 
@@ -465,11 +465,15 @@ The configurable properties of a log record handler are the following:
 
 * `level` (default: `'debug'`): The lowest level of log records that this handler can process.
 * `format` (default: `'%(message)s'`): Format string for the printout of the log record.
-   ReFrame supports all the [format strings](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) from Python's logging library and provides two additional ones:
+   ReFrame supports all the [format strings](https://docs.python.org/3.6/library/logging.html#logrecord-attributes) from Python's logging library and provides the following additional ones:
      - `check_name`: Prints the name of the regression test on behalf of which ReFrame is currently executing.
        If ReFrame is not in the context of regression test, `reframe` will be printed.
      - `check_jobid`: Prints the job or process id of the job or process associated with currently executing regression test.
        If a job or process is not yet created, `-1` will be printed.
+     - `testcase_name`: Print the name of the test case that is currently executing.
+        Test case is essentially a tuple consisting of the test name, the current system and partition and the current programming envrinoment.
+        This format string prints out like `<test-name>@<partition> using <environ>`.
+
 * `datefmt` (default: `'%FT%T'`) The format that will be used for outputting timestamps (i.e., the `%(asctime)s` field).
   Acceptable formats must conform to standard library's [time.strftime()](https://docs.python.org/3.6/library/time.html#time.strftime) function.
 * `append` (default: `False`) Controls whether ReFrame should append to this file or not.
@@ -483,15 +487,14 @@ The configurable properties of a log record handler are the following:
 
 ReFrame supports additional logging for performance tests specifically, in order to record historical performance data.
 For each performance test, a log file of the form `<test-name>.log` is created under the ReFrame's [log directory](#configuring-reframe-directories) where the test's performance is recorded.
-The default format used for this file is `'[%(asctime)s] %(check_name)s (jobid=%(check_jobid)s): %(message)s'` and ReFrame always appends to this file.
+The default format used for this file is `'[%(asctime)s] %(testcase_name)s (jobid=%(check_jobid)s): %(message)s'` and ReFrame always appends to this file.
 Currently, it is not possible for users to configure performance logging.
 
 The resulting log file looks like the following:
 ```
-[2017-10-21T00:48:42] example7_check (jobid=4073910): value: 49.253851, reference: (50.0, -0.1, 0.1)
-[2017-10-24T21:19:21] example7_check (jobid=4163846): value: 49.690761, reference: (50.0, -0.1, 0.1)
-[2017-10-24T21:19:33] example7_check (jobid=4163852): value: 50.037254, reference: (50.0, -0.1, 0.1)
-[2017-10-24T21:20:00] example7_check (jobid=4163856): value: 49.622199, reference: (50.0, -0.1, 0.1)
+[2017-12-01T15:31:20] example7_check@daint:gpu using PrgEnv-cray (jobid=649790): value: 47.797996, reference: (50.0, -0.1, 0.1)
+[2017-12-01T15:31:24] example7_check@daint:gpu using PrgEnv-gnu (jobid=649791): value: 49.048228, reference: (50.0, -0.1, 0.1)
+[2017-12-01T15:31:24] example7_check@daint:gpu using PrgEnv-pgi (jobid=649792): value: 48.575334, reference: (50.0, -0.1, 0.1)
 ```
 
 The interpretation of the performance values depends on the individual tests.
@@ -516,7 +519,7 @@ Here is an example output of ReFrame using asynchronous execution policy:
 
 ```
 ommand line: ./reframe.py -c tutorial/ --exec-policy=async -r
-Reframe version: 2.6.1
+Reframe version: 2.7
 Launched by user: karakasv
 Launched on host: daint104
 Reframe paths