[feat] Introduce new naming scheme for tests

cscs-checks/system/io/ior_check.py

@@ -5,7 +5,6 @@
 
 import getpass
 import os
-import re
 
 import reframe as rfm
 import reframe.utility.sanity as sn
@@ -173,5 +172,6 @@ def set_perf_patterns(self):
         }
 
     @run_after('init')
-    def set_dependency(self):
-        self.depends_on(re.sub(r'IorReadCheck', 'IorWriteCheck', self.name))
+    def set_deps(self):
+        variant = IorWriteCheck.get_variant_nums(base_dir=self.base_dir)[0]
+        self.depends_on(IorWriteCheck.variant_name(variant))

cscs-checks/tools/profiling_and_debugging/notool.py

@@ -32,7 +32,6 @@ class JacobiNoToolHybrid(rfm.RegressionTest):
     @run_after('init')
     def set_descr_name(self):
         self.descr = f'Jacobi (without tool) {self.lang} check'
-        self.name = f'{type(self).__name__}_{self.lang.replace("+", "p")}'
 
     @run_after('init')
     def remove_buggy_prgenv(self):

docs/listings/alltests_daint.txt

@@ -0,0 +1,130 @@
+[ReFrame Setup]
+  version:           3.10.0-dev.3+605af31a
+  command:           './bin/reframe -c tutorials/basics/ -R -n HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest --performance-report -r'
+  launched by:       user@host
+  working directory: '/home/user/Devel/reframe'
+  settings file:     '/home/user/Devel/reframe/tutorials/config/settings.py'
+  check search path: (R) '/home/user/Devel/reframe/tutorials/basics'
+  stage directory:   '/home/user/Devel/reframe/stage'
+  output directory:  '/home/user/Devel/reframe/output'
+
+[==========] Running 4 check(s)
+[==========] Started on Sat Jan 22 22:43:38 2022 
+
+[----------] start processing checks
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:login+builtin
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:login+gnu
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:login+intel
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:login+pgi
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:login+cray
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:gpu+gnu
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:gpu+intel
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:gpu+pgi
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:gpu+cray
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:mc+gnu
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:mc+intel
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:mc+pgi
+[ RUN      ] HelloMultiLangTest %lang=cpp @daint:mc+cray
+[ RUN      ] HelloMultiLangTest %lang=c @daint:login+builtin
+[ RUN      ] HelloMultiLangTest %lang=c @daint:login+gnu
+[ RUN      ] HelloMultiLangTest %lang=c @daint:login+intel
+[ RUN      ] HelloMultiLangTest %lang=c @daint:login+pgi
+[ RUN      ] HelloMultiLangTest %lang=c @daint:login+cray
+[ RUN      ] HelloMultiLangTest %lang=c @daint:gpu+gnu
+[ RUN      ] HelloMultiLangTest %lang=c @daint:gpu+intel
+[ RUN      ] HelloMultiLangTest %lang=c @daint:gpu+pgi
+[ RUN      ] HelloMultiLangTest %lang=c @daint:gpu+cray
+[ RUN      ] HelloMultiLangTest %lang=c @daint:mc+gnu
+[ RUN      ] HelloMultiLangTest %lang=c @daint:mc+intel
+[ RUN      ] HelloMultiLangTest %lang=c @daint:mc+pgi
+[ RUN      ] HelloMultiLangTest %lang=c @daint:mc+cray
+[ RUN      ] HelloThreadedExtended2Test @daint:login+builtin
+[ RUN      ] HelloThreadedExtended2Test @daint:login+gnu
+[ RUN      ] HelloThreadedExtended2Test @daint:login+intel
+[ RUN      ] HelloThreadedExtended2Test @daint:login+pgi
+[ RUN      ] HelloThreadedExtended2Test @daint:login+cray
+[ RUN      ] HelloThreadedExtended2Test @daint:gpu+gnu
+[ RUN      ] HelloThreadedExtended2Test @daint:gpu+intel
+[ RUN      ] HelloThreadedExtended2Test @daint:gpu+pgi
+[ RUN      ] HelloThreadedExtended2Test @daint:gpu+cray
+[ RUN      ] HelloThreadedExtended2Test @daint:mc+gnu
+[ RUN      ] HelloThreadedExtended2Test @daint:mc+intel
+[ RUN      ] HelloThreadedExtended2Test @daint:mc+pgi
+[ RUN      ] HelloThreadedExtended2Test @daint:mc+cray
+[ RUN      ] StreamWithRefTest @daint:login+gnu
+[ RUN      ] StreamWithRefTest @daint:gpu+gnu
+[ RUN      ] StreamWithRefTest @daint:mc+gnu
+[       OK ] ( 1/42) HelloMultiLangTest %lang=cpp @daint:login+builtin [compile: 4.053s run: 36.016s total: 43.208s]
+[       OK ] ( 2/42) HelloMultiLangTest %lang=cpp @daint:login+gnu [compile: 4.047s run: 36.009s total: 43.203s]
+[       OK ] ( 3/42) HelloMultiLangTest %lang=cpp @daint:login+intel [compile: 3.431s run: 35.376s total: 43.206s]
+[       OK ] ( 4/42) HelloMultiLangTest %lang=cpp @daint:login+pgi [compile: 2.758s run: 34.675s total: 43.208s]
+[       OK ] ( 5/42) HelloMultiLangTest %lang=cpp @daint:login+cray [compile: 2.149s run: 34.052s total: 43.211s]
+[       OK ] ( 6/42) HelloMultiLangTest %lang=cpp @daint:gpu+gnu [compile: 2.139s run: 60.830s total: 69.995s]
+[       OK ] ( 7/42) HelloMultiLangTest %lang=cpp @daint:gpu+intel [compile: 8.863s run: 55.184s total: 70.004s]
+[       OK ] ( 8/42) HelloMultiLangTest %lang=c @daint:login+builtin [compile: 32.460s run: 18.053s total: 69.949s]
+[       OK ] ( 9/42) HelloMultiLangTest %lang=c @daint:login+gnu [compile: 27.081s run: 18.051s total: 69.954s]
+[       OK ] (10/42) HelloMultiLangTest %lang=c @daint:login+intel [compile: 39.615s run: 32.065s total: 87.922s]
+[       OK ] (11/42) HelloMultiLangTest %lang=c @daint:login+pgi [compile: 38.873s run: 31.356s total: 87.926s]
+[       OK ] (12/42) HelloMultiLangTest %lang=c @daint:login+cray [compile: 38.265s run: 30.731s total: 87.931s]
+[       OK ] (13/42) HelloThreadedExtended2Test @daint:login+builtin [compile: 12.837s run: 7.254s total: 92.404s]
+[       OK ] (14/42) HelloThreadedExtended2Test @daint:login+gnu [compile: 31.377s run: 31.894s total: 119.747s]
+[       OK ] (15/42) HelloThreadedExtended2Test @daint:login+intel [compile: 30.708s run: 31.252s total: 119.749s]
+[       OK ] (16/42) HelloThreadedExtended2Test @daint:login+pgi [compile: 18.581s run: 30.571s total: 119.753s]
+[       OK ] (17/42) HelloThreadedExtended2Test @daint:login+cray [compile: 17.981s run: 29.963s total: 119.756s]
+[       OK ] (18/42) HelloMultiLangTest %lang=cpp @daint:mc+intel [compile: 33.792s run: 87.427s total: 130.572s]
+[       OK ] (19/42) HelloMultiLangTest %lang=cpp @daint:mc+pgi [compile: 33.120s run: 84.192s total: 130.591s]
+[       OK ] (20/42) HelloMultiLangTest %lang=cpp @daint:mc+cray [compile: 32.474s run: 81.119s total: 130.609s]
+[       OK ] (21/42) HelloMultiLangTest %lang=c @daint:mc+pgi [compile: 13.468s run: 51.389s total: 130.540s]
+[       OK ] (22/42) HelloMultiLangTest %lang=c @daint:mc+cray [compile: 12.847s run: 48.146s total: 130.559s]
+[       OK ] (23/42) HelloMultiLangTest %lang=cpp @daint:gpu+pgi [compile: 8.167s run: 120.870s total: 138.874s]
+[       OK ] (24/42) HelloMultiLangTest %lang=cpp @daint:gpu+cray [compile: 7.412s run: 109.470s total: 138.883s]
+[       OK ] (25/42) HelloMultiLangTest %lang=c @daint:gpu+gnu [compile: 13.293s run: 81.519s total: 138.729s]
+[       OK ] (26/42) HelloMultiLangTest %lang=c @daint:gpu+cray [compile: 11.378s run: 74.651s total: 138.736s]
+[       OK ] (27/42) HelloMultiLangTest %lang=c @daint:mc+gnu [compile: 25.399s run: 65.789s total: 138.749s]
+[       OK ] (28/42) HelloMultiLangTest %lang=c @daint:gpu+intel [compile: 12.677s run: 79.097s total: 139.421s]
+[       OK ] (29/42) HelloMultiLangTest %lang=c @daint:gpu+pgi [compile: 23.579s run: 69.505s total: 139.432s]
+[       OK ] (30/42) HelloThreadedExtended2Test @daint:gpu+gnu [compile: 22.616s run: 46.878s total: 139.268s]
+[       OK ] (31/42) HelloThreadedExtended2Test @daint:gpu+pgi [compile: 21.265s run: 40.181s total: 139.267s]
+[       OK ] (32/42) HelloThreadedExtended2Test @daint:gpu+cray [compile: 20.642s run: 37.158s total: 139.275s]
+[       OK ] (33/42) HelloThreadedExtended2Test @daint:mc+gnu [compile: 4.691s run: 30.273s total: 139.280s]
+[       OK ] (34/42) HelloThreadedExtended2Test @daint:mc+intel [compile: 28.304s run: 19.597s total: 139.281s]
+[       OK ] (35/42) StreamWithRefTest @daint:login+gnu [compile: 24.257s run: 10.594s total: 139.286s]
+[       OK ] (36/42) HelloMultiLangTest %lang=c @daint:mc+intel [compile: 14.135s run: 70.976s total: 146.961s]
+[       OK ] (37/42) HelloMultiLangTest %lang=cpp @daint:mc+gnu [compile: 7.397s run: 194.065s total: 229.737s]
+[       OK ] (38/42) HelloThreadedExtended2Test @daint:gpu+intel [compile: 21.956s run: 133.885s total: 229.342s]
+[       OK ] (39/42) HelloThreadedExtended2Test @daint:mc+pgi [compile: 27.596s run: 106.403s total: 229.264s]
+[       OK ] (40/42) HelloThreadedExtended2Test @daint:mc+cray [compile: 26.958s run: 103.318s total: 229.274s]
+[       OK ] (41/42) StreamWithRefTest @daint:gpu+gnu [compile: 38.940s run: 98.873s total: 229.279s]
+[       OK ] (42/42) StreamWithRefTest @daint:mc+gnu [compile: 38.304s run: 94.811s total: 229.299s]
+[----------] all spawned checks have finished
+
+[  PASSED  ] Ran 42/42 test case(s) from 4 check(s) (0 failure(s), 0 skipped)
+[==========] Finished on Sat Jan 22 22:47:28 2022 
+==============================================================================
+PERFORMANCE REPORT
+------------------------------------------------------------------------------
+StreamWithRefTest
+- daint:login
+   - gnu
+      * num_tasks: 1
+      * Copy: 67915.3 MB/s
+      * Scale: 37485.6 MB/s
+      * Add: 39545.5 MB/s
+      * Triad: 39906.2 MB/s
+- daint:gpu
+   - gnu
+      * num_tasks: 1
+      * Copy: 50553.4 MB/s
+      * Scale: 34780.1 MB/s
+      * Add: 38043.6 MB/s
+      * Triad: 38522.2 MB/s
+- daint:mc
+   - gnu
+      * num_tasks: 1
+      * Copy: 48200.9 MB/s
+      * Scale: 31370.4 MB/s
+      * Add: 33000.2 MB/s
+      * Triad: 33205.5 MB/s
+------------------------------------------------------------------------------
+Run report saved in '/home/user/.reframe/reports/run-report.json'
+Log file(s) saved in '/tmp/rfm-n3d18lq9.log'

docs/listings/deps_complex_run.txt

@@ -0,0 +1,117 @@
+[ReFrame Setup]
+  version:           3.10.0-dev.3+149af549
+  command:           './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -r'
+  launched by:       user@host
+  working directory: '/home/user/Repositories/reframe'
+  settings file:     '<builtin>'
+  check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
+  stage directory:   '/home/user/Repositories/reframe/stage'
+  output directory:  '/home/user/Repositories/reframe/output'
+
+[==========] Running 10 check(s)
+[==========] Started on Sat Jan 22 23:44:18 2022 
+
+[----------] start processing checks
+[ RUN      ] T0 @generic:default+builtin
+[       OK ] ( 1/10) T0 @generic:default+builtin [compile: 0.018s run: 0.292s total: 0.336s]
+[ RUN      ] T4 @generic:default+builtin
+[       OK ] ( 2/10) T4 @generic:default+builtin [compile: 0.016s run: 0.336s total: 0.380s]
+[ RUN      ] T5 @generic:default+builtin
+[       OK ] ( 3/10) T5 @generic:default+builtin [compile: 0.016s run: 0.389s total: 0.446s]
+[ RUN      ] T1 @generic:default+builtin
+[       OK ] ( 4/10) T1 @generic:default+builtin [compile: 0.016s run: 0.459s total: 0.501s]
+[ RUN      ] T8 @generic:default+builtin
+[     FAIL ] ( 5/10) T8 @generic:default+builtin [compile: n/a run: n/a total: 0.006s]
+==> test failed during 'setup': test staged in '/home/user/Repositories/reframe/stage/generic/default/builtin/T8'
+[     FAIL ] ( 6/10) T9 @generic:default+builtin [compile: n/a run: n/a total: n/a]
+==> test failed during 'startup': test staged in None
+[ RUN      ] T6 @generic:default+builtin
+[       OK ] ( 7/10) T6 @generic:default+builtin [compile: 0.016s run: 0.530s total: 0.584s]
+[ RUN      ] T2 @generic:default+builtin
+[ RUN      ] T3 @generic:default+builtin
+[     FAIL ] ( 8/10) T2 @generic:default+builtin [compile: 0.019s run: 0.324s total: 0.424s]
+==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/generic/default/builtin/T2'
+[     FAIL ] ( 9/10) T7 @generic:default+builtin [compile: n/a run: n/a total: n/a]
+==> test failed during 'startup': test staged in None
+[       OK ] (10/10) T3 @generic:default+builtin [compile: 0.017s run: 0.328s total: 0.403s]
+[----------] all spawned checks have finished
+
+[  FAILED  ] Ran 10/10 test case(s) from 10 check(s) (4 failure(s), 0 skipped)
+[==========] Finished on Sat Jan 22 23:44:21 2022 
+
+==============================================================================
+SUMMARY OF FAILURES
+------------------------------------------------------------------------------
+FAILURE INFO for T8 
+  * Expanded name: T8
+  * Description: T8
+  * System partition: generic:default
+  * Environment: builtin
+  * Stage directory: /home/user/Repositories/reframe/stage/generic/default/builtin/T8
+  * Node list: 
+  * Job type: local (id=None)
+  * Dependencies (conceptual): ['T1']
+  * Dependencies (actual): [('T1', 'generic:default', 'builtin')]
+  * Maintainers: []
+  * Failing phase: setup
+  * Rerun with '-n T8 -p builtin --system generic:default -r'
+  * Reason: exception
+Traceback (most recent call last):
+  File "/home/user/Repositories/reframe/reframe/frontend/executors/__init__.py", line 291, in _safe_call
+    return fn(*args, **kwargs)
+  File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 82, in _fn
+    getattr(obj, h.__name__)()
+  File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 32, in _fn
+    func(*args, **kwargs)
+  File "/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py", line 180, in fail
+    raise Exception
+Exception
+
+------------------------------------------------------------------------------
+FAILURE INFO for T9 
+  * Expanded name: T9
+  * Description: T9
+  * System partition: generic:default
+  * Environment: builtin
+  * Stage directory: None
+  * Node list: 
+  * Job type: local (id=None)
+  * Dependencies (conceptual): ['T8']
+  * Dependencies (actual): [('T8', 'generic:default', 'builtin')]
+  * Maintainers: []
+  * Failing phase: startup
+  * Rerun with '-n T9 -p builtin --system generic:default -r'
+  * Reason: task dependency error: dependencies failed
+------------------------------------------------------------------------------
+FAILURE INFO for T2 
+  * Expanded name: T2
+  * Description: T2
+  * System partition: generic:default
+  * Environment: builtin
+  * Stage directory: /home/user/Repositories/reframe/stage/generic/default/builtin/T2
+  * Node list: tresa.localNone
+  * Job type: local (id=49427)
+  * Dependencies (conceptual): ['T6']
+  * Dependencies (actual): [('T6', 'generic:default', 'builtin')]
+  * Maintainers: []
+  * Failing phase: sanity
+  * Rerun with '-n T2 -p builtin --system generic:default -r'
+  * Reason: sanity error: 31 != 30
+------------------------------------------------------------------------------
+FAILURE INFO for T7 
+  * Expanded name: T7
+  * Description: T7
+  * System partition: generic:default
+  * Environment: builtin
+  * Stage directory: None
+  * Node list: 
+  * Job type: local (id=None)
+  * Dependencies (conceptual): ['T2']
+  * Dependencies (actual): [('T2', 'generic:default', 'builtin')]
+  * Maintainers: []
+  * Failing phase: startup
+  * Rerun with '-n T7 -p builtin --system generic:default -r'
+  * Reason: task dependency error: dependencies failed
+------------------------------------------------------------------------------
+Run report saved in '/home/user/.reframe/reports/run-report.json'
+Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-92y3fr5s.log'

docs/listings/deps_rerun_t6.txt

@@ -0,0 +1,22 @@
+[ReFrame Setup]
+  version:           3.10.0-dev.3+149af549
+  command:           './bin/reframe --restore-session --keep-stage-files -n T6 -r'
+  launched by:       user@host
+  working directory: '/home/user/Repositories/reframe'
+  settings file:     '<builtin>'
+  check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
+  stage directory:   '/home/user/Repositories/reframe/stage'
+  output directory:  '/home/user/Repositories/reframe/output'
+
+[==========] Running 1 check(s)
+[==========] Started on Sat Jan 22 23:44:25 2022 
+
+[----------] start processing checks
+[ RUN      ] T6 @generic:default+builtin
+[       OK ] (1/1) T6 @generic:default+builtin [compile: 0.017s run: 0.286s total: 0.330s]
+[----------] all spawned checks have finished
+
+[  PASSED  ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped)
+[==========] Finished on Sat Jan 22 23:44:25 2022 
+Run report saved in '/home/user/.reframe/reports/run-report.json'
+Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-mug0a4cb.log'

docs/listings/deps_run_t6.txt

@@ -0,0 +1,30 @@
+[ReFrame Setup]
+  version:           3.10.0-dev.3+149af549
+  command:           './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -n T6 -r'
+  launched by:       user@host
+  working directory: '/home/user/Repositories/reframe'
+  settings file:     '<builtin>'
+  check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py'
+  stage directory:   '/home/user/Repositories/reframe/stage'
+  output directory:  '/home/user/Repositories/reframe/output'
+
+[==========] Running 5 check(s)
+[==========] Started on Sat Jan 22 23:44:25 2022 
+
+[----------] start processing checks
+[ RUN      ] T0 @generic:default+builtin
+[       OK ] (1/5) T0 @generic:default+builtin [compile: 0.017s run: 0.289s total: 0.331s]
+[ RUN      ] T4 @generic:default+builtin
+[       OK ] (2/5) T4 @generic:default+builtin [compile: 0.018s run: 0.330s total: 0.374s]
+[ RUN      ] T5 @generic:default+builtin
+[       OK ] (3/5) T5 @generic:default+builtin [compile: 0.018s run: 0.384s total: 0.442s]
+[ RUN      ] T1 @generic:default+builtin
+[       OK ] (4/5) T1 @generic:default+builtin [compile: 0.018s run: 0.452s total: 0.494s]
+[ RUN      ] T6 @generic:default+builtin
+[       OK ] (5/5) T6 @generic:default+builtin [compile: 0.018s run: 0.525s total: 0.582s]
+[----------] all spawned checks have finished
+
+[  PASSED  ] Ran 5/5 test case(s) from 5 check(s) (0 failure(s), 0 skipped)
+[==========] Finished on Sat Jan 22 23:44:28 2022 
+Run report saved in '/home/user/.reframe/reports/run-report.json'
+Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-ktylyaqk.log'

docs/listings/hello1.txt

@@ -0,0 +1,22 @@
+[ReFrame Setup]
+  version:           3.10.0-dev.3+c22440c1
+  command:           './bin/reframe -c tutorials/basics/hello/hello1.py -r'
+  launched by:       user@host
+  working directory: '/path/to/reframe'
+  settings file:     '<builtin>'
+  check search path: '/path/to/reframe/tutorials/basics/hello/hello1.py'
+  stage directory:   '/path/to/reframe/stage'
+  output directory:  '/path/to/reframe/output'
+
+[==========] Running 1 check(s)
+[==========] Started on Sat Jan 22 13:21:50 2022 
+
+[----------] start processing checks
+[ RUN      ] HelloTest @generic:default+builtin
+[       OK ] (1/1) HelloTest @generic:default+builtin [compile: 0.272s run: 0.359s total: 0.784s]
+[----------] all spawned checks have finished
+
+[  PASSED  ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped)
+[==========] Finished on Sat Jan 22 13:21:51 2022 
+Run report saved in '/home/user/.reframe/reports/run-report.json'
+Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-8c6ybdvg.log'