From d9516f055a8836fea44e0c50555442464ecfc282 Mon Sep 17 00:00:00 2001
From: Luca <lucamar@cscs.ch>
Date: Fri, 11 Mar 2022 19:35:55 +0100
Subject: [PATCH 1/4] Update CPMD check

---
 cscs-checks/apps/cpmd/cpmd_check.py | 124 +++++++++++++++++-----------
 1 file changed, 74 insertions(+), 50 deletions(-)

diff --git a/cscs-checks/apps/cpmd/cpmd_check.py b/cscs-checks/apps/cpmd/cpmd_check.py
index 9c35e341cc..c38171c96b 100644
--- a/cscs-checks/apps/cpmd/cpmd_check.py
+++ b/cscs-checks/apps/cpmd/cpmd_check.py
@@ -3,80 +3,104 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import contextlib
 import reframe as rfm
 import reframe.utility.sanity as sn
 
 
 @rfm.simple_test
 class CPMDCheck(rfm.RunOnlyRegressionTest):
-    scale = parameter(['small', 'large'])
-    descr = 'CPMD check (C4H6 metadynamics)'
-    maintainers = ['AJ', 'LM']
-    tags = {'production'}
-    valid_systems = ['daint:gpu']
-    num_tasks_per_node = 1
-    valid_prog_environs = ['builtin']
     modules = ['CPMD']
     executable = 'cpmd.x'
-    executable_opts = ['ana_c4h6.in > stdout.txt']
-    readonly_files = ['ana_c4h6.in', 'C_MT_BLYP', 'H_MT_BLYP']
-    use_multithreading = True
-    strict_check = False
+    executable_opts = ['ana_c4h6.in']
     extra_resources = {
         'switches': {
             'num_switches': 1
         }
     }
-    allref = {
-        '9': {
-            'p100': {
-                'time': (284, None, 0.15, 's')
+    strict_check = False
+    use_multithreading = False
+    tags = {'maintenance', 'production'}
+    maintainers = ['AJ', 'LM']
+
+    num_nodes = parameter([6, 16], loggable=True)
+    references = {
+        6: {
+            'sm_60': {
+                'dom:gpu': {'time': (120, None, 0.15, 's')},
+                'daint:gpu': {'time': (120, None, 0.15, 's')},
+            },
+            'broadwell': {
+                'dom:mc': {'time': (150.0, None, 0.15, 's')},
+                'daint:mc': {'time': (150.0, None, 0.15, 's')},
             },
         },
-        '16': {
-            'p100': {
-                'time': (226, None, 0.15, 's')
-            }
+        16: {
+            'sm_60': {
+                'daint:gpu': {'time': (120, None, 0.15, 's')}
+            },
+            'broadwell': {
+                'daint:mc': {'time': (150.0, None, 0.15, 's')},
+            },
         }
     }
 
-    @run_after('init')
-    def setup_by_scale(self):
-        if self.scale == 'small':
-            self.valid_systems += ['dom:gpu']
-            self.num_tasks = 9
-        else:
-            self.num_tasks = 16
-
-    @run_before('performance')
-    def set_perf_reference(self):
-        proc = self.current_partition.processor
-        pname = self.current_partition.fullname
-        if pname in ('daint:gpu', 'dom:gpu'):
-            arch = 'p100'
-        else:
-            arch = proc.arch
-
-        with contextlib.suppress(KeyError):
-            self.reference = {
-                pname: {
-                    'perf': self.allref[self.num_tasks][arch][self.benchmark]
-                }
-            }
+    @performance_function('s')
+    def elapsed_time(self):
+        return sn.extractsingle(r'^ cpmd(\s+[\d\.]+){3}\s+(?P<time>\S+)',
+                                self.stdout, 'time', float)
 
     @sanity_function
     def assert_energy_diff(self):
-        #  OpenMP version of CPMD segfaults
-        #  self.variables = { 'OMP_NUM_THREADS' : '8' }
+        # OpenMP version of CPMD segfaults
         energy = sn.extractsingle(
             r'CLASSICAL ENERGY\s+-(?P<result>\S+)',
-            'stdout.txt', 'result', float)
+            self.stdout, 'result', float)
         energy_reference = 25.81
         energy_diff = sn.abs(energy - energy_reference)
         return sn.assert_lt(energy_diff, 0.26)
 
-    @performance_function('s')
-    def time(self):
-        return sn.extractsingle(r'^ cpmd(\s+[\d\.]+){3}\s+(?P<perf>\S+)',
-                                'stdout.txt', 'perf', float)
+    @run_after('init')
+    def setup_system_filtering(self):
+        self.descr = f'CPMD check ({self.num_nodes} node(s))'
+
+        # setup system filter
+        valid_systems = {
+            6: ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc'],
+            16: ['daint:gpu', 'daint:mc']
+        }
+
+        self.skip_if(self.num_nodes not in valid_systems,
+                     f'No valid systems found for {self.num_nodes}(s)')
+        self.valid_systems = valid_systems[self.num_nodes]
+
+        # setup programming environment filter
+        self.valid_prog_environs = ['builtin']
+
+    @run_before('run')
+    def setup_run(self):
+        # retrieve processor data
+        self.skip_if_no_procinfo()
+        proc = self.current_partition.processor
+
+        # set architecture for GPU partition (no auto-detection)
+        if self.current_partition.fullname in ['daint:gpu', 'dom:gpu']:
+            arch = 'sm_60'
+            self.variables = {
+                'CRAY_CUDA_MPS': '1'
+            }
+        else:
+            arch = proc.arch
+
+        # common setup for every architecture
+        self.job.launcher.options = ['--cpu-bind=cores']
+        self.job.options = ['--distribution=block:block']
+        self.num_tasks = 72
+
+        try:
+            found = self.references[self.num_nodes][arch]
+        except KeyError:
+            self.skip(f'Configuration with {self.num_nodes} node(s) '
+                      f'is not supported on {arch!r}')
+
+        # setup performance references
+        self.reference = self.references[self.num_nodes][arch]

From bf334cc2915b9fea6f705b67cf5d1548b5eb3ac8 Mon Sep 17 00:00:00 2001
From: Luca <lucamar@cscs.ch>
Date: Mon, 14 Mar 2022 12:22:07 +0100
Subject: [PATCH 2/4] Remove comment

---
 cscs-checks/apps/cpmd/cpmd_check.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cscs-checks/apps/cpmd/cpmd_check.py b/cscs-checks/apps/cpmd/cpmd_check.py
index c38171c96b..9eb2b7acaf 100644
--- a/cscs-checks/apps/cpmd/cpmd_check.py
+++ b/cscs-checks/apps/cpmd/cpmd_check.py
@@ -51,7 +51,6 @@ def elapsed_time(self):
 
     @sanity_function
     def assert_energy_diff(self):
-        # OpenMP version of CPMD segfaults
         energy = sn.extractsingle(
             r'CLASSICAL ENERGY\s+-(?P<result>\S+)',
             self.stdout, 'result', float)

From 24eb77ddecc71abc85c8bb420a685d70de5160a3 Mon Sep 17 00:00:00 2001
From: Luca <lucamar@cscs.ch>
Date: Mon, 14 Mar 2022 15:33:11 +0100
Subject: [PATCH 3/4] Comment about num_tasks

---
 cscs-checks/apps/cpmd/cpmd_check.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cscs-checks/apps/cpmd/cpmd_check.py b/cscs-checks/apps/cpmd/cpmd_check.py
index 9eb2b7acaf..815b6a962e 100644
--- a/cscs-checks/apps/cpmd/cpmd_check.py
+++ b/cscs-checks/apps/cpmd/cpmd_check.py
@@ -93,6 +93,9 @@ def setup_run(self):
         # common setup for every architecture
         self.job.launcher.options = ['--cpu-bind=cores']
         self.job.options = ['--distribution=block:block']
+        # FIXME: the current test case does not scale beyond 72 MPI tasks,
+        # so the last node in 16-nodes jobs will be used only partially. 
+        # The test case needs to be updated (warning about XC_DRIVER IN &DFT)
         self.num_tasks = 72
 
         try:

From c835daf8c1dcb3d9e5bf44a1b2462816dee6c4fc Mon Sep 17 00:00:00 2001
From: Luca <lucamar@cscs.ch>
Date: Mon, 14 Mar 2022 15:45:09 +0100
Subject: [PATCH 4/4] Remove trailing whitespace Line 97:73

---
 cscs-checks/apps/cpmd/cpmd_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cscs-checks/apps/cpmd/cpmd_check.py b/cscs-checks/apps/cpmd/cpmd_check.py
index 815b6a962e..9ddd653cb4 100644
--- a/cscs-checks/apps/cpmd/cpmd_check.py
+++ b/cscs-checks/apps/cpmd/cpmd_check.py
@@ -94,7 +94,7 @@ def setup_run(self):
         self.job.launcher.options = ['--cpu-bind=cores']
         self.job.options = ['--distribution=block:block']
         # FIXME: the current test case does not scale beyond 72 MPI tasks,
-        # so the last node in 16-nodes jobs will be used only partially. 
+        # so the last node in 16-nodes jobs will be used only partially.
         # The test case needs to be updated (warning about XC_DRIVER IN &DFT)
         self.num_tasks = 72