From bce800d1d1643595b086cfe54fd3a9d1446f3f43 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Thu, 9 Sep 2021 18:19:46 +0300 Subject: [PATCH 01/11] add redesigned python tests --- cscs-checks/apps/python/numpy_check.py | 128 +++++++----------- hpctestlib/apps/python/base_check.py | 76 +++++++++++ .../apps/python/src/np_ops.py | 0 3 files changed, 127 insertions(+), 77 deletions(-) create mode 100644 hpctestlib/apps/python/base_check.py rename {cscs-checks => hpctestlib}/apps/python/src/np_ops.py (100%) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index 260d5f4f51..fa84383567 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -5,86 +5,60 @@ import reframe as rfm import reframe.utility.sanity as sn +from hpctestlib.apps.python.base_check import Numpy_BaseCheck -class NumpyBaseTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.descr = 'Test a few typical numpy operations' - self.valid_prog_environs = ['builtin'] - self.modules = ['numpy'] - self.reference = { - 'daint:gpu': { - 'dot': (0.4, None, 0.05, 'seconds'), - 'svd': (0.37, None, 0.05, 'seconds'), - 'cholesky': (0.12, None, 0.05, 'seconds'), - 'eigendec': (3.5, None, 0.05, 'seconds'), - 'inv': (0.21, None, 0.05, 'seconds'), - }, - 'daint:mc': { - 'dot': (0.3, None, 0.05, 'seconds'), - 'svd': (0.35, None, 0.05, 'seconds'), - 'cholesky': (0.1, None, 0.05, 'seconds'), - 'eigendec': (4.14, None, 0.05, 'seconds'), - 'inv': (0.16, None, 0.05, 'seconds'), - }, - 'dom:gpu': { - 'dot': (0.4, None, 0.05, 'seconds'), - 'svd': (0.37, None, 0.05, 'seconds'), - 'cholesky': (0.12, None, 0.05, 'seconds'), - 'eigendec': (3.5, None, 0.05, 'seconds'), - 'inv': (0.21, None, 0.05, 'seconds'), - }, - 'dom:mc': { - 'dot': (0.3, None, 0.05, 'seconds'), - 'svd': (0.35, None, 0.05, 'seconds'), - 'cholesky': (0.1, None, 0.05, 'seconds'), - 'eigendec': (4.14, None, 0.05, 'seconds'), - 'inv': (0.16, None, 0.05, 'seconds'), - }, - } - self.perf_patterns = { - 'dot': sn.extractsingle( - r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', - self.stdout, 'dot', float), - 'svd': sn.extractsingle( - r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', - self.stdout, 'svd', float), - 'cholesky': sn.extractsingle( - r'^Cholesky decomposition of a 2048x2048 matrix in' - r'\s+(?P\S+)\s+s', - self.stdout, 'cholesky', float), - 'eigendec': sn.extractsingle( - r'^Eigendecomposition of a 2048x2048 matrix in' - r'\s+(?P\S+)\s+s', - self.stdout, 'eigendec', float), - 'inv': sn.extractsingle( - r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', - self.stdout, 'inv', float) - } - self.sanity_patterns = sn.assert_found(r'Numpy version:\s+\S+', - self.stdout) - self.variables = { - 'OMP_NUM_THREADS': '$SLURM_CPUS_PER_TASK', - } - self.executable = 'python' - self.executable_opts = ['np_ops.py'] - self.num_tasks_per_node = 1 - self.use_multithreading = False - self.tags = {'production'} - self.maintainers = ['RS', 'TR'] +REFERENCE_PERFOMANCE = { + 'daint:gpu': { + 'dot': (0.4, None, 0.05, 'seconds'), + 'svd': (0.37, None, 0.05, 'seconds'), + 'cholesky': (0.12, None, 0.05, 'seconds'), + 'eigendec': (3.5, None, 0.05, 'seconds'), + 'inv': (0.21, None, 0.05, 'seconds'), + }, + 'daint:mc': { + 'dot': (0.3, None, 0.05, 'seconds'), + 'svd': (0.35, None, 0.05, 'seconds'), + 'cholesky': (0.1, None, 0.05, 'seconds'), + 'eigendec': (4.14, None, 0.05, 'seconds'), + 'inv': (0.16, None, 0.05, 'seconds'), + }, + 'dom:gpu': { + 'dot': (0.4, None, 0.05, 'seconds'), + 'svd': (0.37, None, 0.05, 'seconds'), + 'cholesky': (0.12, None, 0.05, 'seconds'), + 'eigendec': (3.5, None, 0.05, 'seconds'), + 'inv': (0.21, None, 0.05, 'seconds'), + }, + 'dom:mc': { + 'dot': (0.3, None, 0.05, 'seconds'), + 'svd': (0.35, None, 0.05, 'seconds'), + 'cholesky': (0.1, None, 0.05, 'seconds'), + 'eigendec': (4.14, None, 0.05, 'seconds'), + 'inv': (0.16, None, 0.05, 'seconds'), + }, +} @rfm.simple_test -class NumpyHaswellTest(NumpyBaseTest): - def __init__(self): - super().__init__() - self.valid_systems = ['daint:gpu', 'dom:gpu'] - self.num_cpus_per_task = 12 +class Numpy_TestCSCS(Numpy_BaseCheck): + valid_prog_environs = ['builtin'] + modules = ['numpy'] + variables = { + 'OMP_NUM_THREADS': '$SLURM_CPUS_PER_TASK', + } + num_tasks_per_node = 1 + use_multithreading = False + tags = {'production'} + maintainers = ['RS', 'TR'] + reference = REFERENCE_PERFOMANCE + platform = parameter(['gpu', 'cpu']) - -@rfm.simple_test -class NumpyBroadwellTest(NumpyBaseTest): - def __init__(self): - super().__init__() - self.valid_systems = ['daint:mc', 'dom:mc'] - self.num_cpus_per_task = 36 + @run_after('init') + def set_system(self): + if self.platform == 'gpu': + self.valid_systems = ['daint:gpu', 'dom:gpu'] + self.num_cpus_per_task = 12 + else: + self.valid_systems = ['daint:mc', 'dom:mc'] + self.num_cpus_per_task = 36 diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/base_check.py new file mode 100644 index 0000000000..01635d90b9 --- /dev/null +++ b/hpctestlib/apps/python/base_check.py @@ -0,0 +1,76 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +class Numpy_BaseCheck(rfm.RunOnlyRegressionTest, pin_prefix=True): + '''Base class for the NumPy Test. + NumPy is the fundamental package for scientific computing in Python. + It is a Python library that provides a multidimensional array object, + various derived objects (such as masked arrays and matrices), and an + assortment of routines for fast operations on arrays, including + mathematical, logical, shape manipulation, sorting, selecting, I/O, + discrete Fourier transforms, basic linear algebra, basic statistical + operations, random simulation and much more (see numpy.org). + + The presented abstract run-only class checks the numpy perfomance. + This test checks whether some basic operations (such as matrix product, + SVD decomposition, Cholesky decomposition, eigendecomposition, and + inverse matrix calculation) are performed, and also checks the + execution time of these operations. The default assumption is that + NumPy is already installed on the device under test. + ''' + + #: :default: :class:`required` + executable = required + + #: :default: :class:`required` + num_tasks_per_node = required + + executable = 'python' + executable_opts = ['np_ops.py'] + + @run_after('init') + def set_description(self): + self.mydescr = 'Test a few typical numpy operations' + + @performance_function('seconds', perf_key='dot') + def set_perf_dot(self): + return sn.extractsingle( + r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', + self.stdout, 'dot', float) + + @performance_function('seconds', perf_key='svd') + def set_perf_svd(self): + return sn.extractsingle( + r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', + self.stdout, 'svd', float) + + @performance_function('seconds', perf_key='cholesky') + def set_perf_cholesky(self): + return sn.extractsingle( + r'^Cholesky decomposition of a 2048x2048 matrix in' + r'\s+(?P\S+)\s+s', + self.stdout, 'cholesky', float) + + @performance_function('seconds', perf_key='eigendec') + def set_perf_eigendec(self): + return sn.extractsingle( + r'^Eigendecomposition of a 2048x2048 matrix in' + r'\s+(?P\S+)\s+s', + self.stdout, 'eigendec', float) + + @performance_function('seconds', perf_key='inv') + def set_perf_inv(self): + return sn.extractsingle( + r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', + self.stdout, 'inv', float) + + @sanity_function + def assert_energy_readout(self): + return sn.assert_found(r'Numpy version:\s+\S+', + self.stdout) diff --git a/cscs-checks/apps/python/src/np_ops.py b/hpctestlib/apps/python/src/np_ops.py similarity index 100% rename from cscs-checks/apps/python/src/np_ops.py rename to hpctestlib/apps/python/src/np_ops.py From b0f12a721a6e12dca4db34962d6644656fae6868 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Thu, 9 Sep 2021 18:23:22 +0300 Subject: [PATCH 02/11] fix problems with indents --- hpctestlib/apps/python/base_check.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/base_check.py index 01635d90b9..1329373734 100644 --- a/hpctestlib/apps/python/base_check.py +++ b/hpctestlib/apps/python/base_check.py @@ -41,34 +41,34 @@ def set_description(self): @performance_function('seconds', perf_key='dot') def set_perf_dot(self): return sn.extractsingle( - r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', - self.stdout, 'dot', float) + r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', + self.stdout, 'dot', float) @performance_function('seconds', perf_key='svd') def set_perf_svd(self): return sn.extractsingle( - r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', - self.stdout, 'svd', float) + r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', + self.stdout, 'svd', float) @performance_function('seconds', perf_key='cholesky') def set_perf_cholesky(self): return sn.extractsingle( - r'^Cholesky decomposition of a 2048x2048 matrix in' - r'\s+(?P\S+)\s+s', - self.stdout, 'cholesky', float) + r'^Cholesky decomposition of a 2048x2048 matrix in' + r'\s+(?P\S+)\s+s', + self.stdout, 'cholesky', float) @performance_function('seconds', perf_key='eigendec') def set_perf_eigendec(self): return sn.extractsingle( - r'^Eigendecomposition of a 2048x2048 matrix in' - r'\s+(?P\S+)\s+s', - self.stdout, 'eigendec', float) + r'^Eigendecomposition of a 2048x2048 matrix in' + r'\s+(?P\S+)\s+s', + self.stdout, 'eigendec', float) @performance_function('seconds', perf_key='inv') def set_perf_inv(self): return sn.extractsingle( - r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', - self.stdout, 'inv', float) + r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', + self.stdout, 'inv', float) @sanity_function def assert_energy_readout(self): From 3220f2d5f81e7c9e8ff893a51f1f136372618f7c Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Fri, 10 Sep 2021 16:52:18 +0300 Subject: [PATCH 03/11] delete unused import --- cscs-checks/apps/python/numpy_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index fa84383567..5c12ab91a8 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause import reframe as rfm -import reframe.utility.sanity as sn from hpctestlib.apps.python.base_check import Numpy_BaseCheck From cc5adbfb261d199001acade4e1bae992d806c32f Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Fri, 10 Sep 2021 17:28:29 +0300 Subject: [PATCH 04/11] delete unnessessary comments, add blank line --- cscs-checks/apps/python/numpy_check.py | 1 + hpctestlib/apps/python/base_check.py | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index 5c12ab91a8..2a8f4b5b47 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import reframe as rfm + from hpctestlib.apps.python.base_check import Numpy_BaseCheck diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/base_check.py index 1329373734..ef15428d5a 100644 --- a/hpctestlib/apps/python/base_check.py +++ b/hpctestlib/apps/python/base_check.py @@ -25,10 +25,6 @@ class Numpy_BaseCheck(rfm.RunOnlyRegressionTest, pin_prefix=True): NumPy is already installed on the device under test. ''' - #: :default: :class:`required` - executable = required - - #: :default: :class:`required` num_tasks_per_node = required executable = 'python' @@ -72,5 +68,4 @@ def set_perf_inv(self): @sanity_function def assert_energy_readout(self): - return sn.assert_found(r'Numpy version:\s+\S+', - self.stdout) + return sn.assert_found(r'Numpy version:\s+\S+', self.stdout) From c6dd20378c6946b25470e0833bab98f8cfa8c760 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Wed, 15 Sep 2021 02:31:38 +0300 Subject: [PATCH 05/11] delete unnessesary requirement for num_task_per_node --- hpctestlib/apps/python/base_check.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/base_check.py index ef15428d5a..7d27b637f0 100644 --- a/hpctestlib/apps/python/base_check.py +++ b/hpctestlib/apps/python/base_check.py @@ -25,8 +25,6 @@ class Numpy_BaseCheck(rfm.RunOnlyRegressionTest, pin_prefix=True): NumPy is already installed on the device under test. ''' - num_tasks_per_node = required - executable = 'python' executable_opts = ['np_ops.py'] From 3d2e90b4e5cab0b8219a329116f0bb3e54a2b973 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Mon, 20 Sep 2021 20:16:24 +0200 Subject: [PATCH 06/11] change the name of the sanity function --- hpctestlib/apps/python/base_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/base_check.py index 7d27b637f0..b03aa38ec6 100644 --- a/hpctestlib/apps/python/base_check.py +++ b/hpctestlib/apps/python/base_check.py @@ -65,5 +65,5 @@ def set_perf_inv(self): self.stdout, 'inv', float) @sanity_function - def assert_energy_readout(self): + def assert_numpy_version(self): return sn.assert_found(r'Numpy version:\s+\S+', self.stdout) From c5333dcdd9f91c93ed55fda3c156fe6e4d61a2f3 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Wed, 6 Oct 2021 17:25:30 +0200 Subject: [PATCH 07/11] Use number of cores to specify cpus per task --- cscs-checks/apps/python/numpy_check.py | 17 ++++++----------- .../apps/python/{ => numpy}/base_check.py | 0 2 files changed, 6 insertions(+), 11 deletions(-) rename hpctestlib/apps/python/{ => numpy}/base_check.py (100%) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index 2a8f4b5b47..ace184dac6 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -5,7 +5,7 @@ import reframe as rfm -from hpctestlib.apps.python.base_check import Numpy_BaseCheck +from hpctestlib.apps.python.numpy.base_check import Numpy_BaseCheck REFERENCE_PERFOMANCE = { @@ -43,22 +43,17 @@ @rfm.simple_test class Numpy_TestCSCS(Numpy_BaseCheck): valid_prog_environs = ['builtin'] + valid_systems = ['daint:gpu', 'daint-mc', 'dom:gpu', 'dom:mc'] modules = ['numpy'] variables = { 'OMP_NUM_THREADS': '$SLURM_CPUS_PER_TASK', } num_tasks_per_node = 1 use_multithreading = False + reference = REFERENCE_PERFOMANCE tags = {'production'} maintainers = ['RS', 'TR'] - reference = REFERENCE_PERFOMANCE - platform = parameter(['gpu', 'cpu']) - @run_after('init') - def set_system(self): - if self.platform == 'gpu': - self.valid_systems = ['daint:gpu', 'dom:gpu'] - self.num_cpus_per_task = 12 - else: - self.valid_systems = ['daint:mc', 'dom:mc'] - self.num_cpus_per_task = 36 + @run_after('setup') + def set_num_cpus_per_task(self): + self.num_cpus_per_task = self.current_partition.processor.num_cores diff --git a/hpctestlib/apps/python/base_check.py b/hpctestlib/apps/python/numpy/base_check.py similarity index 100% rename from hpctestlib/apps/python/base_check.py rename to hpctestlib/apps/python/numpy/base_check.py From 5ac2bdc53da1044d258951a41382a15b33c0cad7 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Wed, 6 Oct 2021 17:54:56 +0200 Subject: [PATCH 08/11] Fix typo --- cscs-checks/apps/python/numpy_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index ace184dac6..f708483b70 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -43,7 +43,7 @@ @rfm.simple_test class Numpy_TestCSCS(Numpy_BaseCheck): valid_prog_environs = ['builtin'] - valid_systems = ['daint:gpu', 'daint-mc', 'dom:gpu', 'dom:mc'] + valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc'] modules = ['numpy'] variables = { 'OMP_NUM_THREADS': '$SLURM_CPUS_PER_TASK', From a3fa24e7dc0a77eeadc4442324e1052a288aa235 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 7 Oct 2021 10:18:21 +0200 Subject: [PATCH 09/11] Move the numpy script to the new directory hierarchy --- hpctestlib/apps/python/{ => numpy}/src/np_ops.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hpctestlib/apps/python/{ => numpy}/src/np_ops.py (100%) diff --git a/hpctestlib/apps/python/src/np_ops.py b/hpctestlib/apps/python/numpy/src/np_ops.py similarity index 100% rename from hpctestlib/apps/python/src/np_ops.py rename to hpctestlib/apps/python/numpy/src/np_ops.py From e56abd5da5e078a781a578eb7a0b20a723aaa8b7 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 12 Oct 2021 14:41:23 +0200 Subject: [PATCH 10/11] Address PR comments --- cscs-checks/apps/python/numpy_check.py | 67 ++++++++----------- .../numpy/{base_check.py => numpy_ops.py} | 55 ++++++++------- 2 files changed, 61 insertions(+), 61 deletions(-) rename hpctestlib/apps/python/numpy/{base_check.py => numpy_ops.py} (51%) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index f708483b70..a916b35dd4 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -5,55 +5,46 @@ import reframe as rfm -from hpctestlib.apps.python.numpy.base_check import Numpy_BaseCheck - - -REFERENCE_PERFOMANCE = { - 'daint:gpu': { - 'dot': (0.4, None, 0.05, 'seconds'), - 'svd': (0.37, None, 0.05, 'seconds'), - 'cholesky': (0.12, None, 0.05, 'seconds'), - 'eigendec': (3.5, None, 0.05, 'seconds'), - 'inv': (0.21, None, 0.05, 'seconds'), - }, - 'daint:mc': { - 'dot': (0.3, None, 0.05, 'seconds'), - 'svd': (0.35, None, 0.05, 'seconds'), - 'cholesky': (0.1, None, 0.05, 'seconds'), - 'eigendec': (4.14, None, 0.05, 'seconds'), - 'inv': (0.16, None, 0.05, 'seconds'), - }, - 'dom:gpu': { - 'dot': (0.4, None, 0.05, 'seconds'), - 'svd': (0.37, None, 0.05, 'seconds'), - 'cholesky': (0.12, None, 0.05, 'seconds'), - 'eigendec': (3.5, None, 0.05, 'seconds'), - 'inv': (0.21, None, 0.05, 'seconds'), - }, - 'dom:mc': { - 'dot': (0.3, None, 0.05, 'seconds'), - 'svd': (0.35, None, 0.05, 'seconds'), - 'cholesky': (0.1, None, 0.05, 'seconds'), - 'eigendec': (4.14, None, 0.05, 'seconds'), - 'inv': (0.16, None, 0.05, 'seconds'), - }, -} +from hpctestlib.apps.python.numpy.numpy_ops import numpy_ops_check @rfm.simple_test -class Numpy_TestCSCS(Numpy_BaseCheck): +class cscs_numpy_test(numpy_ops_check): valid_prog_environs = ['builtin'] valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc'] modules = ['numpy'] - variables = { - 'OMP_NUM_THREADS': '$SLURM_CPUS_PER_TASK', - } num_tasks_per_node = 1 use_multithreading = False - reference = REFERENCE_PERFOMANCE + all_ref = { + 'haswell': { + 'dot': (0.4, None, 0.05, 'seconds'), + 'svd': (0.37, None, 0.05, 'seconds'), + 'cholesky': (0.12, None, 0.05, 'seconds'), + 'eigendec': (3.5, None, 0.05, 'seconds'), + 'inv': (0.21, None, 0.05, 'seconds'), + }, + 'broadwell': { + 'dot': (0.3, None, 0.05, 'seconds'), + 'svd': (0.35, None, 0.05, 'seconds'), + 'cholesky': (0.1, None, 0.05, 'seconds'), + 'eigendec': (4.14, None, 0.05, 'seconds'), + 'inv': (0.16, None, 0.05, 'seconds'), + } + } tags = {'production'} maintainers = ['RS', 'TR'] @run_after('setup') def set_num_cpus_per_task(self): self.num_cpus_per_task = self.current_partition.processor.num_cores + variables = { + 'OMP_NUM_THREADS': self.num_cpus_per_task + } + + @run_before('performance') + def set_perf_ref(self): + arch = self.current_partition.processor.arch + pname = self.current_partition.fullname + self.reference = { + pname: self.all_ref[arch] + } diff --git a/hpctestlib/apps/python/numpy/base_check.py b/hpctestlib/apps/python/numpy/numpy_ops.py similarity index 51% rename from hpctestlib/apps/python/numpy/base_check.py rename to hpctestlib/apps/python/numpy/numpy_ops.py index b03aa38ec6..6251d73511 100644 --- a/hpctestlib/apps/python/numpy/base_check.py +++ b/hpctestlib/apps/python/numpy/numpy_ops.py @@ -7,59 +7,68 @@ import reframe.utility.sanity as sn -class Numpy_BaseCheck(rfm.RunOnlyRegressionTest, pin_prefix=True): - '''Base class for the NumPy Test. - NumPy is the fundamental package for scientific computing in Python. - It is a Python library that provides a multidimensional array object, - various derived objects (such as masked arrays and matrices), and an - assortment of routines for fast operations on arrays, including - mathematical, logical, shape manipulation, sorting, selecting, I/O, - discrete Fourier transforms, basic linear algebra, basic statistical - operations, random simulation and much more (see numpy.org). +class numpy_ops_check(rfm.RunOnlyRegressionTest, pin_prefix=True): + '''NumPy basic operations test. - The presented abstract run-only class checks the numpy perfomance. - This test checks whether some basic operations (such as matrix product, - SVD decomposition, Cholesky decomposition, eigendecomposition, and - inverse matrix calculation) are performed, and also checks the - execution time of these operations. The default assumption is that - NumPy is already installed on the device under test. + `NumPy `__ is the fundamental package for scientific + computing in Python. + It provides a multidimensional array object, various derived objects + (such as masked arrays and matrices), and an assortment of routines + for fast operations on arrays, including mathematical, logical, shape + manipulation, sorting, selecting, I/O, discrete Fourier transforms, + basic linear algebra, basic statistical operations, random simulation + and much more. + + This test test performs some fundamental NumPy linear algebra operations + (matrix product, SVD, Cholesky decomposition, eigendecomposition, and + inverse matrix calculation) and users the execution time as a performance + metric. The default assumption is that NumPy is already installed on the + currest system. ''' + #: See :attr:`~reframe.core.pipeline.RegressionTest.executable`. + #: + #: :required: No executable = 'python' + + #: See :attr:`~reframe.core.pipeline.RegressionTest.executable_opts`. + #: + #: :required: No executable_opts = ['np_ops.py'] - @run_after('init') - def set_description(self): - self.mydescr = 'Test a few typical numpy operations' + #: See :attr:`~reframe.core.pipeline.RegressionTest.descr`. + #: + #: :required: No + descr = 'Test NumPy operations: dot, svd, cholesky, eigen and inv' @performance_function('seconds', perf_key='dot') - def set_perf_dot(self): + def time_dot(self): return sn.extractsingle( r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', self.stdout, 'dot', float) @performance_function('seconds', perf_key='svd') - def set_perf_svd(self): + def time_svd(self): return sn.extractsingle( r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', self.stdout, 'svd', float) @performance_function('seconds', perf_key='cholesky') - def set_perf_cholesky(self): + def time_cholesky(self): return sn.extractsingle( r'^Cholesky decomposition of a 2048x2048 matrix in' r'\s+(?P\S+)\s+s', self.stdout, 'cholesky', float) @performance_function('seconds', perf_key='eigendec') - def set_perf_eigendec(self): + def time_eigendec(self): return sn.extractsingle( r'^Eigendecomposition of a 2048x2048 matrix in' r'\s+(?P\S+)\s+s', self.stdout, 'eigendec', float) @performance_function('seconds', perf_key='inv') - def set_perf_inv(self): + def time_inv(self): return sn.extractsingle( r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', self.stdout, 'inv', float) From afdb10d3fa6a1b1360d17d5ea7bbeff7d47962df Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 13 Oct 2021 21:49:31 +0200 Subject: [PATCH 11/11] Fine tune and move NumPy check --- cscs-checks/apps/python/numpy_check.py | 29 +++++++++-------- .../{apps => }/python/numpy/numpy_ops.py | 32 +++++++++---------- .../{apps => }/python/numpy/src/np_ops.py | 0 3 files changed, 31 insertions(+), 30 deletions(-) rename hpctestlib/{apps => }/python/numpy/numpy_ops.py (80%) rename hpctestlib/{apps => }/python/numpy/src/np_ops.py (100%) diff --git a/cscs-checks/apps/python/numpy_check.py b/cscs-checks/apps/python/numpy_check.py index a916b35dd4..31abb53272 100644 --- a/cscs-checks/apps/python/numpy_check.py +++ b/cscs-checks/apps/python/numpy_check.py @@ -5,7 +5,7 @@ import reframe as rfm -from hpctestlib.apps.python.numpy.numpy_ops import numpy_ops_check +from hpctestlib.python.numpy.numpy_ops import numpy_ops_check @rfm.simple_test @@ -16,19 +16,19 @@ class cscs_numpy_test(numpy_ops_check): num_tasks_per_node = 1 use_multithreading = False all_ref = { - 'haswell': { - 'dot': (0.4, None, 0.05, 'seconds'), - 'svd': (0.37, None, 0.05, 'seconds'), - 'cholesky': (0.12, None, 0.05, 'seconds'), - 'eigendec': (3.5, None, 0.05, 'seconds'), - 'inv': (0.21, None, 0.05, 'seconds'), + 'haswell@12c': { + 'dot': (0.4, None, 0.05, 's'), + 'svd': (0.37, None, 0.05, 's'), + 'cholesky': (0.12, None, 0.05, 's'), + 'eigendec': (3.5, None, 0.05, 's'), + 'inv': (0.21, None, 0.05, 's'), }, - 'broadwell': { - 'dot': (0.3, None, 0.05, 'seconds'), - 'svd': (0.35, None, 0.05, 'seconds'), - 'cholesky': (0.1, None, 0.05, 'seconds'), - 'eigendec': (4.14, None, 0.05, 'seconds'), - 'inv': (0.16, None, 0.05, 'seconds'), + 'broadwell@36c': { + 'dot': (0.3, None, 0.05, 's'), + 'svd': (0.35, None, 0.05, 's'), + 'cholesky': (0.1, None, 0.05, 's'), + 'eigendec': (4.14, None, 0.05, 's'), + 'inv': (0.16, None, 0.05, 's'), } } tags = {'production'} @@ -45,6 +45,7 @@ def set_num_cpus_per_task(self): def set_perf_ref(self): arch = self.current_partition.processor.arch pname = self.current_partition.fullname + num_cores = self.current_partition.processor.num_cores self.reference = { - pname: self.all_ref[arch] + pname: self.all_ref[f'{arch}@{num_cores}c'] } diff --git a/hpctestlib/apps/python/numpy/numpy_ops.py b/hpctestlib/python/numpy/numpy_ops.py similarity index 80% rename from hpctestlib/apps/python/numpy/numpy_ops.py rename to hpctestlib/python/numpy/numpy_ops.py index 6251d73511..5e438b9017 100644 --- a/hpctestlib/apps/python/numpy/numpy_ops.py +++ b/hpctestlib/python/numpy/numpy_ops.py @@ -7,6 +7,7 @@ import reframe.utility.sanity as sn +@rfm.simple_test class numpy_ops_check(rfm.RunOnlyRegressionTest, pin_prefix=True): '''NumPy basic operations test. @@ -26,49 +27,48 @@ class numpy_ops_check(rfm.RunOnlyRegressionTest, pin_prefix=True): currest system. ''' - #: See :attr:`~reframe.core.pipeline.RegressionTest.executable`. - #: - #: :required: No executable = 'python' - - #: See :attr:`~reframe.core.pipeline.RegressionTest.executable_opts`. - #: - #: :required: No executable_opts = ['np_ops.py'] - - #: See :attr:`~reframe.core.pipeline.RegressionTest.descr`. - #: - #: :required: No descr = 'Test NumPy operations: dot, svd, cholesky, eigen and inv' - @performance_function('seconds', perf_key='dot') + @performance_function('s', perf_key='dot') def time_dot(self): + '''Time of the ``dot`` kernel in seconds.''' + return sn.extractsingle( r'^Dotted two 4096x4096 matrices in\s+(?P\S+)\s+s', self.stdout, 'dot', float) - @performance_function('seconds', perf_key='svd') + @performance_function('s', perf_key='svd') def time_svd(self): + '''Time of the ``svd`` kernel in seconds.''' + return sn.extractsingle( r'^SVD of a 2048x1024 matrix in\s+(?P\S+)\s+s', self.stdout, 'svd', float) - @performance_function('seconds', perf_key='cholesky') + @performance_function('s', perf_key='cholesky') def time_cholesky(self): + '''Time of the ``cholesky`` kernel in seconds.''' + return sn.extractsingle( r'^Cholesky decomposition of a 2048x2048 matrix in' r'\s+(?P\S+)\s+s', self.stdout, 'cholesky', float) - @performance_function('seconds', perf_key='eigendec') + @performance_function('s', perf_key='eigendec') def time_eigendec(self): + '''Time of the ``eigendec`` kernel in seconds.''' + return sn.extractsingle( r'^Eigendecomposition of a 2048x2048 matrix in' r'\s+(?P\S+)\s+s', self.stdout, 'eigendec', float) - @performance_function('seconds', perf_key='inv') + @performance_function('s', perf_key='inv') def time_inv(self): + '''Time of the ``inv`` kernel in seconds.''' + return sn.extractsingle( r'^Inversion of a 2048x2048 matrix in\s+(?P\S+)\s+s', self.stdout, 'inv', float) diff --git a/hpctestlib/apps/python/numpy/src/np_ops.py b/hpctestlib/python/numpy/src/np_ops.py similarity index 100% rename from hpctestlib/apps/python/numpy/src/np_ops.py rename to hpctestlib/python/numpy/src/np_ops.py