diff --git a/cscs-checks/microbenchmarks/hpcg/hpcg_benchmark.py b/cscs-checks/microbenchmarks/hpcg/hpcg_benchmark.py new file mode 100644 index 0000000000..9149aacef4 --- /dev/null +++ b/cscs-checks/microbenchmarks/hpcg/hpcg_benchmark.py @@ -0,0 +1,165 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.required_version('>=2.16-dev0') +@rfm.simple_test +class HPCGCheckRef(rfm.RegressionTest): + def __init__(self): + super().__init__() + + self.descr = 'HPCG reference benchmark' + self.valid_systems = ['daint:mc', 'daint:gpu', 'dom:gpu', 'dom:mc'] + self.valid_prog_environs = ['PrgEnv-gnu'] + self.modules = ['craype-hugepages8M'] + self.build_system = 'Make' + self.build_system.options = ['arch=MPI_GCC_OMP'] + self.sourcesdir = 'https://github.com/hpcg-benchmark/hpcg.git' + + self.executable = 'bin/xhpcg' + self.executable_opts = ['--nx=104', '--ny=104', '--nz=104', '-t2'] + # use glob to catch the output file suffix dependent on execution time + self.output_file = sn.getitem(sn.glob('HPCG*.txt'), 0) + + self.num_tasks = 0 + self.num_cpus_per_task = 1 + self.system_num_tasks = { + 'daint:mc': 36, + 'daint:gpu': 12, + 'dom:mc': 36, + 'dom:gpu': 12, + } + + self.reference = { + 'daint:gpu': { + 'gflops': (7.6, -0.1, None, 'Gflop/s') + }, + 'daint:mc': { + 'gflops': (13.4, -0.1, None, 'Gflop/s') + }, + 'dom:gpu': { + 'gflops': (7.6, -0.1, None, 'Gflop/s') + }, + 'dom:mc': { + 'gflops': (13.4, -0.1, None, 'Gflop/s') + }, + } + + self.maintainers = ['SK'] + self.tags = {'diagnostic'} + + @property + @sn.sanity_function + def num_tasks_assigned(self): + return self.job.num_tasks + + def setup(self, partition, environ, **job_opts): + self.num_tasks_per_node = self.system_num_tasks[partition.fullname] + + num_nodes = self.num_tasks_assigned / self.num_tasks_per_node + self.perf_patterns = { + 'gflops': sn.extractsingle( + r'HPCG result is VALID with a GFLOP\/s rating of=\s*' + r'(?P\S+)', + self.output_file, 'perf', float) / num_nodes + } + + self.sanity_patterns = sn.all([ + sn.assert_eq(4, sn.count( + sn.findall(r'PASSED', self.output_file))), + sn.assert_eq(0, self.num_tasks_assigned % self.num_tasks_per_node) + ]) + + super().setup(partition, environ, **job_opts) + + +@rfm.required_version('>=2.16-dev0') +@rfm.simple_test +class HPCGCheckMKL(rfm.RegressionTest): + def __init__(self): + super().__init__() + + self.descr = 'HPCG benchmark Intel MKL implementation' + self.valid_systems = ['daint:mc', 'dom:mc', 'daint:gpu', 'dom:gpu'] + self.valid_prog_environs = ['PrgEnv-intel'] + self.modules = ['craype-hugepages8M'] + self.build_system = 'Make' + self.prebuild_cmd = ['cp -r ${MKLROOT}/benchmarks/hpcg/* .', + 'mv Make.CrayXC setup', + './configure CrayXC'] + + self.num_tasks = 0 + self.num_tasks_per_core = 2 + self.problem_size = 104 + + self.variables = { + 'HUGETLB_VERBOSE': '0', + 'MPICH_MAX_THREAD_SAFETY': 'multiple', + 'MPICH_USE_DMAPP_COLL': '1', + 'PMI_NO_FORK': '1', + 'KMP_HW_SUBSET': '9c,2t', + 'KMP_AFFINITY': 'granularity=fine,compact' + } + + self.executable = 'bin/xhpcg_avx2' + self.executable_opts = ['--nx=%d' % self.problem_size, + '--ny=%d' % self.problem_size, + '--nz=%d' % self.problem_size, '-t2'] + + self.reference = { + 'dom:mc': { + 'gflops': (22, -0.1, None, 'Gflop/s') + }, + 'daint:mc': { + 'gflops': (22, -0.1, None, 'Gflop/s') + }, + 'dom:gpu': { + 'gflops': (10.7, -0.1, None, 'Gflop/s') + }, + 'daint:gpu': { + 'gflops': (10.7, -0.1, None, 'Gflop/s') + }, + } + + self.maintainers = ['SK'] + self.tags = {'diagnostic'} + + @property + @sn.sanity_function + def num_tasks_assigned(self): + return self.job.num_tasks + + @property + @sn.sanity_function + def outfile_lazy(self): + pattern = 'n%d-%dp-%dt-*.yaml' % (self.problem_size, + self.job.num_tasks, + self.num_cpus_per_task) + return sn.getitem(sn.glob(pattern), 0) + + def setup(self, partition, environ, **job_opts): + if partition.fullname in ['daint:gpu', 'dom:gpu']: + self.num_tasks_per_node = 2 + self.num_cpus_per_task = 12 + else: + self.num_tasks_per_node = 4 + self.num_cpus_per_task = 18 + + # since this is a flexible test, we divide the extracted + # performance by the number of nodes and compare + # against a single reference + num_nodes = self.num_tasks_assigned / self.num_tasks_per_node + self.perf_patterns = { + 'gflops': sn.extractsingle( + r'HPCG result is VALID with a GFLOP\/s rating of:\s*' + r'(?P\S+)', + self.outfile_lazy, 'perf', float) / num_nodes + } + + self.sanity_patterns = sn.all([ + sn.assert_eq(4, sn.count( + sn.findall(r'PASSED', self.outfile_lazy))), + sn.assert_eq(0, self.num_tasks_assigned % self.num_tasks_per_node) + ]) + + super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/microbenchmarks/hpcg/src/Make.CrayXC b/cscs-checks/microbenchmarks/hpcg/src/Make.CrayXC new file mode 100644 index 0000000000..20195a2c43 --- /dev/null +++ b/cscs-checks/microbenchmarks/hpcg/src/Make.CrayXC @@ -0,0 +1,149 @@ +#=============================================================================== +# Copyright 2014-2018 Intel Corporation All Rights Reserved. +# +# The source code, information and material ("Material") contained herein is +# owned by Intel Corporation or its suppliers or licensors, and title to such +# Material remains with Intel Corporation or its suppliers or licensors. The +# Material contains proprietary information of Intel or its suppliers and +# licensors. The Material is protected by worldwide copyright laws and treaty +# provisions. No part of the Material may be used, copied, reproduced, +# modified, published, uploaded, posted, transmitted, distributed or disclosed +# in any way without Intel's prior express written permission. No license under +# any patent, copyright or other intellectual property rights in the Material +# is granted to or conferred upon you, either expressly, by implication, +# inducement, estoppel or otherwise. Any license under such intellectual +# property rights must be express and approved by Intel in writing. +# +# Unless otherwise agreed by Intel in writing, you may not remove or alter this +# notice or any other notice embedded in Materials by Intel or Intel's +# suppliers or licensors in any way. +#=============================================================================== + +# -- High Performance Conjugate Gradient Benchmark (HPCG) +# HPCG - 2.1 - January 31, 2014 +# +# Michael A. Heroux +# Scalable Algorithms Group, Computing Research Center +# Sandia National Laboratories, Albuquerque, NM +# +# Piotr Luszczek +# Jack Dongarra +# University of Tennessee, Knoxville +# Innovative Computing Laboratory +# (C) Copyright 2013 All Rights Reserved +# +# -- Copyright notice and Licensing terms: +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. The name of the University, the name of the Laboratory, or the +# names of its contributors may not be used to endorse or promote +# products derived from this software without specific written +# permission. +# +# -- Disclaimer: +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#****************************************************************************** + +# ---------------------------------------------------------------------- +# - shell -------------------------------------------------------------- +# ---------------------------------------------------------------------- +# +SHELL = /bin/sh +# +CD = cd +CP = cp +LN_S = ln -s -f +MKDIR = mkdir -p +RM = /bin/rm -f +TOUCH = touch +# +# ---------------------------------------------------------------------- +# - HPCG Directory Structure / HPCG library ------------------------------ +# ---------------------------------------------------------------------- +# +TOPdir = . +SRCdir = $(TOPdir)/src +INCdir = $(TOPdir)/src +BINdir = $(TOPdir)/bin +# +# ---------------------------------------------------------------------- +# - Message Passing library (MPI) -------------------------------------- +# ---------------------------------------------------------------------- +# MPinc tells the C compiler where to find the Message Passing library +# header files, MPlib is defined to be the name of the library to be +# used. The variable MPdir is only used for defining MPinc and MPlib. +# +MPdir = +MPinc = +MPlib = -Wl,--whole-archive,-ldmapp,--no-whole-archive +# +# +# ---------------------------------------------------------------------- +# - HPCG includes / libraries / specifics ------------------------------- +# ---------------------------------------------------------------------- +# +HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) +HPCG_LIBS = $(MPlib) +# +# - Compile time options ----------------------------------------------- +# +# -DHPCG_NO_MPI Define to disable MPI +# -DHPCG_NO_OPENMP Define to disable OPENMP +# -DHPCG_DEBUG Define to enable debugging output +# -DHPCG_DETAILED_DEBUG Define to enable very detailed debugging output +# +# By default HPCG will: +# *) Build with MPI enabled. +# *) Build with OpenMP enabled. +# *) Not generate debugging output. +# +HPCG_OPTS = +# +# ---------------------------------------------------------------------- +# +HPCG_DEFS = -DMPICH_IGNORE_CXX_SEEK $(HPCG_OPTS) $(HPCG_INCLUDES) +# +# ---------------------------------------------------------------------- +# - Compilers / linkers - Optimization flags --------------------------- +# ---------------------------------------------------------------------- +# +CXX = CC +CXXFLAGS = -xCORE-AVX2 -qopenmp -std=c++11 $(HPCG_DEFS) +ifeq (yes, $(DBG)) + CXXFLAGS += -O0 -g -DHPCG_DEBUG +else + CXXFLAGS += -O3 -DNDEBUG +endif +# +LINKER = $(CXX) +MKL_LIB=$(MKLROOT)/lib/intel64 + +LINKFLAGS = -z relro -z now -Wl,-R'$$ORIGIN/lib/intel64' -liomp5 -L$(MKL_LIB) -liomp5 -static-intel -mkl +# +ARCHIVER = ar +ARFLAGS = r +RANLIB = echo +# +# ---------------------------------------------------------------------- +xhpcg_suff = _avx2