[test] Add flexible HPCG test

cscs-checks/microbenchmarks/hpcg/hpcg_benchmark.py

@@ -0,0 +1,165 @@
+import reframe as rfm
+import reframe.utility.sanity as sn
+
+
+@rfm.required_version('>=2.16-dev0')
+@rfm.simple_test
+class HPCGCheckRef(rfm.RegressionTest):
+    def __init__(self):
+        super().__init__()
+
+        self.descr = 'HPCG reference benchmark'
+        self.valid_systems = ['daint:mc', 'daint:gpu', 'dom:gpu', 'dom:mc']
+        self.valid_prog_environs = ['PrgEnv-gnu']
+        self.modules = ['craype-hugepages8M']
+        self.build_system = 'Make'
+        self.build_system.options = ['arch=MPI_GCC_OMP']
+        self.sourcesdir = 'https://github.com/hpcg-benchmark/hpcg.git'
+
+        self.executable = 'bin/xhpcg'
+        self.executable_opts = ['--nx=104', '--ny=104', '--nz=104', '-t2']
+        # use glob to catch the output file suffix dependent on execution time
+        self.output_file = sn.getitem(sn.glob('HPCG*.txt'), 0)
+
+        self.num_tasks = 0
+        self.num_cpus_per_task = 1
+        self.system_num_tasks = {
+            'daint:mc':  36,
+            'daint:gpu': 12,
+            'dom:mc':  36,
+            'dom:gpu': 12,
+        }
+
+        self.reference = {
+            'daint:gpu': {
+                'gflops': (7.6, -0.1, None, 'Gflop/s')
+            },
+            'daint:mc': {
+                'gflops': (13.4, -0.1, None, 'Gflop/s')
+            },
+            'dom:gpu': {
+                'gflops': (7.6, -0.1, None, 'Gflop/s')
+            },
+            'dom:mc': {
+                'gflops': (13.4, -0.1, None, 'Gflop/s')
+            },
+        }
+
+        self.maintainers = ['SK']
+        self.tags = {'diagnostic'}
+
+    @property
+    @sn.sanity_function
+    def num_tasks_assigned(self):
+        return self.job.num_tasks
+
+    def setup(self, partition, environ, **job_opts):
+        self.num_tasks_per_node = self.system_num_tasks[partition.fullname]
+
+        num_nodes = self.num_tasks_assigned / self.num_tasks_per_node
+        self.perf_patterns = {
+            'gflops': sn.extractsingle(
+                r'HPCG result is VALID with a GFLOP\/s rating of=\s*'
+                r'(?P<perf>\S+)',
+                self.output_file, 'perf',  float) / num_nodes
+        }
+
+        self.sanity_patterns = sn.all([
+            sn.assert_eq(4, sn.count(
+                sn.findall(r'PASSED', self.output_file))),
+            sn.assert_eq(0, self.num_tasks_assigned % self.num_tasks_per_node)
+        ])
+
+        super().setup(partition, environ, **job_opts)
+
+
+@rfm.required_version('>=2.16-dev0')
+@rfm.simple_test
+class HPCGCheckMKL(rfm.RegressionTest):
+    def __init__(self):
+        super().__init__()
+
+        self.descr = 'HPCG benchmark Intel MKL implementation'
+        self.valid_systems = ['daint:mc', 'dom:mc', 'daint:gpu', 'dom:gpu']
+        self.valid_prog_environs = ['PrgEnv-intel']
+        self.modules = ['craype-hugepages8M']
+        self.build_system = 'Make'
+        self.prebuild_cmd = ['cp -r ${MKLROOT}/benchmarks/hpcg/* .',
+                             'mv Make.CrayXC setup',
+                             './configure CrayXC']
+
+        self.num_tasks = 0
+        self.num_tasks_per_core = 2
+        self.problem_size = 104
+
+        self.variables = {
+            'HUGETLB_VERBOSE': '0',
+            'MPICH_MAX_THREAD_SAFETY': 'multiple',
+            'MPICH_USE_DMAPP_COLL': '1',
+            'PMI_NO_FORK': '1',
+            'KMP_HW_SUBSET': '9c,2t',
+            'KMP_AFFINITY': 'granularity=fine,compact'
+        }
+
+        self.executable = 'bin/xhpcg_avx2'
+        self.executable_opts = ['--nx=%d' % self.problem_size,
+                                '--ny=%d' % self.problem_size,
+                                '--nz=%d' % self.problem_size, '-t2']
+
+        self.reference = {
+            'dom:mc': {
+                'gflops': (22, -0.1, None, 'Gflop/s')
+            },
+            'daint:mc': {
+                'gflops': (22, -0.1, None, 'Gflop/s')
+            },
+            'dom:gpu': {
+                'gflops': (10.7, -0.1, None, 'Gflop/s')
+            },
+            'daint:gpu': {
+                'gflops': (10.7, -0.1, None, 'Gflop/s')
+            },
+        }
+
+        self.maintainers = ['SK']
+        self.tags = {'diagnostic'}
+
+    @property
+    @sn.sanity_function
+    def num_tasks_assigned(self):
+        return self.job.num_tasks
+
+    @property
+    @sn.sanity_function
+    def outfile_lazy(self):
+        pattern = 'n%d-%dp-%dt-*.yaml' % (self.problem_size,
+                                          self.job.num_tasks,
+                                          self.num_cpus_per_task)
+        return sn.getitem(sn.glob(pattern), 0)
+
+    def setup(self, partition, environ, **job_opts):
+        if partition.fullname in ['daint:gpu', 'dom:gpu']:
+            self.num_tasks_per_node = 2
+            self.num_cpus_per_task = 12
+        else:
+            self.num_tasks_per_node = 4
+            self.num_cpus_per_task = 18
+
+        # since this is a flexible test, we divide the extracted
+        # performance by the number of nodes and compare
+        # against a single reference
+        num_nodes = self.num_tasks_assigned / self.num_tasks_per_node
+        self.perf_patterns = {
+            'gflops': sn.extractsingle(
+                r'HPCG result is VALID with a GFLOP\/s rating of:\s*'
+                r'(?P<perf>\S+)',
+                self.outfile_lazy, 'perf',  float) / num_nodes
+        }
+
+        self.sanity_patterns = sn.all([
+            sn.assert_eq(4, sn.count(
+                sn.findall(r'PASSED', self.outfile_lazy))),
+            sn.assert_eq(0, self.num_tasks_assigned % self.num_tasks_per_node)
+        ])
+
+        super().setup(partition, environ, **job_opts)

cscs-checks/microbenchmarks/hpcg/src/Make.CrayXC

@@ -0,0 +1,149 @@
+#===============================================================================
+# Copyright 2014-2018 Intel Corporation All Rights Reserved.
+#
+# The source code,  information  and material  ("Material") contained  herein is
+# owned by Intel Corporation or its  suppliers or licensors,  and  title to such
+# Material remains with Intel  Corporation or its  suppliers or  licensors.  The
+# Material  contains  proprietary  information  of  Intel or  its suppliers  and
+# licensors.  The Material is protected by  worldwide copyright  laws and treaty
+# provisions.  No part  of  the  Material   may  be  used,  copied,  reproduced,
+# modified, published,  uploaded, posted, transmitted,  distributed or disclosed
+# in any way without Intel's prior express written permission.  No license under
+# any patent,  copyright or other  intellectual property rights  in the Material
+# is granted to  or  conferred  upon  you,  either   expressly,  by implication,
+# inducement,  estoppel  or  otherwise.  Any  license   under such  intellectual
+# property rights must be express and approved by Intel in writing.
+#
+# Unless otherwise agreed by Intel in writing,  you may not remove or alter this
+# notice or  any  other  notice   embedded  in  Materials  by  Intel  or Intel's
+# suppliers or licensors in any way.
+#===============================================================================
+
+# -- High Performance Conjugate Gradient Benchmark (HPCG)
+#    HPCG - 2.1 - January 31, 2014
+#
+#    Michael A. Heroux
+#    Scalable Algorithms Group, Computing Research Center
+#    Sandia National Laboratories, Albuquerque, NM
+#
+#    Piotr Luszczek
+#    Jack Dongarra
+#    University of Tennessee, Knoxville
+#    Innovative Computing Laboratory
+#    (C) Copyright 2013 All Rights Reserved
+#
+# -- Copyright notice and Licensing terms:
+#
+# Redistribution  and  use in  source and binary forms, with or without
+# modification, are  permitted provided  that the following  conditions
+# are met:
+#
+# 1. Redistributions  of  source  code  must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce  the above copyright
+# notice, this list of conditions,  and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# 3. The name of the  University,  the name of the  Laboratory,  or the
+# names  of  its  contributors  may  not  be used to endorse or promote
+# products  derived   from   this  software  without  specific  written
+# permission.
+#
+# -- Disclaimer:
+#
+# THIS  SOFTWARE  IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,  INCLUDING,  BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY
+# OR  CONTRIBUTORS  BE  LIABLE FOR ANY  DIRECT,  INDIRECT,  INCIDENTAL,
+# SPECIAL,  EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES  (INCLUDING,  BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA OR PROFITS; OR BUSINESS INTERRUPTION)  HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT,  STRICT LIABILITY,  OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#******************************************************************************
+
+# ----------------------------------------------------------------------
+# - shell --------------------------------------------------------------
+# ----------------------------------------------------------------------
+#
+SHELL        = /bin/sh
+#
+CD           = cd
+CP           = cp
+LN_S         = ln -s -f
+MKDIR        = mkdir -p
+RM           = /bin/rm -f
+TOUCH        = touch
+#
+# ----------------------------------------------------------------------
+# - HPCG Directory Structure / HPCG library ------------------------------
+# ----------------------------------------------------------------------
+#
+TOPdir       = .
+SRCdir       = $(TOPdir)/src
+INCdir       = $(TOPdir)/src
+BINdir       = $(TOPdir)/bin
+#
+# ----------------------------------------------------------------------
+# - Message Passing library (MPI) --------------------------------------
+# ----------------------------------------------------------------------
+# MPinc tells the  C  compiler where to find the Message Passing library
+# header files,  MPlib  is defined  to be the name of  the library to be
+# used. The variable MPdir is only used for defining MPinc and MPlib.
+#
+MPdir        = 
+MPinc        = 
+MPlib        = -Wl,--whole-archive,-ldmapp,--no-whole-archive
+#
+#
+# ----------------------------------------------------------------------
+# - HPCG includes / libraries / specifics -------------------------------
+# ----------------------------------------------------------------------
+#
+HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc)
+HPCG_LIBS     = $(MPlib)
+#
+# - Compile time options -----------------------------------------------
+#
+# -DHPCG_NO_MPI		Define to disable MPI
+# -DHPCG_NO_OPENMP	Define to disable OPENMP
+# -DHPCG_DEBUG       	Define to enable debugging output
+# -DHPCG_DETAILED_DEBUG Define to enable very detailed debugging output
+#
+# By default HPCG will:
+#    *) Build with MPI enabled.
+#    *) Build with OpenMP enabled.
+#    *) Not generate debugging output.
+#
+HPCG_OPTS     =
+#
+# ----------------------------------------------------------------------
+#
+HPCG_DEFS     = -DMPICH_IGNORE_CXX_SEEK $(HPCG_OPTS) $(HPCG_INCLUDES)
+#
+# ----------------------------------------------------------------------
+# - Compilers / linkers - Optimization flags ---------------------------
+# ----------------------------------------------------------------------
+#
+CXX          = CC
+CXXFLAGS     = -xCORE-AVX2 -qopenmp -std=c++11 $(HPCG_DEFS)
+ifeq (yes, $(DBG))
+  CXXFLAGS  += -O0 -g -DHPCG_DEBUG
+else
+  CXXFLAGS  += -O3 -DNDEBUG
+endif
+#
+LINKER       = $(CXX)
+MKL_LIB=$(MKLROOT)/lib/intel64
+
+LINKFLAGS    = -z relro -z now -Wl,-R'$$ORIGIN/lib/intel64' -liomp5 -L$(MKL_LIB) -liomp5 -static-intel -mkl 
+#
+ARCHIVER     = ar
+ARFLAGS      = r
+RANLIB       = echo
+#
+# ----------------------------------------------------------------------
+xhpcg_suff = _avx2