Tim hpl

hpcbench/api.py

@@ -24,6 +24,9 @@ class Metrics(object):  # pragma pylint: disable=too-few-public-methods
     Milisecond = Metric('ms', float)
     Second = Metric('s', float)
     MegaBytesPerSecond = Metric('MB/s', float)
+    Cardinal = Metric('#', int)
+    Flops = Metric('flop/s', float)
+    Validity = Metric('boolean', str)
 
 
 class MetricsExtractor(with_metaclass(ABCMeta, object)):

hpcbench/benchmark/hpl.py

@@ -0,0 +1,149 @@
+"""the High-Performance Linpack Benchmark for Distributed-Memory Computers
+    http://www.netlib.org/benchmark/hpl/
+"""
+import os.path as osp
+import re
+import shutil
+
+from cached_property import cached_property
+
+from hpcbench.api import (
+    Benchmark,
+    Metric,
+    Metrics,
+    MetricsExtractor,
+)
+from hpcbench.toolbox.process import find_executable
+
+
+PRECISION_FORMULA = "||Ax-b||_oo/(eps*(||A||_oo*||x||_oo+||b||_oo)*N)"
+
+
+def get_precision_regex():
+    """Build regular expression used to extract precision
+    metric from command output"""
+    expr = re.escape(PRECISION_FORMULA)
+    expr += r'=\s*(\S*)\s.*\s([A-Z]*)'
+    return re.compile(expr)
+
+
+class HPLExtractor(MetricsExtractor):
+    """Ignore stdout until this line"""
+    STDOUT_IGNORE_PRIOR = (
+        "T/V                N    NB"
+        "     P     Q               Time                 Gflops"
+    )
+    SECTIONS = ['flops', 'precision']
+
+    REGEX = dict(
+        flops=re.compile(
+            r'^[\w]+[\s]+([\d]+)[\s]+([\d]+)[\s]+([\d]+)[\s]+'
+            r'([\d]+)[\s]+([\d.]+)[\s]+([\d.]+e[+-][\d]+)'
+        ),
+        precision=get_precision_regex()
+    )
+
+    METRICS = dict(
+        size_n=Metrics.Cardinal,
+        size_nb=Metrics.Cardinal,
+        size_p=Metrics.Cardinal,
+        size_q=Metrics.Cardinal,
+        time=Metrics.Second,
+        flops=Metrics.Flops,
+        validity=Metrics.Validity,
+        precision=Metric(unit='', type=float)
+    )
+
+    METRICS_NAMES = set(METRICS)
+
+    @property
+    def metrics(self):
+        """ The metrics to be extracted.
+            This property can not be replaced, but can be mutated as required
+        """
+        return HPLExtractor.METRICS
+
+    def extract(self, outdir, metas):
+        metrics = {}
+        # parse stdout and extract desired metrics
+        with open(self.stdout(outdir)) as istr:
+            for line in istr:
+                if line.strip() == HPLExtractor.STDOUT_IGNORE_PRIOR:
+                    break
+            for line in istr:
+                line = line.strip()
+
+                for sect in self.SECTIONS:
+                    search = HPLExtractor.REGEX[sect].search(line)
+                    if search:
+                        if sect == 'flops':
+                            metrics["size_n"] = int(search.group(1))
+                            metrics["size_nb"] = int(search.group(2))
+                            metrics["size_p"] = int(search.group(3))
+                            metrics["size_q"] = int(search.group(4))
+                            metrics["time"] = float(search.group(5))
+                            metrics["flops"] = float(search.group(6))
+                        elif sect == 'precision':
+                            metrics["precision"] = float(search.group(1))
+                            metrics["validity"] = str(search.group(2))
+
+        # ensure all metrics have been extracted
+        unset_attributes = HPLExtractor.METRICS_NAMES - set(metrics)
+        if any(unset_attributes):
+            error = \
+                'Could not extract some metrics: %s\n' \
+                'metrics setted are: %s'
+            raise Exception(error % (' ,'.join(unset_attributes),
+                                     ' ,'.join(set(metrics))))
+        return metrics
+
+
+class HPL(Benchmark):
+    """Benchmark wrapper for the HPLbench utility
+    """
+    DEFAULT_THREADS = [1]
+    DEFAULT_DEVICE = 'cpu'
+    DEFAULT_EXECUTABLE = 'xhpl'
+
+    def __init__(self):
+        # locate `stream_c` executable
+        super(HPL, self).__init__(
+            attributes=dict(
+                threads=HPL.DEFAULT_THREADS,
+                data="",
+                device=HPL.DEFAULT_DEVICE,
+                executable=HPL.DEFAULT_EXECUTABLE
+            )
+        )
+    name = 'hpl'
+
+    description = "Provides Intensive FLOPS benchmark."
+
+    @cached_property
+    def executable(self):
+        """Get absolute path to executable
+        """
+        return find_executable(self.attributes['executable'])
+
+    @property
+    def execution_matrix(self):
+        yield dict(
+            category=HPL.DEFAULT_DEVICE,
+            command=[
+                './' + osp.basename(self.executable),
+            ],
+            environment=dict(
+                OMP_NUM_THREADS=str(self.attributes['threads'][0]),
+                KMP_AFFINITY='scatter'
+            ),
+        )
+
+    @cached_property
+    def metrics_extractors(self):
+        return HPLExtractor()
+
+    def pre_execute(self, execution):
+        data = self.attributes['data']
+        with open('HPL.dat', 'w') as ostr:
+            ostr.write(data)
+        shutil.copy(self.executable, '.')

setup.py

@@ -65,5 +65,6 @@
         [hpcbench.benchmarks]
         sysbench = hpcbench.benchmark.sysbench
         stream = hpcbench.benchmark.stream
+        hpl = hpcbench.benchmark.hpl
     """
 )

tests/benchmark/benchmark.py

@@ -12,6 +12,7 @@
     MetricsExtractor,
 )
 from hpcbench.driver import YAML_REPORT_FILE, MetricsDriver
+from hpcbench.toolbox.collections_ext import dict_merge
 from hpcbench.toolbox.contextlib_ext import (
     mkdtemp,
     pushd,
@@ -43,6 +44,14 @@ def get_expected_metrics(self, category):
         """
         raise NotImplementedError
 
+    @property
+    def attributes(self):
+        """
+        :return: attributes to merge on the Benchmark instance
+        :rtype: dictionary
+        """
+        return {}
+
     def create_sample_run(self, category):
         pyfile = inspect.getfile(self.__class__)
         for output in ['stdout', 'stderr']:
@@ -86,7 +95,7 @@ def check_category_metrics(self, category):
                 report = md()
                 parsed_metrics = report.get('metrics', {})
                 expected_metrics = self.get_expected_metrics(category)
-                assert parsed_metrics == expected_metrics
+                self.assertEqual(parsed_metrics, expected_metrics)
 
     def test_has_description(self):
         clazz = self.get_benchmark_clazz()
@@ -95,6 +104,10 @@ def test_has_description(self):
     def test_execution_matrix(self):
         clazz = self.get_benchmark_clazz()
         benchmark = clazz()
+        dict_merge(
+            benchmark.attributes,
+            self.attributes
+        )
         exec_matrix = benchmark.execution_matrix
         exec_matrix = list(exec_matrix)
         assert isinstance(exec_matrix, list)

tests/benchmark/test_hpl.cpu.stdout

@@ -0,0 +1,75 @@
+================================================================================
+HPLinpack 2.2  --  High-Performance Linpack benchmark  --   February 24, 2016
+Written by A. Petitet and R. Clint Whaley,  Innovative Computing Laboratory, UTK
+Modified by Piotr Luszczek, Innovative Computing Laboratory, UTK
+Modified by Julien Langou, University of Colorado Denver
+================================================================================
+
+An explanation of the input/output parameters follows:
+T/V    : Wall time / encoded variant.
+N      : The order of the coefficient matrix A.
+NB     : The partitioning blocking factor.
+P      : The number of process rows.
+Q      : The number of process columns.
+Time   : Time in seconds to solve the linear system.
+Gflops : Rate of execution for solving the linear system.
+
+The following parameter values will be used:
+
+N      :    2096 
+NB     :     192 
+PMAP   : Row-major process mapping
+P      :       2 
+Q      :       2 
+PFACT  :   Right 
+NBMIN  :       4 
+NDIV   :       2 
+RFACT  :   Crout 
+BCAST  :  1ringM 
+DEPTH  :       1 
+SWAP   : Mix (threshold = 64)
+L1     : transposed form
+U      : transposed form
+EQUIL  : yes
+ALIGN  : 8 double precision words
+
+--------------------------------------------------------------------------------
+
+- The matrix A is randomly generated for each test.
+- The following scaled residual check will be computed:
+      ||Ax-b||_oo / ( eps * ( || x ||_oo * || A ||_oo + || b ||_oo ) * N )
+- The relative machine precision (eps) is taken to be               1.110223e-16
+- Computational tests pass if scaled residuals are less than                16.0
+
+Column=000000192 Fraction= 9.2% Gflops=1.637e+02
+Column=000000384 Fraction=18.3% Gflops=5.360e+01
+Column=000000576 Fraction=27.5% Gflops=2.771e+01
+Column=000000768 Fraction=36.6% Gflops=2.919e+01
+Column=000000960 Fraction=45.8% Gflops=2.504e+01
+Column=000001152 Fraction=55.0% Gflops=2.526e+01
+Column=000001344 Fraction=64.1% Gflops=2.312e+01
+Column=000001536 Fraction=73.3% Gflops=2.274e+01
+Column=000001728 Fraction=82.4% Gflops=2.180e+01
+Column=000001920 Fraction=91.6% Gflops=2.137e+01
+================================================================================
+T/V                N    NB     P     Q               Time                 Gflops
+--------------------------------------------------------------------------------
+WR11C2R4        2096   192     2     2               0.29              2.098e+01
+HPL_pdgesv() start time Wed Aug 23 13:31:46 2017
+
+HPL_pdgesv() end time   Wed Aug 23 13:31:46 2017
+
+--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV-
+Max aggregated wall time rfact . . . :               0.09
++ Max aggregated wall time pfact . . :               0.08
++ Max aggregated wall time mxswp . . :               0.07
+Max aggregated wall time update  . . :               0.21
++ Max aggregated wall time laswp . . :               0.08
+Max aggregated wall time up tr sv  . :               0.00
+--------------------------------------------------------------------------------
+||Ax-b||_oo/(eps*(||A||_oo*||x||_oo+||b||_oo)*N)=        0.0051555 ...... PASSED
+================================================================================
+
+Finished      1 tests with the following results:
+              1 tests completed and passed residual checks,
+              0 tests completed and failed residual checks,

tests/benchmark/test_hpl.py

@@ -0,0 +1,32 @@
+import unittest
+
+from hpcbench.benchmark.hpl import HPL
+from . benchmark import AbstractBenchmarkTest
+
+
+class TestHpl(AbstractBenchmarkTest, unittest.TestCase):
+    EXPECTED_METRICS = dict(
+        size_n=2096,
+        size_nb=192,
+        size_p=2,
+        size_q=2,
+        time=0.29,
+        flops=2.098e+01,
+        validity="PASSED",
+        precision=0.0051555,
+    )
+
+    def get_benchmark_clazz(self):
+        return HPL
+
+    def get_expected_metrics(self, category):
+        return TestHpl.EXPECTED_METRICS
+
+    def get_benchmark_categories(self):
+        return [self.get_benchmark_clazz().DEFAULT_DEVICE]
+
+    @property
+    def attributes(self):
+        return dict(
+            executable='/path/to/fake'
+        )