diff --git a/hpcbench/api.py b/hpcbench/api.py index 12ef0be..9e8b72e 100644 --- a/hpcbench/api.py +++ b/hpcbench/api.py @@ -24,6 +24,9 @@ class Metrics(object): # pragma pylint: disable=too-few-public-methods Milisecond = Metric('ms', float) Second = Metric('s', float) MegaBytesPerSecond = Metric('MB/s', float) + Cardinal = Metric('#', int) + Flops = Metric('flop/s', float) + Validity = Metric('boolean', str) class MetricsExtractor(with_metaclass(ABCMeta, object)): diff --git a/hpcbench/benchmark/hpl.py b/hpcbench/benchmark/hpl.py new file mode 100644 index 0000000..7aa6634 --- /dev/null +++ b/hpcbench/benchmark/hpl.py @@ -0,0 +1,149 @@ +"""the High-Performance Linpack Benchmark for Distributed-Memory Computers + http://www.netlib.org/benchmark/hpl/ +""" +import os.path as osp +import re +import shutil + +from cached_property import cached_property + +from hpcbench.api import ( + Benchmark, + Metric, + Metrics, + MetricsExtractor, +) +from hpcbench.toolbox.process import find_executable + + +PRECISION_FORMULA = "||Ax-b||_oo/(eps*(||A||_oo*||x||_oo+||b||_oo)*N)" + + +def get_precision_regex(): + """Build regular expression used to extract precision + metric from command output""" + expr = re.escape(PRECISION_FORMULA) + expr += r'=\s*(\S*)\s.*\s([A-Z]*)' + return re.compile(expr) + + +class HPLExtractor(MetricsExtractor): + """Ignore stdout until this line""" + STDOUT_IGNORE_PRIOR = ( + "T/V N NB" + " P Q Time Gflops" + ) + SECTIONS = ['flops', 'precision'] + + REGEX = dict( + flops=re.compile( + r'^[\w]+[\s]+([\d]+)[\s]+([\d]+)[\s]+([\d]+)[\s]+' + r'([\d]+)[\s]+([\d.]+)[\s]+([\d.]+e[+-][\d]+)' + ), + precision=get_precision_regex() + ) + + METRICS = dict( + size_n=Metrics.Cardinal, + size_nb=Metrics.Cardinal, + size_p=Metrics.Cardinal, + size_q=Metrics.Cardinal, + time=Metrics.Second, + flops=Metrics.Flops, + validity=Metrics.Validity, + precision=Metric(unit='', type=float) + ) + + METRICS_NAMES = set(METRICS) + + @property + def metrics(self): + """ The metrics to be extracted. + This property can not be replaced, but can be mutated as required + """ + return HPLExtractor.METRICS + + def extract(self, outdir, metas): + metrics = {} + # parse stdout and extract desired metrics + with open(self.stdout(outdir)) as istr: + for line in istr: + if line.strip() == HPLExtractor.STDOUT_IGNORE_PRIOR: + break + for line in istr: + line = line.strip() + + for sect in self.SECTIONS: + search = HPLExtractor.REGEX[sect].search(line) + if search: + if sect == 'flops': + metrics["size_n"] = int(search.group(1)) + metrics["size_nb"] = int(search.group(2)) + metrics["size_p"] = int(search.group(3)) + metrics["size_q"] = int(search.group(4)) + metrics["time"] = float(search.group(5)) + metrics["flops"] = float(search.group(6)) + elif sect == 'precision': + metrics["precision"] = float(search.group(1)) + metrics["validity"] = str(search.group(2)) + + # ensure all metrics have been extracted + unset_attributes = HPLExtractor.METRICS_NAMES - set(metrics) + if any(unset_attributes): + error = \ + 'Could not extract some metrics: %s\n' \ + 'metrics setted are: %s' + raise Exception(error % (' ,'.join(unset_attributes), + ' ,'.join(set(metrics)))) + return metrics + + +class HPL(Benchmark): + """Benchmark wrapper for the HPLbench utility + """ + DEFAULT_THREADS = [1] + DEFAULT_DEVICE = 'cpu' + DEFAULT_EXECUTABLE = 'xhpl' + + def __init__(self): + # locate `stream_c` executable + super(HPL, self).__init__( + attributes=dict( + threads=HPL.DEFAULT_THREADS, + data="", + device=HPL.DEFAULT_DEVICE, + executable=HPL.DEFAULT_EXECUTABLE + ) + ) + name = 'hpl' + + description = "Provides Intensive FLOPS benchmark." + + @cached_property + def executable(self): + """Get absolute path to executable + """ + return find_executable(self.attributes['executable']) + + @property + def execution_matrix(self): + yield dict( + category=HPL.DEFAULT_DEVICE, + command=[ + './' + osp.basename(self.executable), + ], + environment=dict( + OMP_NUM_THREADS=str(self.attributes['threads'][0]), + KMP_AFFINITY='scatter' + ), + ) + + @cached_property + def metrics_extractors(self): + return HPLExtractor() + + def pre_execute(self, execution): + data = self.attributes['data'] + with open('HPL.dat', 'w') as ostr: + ostr.write(data) + shutil.copy(self.executable, '.') diff --git a/setup.py b/setup.py index 1264650..5fa6216 100755 --- a/setup.py +++ b/setup.py @@ -65,5 +65,6 @@ [hpcbench.benchmarks] sysbench = hpcbench.benchmark.sysbench stream = hpcbench.benchmark.stream + hpl = hpcbench.benchmark.hpl """ ) diff --git a/tests/benchmark/benchmark.py b/tests/benchmark/benchmark.py index 8fb3827..cdbc481 100644 --- a/tests/benchmark/benchmark.py +++ b/tests/benchmark/benchmark.py @@ -12,6 +12,7 @@ MetricsExtractor, ) from hpcbench.driver import YAML_REPORT_FILE, MetricsDriver +from hpcbench.toolbox.collections_ext import dict_merge from hpcbench.toolbox.contextlib_ext import ( mkdtemp, pushd, @@ -43,6 +44,14 @@ def get_expected_metrics(self, category): """ raise NotImplementedError + @property + def attributes(self): + """ + :return: attributes to merge on the Benchmark instance + :rtype: dictionary + """ + return {} + def create_sample_run(self, category): pyfile = inspect.getfile(self.__class__) for output in ['stdout', 'stderr']: @@ -86,7 +95,7 @@ def check_category_metrics(self, category): report = md() parsed_metrics = report.get('metrics', {}) expected_metrics = self.get_expected_metrics(category) - assert parsed_metrics == expected_metrics + self.assertEqual(parsed_metrics, expected_metrics) def test_has_description(self): clazz = self.get_benchmark_clazz() @@ -95,6 +104,10 @@ def test_has_description(self): def test_execution_matrix(self): clazz = self.get_benchmark_clazz() benchmark = clazz() + dict_merge( + benchmark.attributes, + self.attributes + ) exec_matrix = benchmark.execution_matrix exec_matrix = list(exec_matrix) assert isinstance(exec_matrix, list) diff --git a/tests/benchmark/test_hpl.cpu.stdout b/tests/benchmark/test_hpl.cpu.stdout new file mode 100644 index 0000000..9c94471 --- /dev/null +++ b/tests/benchmark/test_hpl.cpu.stdout @@ -0,0 +1,75 @@ +================================================================================ +HPLinpack 2.2 -- High-Performance Linpack benchmark -- February 24, 2016 +Written by A. Petitet and R. Clint Whaley, Innovative Computing Laboratory, UTK +Modified by Piotr Luszczek, Innovative Computing Laboratory, UTK +Modified by Julien Langou, University of Colorado Denver +================================================================================ + +An explanation of the input/output parameters follows: +T/V : Wall time / encoded variant. +N : The order of the coefficient matrix A. +NB : The partitioning blocking factor. +P : The number of process rows. +Q : The number of process columns. +Time : Time in seconds to solve the linear system. +Gflops : Rate of execution for solving the linear system. + +The following parameter values will be used: + +N : 2096 +NB : 192 +PMAP : Row-major process mapping +P : 2 +Q : 2 +PFACT : Right +NBMIN : 4 +NDIV : 2 +RFACT : Crout +BCAST : 1ringM +DEPTH : 1 +SWAP : Mix (threshold = 64) +L1 : transposed form +U : transposed form +EQUIL : yes +ALIGN : 8 double precision words + +-------------------------------------------------------------------------------- + +- The matrix A is randomly generated for each test. +- The following scaled residual check will be computed: + ||Ax-b||_oo / ( eps * ( || x ||_oo * || A ||_oo + || b ||_oo ) * N ) +- The relative machine precision (eps) is taken to be 1.110223e-16 +- Computational tests pass if scaled residuals are less than 16.0 + +Column=000000192 Fraction= 9.2% Gflops=1.637e+02 +Column=000000384 Fraction=18.3% Gflops=5.360e+01 +Column=000000576 Fraction=27.5% Gflops=2.771e+01 +Column=000000768 Fraction=36.6% Gflops=2.919e+01 +Column=000000960 Fraction=45.8% Gflops=2.504e+01 +Column=000001152 Fraction=55.0% Gflops=2.526e+01 +Column=000001344 Fraction=64.1% Gflops=2.312e+01 +Column=000001536 Fraction=73.3% Gflops=2.274e+01 +Column=000001728 Fraction=82.4% Gflops=2.180e+01 +Column=000001920 Fraction=91.6% Gflops=2.137e+01 +================================================================================ +T/V N NB P Q Time Gflops +-------------------------------------------------------------------------------- +WR11C2R4 2096 192 2 2 0.29 2.098e+01 +HPL_pdgesv() start time Wed Aug 23 13:31:46 2017 + +HPL_pdgesv() end time Wed Aug 23 13:31:46 2017 + +--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV--VVV- +Max aggregated wall time rfact . . . : 0.09 ++ Max aggregated wall time pfact . . : 0.08 ++ Max aggregated wall time mxswp . . : 0.07 +Max aggregated wall time update . . : 0.21 ++ Max aggregated wall time laswp . . : 0.08 +Max aggregated wall time up tr sv . : 0.00 +-------------------------------------------------------------------------------- +||Ax-b||_oo/(eps*(||A||_oo*||x||_oo+||b||_oo)*N)= 0.0051555 ...... PASSED +================================================================================ + +Finished 1 tests with the following results: + 1 tests completed and passed residual checks, + 0 tests completed and failed residual checks, diff --git a/tests/benchmark/test_hpl.py b/tests/benchmark/test_hpl.py new file mode 100644 index 0000000..78a6230 --- /dev/null +++ b/tests/benchmark/test_hpl.py @@ -0,0 +1,32 @@ +import unittest + +from hpcbench.benchmark.hpl import HPL +from . benchmark import AbstractBenchmarkTest + + +class TestHpl(AbstractBenchmarkTest, unittest.TestCase): + EXPECTED_METRICS = dict( + size_n=2096, + size_nb=192, + size_p=2, + size_q=2, + time=0.29, + flops=2.098e+01, + validity="PASSED", + precision=0.0051555, + ) + + def get_benchmark_clazz(self): + return HPL + + def get_expected_metrics(self, category): + return TestHpl.EXPECTED_METRICS + + def get_benchmark_categories(self): + return [self.get_benchmark_clazz().DEFAULT_DEVICE] + + @property + def attributes(self): + return dict( + executable='/path/to/fake' + )