babelstream: new benchmark

hpcbench/benchmark/babelstream.py

@@ -0,0 +1,139 @@
+"""https://github.com/UoB-HPC/BabelStream
+
+Usage: ./cuda-stream [OPTIONS]
+
+Options:
+  -h  --help               Print the message
+      --list               List available devices
+      --device     INDEX   Select device at INDEX
+  -s  --arraysize  SIZE    Use SIZE elements in the array
+  -n  --numtimes   NUM     Run the test NUM times (NUM >= 2)
+      --float              Use floats (rather than doubles)
+      --triad-only         Only run triad
+      --csv                Output as csv table
+
+
+"""
+
+import csv
+import subprocess
+
+from cached_property import cached_property
+
+from hpcbench.api import (
+    Benchmark,
+    Metrics,
+    MetricsExtractor,
+)
+from hpcbench.toolbox.functools_ext import listify
+from hpcbench.toolbox.process import find_executable
+
+
+class BabelStreamExtractor(MetricsExtractor):
+    OPERATIONS = {
+        'Copy',
+        'Mul',
+        'Add',
+        'Triad',
+        'Dot',
+    }
+    METRICS = dict(
+        max_mbytes_per_sec=('bandwidth', Metrics.MegaBytesPerSecond),
+    )
+
+    @property
+    def metrics(self):
+        eax = {}
+        for metric in BabelStreamExtractor.METRICS.values():
+            for op in BabelStreamExtractor.OPERATIONS:
+                name = op.lower() + '_' + metric[0]
+                eax[name] = metric[1]
+        return eax
+
+    @listify(wrapper=dict)
+    def extract_metrics(self, outdir, metas):
+        with open(self.stdout(outdir)) as istr:
+            istr.readline()
+            istr.readline()
+            reader = csv.DictReader(istr, delimiter=',')
+            for row in reader:
+                op = row['function']
+                for metric, desc in BabelStreamExtractor.METRICS.items():
+                    value = float(row[metric])
+                    name = op.lower() + '_' + desc[0]
+                    yield name, value
+
+
+class BabelStream(Benchmark):
+    DEFAULT_EXECUTABLE = 'cuda-stream'
+    CATEGORY = 'stream'
+
+    name = 'babelstream'
+
+    description = 'STREAM, for lots of different devices'
+
+    def __init__(self):
+        super(BabelStream, self).__init__(
+            attributes=dict(
+                executable=BabelStream.DEFAULT_EXECUTABLE
+            )
+        )
+
+    @property
+    def devices(self):
+        """List of devices to test
+        """
+        eax = self.attributes.get('devices')
+        if eax is None:
+            eax = self._all_devices
+        if not isinstance(eax, list):
+            eax = [eax]
+        return [str(dev) for dev in eax]
+
+    @cached_property
+    def executable(self):
+        """babel-stream executable to use, "cuda-stream" for instance"""
+        return self.attributes['executable']
+
+    @cached_property
+    def options(self):
+        """Additional optional command line arguments"""
+        return self.attributes.get('options', [])
+
+    @property
+    @listify
+    def _all_devices(self):
+        binary = find_executable(self.executable)
+        listing_devices = False
+        for line in subprocess.check_output([binary, '--list']).splitlines():
+            line = line.strip()
+            if line == 'Devices:':
+                listing_devices = True
+                continue
+            if listing_devices:
+                device = line.split(':', 1)[0]
+                yield device
+
+    def execution_matrix(self, context):
+        for device in self.devices:
+            yield dict(
+                category=BabelStream.CATEGORY,
+                command=self._command(device),
+                metas=dict(
+                    device=device,
+                )
+            )
+
+    def _command(self, device):
+        cmd = [
+            find_executable(self.executable),
+            '--device',
+            device
+        ] + self.options
+        if '--csv' not in cmd:
+            cmd.append('--csv')
+        return cmd
+
+    @cached_property
+    def metrics_extractors(self):
+        return BabelStreamExtractor()

setup.py

@@ -44,6 +44,7 @@
         'docopt==0.6.2',
         'elasticsearch==6.0.0',
         'jinja2==2.10',
+        'mock==2.0.0',
         'numpy==1.13.3',
         'PyYAML>=3.12',
         'six==1.11',
@@ -76,6 +77,7 @@
         ben-umb = hpcbench.cli.benumb:main
 
         [hpcbench.benchmarks]
+        babelstream = hpcbench.benchmark.babelstream
         hpl = hpcbench.benchmark.hpl
         imb = hpcbench.benchmark.imb
         ior = hpcbench.benchmark.ior

tests/benchmark/test_babelstream.py

@@ -0,0 +1,92 @@
+import textwrap
+import unittest
+
+import mock
+
+from hpcbench.benchmark.babelstream import BabelStream
+from . benchmark import AbstractBenchmarkTest
+
+
+class TestBabelStream(AbstractBenchmarkTest, unittest.TestCase):
+    EXPECTED_METRICS = dict(
+        copy_bandwidth=152212.0,
+        mul_bandwidth=152026.0,
+        add_bandwidth=150275.0,
+        triad_bandwidth=150272.0,
+        dot_bandwidth=144512.0,
+    )
+
+    def get_benchmark_clazz(self):
+        return BabelStream
+
+    def get_expected_metrics(self, category):
+        del category
+        return TestBabelStream.EXPECTED_METRICS
+
+    @property
+    def attributes(self):
+        return dict(
+            executable='/fake-stream',
+            devices=["0"],
+        )
+
+    def get_benchmark_categories(self):
+        return [BabelStream.CATEGORY]
+
+    def test_custom_attributes(self):
+        self.assertExecutionMatrix(
+            dict(
+                devices=42,
+                executable='/foo',
+                options=[
+                    '--csv',
+                ],
+            ),
+            [
+                dict(
+                    category='stream',
+                    command=[
+                        '/foo',
+                        '--device',
+                        '42',
+                        '--csv'
+                    ],
+                    metas=dict(device='42')
+                )
+            ]
+        )
+
+    DEVICES_MOCK_OUTPUT = textwrap.dedent("""\
+    Devices:
+    device01: Tesla K20m
+    device02: Tesla K20m
+    """)
+
+    @mock.patch('subprocess.check_output')
+    def test_list_devices(self, mock_co):
+        mock_co.return_value = TestBabelStream.DEVICES_MOCK_OUTPUT
+        self.assertExecutionMatrix(
+            dict(executable='/bar'),
+            [
+                dict(
+                    category='stream',
+                    command=[
+                        '/bar',
+                        '--device',
+                        'device01',
+                        '--csv'
+                    ],
+                    metas=dict(device='device01'),
+                ),
+                dict(
+                    category='stream',
+                    command=[
+                        '/bar',
+                        '--device',
+                        'device02',
+                        '--csv'
+                    ],
+                    metas=dict(device='device02'),
+                ),
+            ]
+        )

tests/benchmark/test_babelstream.stream.stdout

@@ -0,0 +1,8 @@
+Using CUDA device Tesla K20m
+Driver: 8000
+function,num_times,n_elements,sizeof,max_mbytes_per_sec,min_runtime,max_runtime,avg_runtime
+Copy,100,33554432,8,152212,0.00352713,0.00354357,0.00353609,
+Mul,100,33554432,8,152026,0.00353144,0.00354848,0.00354003,
+Add,100,33554432,8,150275,0.00535889,0.00538235,0.00537029,
+Triad,100,33554432,8,150272,0.00535899,0.0053795,0.00536942,
+Dot,100,33554432,8,144512,0.00371507,0.00376136,0.00373652,