Merge 413fc53 into 4f059a3

rebench/model/data_point.py

@@ -76,3 +76,6 @@ def measurements_as_dict(self, criteria):
             'it': iteration,
             'm': data
         }
+
+    def __repr__(self):
+        return "DataPoint(" + str(self.run_id) + ", " + str(self._measurements) + ")"

rebench/model/measurement.py

@@ -71,3 +71,6 @@ def as_dict(self):
             'u': self.unit,
             'v': self.value
         }
+
+    def __repr__(self):
+        return "Measurement(%s, %s, %s)" % (self.value, self.unit, self.criterion)

rebench/model/run_id.py

@@ -327,13 +327,14 @@ def as_str_list(self):
         return result
 
     def as_dict(self):
+        extra_args = self.benchmark.extra_args
         return {
             'benchmark': self.benchmark.as_dict(),
             'cores': self.cores,
             'inputSize': self.input_size,
             'varValue': self.var_value,
             'machine': self.machine,
-            'extraArgs': str(self.benchmark.extra_args),
+            'extraArgs': extra_args if extra_args is None else str(extra_args),
             'cmdline': self.cmdline(),
             'location': self.location
         }

rebench/tests/features/issue_16_multiple_data_points_test.py

@@ -35,8 +35,10 @@ def setUp(self):
         super(Issue16MultipleDataPointsTest, self).setUp()
         self._set_path(__file__)
 
-    def _records_data_points(self, exp_name, num_data_points):
-        cnf = Configurator(load_config(self._path + '/issue_16.conf'), DataStore(self.ui),
+    def _records_data_points(self, exp_name, num_data_points, yaml=None):
+        if yaml is None:
+            yaml = load_config(self._path + '/issue_16.conf')
+        cnf = Configurator(yaml, DataStore(self.ui),
                            self.ui, exp_name=exp_name,
                            data_file=self._tmp_file)
         runs = cnf.get_runs()
@@ -65,3 +67,21 @@ def test_associates_measurements_and_data_points_correctly(self):
                                               point.get_measurements()):
                 self.assertEqual(criterion, measurement.criterion)
                 self.assertEqual(i, int(measurement.value))
+
+    def test_not_every_criterion_has_to_be_present_for_all_iterations(self):
+        yaml = load_config(self._path + '/issue_16.conf')
+        yaml['executors']['TestRunner']['executable'] = 'issue_16_vm2.py'
+        data_points = self._records_data_points('Test1', 10, yaml)
+        for point, i in zip(data_points, list(range(0, 10))):
+            criteria = []
+            if i % 2 == 0:
+                criteria.append("bar")
+            if i % 3 == 0:
+                criteria.append("baz")
+            if i % 2 == 1:
+                criteria.append("foo")
+            criteria.append("total")
+
+            for criterion, m in zip(criteria, point.get_measurements()):
+                self.assertEqual(criterion, m.criterion)
+                self.assertEqual(i, int(m.value))

rebench/tests/features/issue_16_vm2.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# simple script emulating an executor generating benchmark results
+import sys
+
+print(sys.argv)
+
+print("Harness Name: ", sys.argv[1])
+print("Bench Name:", sys.argv[2])
+print("Input Size: ", sys.argv[3])
+
+INPUT_SIZE = int(sys.argv[3])
+
+for i in range(0, INPUT_SIZE):
+    if i % 2 == 0:
+        print("RESULT-bar:   %d.%d" % (i, i))
+    if i % 3 == 0:
+        print("RESULT-baz:   %d.%d" % (i, i))
+    if i % 2 == 1:
+        print("RESULT-foo:   %d.%d" % (i, i))
+
+    print("RESULT-total: %d.%d" % (i, i))