Merge 977c170 into 4b81ee1

rebench/model/run_id.py

@@ -178,20 +178,19 @@ def close_files(self):
         for persistence in self._persistence:
             persistence.close()
 
-    def _new_data_point(self, data_point):
+    def _new_data_point(self, data_point, warmup):
         self._max_invocation = max(self._max_invocation, data_point.invocation)
         if self._total_unit is None:
             self._total_unit = data_point.get_total_unit()
+        if not warmup:
+            self._statistics.add_sample(data_point.get_total_value())
 
-    def loaded_data_point(self, data_point):
-        self._new_data_point(data_point)
-        self._statistics.add_sample(data_point.get_total_value())
+    def loaded_data_point(self, data_point, warmup):
+        self._new_data_point(data_point, warmup)
 
     def add_data_point(self, data_point, warmup):
-        self._new_data_point(data_point)
+        self._new_data_point(data_point, warmup)
 
-        if not warmup:
-            self._statistics.add_sample(data_point.get_total_value())
         for persistence in self._persistence:
             persistence.persist_data_point(data_point)
 

rebench/persistence.py

@@ -167,7 +167,9 @@ def _process_lines(self, data_file, runs, filtered_data_file):
                 data_point.add_measurement(measurement)
 
                 if measurement.is_total():
-                    run_id.loaded_data_point(data_point)
+                    run_id.loaded_data_point(data_point,
+                                             (measurement.iteration <= run_id.warmup_iterations
+                                              if run_id.warmup_iterations else False))
                     data_point = DataPoint(run_id)
 
             except ValueError as err:

rebench/tests/bugs/issue_111.conf

@@ -0,0 +1,27 @@
+default_experiment: Test
+
+benchmark_suites:
+    Suite:
+        gauge_adapter: RebenchLog
+        command: " "
+        benchmarks:
+            - Bench
+
+executors:
+    TestRunner:
+        path: .
+        executable: issue_111_vm.py
+
+experiments:
+    test-warmup-0:
+        warmup: 0
+        suites:
+            - Suite
+        executions:
+            - TestRunner
+    test-warmup-2:
+        warmup: 2
+        suites:
+         - Suite
+        executions:
+         - TestRunner

rebench/tests/bugs/issue_111_report_ignores_warmup_test.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2019 Stefan Marr <http://www.stefan-marr.de/>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from ..rebench_test_case import ReBenchTestCase
+
+from ...configurator import Configurator, load_config
+from ...executor import Executor
+from ...persistence import DataStore
+
+
+class Issue111Test(ReBenchTestCase):
+
+    def setUp(self):
+        super(Issue111Test, self).setUp()
+        self._set_path(__file__)
+
+    def test_invocation_and_mean_with_warmup_2(self):
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_111.conf'),
+                           ds, self._ui, exp_name='test-warmup-2', data_file=self._tmp_file)
+        runs = cnf.get_runs()
+        ds.load_data(runs, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(runs, False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 7, 1)
+        run = runs.pop()
+        self.assertEqual(run.get_mean_of_totals(), 10)
+
+        # Reload data from file, and confirm we get the same result
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_111.conf'),
+                           ds, self._ui, exp_name='test-warmup-2', data_file=self._tmp_file)
+        runs = cnf.get_runs()
+        ds.load_data(runs, False)
+
+        self._assert_runs(cnf, 1, 7, 1)
+        run = runs.pop()
+        self.assertEqual(run.get_mean_of_totals(), 10)
+
+    def test_invocation_and_mean_with_warmup_0(self):
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_111.conf'),
+                           ds, self._ui, exp_name='test-warmup-0', data_file=self._tmp_file)
+        runs = cnf.get_runs()
+        ds.load_data(runs, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(runs, False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 9, 1)
+        run = runs.pop()
+        self.assertEqual(run.get_mean_of_totals(), 230)
+
+        # Reload data from file, and confirm we get the same result
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_111.conf'),
+                           ds, self._ui, exp_name='test-warmup-0', data_file=self._tmp_file)
+        runs = cnf.get_runs()
+        ds.load_data(runs, False)
+
+        self._assert_runs(cnf, 1, 9, 1)
+        run = runs.pop()
+        self.assertEqual(run.get_mean_of_totals(), 230)

rebench/tests/bugs/issue_111_vm.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# simple script emulating a failing benchmark
+from __future__ import print_function
+
+print("Bench: iterations=1 runtime: 1000ms")
+print("Bench: iterations=1 runtime: 1000ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")
+print("Bench: iterations=1 runtime: 10ms")

rebench/tests/bugs/issue_112_invocations_setting_ignored_test.py

@@ -1,3 +1,22 @@
+# Copyright (c) 2019 Stefan Marr <http://www.stefan-marr.de/>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 from ..rebench_test_case import ReBenchTestCase
 
 from ...persistence import DataStore

rebench/tests/model/runs_config_test.py

@@ -43,12 +43,12 @@ def test_termination_check_basic(self):
         for i in range(1, 10):
             dp = DataPoint(self._run)
             dp.add_measurement(Measurement(i, 1, 0, 'ms', self._run))
-            self._run.loaded_data_point(dp)
+            self._run.loaded_data_point(dp, False)
         self.assertFalse(check.should_terminate(0))
 
         dp = DataPoint(self._run)
         dp.add_measurement(Measurement(10, 1, 0, 'ms', self._run))
-        self._run.loaded_data_point(dp)
+        self._run.loaded_data_point(dp, False)
         self.assertTrue(check.should_terminate(0))
 
     def test_terminate_not_determine_by_number_of_data_points(self):