Merge PR #117: Fix handling of input size configuration and respect d…

…efault_experiment setting

rebench/model/experiment.py

@@ -70,7 +70,7 @@ def _compile_runs(self, configurator):
         for bench in self._benchmarks:
             if not configurator.run_filter.applies(bench):
                 continue
-            variables = bench.suite.variables
+            variables = bench.variables
             for cores in variables.cores:
                 for input_size in variables.input_sizes:
                     for var_val in variables.variable_values:

rebench/rebench.py

@@ -189,7 +189,7 @@ def shell_options(self):
     def determine_exp_name_and_filters(filters):
         exp_name = filters[0] if filters and (
             not filters[0].startswith("e:") and
-            not filters[0].startswith("s:")) else "all"
+            not filters[0].startswith("s:")) else None
         exp_filter = [f for f in filters if (f.startswith("e:") or
                                              f.startswith("s:"))]
         return exp_name, exp_filter

rebench/tests/bugs/issue_112_invocations_setting_ignored_test.py

@@ -30,11 +30,11 @@ def setUp(self):
         super(Issue112Test, self).setUp()
         self._set_path(__file__)
 
-    def test_invocation_setting_on_experiment(self):
+    def _test(self, exp_name, exp_result):
         # Executes first time
         ds = DataStore(self._ui)
         cnf = Configurator(load_config(self._path + '/issue_112.conf'),
-                           ds, self._ui, exp_name='ExpSetting', data_file=self._tmp_file)
+                           ds, self._ui, exp_name=exp_name, data_file=self._tmp_file)
         ds.load_data(None, False)
 
         # Has not executed yet, check that there is simply
@@ -43,49 +43,16 @@ def test_invocation_setting_on_experiment(self):
         ex = Executor(cnf.get_runs(), False, False, self._ui)
         ex.execute()
 
-        self._assert_runs(cnf, 1, 10, 10)
-
-    def test_invocation_setting_on_experiment_execution_detail(self):
-        # Executes first time
-        ds = DataStore(self._ui)
-        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
-                           ds, self._ui, exp_name='ExecSetting', data_file=self._tmp_file)
-        ds.load_data(None, False)
-
-        # Has not executed yet, check that there is simply
-        self._assert_runs(cnf, 1, 0, 0)
+        self._assert_runs(cnf, 1, exp_result, exp_result)
 
-        ex = Executor(cnf.get_runs(), False, False, self._ui)
-        ex.execute()
+    def test_invocation_setting_on_experiment(self):
+        self._test('ExpSetting', 10)
 
-        self._assert_runs(cnf, 1, 7, 7)
+    def test_invocation_setting_on_experiment_execution_detail(self):
+        self._test('ExecSetting', 7)
 
     def test_invocation_setting_for_global_run_details(self):
-        # Executes first time
-        ds = DataStore(self._ui)
-        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
-                           ds, self._ui, exp_name='GlobalSetting', data_file=self._tmp_file)
-        ds.load_data(None, False)
-
-        # Has not executed yet, check that there is simply
-        self._assert_runs(cnf, 1, 0, 0)
-
-        ex = Executor(cnf.get_runs(), False, False, self._ui)
-        ex.execute()
-
-        self._assert_runs(cnf, 1, 5, 5)
+        self._test('GlobalSetting', 5)
 
     def test_invocation_setting_in_suite(self):
-        # Executes first time
-        ds = DataStore(self._ui)
-        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
-                           ds, self._ui, exp_name='SuiteSetting', data_file=self._tmp_file)
-        ds.load_data(None, False)
-
-        # Has not executed yet, check that there is simply
-        self._assert_runs(cnf, 1, 0, 0)
-
-        ex = Executor(cnf.get_runs(), False, False, self._ui)
-        ex.execute()
-
-        self._assert_runs(cnf, 1, 3, 3)
+        self._test('SuiteSetting', 3)

rebench/tests/bugs/issue_117.conf

@@ -0,0 +1,53 @@
+default_experiment: Test
+
+benchmark_suites:
+    SuiteSettingOnBenchmark:
+        gauge_adapter: TestExecutor
+        command: no-exec1 %(benchmark)s %(input)s
+        benchmarks:
+            - Bench1:
+                input_sizes: [0, 10]
+
+    SuiteWithSetting:
+        input_sizes: [1, 2]
+        gauge_adapter: TestExecutor
+        command: no-exec2 %(benchmark)s %(input)s
+        benchmarks:
+            - Bench2
+
+    Suite:
+        gauge_adapter: TestExecutor
+        command: no-exec3 %(benchmark)s %(input)s
+        benchmarks:
+            - Bench3
+executors:
+    TestRunner:
+        path: .
+        executable: none-existing
+
+experiments:
+    ExpSetting:
+        input_sizes: [3, 4]
+        suites:
+         - SuiteSettingOnBenchmark
+         - SuiteWithSetting
+         - Suite
+        executions:
+         - TestRunner
+
+    ExecSetting:
+        executions:
+            - TestRunner:
+                  input_sizes: [5, 6]
+                  suites:
+                      - SuiteSettingOnBenchmark
+                      - SuiteWithSetting
+                      - Suite
+
+    BaseSetting:
+        suites:
+            - SuiteSettingOnBenchmark
+            - SuiteWithSetting
+            - Suite
+        executions:
+            - TestRunner

rebench/tests/bugs/issue_117_input_size_setting_ignored_test.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2019 Stefan Marr <http://www.stefan-marr.de/>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from ..rebench_test_case import ReBenchTestCase
+
+from ...persistence import DataStore
+from ...configurator import Configurator, load_config
+
+
+class Issue117Test(ReBenchTestCase):
+
+    def setUp(self):
+        super(Issue117Test, self).setUp()
+        self._set_path(__file__)
+
+    def _test(self, exp_name, num_runs, exp_input_sizes):
+        # Executes first time
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_117.conf'),
+                           ds, self._ui, exp_name=exp_name, data_file=self._tmp_file)
+        ds.load_data(None, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, num_runs, 0, 0)
+
+        runs = list(cnf.get_runs())
+        runs = sorted(runs, key=lambda e: e.cmdline())
+        for i in range(num_runs):
+            self.assertEqual(exp_input_sizes[i], runs[i].input_size,
+                             "expected input_size at idx %d" % i)
+
+    def test_input_size_setting_on_experiment(self):
+        self._test('ExpSetting', 6, [0, 10, 1, 2, 3, 4])
+
+    def test_input_size_setting_on_experiment_execution_detail(self):
+        self._test('ExecSetting', 6, [0, 10, 1, 2, 5, 6])
+
+    def test_input_size_setting_on_benchmark(self):
+        self._test('BaseSetting', 5, [0, 10, 1, 2, None])

rebench/tests/executor_test.py

@@ -164,7 +164,7 @@ def test_shell_options_with_executor_filter(self):
     def test_determine_exp_name_and_filters_empty(self):
         empty = []
         exp_name, exp_filter = ReBench.determine_exp_name_and_filters(empty)
-        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_name, None)
         self.assertEqual(exp_filter, [])
 
     def test_determine_exp_name_and_filters_all(self):
@@ -188,7 +188,7 @@ def test_determine_exp_name_and_filters_all_and_other(self):
     def test_determine_exp_name_and_filters_only_others(self):
         filters = ['e:bar', 's:b']
         exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters)
-        self.assertEqual(exp_name, "all")
+        self.assertEqual(exp_name, None)
         self.assertEqual(exp_filter, ['e:bar', 's:b'])