Merge PR #113: Fix how settings are inherited, strictly follow docume…

…ntation

rebench/configurator.py

@@ -22,7 +22,6 @@
 
 from .model.experiment import Experiment
 from .model.exp_run_details import ExpRunDetails
-from .model.exp_variables import ExpVariables
 from .model.reporting import Reporting
 from .model.executor import Executor
 from .ui import UIError, escape_braces
@@ -178,8 +177,7 @@ def __init__(self, raw_config, data_store, ui, cli_options=None, cli_reporter=No
 
         self._run_filter = _RunFilter(run_filter)
 
-        self._executors = self._compile_executors(raw_config.get('executors', {}))
-
+        self._executors = raw_config.get('executors', {})
         self._suites_config = raw_config.get('benchmark_suites', {})
 
         experiments = raw_config.get('experiments', {})
@@ -253,8 +251,11 @@ def data_store(self):
     def has_executor(self, executor_name):
         return executor_name in self._executors
 
-    def get_executor(self, executor_name):
-        return self._executors[executor_name]
+    def get_executor(self, executor_name, run_details, variables):
+        executor = Executor.compile(
+            executor_name, self._executors[executor_name],
+            run_details, variables, self._build_commands)
+        return executor
 
     def get_suite(self, suite_name):
         return self._suites_config[suite_name]
@@ -276,17 +277,6 @@ def get_runs(self):
             runs |= exp.get_runs()
         return runs
 
-    def _compile_executors(self, executors):
-        result = {}
-
-        variables = ExpVariables.empty()
-
-        for executor_name, details in executors.items():
-            result[executor_name] = Executor.compile(
-                executor_name, details, self._root_run_details, variables, self._build_commands)
-
-        return result
-
     def _compile_experiments(self, experiments):
         results = {}
 

rebench/model/benchmark_suite.py

@@ -27,7 +27,7 @@
 class BenchmarkSuite(object):
 
     @classmethod
-    def compile(cls, suite_name, suite, executor, run_details, variables, build_commands):
+    def compile(cls, suite_name, suite, executor, build_commands):
         gauge_adapter = suite.get('gauge_adapter')
         command = suite.get('command')
 
@@ -40,8 +40,8 @@ def compile(cls, suite_name, suite, executor, run_details, variables, build_comm
         description = suite.get('description')
         desc = suite.get('desc')
 
-        run_details = ExpRunDetails.compile(suite, run_details)
-        variables = ExpVariables.compile(suite, variables)
+        run_details = ExpRunDetails.compile(suite, executor.run_details)
+        variables = ExpVariables.compile(suite, executor.variables)
 
         return BenchmarkSuite(suite_name, executor, gauge_adapter, command, location,
                               build, benchmarks_config, description or desc, run_details, variables)

rebench/model/experiment.py

@@ -59,9 +59,7 @@ def __init__(self, name, description, data_file, reporting, run_details,
         self._persistence = self._data_store.get(data_file,
                                                  configurator.discard_old_data)
 
-        self._executors = self._compile_executors(executions, configurator)
-
-        self._suites = self._compile_benchmark_suites(
+        self._suites = self._compile_executors_and_benchmark_suites(
             executions, suites, configurator)
         self._benchmarks = self._compile_benchmarks()
         self._runs = self._compile_runs(configurator)
@@ -84,38 +82,28 @@ def _compile_runs(self, configurator):
                         run.add_persistence(self._persistence)
         return runs
 
-    def _compile_executors(self, executions, configurator):
-        executors = {}
-
-        for executor in executions:
-            executor_name, _ = value_with_optional_details(executor)
-            if not configurator.has_executor(executor_name):
-                raise ValueError("The executor '%s' requested in %s was not found."
-                                 % (executor, self._name))
-
-            executors[executor_name] = configurator.get_executor(executor_name)
+    def _compile_executors_and_benchmark_suites(self, executions, suites, configurator):
+        # we now assemble the executors and the benchmark suites
+        results = []
+        for executor_cfg in executions:
+            executor_name, executor_details = value_with_optional_details(executor_cfg)
 
-        return executors
+            run_details = self._run_details
+            variables = self._variables
 
-    def _compile_benchmark_suites(self, executions, suites, configurator):
-        # for each executor, we now assemble the benchmark suites
-        results = []
-        for executor in executions:
-            executor_name, executor_details = value_with_optional_details(executor)
-            global_executor = self._executors[executor_name]
-            run_details = global_executor.run_details
-            variables = global_executor.variables
             if executor_details:
                 run_details = ExpRunDetails.compile(executor_details, run_details)
                 variables = ExpVariables.compile(executor_details, variables)
                 suites_for_executor = executor_details.get('suites', suites)
             else:
                 suites_for_executor = suites
 
+            executor = configurator.get_executor(executor_name, run_details, variables)
+
             for suite_name in suites_for_executor:
                 suite = BenchmarkSuite.compile(
-                    suite_name, configurator.get_suite(suite_name), global_executor,
-                    run_details, variables, configurator.build_commands)
+                    suite_name, configurator.get_suite(suite_name), executor,
+                    configurator.build_commands)
                 results.append(suite)
 
         return results

rebench/tests/bugs/issue_112.conf

@@ -0,0 +1,49 @@
+default_experiment: Test
+
+runs:
+    invocations: 5
+
+benchmark_suites:
+    Suite:
+        gauge_adapter: TestExecutor
+        command: TestBenchMarks %(benchmark)s
+        benchmarks:
+            - Bench1
+
+    SuiteWithSetting:
+        invocations:  3
+        gauge_adapter: TestExecutor
+        command: TestBenchMarks %(benchmark)s
+        benchmarks:
+            - Bench1
+executors:
+    TestRunner1:
+        path: .
+        executable: vm-one-result.py
+
+experiments:
+    ExpSetting:
+        invocations: 10
+        suites:
+         - Suite
+        executions:
+         - TestRunner1
+
+    ExecSetting:
+        executions:
+            - TestRunner1:
+                  invocations: 7
+                  suites:
+                      - Suite
+
+    GlobalSetting:
+        suites:
+            - Suite
+        executions:
+            - TestRunner1
+
+    SuiteSetting:
+        suites:
+            - SuiteWithSetting
+        executions:
+            - TestRunner1

rebench/tests/bugs/issue_112_invocations_setting_ignored_test.py

@@ -0,0 +1,72 @@
+from ..rebench_test_case import ReBenchTestCase
+
+from ...persistence import DataStore
+from ...configurator import Configurator, load_config
+from ...executor import Executor
+
+
+class Issue112Test(ReBenchTestCase):
+
+    def setUp(self):
+        super(Issue112Test, self).setUp()
+        self._set_path(__file__)
+
+    def test_invocation_setting_on_experiment(self):
+        # Executes first time
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
+                           ds, self._ui, exp_name='ExpSetting', data_file=self._tmp_file)
+        ds.load_data(None, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(cnf.get_runs(), False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 10, 10)
+
+    def test_invocation_setting_on_experiment_execution_detail(self):
+        # Executes first time
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
+                           ds, self._ui, exp_name='ExecSetting', data_file=self._tmp_file)
+        ds.load_data(None, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(cnf.get_runs(), False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 7, 7)
+
+    def test_invocation_setting_for_global_run_details(self):
+        # Executes first time
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
+                           ds, self._ui, exp_name='GlobalSetting', data_file=self._tmp_file)
+        ds.load_data(None, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(cnf.get_runs(), False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 5, 5)
+
+    def test_invocation_setting_in_suite(self):
+        # Executes first time
+        ds = DataStore(self._ui)
+        cnf = Configurator(load_config(self._path + '/issue_112.conf'),
+                           ds, self._ui, exp_name='SuiteSetting', data_file=self._tmp_file)
+        ds.load_data(None, False)
+
+        # Has not executed yet, check that there is simply
+        self._assert_runs(cnf, 1, 0, 0)
+
+        ex = Executor(cnf.get_runs(), False, False, self._ui)
+        ex.execute()
+
+        self._assert_runs(cnf, 1, 3, 3)

rebench/tests/bugs/vm-one-result.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# simple script emulating an executor generating benchmark results
+from __future__ import print_function
+
+import sys
+import random
+
+print(sys.argv)
+
+print("Harness Name: ", sys.argv[1])
+print("Bench Name:", sys.argv[2])
+
+print("RESULT-total: ", random.triangular(700, 850))

rebench/tests/persistency_test.py

@@ -57,14 +57,6 @@ def test_de_serialization(self):
 
         self.assertEqual(deserialized.run_id, measurement.run_id)
 
-    def _assert_runs(self, cnf, num_runs, num_dps, num_invocations):
-        runs = cnf.get_runs()
-        self.assertEqual(num_runs, len(runs))
-        run = list(runs)[0]
-
-        self.assertEqual(num_dps, run.get_number_of_data_points())
-        self.assertEqual(num_invocations, run.completed_invocations)
-
     def test_iteration_invocation_semantics(self):
         # Executes first time
         ds = DataStore(self._ui)

rebench/tests/rebench_test_case.py

@@ -45,3 +45,18 @@ def setUp(self):
     def tearDown(self):
         os.remove(self._tmp_file)
         sys.exit = self._sys_exit
+
+    def _assert_runs(self, cnf, num_runs, num_dps, num_invocations):
+        """
+        :param cnf: Configurator
+        :param num_runs: expected number of runs
+        :param num_dps: expected number of data points
+        :param num_invocations: expected number of invocations
+        :return:
+        """
+        runs = cnf.get_runs()
+        self.assertEqual(num_runs, len(runs))
+        run = list(runs)[0]
+
+        self.assertEqual(num_dps, run.get_number_of_data_points())
+        self.assertEqual(num_invocations, run.completed_invocations)