Merge dd772f4 into 3914b3e

docs/config.md

@@ -404,19 +404,34 @@ all of them.
 
 **gauge_adapter:**
 
-Name of the parser that interprets the output of the benchmark harness.
-For a list of supported options see the list of [extensions](extensions.md#available-harness-support).
+Either the name of the parser that interpreters the output of the benchmark harness,
+or a map with one element, which is the name of the parser and the path
+to the Python file with a custom parser.
+
+For a list of supported parers see the list of [extensions](extensions.md#available-harness-support).
+
+If a custom parser is used, the given path is assumed to be relative to the
+configuration file.
 
 This key is mandatory.
 
-Example:
+Example 1, using a built-in parser:
 
 ```yaml
 benchmark_suites:
   ExampleSuite:
     gauge_adapter: ReBenchLog
 ```
 
+Example 2, using a custom parser:
+
+```yaml
+benchmark_suites:
+  ExampleSuite:
+    gauge_adapter:
+      MyClass: ./my_parser.py
+```
+
 ---
 
 **command:**

rebench/configurator.py

@@ -18,7 +18,7 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 import logging
-from os.path import dirname
+from os.path import dirname, abspath
 from pykwalify.core import Core
 from pykwalify.errors import SchemaError
 import yaml
@@ -130,6 +130,12 @@ def load_config(file_name):
                 schema_files=[dirname(__file__) + "/rebench-schema.yml"])
             try:
                 validator.validate(raise_exception=True)
+                validate_gauge_adapters(data)
+
+                # add file name and directory to config to be able to use it when loading
+                # for instance gauge adapters
+                data['__file__'] = file_name
+                data['__dir__'] = dirname(abspath(file_name))
             except SchemaError as err:
                 errors = [escape_braces(val_err) for val_err in validator.validation_errors]
                 raise UIError(
@@ -147,6 +153,21 @@ def load_config(file_name):
                       + file_name + " failed.\nError " + str(err) + "\n", err)
 
 
+def validate_gauge_adapters(raw_config):
+    benchmark_suites = raw_config.get('benchmark_suites', {})
+    for suite_name, suite in benchmark_suites.items():
+        adapter = suite['gauge_adapter']
+        if not isinstance(adapter, (dict, str)):
+            raise UIError(("Gauge adapter for suite %s must be a string or a dictionary," +
+                           "but is %s.\n") % (suite_name, type(adapter).__name__), None)
+
+        if isinstance(adapter, dict) and len(adapter) != 1:
+            raise UIError("When specifying a custom gauge adapter," +
+                          " exactly one must to be specified." +
+                          " Currently there are %d. (%s)\n" % (len(adapter), adapter), None)
+    return True
+
+
 class Configurator(object):
 
     def __init__(self, raw_config, data_store, ui, cli_options=None, cli_reporter=None,
@@ -157,6 +178,8 @@ def __init__(self, raw_config, data_store, ui, cli_options=None, cli_reporter=No
         self.data_file = data_file or raw_config.get('default_data_file', 'rebench.data')
         self._exp_name = exp_name or raw_config.get('default_experiment', 'all')
         self.artifact_review = raw_config.get('artifact_review', False)
+        self.config_dir = raw_config.get('__dir__', None)
+        self.config_file = raw_config.get('__file__', None)
 
         self._rebench_db_connector = None
 

rebench/executor.py

@@ -262,7 +262,7 @@ class Executor(object):
     def __init__(self, runs, do_builds, ui, include_faulty=False,
                  debug=False, scheduler=BatchScheduler, build_log=None,
                  artifact_review=False, use_nice=False, use_shielding=False,
-                 print_execution_plan=False):
+                 print_execution_plan=False, config_dir=None):
         self._runs = runs
 
         self._use_nice = use_nice
@@ -278,6 +278,7 @@ def __init__(self, runs, do_builds, ui, include_faulty=False,
         self._scheduler = self._create_scheduler(scheduler)
         self.build_log = build_log
         self._artifact_review = artifact_review
+        self.config_dir = config_dir
 
         num_runs = RunScheduler.number_of_uncompleted_runs(runs, ui)
         for run in runs:
@@ -443,14 +444,14 @@ def execute_run(self, run_id):
         return terminate
 
     def _get_gauge_adapter_instance(self, run_id):
-        adapter_name = run_id.get_gauge_adapter_name()
-        adapter = instantiate_adapter(adapter_name,
+        adapter_cfg = run_id.get_gauge_adapter()
+        adapter = instantiate_adapter(adapter_cfg,
                                       self._include_faulty,
                                       self)
 
         if adapter is None:
             run_id.fail_immediately()
-            msg = "{ind}Couldn't find gauge adapter: %s\n" % adapter_name
+            msg = "{ind}Couldn't find gauge adapter: %s\n" % run_id.get_gauge_adapter_name()
             self.ui.error_once(msg, run_id)
 
         return adapter

rebench/interop/adapter.py

@@ -20,6 +20,7 @@
 import re
 import pkgutil
 import sys
+from os.path import join
 
 
 class GaugeAdapter(object):
@@ -82,7 +83,13 @@ class ResultsIndicatedAsInvalid(ExecutionDeliveredNoResults):
     pass
 
 
-def instantiate_adapter(name, include_faulty, executor):
+def instantiate_adapter(adapter_cfg, include_faulty, executor):
+    if isinstance(adapter_cfg, str):
+        return _search_in_rebench_modules(adapter_cfg, include_faulty, executor)
+    return _load_directly(adapter_cfg, include_faulty, executor)
+
+
+def _search_in_rebench_modules(name, include_faulty, executor):
     adapter_name = name + "Adapter"
     root = sys.modules['rebench.interop'].__path__
 
@@ -95,9 +102,33 @@ def instantiate_adapter(name, include_faulty, executor):
                 mod = __import__("interop." + module_name, fromlist=adapter_name)
             except ImportError:
                 mod = None
-        if mod is not None:
-            for key in dir(mod):
-                if key.lower() == adapter_name.lower():
-                    return getattr(mod, key)(include_faulty, executor)
+        adapter_cls = _get_adapter_case_insensitively_from_module(mod, adapter_name)
+        if adapter_cls is not None:
+            return adapter_cls(include_faulty, executor)
+
+    return None
+
+
+def _get_adapter_case_insensitively_from_module(mod, adapter_name):
+    if mod is not None:
+        keys = mod.keys() if isinstance(mod, dict) else dir(mod)
+        for key in keys:
+            if key.lower() == adapter_name.lower():
+                return mod[key] if isinstance(mod, dict) else getattr(mod, key)
+    return None
+
+
+def _load_directly(adapter_cfg, include_faulty, executor):
+    name, path = next(iter(adapter_cfg.items()))
+    full_path = join(executor.config_dir, path)
+    with open(full_path, 'r') as adapter_file:  # pylint: disable=unspecified-encoding
+        file_content = adapter_file.read()
+
+    module_globals = {'__name__': name}
+    code = compile(file_content, full_path, 'exec')
+    exec(code, module_globals)  # pylint: disable=exec-used
 
+    cls = _get_adapter_case_insensitively_from_module(module_globals, name)
+    if cls is not None:
+        return cls(include_faulty, executor)
     return None

rebench/model/run_id.py

@@ -110,13 +110,22 @@ def location(self):
             return None
         return self._expand_vars(self.benchmark.suite.location)
 
-    def get_gauge_adapter_name(self):
+    def get_gauge_adapter(self):
         if self.is_profiling():
             # TODO: needs changing once we want to support different profilers
             perf_profile = self.benchmark.suite.executor.profiler[0]
             return perf_profile.gauge_adapter_name
         return self.benchmark.gauge_adapter
 
+    def get_gauge_adapter_name(self):
+        if self.is_profiling():
+            return self.get_gauge_adapter()
+
+        adapter = self.benchmark.gauge_adapter
+        if isinstance(adapter, str):
+            return adapter
+        return next(iter(adapter))  # get the first key in the dict
+
     def is_profiling(self):
         return self.benchmark.suite.executor.action == "profile"
 

rebench/rebench-schema.yml

@@ -193,10 +193,12 @@ schema;benchmark_suite_type:
   mapping:
     <<: [ *EXP_RUN_DETAILS, *EXP_VARIABLES ]
     gauge_adapter:
-      type: str
+      type: any
       required: true
       desc: |
-        Name of the parser that interpreters the output of the benchmark harness
+        Either the name of the parser that interpreters the output of the benchmark harness,
+        or a map with one element, which is the name of the parser and the path
+        to the Python file with a custom parser.
     command:
       type: str
       required: true

rebench/rebench.py

@@ -287,7 +287,8 @@ def execute_experiment(self, runs, use_nice, use_shielding):
                             self._config.options.debug,
                             scheduler_class,
                             self._config.build_log, self._config.artifact_review,
-                            use_nice, use_shielding, self._config.options.execution_plan)
+                            use_nice, use_shielding, self._config.options.execution_plan,
+                            self._config.config_dir)
 
         if self._config.options.no_execution:
             return True

rebench/tests/configurator_test.py

@@ -19,8 +19,9 @@
 # IN THE SOFTWARE.
 import unittest
 
-from ..configurator           import Configurator, load_config
-from ..persistence            import DataStore
+from ..configurator     import Configurator, load_config, validate_gauge_adapters
+from ..persistence      import DataStore
+from ..ui               import UIError
 from .rebench_test_case import ReBenchTestCase
 
 
@@ -136,6 +137,46 @@ def test_machine_filter_m1_and_m2(self):
         runs = cnf.get_runs()
         self.assertEqual(14 + 24, len(runs))
 
+    def test_validate_gauge_adapters_with_all_correct_settings(self):
+        result = validate_gauge_adapters({
+            'benchmark_suites': {
+                'StandardOne': {
+                    'gauge_adapter': 'ReBenchLog'
+                },
+                'SimpleCustomOne': {
+                    'gauge_adapter': {
+                        'MyClass': './my_class.py'
+                    }
+                }
+            }
+        })
+        self.assertTrue(result)
+
+    def test_validate_gauge_adapters_with_wrong_settings(self):
+        with self.assertRaises(UIError) as ctx:
+            validate_gauge_adapters({
+                'benchmark_suites': {
+                    'TwoAdapters': {
+                        'gauge_adapter': {
+                            'MyClass1': './my_class.py',
+                            'MyClass2': './my_class.py',
+                        }
+                    }
+                }
+            })
+            self.assertIn('exactly one', ctx.exception.message)
+
+    def test_validate_gauge_adapter_with_invalid_value(self):
+        with self.assertRaises(UIError) as ctx:
+            validate_gauge_adapters({
+                'benchmark_suites': {
+                    'InvalidValue': {
+                        'gauge_adapter': 42
+                    }
+                }
+            })
+            self.assertIn('must be a string or a dict', ctx.exception.message)
+
 
 # allow command-line execution
 def test_suite():

rebench/tests/executor_test.py

@@ -74,8 +74,7 @@ def _basic_execution(self, cnf):
 
         runs = cnf.get_runs()
         persistence = TestPersistence()
-        for run in runs:
-            run.add_persistence(persistence)
+        persistence.use_on(runs)
 
         ex = Executor(runs, cnf.do_builds, self.ui)
         ex.execute()

rebench/tests/features/issue_16_multiple_data_points_test.py

@@ -41,8 +41,8 @@ def _records_data_points(self, exp_name, num_data_points):
                            data_file=self._tmp_file)
         runs = cnf.get_runs()
         persistence = TestPersistence()
-        for run in runs:
-            run.add_persistence(persistence)
+        persistence.use_on(runs)
+
         ex = Executor(cnf.get_runs(), False, self.ui)
         ex.execute()
         self.assertEqual(1, len(cnf.get_runs()))

rebench/tests/features/issue_209.conf

@@ -0,0 +1,54 @@
+benchmark_suites:
+  Suite1:
+    command: suite-1 %(benchmark)s %(iterations)s
+    gauge_adapter:
+      MyTestAdapter: issue_209_adapter.py
+    benchmarks:
+      - Bench1
+
+  Suite2:
+    command: suite-2 %(benchmark)s %(iterations)s
+    gauge_adapter:
+      MyTestAdapter: issue_209_adapter2.py
+    benchmarks:
+      - Bench2
+
+  Suite3:
+    command: suite-3 %(benchmark)s %(iterations)s
+    gauge_adapter:
+      MyTestAdapter2: issue_209_adapter2.py
+    benchmarks:
+      - Bench3
+
+  Suite4:
+    command: suite-4 %(benchmark)s %(iterations)s
+    gauge_adapter:
+      NonExisting: issue_209_adapter2.py
+    benchmarks:
+      - Bench4
+
+executors:
+  TestRunner:
+    path: .
+    executable: issue_34_vm.py
+
+experiments:
+
+  Exp1:
+    suites:
+      - Suite1
+    executions:
+      - TestRunner
+
+  Exp2:
+    suites:
+      - Suite2
+      - Suite3
+    executions:
+      - TestRunner
+
+  Exp3:
+    suites:
+      - Suite4
+    executions:
+      - TestRunner