From 5274df3fd84ee271ec357c53a4783a1921526898 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Sun, 19 Mar 2023 21:42:29 +0000 Subject: [PATCH] Expand tests with second adapter file, and multiple classes Signed-off-by: Stefan Marr --- rebench/tests/features/issue_209.conf | 21 +++++++ rebench/tests/features/issue_209_adapter2.py | 58 +++++++++++++++++++ .../features/issue_209_custom_adapter_test.py | 38 +++++++++++- 3 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 rebench/tests/features/issue_209_adapter2.py diff --git a/rebench/tests/features/issue_209.conf b/rebench/tests/features/issue_209.conf index 581e1876..267e97d7 100644 --- a/rebench/tests/features/issue_209.conf +++ b/rebench/tests/features/issue_209.conf @@ -6,6 +6,20 @@ benchmark_suites: benchmarks: - Bench1 + Suite2: + command: suite-2 %(benchmark)s %(iterations)s + gauge_adapter: + MyTestAdapter: issue_209_adapter2.py + benchmarks: + - Bench2 + + Suite3: + command: suite-3 %(benchmark)s %(iterations)s + gauge_adapter: + MyTestAdapter2: issue_209_adapter2.py + benchmarks: + - Bench3 + executors: TestRunner: path: . @@ -18,3 +32,10 @@ experiments: - Suite1 executions: - TestRunner + + Exp2: + suites: + - Suite2 + - Suite3 + executions: + - TestRunner diff --git a/rebench/tests/features/issue_209_adapter2.py b/rebench/tests/features/issue_209_adapter2.py new file mode 100644 index 00000000..f80c03ad --- /dev/null +++ b/rebench/tests/features/issue_209_adapter2.py @@ -0,0 +1,58 @@ +from re import compile as re_compile + +from rebench.interop.adapter import GaugeAdapter, OutputNotParseable,\ + ResultsIndicatedAsInvalid +from rebench.model.data_point import DataPoint +from rebench.model.measurement import Measurement + + +class AbstractAdapter(GaugeAdapter): + """Performance reader for the test case and the definitions + in test/test.conf + """ + + __test__ = False # This is not a test class + re_time = re_compile(r"RESULT-(\w+):\s*(\d+\.\d+)") + + def __init__(self, include_faulty, executor): + super(AbstractAdapter, self).__init__(include_faulty, executor) + self._other_error_definitions = [re_compile("FAILED")] + + def _make_measurement(self, run_id, invocation, iteration, value, criterion): + return Measurement(invocation, iteration, value, 'ms', run_id, criterion) + + def parse_data(self, data, run_id, invocation): + iteration = 1 + data_points = [] + current = DataPoint(run_id) + + for line in data.split("\n"): + if self.check_for_error(line): + raise ResultsIndicatedAsInvalid( + "Output of bench program indicated error.") + + match = MyTestAdapter.re_time.match(line) + if match: + measure = self._make_measurement(run_id, invocation, iteration, + float(match.group(2)), match.group(1)) + current.add_measurement(measure) + + if measure.is_total(): + data_points.append(current) + current = DataPoint(run_id) + iteration += 1 + + if not data_points: + raise OutputNotParseable(data) + + return data_points + + +class MyTestAdapter(AbstractAdapter): + def _make_measurement(self, run_id, invocation, iteration, value, criterion): + return Measurement(invocation, iteration, value + 1, 'ms', run_id, criterion) + + +class MyTestAdapter2(AbstractAdapter): + def _make_measurement(self, run_id, invocation, iteration, value, criterion): + return Measurement(invocation, iteration, value + 2, 'ms', run_id, criterion) diff --git a/rebench/tests/features/issue_209_custom_adapter_test.py b/rebench/tests/features/issue_209_custom_adapter_test.py index 80c10ad3..077b1734 100644 --- a/rebench/tests/features/issue_209_custom_adapter_test.py +++ b/rebench/tests/features/issue_209_custom_adapter_test.py @@ -3,6 +3,7 @@ from ...executor import Executor from ...persistence import DataStore from ...rebench import ReBench +from ..persistence import TestPersistence class Issue209CustomAdapter(ReBenchTestCase): @@ -19,10 +20,41 @@ def test_custom_adapter_gives_data(self): self._data_store, self.ui, self._cli_options, exp_name='Exp1', data_file=self._tmp_file) - self._data_store.load_data(None, False) + runs = cnf.get_runs() + persistence = TestPersistence() + persistence.use_on(runs) + + self.assertEqual(1, len(runs)) + + # self._data_store.load_data(None, False) ex = Executor(cnf.get_runs(), True, self.ui, build_log=cnf.build_log, config_dir=raw_config['__dir__']) - ex.execute() + self.assertTrue(ex.execute()) + + data_points = persistence.get_data_points() + self.assertEqual(1, len(data_points)) + self.assertEqual(1.1, data_points[0].get_total_value()) + + def test_custom_adapters_are_not_confused_and_give_expected_data(self): + raw_config = load_config(self._path + '/issue_209.conf') + cnf = Configurator(raw_config, + self._data_store, self.ui, self._cli_options, + exp_name='Exp2', data_file=self._tmp_file) runs = cnf.get_runs() - self.assertEqual(1, len(runs)) + persistence = TestPersistence() + persistence.use_on(runs) + + self.assertEqual(2, len(runs)) + + # self._data_store.load_data(None, False) + ex = Executor(cnf.get_runs(), True, self.ui, build_log=cnf.build_log, + config_dir=raw_config['__dir__']) + self.assertTrue(ex.execute()) + + data_points = persistence.get_data_points() + data_points.sort(key=lambda x: x.get_total_value()) + + self.assertEqual(2, len(data_points)) + self.assertEqual(2.1, data_points[0].get_total_value()) + self.assertEqual(3.1, data_points[1].get_total_value())