From c5bef8197a0ba7b5efe26b583594917c632f0c82 Mon Sep 17 00:00:00 2001 From: bignamic Date: Thu, 4 Jul 2019 14:03:50 +0200 Subject: [PATCH 01/13] WIP: default reference value implemented --- reframe/core/pipeline.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 6e1faa9c8e..ab3ed55b57 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1129,10 +1129,13 @@ def check_performance(self): for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) - if key not in self.reference: - raise SanityError( - "tag `%s' not resolved in references for `%s'" % - (tag, self._current_partition.fullname)) + try: + if key not in self.reference: + raise SanityError( + "tag `%s' not resolved in references for `%s'" % + (tag, self._current_partition.fullname)) + except SanityError: # TODO: use if-else + self.reference.update({'*':{tag: (0, None, None, 'dummy')}}) self._perfvalues[key] = (value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, From 5a9bc8770361a36400a812ea6d141e6bafdf6544 Mon Sep 17 00:00:00 2001 From: bignamic Date: Tue, 9 Jul 2019 11:54:12 +0200 Subject: [PATCH 02/13] WIP: Default reference performance value adding procedure for all the measuerd variablesimplemented --- reframe/core/pipeline.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ab3ed55b57..9e0888c1ed 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1123,19 +1123,35 @@ def check_performance(self): return with os_ext.change_dir(self._stagedir): + + # Check if default reference perf values are provided and + # stare all the variables tested in the performance check + is_default_present = False + variables = set() + for system, perf_data in self.reference.items(): + variables.add(system.split(":")[-1]) + if system.split(":")[0] == '*': + is_default_present = True + break + + if is_default_present is False: + # If default value is not provided add one for all the + # tested variables + for variable in variables: + self.reference.update( + {'*': {variable: (0, None, None, 'dummy')}}) + print(self.reference.items()) + # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) - try: - if key not in self.reference: - raise SanityError( - "tag `%s' not resolved in references for `%s'" % - (tag, self._current_partition.fullname)) - except SanityError: # TODO: use if-else - self.reference.update({'*':{tag: (0, None, None, 'dummy')}}) + if key not in self.reference: + raise SanityError( + "tag `%s' not resolved in references for `%s'" % + (tag, self._current_partition.fullname)) self._perfvalues[key] = (value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, From a1f7532393afa2b2ff85ee6e260baa24fc56a561 Mon Sep 17 00:00:00 2001 From: bignamic Date: Tue, 9 Jul 2019 15:30:10 +0200 Subject: [PATCH 03/13] unit storage added --- reframe/core/pipeline.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 9e0888c1ed..09cb51ec74 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1125,11 +1125,11 @@ def check_performance(self): with os_ext.change_dir(self._stagedir): # Check if default reference perf values are provided and - # stare all the variables tested in the performance check + # store all the variables tested in the performance check is_default_present = False variables = set() for system, perf_data in self.reference.items(): - variables.add(system.split(":")[-1]) + variables.add((system.split(":")[-1],perf_data[-1])) if system.split(":")[0] == '*': is_default_present = True break @@ -1139,8 +1139,7 @@ def check_performance(self): # tested variables for variable in variables: self.reference.update( - {'*': {variable: (0, None, None, 'dummy')}}) - print(self.reference.items()) + {'*': {variable[0]: (0, None, None, variable[1])}}) # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them From 6d5ed52d06d2e6c2e87001b6058366152f6bd8b0 Mon Sep 17 00:00:00 2001 From: bignamic Date: Tue, 9 Jul 2019 15:57:20 +0200 Subject: [PATCH 04/13] Coding style fix --- reframe/core/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 4c14cf5175..915c287cba 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1130,7 +1130,7 @@ def check_performance(self): is_default_present = False variables = set() for system, perf_data in self.reference.items(): - variables.add((system.split(":")[-1],perf_data[-1])) + variables.add((system.split(":")[-1], perf_data[-1])) if system.split(":")[0] == '*': is_default_present = True break From 86b2e23829a2e603d8946d3b4f8f03e503f7eb4a Mon Sep 17 00:00:00 2001 From: bignamic Date: Wed, 10 Jul 2019 15:14:45 +0200 Subject: [PATCH 05/13] unit test fix --- unittests/test_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 8d6ba9a6dd..20e1e80e17 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -573,7 +573,7 @@ def test_unknown_system(self): 'value3': (3.1, -0.1, 0.1), } } - self.assertRaises(SanityError, self.test.check_performance) + self.test.check_performance() def test_default_reference(self): self.write_performance_output(performance1=1.3, From f91645f8a767e7d2550fc50e3f31fe935ba76a41 Mon Sep 17 00:00:00 2001 From: bignamic Date: Wed, 10 Jul 2019 16:22:28 +0200 Subject: [PATCH 06/13] Variable collecting from multiple systems for default case unit test added --- unittests/test_pipeline.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 20e1e80e17..5d154b72ad 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -575,6 +575,23 @@ def test_unknown_system(self): } self.test.check_performance() + def test_unknown_multiple_systems(self): + self.write_performance_output(performance1=1.3, + performance2=1.8, + performance3=3.3) + self.test.reference = { + 'testsys:part1': { + 'value1': (1.4, -0.1, 0.1) + }, + 'testsys:part2': { + 'value2': (1.7, -0.1, 0.1) + }, + 'testsys:part3': { + 'value3': (3.1, -0.1, 0.1) + } + } + self.test.check_performance() + def test_default_reference(self): self.write_performance_output(performance1=1.3, performance2=1.8, From fea8f8b252e2065c39b0a3c6e02d307262d05add Mon Sep 17 00:00:00 2001 From: bignamic Date: Thu, 11 Jul 2019 09:43:16 +0200 Subject: [PATCH 07/13] Code cleaning --- reframe/core/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 915c287cba..5214a131db 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1127,15 +1127,15 @@ def check_performance(self): # Check if default reference perf values are provided and # store all the variables tested in the performance check - is_default_present = False + is_default_not_present = True variables = set() for system, perf_data in self.reference.items(): variables.add((system.split(":")[-1], perf_data[-1])) if system.split(":")[0] == '*': - is_default_present = True + is_default_not_present = False break - if is_default_present is False: + if is_default_not_present is True: # If default value is not provided add one for all the # tested variables for variable in variables: From 368cba60dd10686c17e645cd062e470af10e4eb7 Mon Sep 17 00:00:00 2001 From: bignamic Date: Thu, 11 Jul 2019 09:55:53 +0200 Subject: [PATCH 08/13] Code cleaning --- reframe/core/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 5214a131db..26aa548147 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1135,7 +1135,7 @@ def check_performance(self): is_default_not_present = False break - if is_default_not_present is True: + if is_default_not_present: # If default value is not provided add one for all the # tested variables for variable in variables: From 6bf964be57c5850dd77cc9f696b456c3e3f0819d Mon Sep 17 00:00:00 2001 From: bignamic Date: Mon, 22 Jul 2019 15:10:14 +0200 Subject: [PATCH 09/13] Fix for no unit case --- reframe/core/pipeline.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 26aa548147..f5e9959dcb 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1124,23 +1124,34 @@ def check_performance(self): return with os_ext.change_dir(self._stagedir): - # Check if default reference perf values are provided and # store all the variables tested in the performance check - is_default_not_present = True + has_default = False variables = set() - for system, perf_data in self.reference.items(): - variables.add((system.split(":")[-1], perf_data[-1])) - if system.split(":")[0] == '*': - is_default_not_present = False + for key, ref in self.reference.items(): + keyparts = key.split(self.reference.scope_separator) + system = keyparts[0] + varname = keyparts[-1] + try: + unit = ref[3] + except IndexError: + unit = None + variables.add((varname, unit)) + if system == '*': + has_default = True break - if is_default_not_present: + if not has_default: # If default value is not provided add one for all the # tested variables - for variable in variables: - self.reference.update( - {'*': {variable[0]: (0, None, None, variable[1])}}) + for var in variables: + name, unit = var + if unit: + self.reference.update( + {'*': {name: (0, None, None, unit)}}) + else: + self.reference.update( + {'*': {name: (0, None, None)}}) # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them From aa2a0130660544e228da209b7ae4344fb14feedf Mon Sep 17 00:00:00 2001 From: bignamic Date: Mon, 22 Jul 2019 15:24:32 +0200 Subject: [PATCH 10/13] Multiple unknown system test merged in existing unit test --- unittests/test_pipeline.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 395a0b5166..68cdae7e8c 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -578,25 +578,10 @@ def test_unknown_system(self): self.test.reference = { 'testsys:login': { 'value1': (1.4, -0.1, 0.1), - 'value2': (1.7, -0.1, 0.1), 'value3': (3.1, -0.1, 0.1), - } - } - self.test.check_performance() - - def test_unknown_multiple_systems(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.test.reference = { - 'testsys:part1': { - 'value1': (1.4, -0.1, 0.1) }, - 'testsys:part2': { + 'testsys:login2': { 'value2': (1.7, -0.1, 0.1) - }, - 'testsys:part3': { - 'value3': (3.1, -0.1, 0.1) } } self.test.check_performance() From 40090470fbb5517302aab554a800e31db3cf067a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 23 Jul 2019 14:44:40 +0200 Subject: [PATCH 11/13] Fix coding style issues --- reframe/core/pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a4d9fcc566..bc20cbfc6f 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1139,6 +1139,7 @@ def check_performance(self): unit = ref[3] except IndexError: unit = None + variables.add((varname, unit)) if system == '*': has_default = True @@ -1149,12 +1150,11 @@ def check_performance(self): # tested variables for var in variables: name, unit = var + ref_tuple = (0, None, None) if unit: - self.reference.update( - {'*': {name: (0, None, None, unit)}}) - else: - self.reference.update( - {'*': {name: (0, None, None)}}) + ref_tuple += (unit,) + + self.reference.update({'*': {name: ref_tuple}}) # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them From 58368cdec2f21e0c0b1a177bd98988ea3fb68873 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 23 Jul 2019 15:09:06 +0200 Subject: [PATCH 12/13] Insert default reference even if references are not defined --- reframe/core/pipeline.py | 8 ++++++-- unittests/test_pipeline.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index bc20cbfc6f..a8ae56579c 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1146,8 +1146,12 @@ def check_performance(self): break if not has_default: - # If default value is not provided add one for all the - # tested variables + if not variables: + # If empty, it means that self.reference was empty, so try + # to infer their name from perf_patterns + variables = {(name, None) + for name in self.perf_patterns.keys()} + for var in variables: name, unit = var ref_tuple = (0, None, None) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 68cdae7e8c..d65bd9a5dc 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -586,6 +586,13 @@ def test_unknown_system(self): } self.test.check_performance() + def test_empty_reference(self): + self.write_performance_output(performance1=1.3, + performance2=1.8, + performance3=3.3) + self.test.reference = {} + self.test.check_performance() + def test_default_reference(self): self.write_performance_output(performance1=1.3, performance2=1.8, From bced3d561ca80e2c872f4f37097d47e5e3bf603e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 23 Jul 2019 15:21:54 +0200 Subject: [PATCH 13/13] Update docs on the default reference entry --- docs/tutorial.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 041c0e4d40..9594f1e618 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -399,7 +399,7 @@ The ``setup()`` method is now very simple: it gets the correct compilation flags from the ``prgenv_flags`` dictionary and applies them to the build system. .. literalinclude:: ../tutorial/example2.py - :lines: 1-4,36-63 + :lines: 1-4,36-62 .. tip:: A regression test is like any other Python class, so you can freely define your own attributes. @@ -600,7 +600,18 @@ Thresholds are specified as decimal fractions of the reference value. For nonneg In our example, the reference value for this test on ``daint:gpu`` is 50 Gflop/s ±10%. Setting a threshold value to :class:`None` disables the threshold. If you specify a measurement unit as well, you will be able to log it the performance logs of the test; this is handy when you are inspecting or plotting the performance values. +ReFrame will always add a default ``*`` entry in the ``reference`` dictionary, if it does not exist, with the reference value of ``(0, None, None, )``, where ``unit`` is derived from the unit of each respective performance variable. +This is useful when using ReFrame for benchmarking purposes and you would like to run a test on an unknown system. +.. note:: + Reference tuples may now optionally contain units. + + .. versionadded:: 2.16 + +.. note:: + A default ``*`` entry is now always added to the reference dictionary. + + .. versionadded:: 2.19 Combining It All Together -------------------------