From 120eaa4200b1f4301f23fee5036f3aa10a7a412a Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 20 May 2019 11:31:30 +0200 Subject: [PATCH 1/4] Print the tag of the performance in failure message * Allow `assert_reference` to accept a `tag` parameter. * Use the performance tag in `assert_reference` at the `check_performance` stage. --- reframe/core/pipeline.py | 9 +++++---- reframe/utility/sanity.py | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 7f75eefdd3..058f0bbdcc 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1123,14 +1123,15 @@ def check_performance(self): "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) - self._perfvalues[key] = (value, *self.reference[key]) + self._perfvalues[key] = (tag, value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) - for val, *reference in self._perfvalues.values(): - ref, low_thres, high_thres, *_ = reference + for values in self._perfvalues.values(): + tag, val, ref, low_thres, high_thres, *_ = values try: - evaluate(assert_reference(val, ref, low_thres, high_thres)) + evaluate(assert_reference(val, ref, low_thres, high_thres, + tag)) except SanityError as e: raise PerformanceError(e) diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index 687edd7abb..d72cb35cf9 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -466,7 +466,7 @@ def assert_bounded(val, lower=None, upper=None, msg=None): @deferrable -def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None): +def assert_reference(val, ref, lower_thres=None, upper_thres=None, tag=None): """Assert that value ``val`` respects the reference value ``ref``. :arg val: The value to check. @@ -479,6 +479,7 @@ def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None): of the reference value. Must be in [0, inf] for ref >= 0.0 and in [0, 1] for ref < 0.0. If ``None``, no upper thresholds is applied. + :arg tag: The tag of the reference value. :returns: ``True`` on success. :raises reframe.core.exceptions.SanityError: if assertion fails or if the lower and upper thresholds do not have appropriate values. @@ -515,7 +516,10 @@ def calc_bound(thres): evaluate(assert_bounded(val, lower, upper)) except SanityError: error_msg = '{0} is beyond reference value {1} (l={2}, u={3})' - raise SanityError(_format(error_msg, val, ref, lower, upper)) + if tag: + error_msg = "'%s' " % tag + error_msg + + raise SanityError(_format(error_msg, val, ref, lower, upper,)) else: return True From 84987c0dc57b48f9364a68d4bce3e99bd3234c79 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 24 May 2019 11:37:46 +0200 Subject: [PATCH 2/4] Address PR comments --- reframe/core/pipeline.py | 2 +- reframe/utility/sanity.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 058f0bbdcc..fbfcdf9461 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1131,7 +1131,7 @@ def check_performance(self): tag, val, ref, low_thres, high_thres, *_ = values try: evaluate(assert_reference(val, ref, low_thres, high_thres, - tag)) + "'%s'" % tag)) except SanityError as e: raise PerformanceError(e) diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index d72cb35cf9..cb46fec4e3 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -466,7 +466,7 @@ def assert_bounded(val, lower=None, upper=None, msg=None): @deferrable -def assert_reference(val, ref, lower_thres=None, upper_thres=None, tag=None): +def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None): """Assert that value ``val`` respects the reference value ``ref``. :arg val: The value to check. @@ -479,7 +479,6 @@ def assert_reference(val, ref, lower_thres=None, upper_thres=None, tag=None): of the reference value. Must be in [0, inf] for ref >= 0.0 and in [0, 1] for ref < 0.0. If ``None``, no upper thresholds is applied. - :arg tag: The tag of the reference value. :returns: ``True`` on success. :raises reframe.core.exceptions.SanityError: if assertion fails or if the lower and upper thresholds do not have appropriate values. @@ -515,11 +514,9 @@ def calc_bound(thres): try: evaluate(assert_bounded(val, lower, upper)) except SanityError: - error_msg = '{0} is beyond reference value {1} (l={2}, u={3})' - if tag: - error_msg = "'%s' " % tag + error_msg - - raise SanityError(_format(error_msg, val, ref, lower, upper,)) + error_msg = '%s ' % msg or '' + error_msg += '{0} is beyond reference value {1} (l={2}, u={3})' + raise SanityError(_format(error_msg, val, ref, lower, upper)) else: return True From 34a67ec3335d84659aff4b5d1977e422b1c585d4 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 24 May 2019 11:50:53 +0200 Subject: [PATCH 3/4] Fix the assignment of message --- reframe/utility/sanity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index cb46fec4e3..1d3e2a2d26 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -514,7 +514,7 @@ def calc_bound(thres): try: evaluate(assert_bounded(val, lower, upper)) except SanityError: - error_msg = '%s ' % msg or '' + error_msg = '%s ' % msg if msg else '' error_msg += '{0} is beyond reference value {1} (l={2}, u={3})' raise SanityError(_format(error_msg, val, ref, lower, upper)) else: From f67c8a77f894dd93f8897f2dc8fc72b70c948ca8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 27 May 2019 00:08:53 +0200 Subject: [PATCH 4/4] Correct behaviour of msg arg. in assert_reference - Message in case of performance failures is set by check_performance() including also the name of the performance variable that failed. --- reframe/core/pipeline.py | 9 +++++++-- reframe/utility/sanity.py | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index fbfcdf9461..ca7de0e0ea 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1130,8 +1130,13 @@ def check_performance(self): for values in self._perfvalues.values(): tag, val, ref, low_thres, high_thres, *_ = values try: - evaluate(assert_reference(val, ref, low_thres, high_thres, - "'%s'" % tag)) + evaluate( + assert_reference( + val, ref, low_thres, high_thres, + msg=('failed to meet reference: %s={0}, ' + 'expected {1} (l={2}, u={3})' % tag), + ) + ) except SanityError as e: raise PerformanceError(e) diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index 1d3e2a2d26..d6048c7d0d 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -514,8 +514,7 @@ def calc_bound(thres): try: evaluate(assert_bounded(val, lower, upper)) except SanityError: - error_msg = '%s ' % msg if msg else '' - error_msg += '{0} is beyond reference value {1} (l={2}, u={3})' + error_msg = msg or '{0} is beyond reference value {1} (l={2}, u={3})' raise SanityError(_format(error_msg, val, ref, lower, upper)) else: return True