diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index afceb0ac60..8ad4a211c2 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -15,6 +15,7 @@ import functools import inspect import itertools +import numbers import os import shutil @@ -37,7 +38,6 @@ from reframe.core.schedulers import Job from reframe.core.schedulers.registry import getscheduler from reframe.core.systems import SystemPartition -from reframe.utility.sanity import assert_reference # Dependency kinds @@ -1321,10 +1321,18 @@ def check_performance(self): for key, values in self._perfvalues.items(): val, ref, low_thres, high_thres, *_ = values + + # Verify that val is a number + if not isinstance(val, numbers.Number): + raise SanityError( + "the value extracted for performance variable '%s' " + "is not a number: %s" % (key, val) + ) + tag = key.split(':')[-1] try: sn.evaluate( - assert_reference( + sn.assert_reference( val, ref, low_thres, high_thres, msg=('failed to meet reference: %s={0}, ' 'expected {1} (l={2}, u={3})' % tag)) diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index 3121440ba1..75d82444d8 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -63,6 +63,7 @@ computing statistical information on series of data etc. ''' + import builtins import glob as pyglob import itertools @@ -551,7 +552,7 @@ def calc_bound(thres): evaluate(assert_bounded(val, lower, upper)) except SanityError: error_msg = msg or '{0} is beyond reference value {1} (l={2}, u={3})' - raise SanityError(_format(error_msg, val, ref, lower, upper)) + raise SanityError(_format(error_msg, val, ref, lower, upper)) from None else: return True diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 75da6c222b..48550abffe 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -740,7 +740,6 @@ def setUp(self): 'value3': sn.extractsingle(r'performance3 = (\S+)', self.perf_file.name, 1, float) } - self.test.sanity_patterns = sn.assert_found(r'result = success', self.output_file.name) @@ -896,6 +895,21 @@ def test_tag_resolution(self): } self.test.check_performance() + def test_invalid_perf_value(self): + self.test.perf_patterns = { + 'value1': sn.extractsingle(r'performance1 = (\S+)', + self.perf_file.name, 1, float), + 'value2': sn.extractsingle(r'performance2 = (\S+)', + self.perf_file.name, 1, str), + 'value3': sn.extractsingle(r'performance3 = (\S+)', + self.perf_file.name, 1, float) + } + self.write_performance_output(performance1=1.3, + performance2='foo', + performance3=3.3) + with pytest.raises(SanityError, match='not a number'): + self.test.check_performance() + def test_perf_var_evaluation(self): # All performance values must be evaluated, despite the first one # failing To test this, we need an extract function that will have a