Merge pull request #704 from teojgo/bugfix/performance_error_exception

[bugfix] Fix error message in case of performance errors

reframe/core/exceptions.py

@@ -8,34 +8,40 @@
 import warnings
 import sys
 
+import reframe.utility as utility
 
-class ReframeError(Exception):
-    """Base exception for soft errors.
 
-    Soft errors may be treated by simply printing the exception's message and
-    trying to continue execution if possible.
-    """
+class ReframeBaseError(BaseException):
+    """Base exception for any ReFrame error."""
+
+    def __init__(self, *args):
+        self._message = str(args[0]) if args else None
+
+    @property
+    def message(self):
+        return self._message
 
     def __str__(self):
-        ret = super().__str__()
+        ret = self._message or ''
         if self.__cause__ is not None:
             ret += ': ' + str(self.__cause__)
 
         return ret
 
 
-class ReframeFatalError(BaseException):
-    """A fatal framework error.
+class ReframeError(ReframeBaseError, Exception):
+    """Base exception for soft errors.
 
-    Execution must be aborted.
+    Soft errors may be treated by simply printing the exception's message and
+    trying to continue execution if possible.
     """
 
-    def __str__(self):
-        ret = super().__str__()
-        if self.__cause__ is not None:
-            ret += ': ' + str(self.__cause__)
 
-        return ret
+class ReframeFatalError(ReframeBaseError):
+    """A fatal framework error.
+
+    Execution must be aborted.
+    """
 
 
 class ReframeSyntaxError(ReframeError):
@@ -81,7 +87,11 @@ class EnvironError(ReframeError):
 
 
 class SanityError(ReframeError):
-    """Raised to denote an error in sanity or performance checking."""
+    """Raised to denote an error in sanity checking."""
+
+
+class PerformanceError(ReframeError):
+    """Raised to denote an error in performance checking."""
 
 
 class PipelineError(ReframeError):
@@ -102,37 +112,36 @@ class BuildError(ReframeError):
     """Raised when a build fails."""
 
     def __init__(self, stdout, stderr):
-        self._stdout = stdout
-        self._stderr = stderr
-
-    def __str__(self):
-        return ("standard error can be found in `%s', "
-                "standard output can be found in `%s'" % (self._stderr,
-                                                          self._stdout))
+        super().__init__()
+        self._message = (
+            "standard error can be found in `%s', "
+            "standard output can be found in `%s'" % (stdout, stderr)
+        )
 
 
 class SpawnedProcessError(ReframeError):
     """Raised when a spawned OS command has failed."""
 
     def __init__(self, command, stdout, stderr, exitcode):
-        super().__init__(command, stdout, stderr, exitcode)
-        self._command = command
-        self._stdout = stdout
-        self._stderr = stderr
-        self._exitcode = exitcode
+        super().__init__()
 
-    def __str__(self):
-        lines = ["command '{0}' failed with exit code {1}:".format(
-            self._command, self._exitcode)]
+        # Format message and put it in args
+        lines = [
+            "command '%s' failed with exit code %s:" % (command, exitcode)
+        ]
         lines.append('=== STDOUT ===')
-        if self._stdout:
-            lines.append(self._stdout)
+        if stdout:
+            lines.append(stdout)
 
         lines.append('=== STDERR ===')
-        if self._stderr:
-            lines.append(self._stderr)
+        if stderr:
+            lines.append(stderr)
 
-        return '\n'.join(lines)
+        self._message = '\n'.join(lines)
+        self._command = command
+        self._stdout = stdout
+        self._stderr = stderr
+        self._exitcode = exitcode
 
     @property
     def command(self):
@@ -156,14 +165,8 @@ class SpawnedProcessTimeout(SpawnedProcessError):
 
     def __init__(self, command, stdout, stderr, timeout):
         super().__init__(command, stdout, stderr, None)
-        self._timeout = timeout
-
-        # Reset the args to match the real ones passed to this exception
-        self.args = command, stdout, stderr, timeout
 
-    def __str__(self):
-        lines = ["command '{0}' timed out after {1}s:".format(self._command,
-                                                              self._timeout)]
+        lines = ["command '%s' timed out after %ss:" % (command, timeout)]
         lines.append('=== STDOUT ===')
         if self._stdout:
             lines.append(self._stdout)
@@ -172,7 +175,8 @@ def __str__(self):
         if self._stderr:
             lines.append(self._stderr)
 
-        return '\n'.join(lines)
+        self._message = '\n'.join(lines)
+        self._timeout = timeout
 
     @property
     def timeout(self):
@@ -182,22 +186,18 @@ def timeout(self):
 class JobError(ReframeError):
     """Job related errors."""
 
-    def __init__(self, *args, jobid=None):
-        super().__init__(*args)
+    def __init__(self, msg=None, jobid=None):
+        message = '[jobid=%s]' % jobid
+        if msg:
+            message += ' ' + msg
+
+        super().__init__(message)
         self._jobid = jobid
 
     @property
     def jobid(self):
         return self._jobid
 
-    def __str__(self):
-        prefix = '(jobid=%s)' % self._jobid
-        msg = super().__str__()
-        if self.args:
-            return prefix + ' ' + msg
-        else:
-            return prefix + msg
-
 
 class JobBlockedError(JobError):
     """Raised by job schedulers when a job is blocked indefinitely."""
@@ -235,21 +235,18 @@ def format_user_frame(frame):
     if exc_type is None:
         return ''
 
-    if isinstance(exc_value, SanityError):
-        return 'sanity error: %s' % exc_value
-
-    if isinstance(exc_value, ReframeFatalError):
-        exc_str = ''.join(traceback.format_exception(exc_type, exc_value, tb))
-        return 'fatal error: %s\n%s' % (exc_value, exc_str)
-
     if isinstance(exc_value, AbortTaskError):
         return 'aborted due to %s' % type(exc_value.__cause__).__name__
 
-    if isinstance(exc_value, BuildError):
-        return 'build failure: %s' % exc_value
-
     if isinstance(exc_value, ReframeError):
-        return 'caught framework exception: %s' % exc_value
+        return '%s: %s' % (utility.decamelize(exc_type.__name__, ' '),
+                           exc_value)
+
+    if isinstance(exc_value, ReframeFatalError):
+        exc_str = '%s: %s' % (utility.decamelize(exc_type.__name__, ' '),
+                              exc_value)
+        tb_str = ''.join(traceback.format_exception(exc_type, exc_value, tb))
+        return '%s\n%s' % (exc_str, tb_str)
 
     if isinstance(exc_value, KeyboardInterrupt):
         return 'cancelled by user'

reframe/core/pipeline.py

@@ -23,7 +23,8 @@
 from reframe.core.buildsystems import BuildSystem, BuildSystemField
 from reframe.core.deferrable import deferrable, _DeferredExpression, evaluate
 from reframe.core.environments import Environment, EnvironmentSnapshot
-from reframe.core.exceptions import BuildError, PipelineError, SanityError
+from reframe.core.exceptions import (BuildError, PipelineError, SanityError,
+                                     PerformanceError)
 from reframe.core.launchers.registry import getlauncher
 from reframe.core.schedulers import Job
 from reframe.core.schedulers.registry import getscheduler
@@ -1075,7 +1076,7 @@ def sanity(self):
     def performance(self):
         try:
             self.check_performance()
-        except SanityError:
+        except PerformanceError:
             if self.strict_check:
                 raise
 
@@ -1090,7 +1091,7 @@ def check_sanity(self):
         with os_ext.change_dir(self._stagedir):
             success = evaluate(self.sanity_patterns)
             if not success:
-                raise SanityError('sanity failure')
+                raise SanityError()
 
     def check_performance(self):
         """The performance checking phase of the regression test pipeline.
@@ -1120,7 +1121,10 @@ def check_performance(self):
 
             for val, reference in perf_values:
                 ref, low_thres, high_thres, *_ = reference
-                evaluate(assert_reference(val, ref, low_thres, high_thres))
+                try:
+                    evaluate(assert_reference(val, ref, low_thres, high_thres))
+                except SanityError as e:
+                    raise PerformanceError(e)
 
     def _copy_job_files(self, job, dst):
         if job is None:

reframe/utility/__init__.py

@@ -83,10 +83,11 @@ def allx(iterable):
     return all(iterable) if iterable else False
 
 
-def decamelize(s):
+def decamelize(s, delim='_'):
     """Decamelize the string ``s``.
 
     For example, ``MyBaseClass`` will be converted to ``my_base_class``.
+    The delimiter may be changed by setting the ``delim`` argument.
     """
 
     if not isinstance(s, str):
@@ -95,7 +96,7 @@ def decamelize(s):
     if not s:
         return ''
 
-    return re.sub(r'([a-z])([A-Z])', r'\1_\2', s).lower()
+    return re.sub(r'([a-z])([A-Z])', r'\1%s\2' % delim, s).lower()
 
 
 def toalphanum(s):

unittests/resources/checks/frontend_checks.py

@@ -4,7 +4,7 @@
 
 import reframe as rfm
 import reframe.utility.sanity as sn
-from reframe.core.exceptions import ReframeError, SanityError
+from reframe.core.exceptions import ReframeError, PerformanceError
 
 
 class BaseFrontendCheck(rfm.RunOnlyRegressionTest):
@@ -91,7 +91,7 @@ def __init__(self):
         self.strict_check = False
 
     def check_performance(self):
-        raise SanityError('performance failure')
+        raise PerformanceError('performance failure')
 
 
 class KeyboardInterruptCheck(BaseFrontendCheck):

unittests/test_exceptions.py

@@ -114,7 +114,7 @@ def test_job_error(self):
         exc_args = ('some error',)
         e = exc.JobError(*exc_args, jobid=1234)
         self.assertEqual(1234, e.jobid)
-        self.assertRaisesRegex(exc.JobError, r'\(jobid=1234\) some error',
+        self.assertRaisesRegex(exc.JobError, r'\[jobid=1234\] some error',
                                raise_exc, e)
         self.assertEqual(exc_args, e.args)
 
@@ -125,7 +125,7 @@ def test_reraise_job_error(self):
             except ValueError as e:
                 raise exc.JobError('some error', jobid=1234) from e
         except exc.JobError as e:
-            self.assertEqual('(jobid=1234) some error: random value error',
+            self.assertEqual('[jobid=1234] some error: random value error',
                              str(e))
 
     def test_reraise_job_error_no_message(self):
@@ -135,5 +135,5 @@ def test_reraise_job_error_no_message(self):
             except ValueError as e:
                 raise exc.JobError(jobid=1234) from e
         except exc.JobError as e:
-            self.assertEqual('(jobid=1234): random value error',
+            self.assertEqual('[jobid=1234]: random value error',
                              str(e))

unittests/test_pipeline.py

@@ -7,7 +7,8 @@
 import reframe.utility.sanity as sn
 import unittests.fixtures as fixtures
 from reframe.core.exceptions import (BuildError, PipelineError, ReframeError,
-                                     ReframeSyntaxError, SanityError)
+                                     ReframeSyntaxError, PerformanceError,
+                                     SanityError)
 from reframe.core.pipeline import (CompileOnlyRegressionTest, RegressionTest,
                                    RunOnlyRegressionTest)
 from reframe.frontend.loader import RegressionCheckLoader
@@ -551,7 +552,7 @@ def test_performance_failure(self):
         self.output_file.write('result = success\n')
         self.output_file.close()
         self.test.check_sanity()
-        self.assertRaises(SanityError, self.test.check_performance)
+        self.assertRaises(PerformanceError, self.test.check_performance)
 
     def test_unknown_tag(self):
         self.test.reference = {
@@ -634,7 +635,7 @@ def extract_perf(patt, tag):
         self.write_performance_output(performance1=1.0,
                                       performance2=1.8,
                                       performance3=3.3)
-        with self.assertRaises(SanityError) as cm:
+        with self.assertRaises(PerformanceError) as cm:
             self.test.check_performance()
 
         logfile = os.path.join(self.test.stagedir, logfile)