Support for version compatibility check

reframe/core/decorators.py

@@ -2,20 +2,30 @@
 # Decorators for registering tests with the framework
 #
 
-__all__ = ['parameterized_test', 'simple_test']
+__all__ = ['parameterized_test', 'simple_test', 'required_version']
 
 
 import collections
 import inspect
 
+import reframe
 from reframe.core.exceptions import ReframeSyntaxError
+from reframe.core.logging import getlogger
 from reframe.core.pipeline import RegressionTest
+from reframe.utility.versioning import Version, VersionValidator
 
 
 def _register_test(cls, args=None):
     def _instantiate():
         ret = []
         for cls, args in mod.__rfm_test_registry:
+            try:
+                if cls in mod.__rfm_skip_tests:
+                    continue
+
+            except AttributeError:
+                mod.__rfm_skip_tests = set()
+
             if isinstance(args, collections.Sequence):
                 ret.append(cls(*args))
             elif isinstance(args, collections.Mapping):
@@ -83,3 +93,52 @@ def _do_register(cls):
         return cls
 
     return _do_register
+
+
+def required_version(*versions):
+    """Class decorator for specifying the required ReFrame versions for the
+    following test.
+
+    If the test is not compatible with the current ReFrame version it will be
+    skipped.
+
+    :arg versions: A list of ReFrame version specifications that this test is
+      allowed to run. A version specification string can have one of the
+      following formats:
+
+      1. ``VERSION``: Specifies a single version.
+
+      2. ``{OP}VERSION``, where ``{OP}`` can be any of ``>``, ``>=``, ``<``,
+      ``<=``, ``==`` and ``!=``. For example, the version specification string
+      ``'>=2.15'`` will only allow the following test to be loaded only by
+      ReFrame 2.15 and higher. The ``==VERSION`` specification is the
+      equivalent of ``VERSION``.
+
+      3. ``V1..V2``: Specifies a range of versions.
+
+      You can specify multiple versions with this decorator, such as
+      ``@required_version('2.13', '>=2.16')``, in which case the test will be
+      selected if *any* of the versions is satisfied, even if the versions
+      specifications are conflicting.
+
+    .. versionadded:: 2.13
+
+    """
+    if not versions:
+        raise ValueError('no versions specified')
+
+    conditions = [VersionValidator(v) for v in versions]
+
+    def _skip_tests(cls):
+        mod = inspect.getmodule(cls)
+        if not hasattr(mod, '__rfm_skip_tests'):
+            mod.__rfm_skip_tests = set()
+
+        if not any(c.validate(reframe.VERSION) for c in conditions):
+            getlogger().info('skipping incompatible test defined'
+                             ' in class: %s' % cls.__name__)
+            mod.__rfm_skip_tests.add(cls)
+
+        return cls
+
+    return _skip_tests

reframe/core/exceptions.py

@@ -3,8 +3,10 @@
 #
 
 import inspect
+import os
 import traceback
 import warnings
+import sys
 
 
 class ReframeError(Exception):
@@ -200,19 +202,17 @@ def user_frame(tb):
         raise ValueError('could not retrieve frame: argument not a traceback')
 
     for finfo in reversed(inspect.getinnerframes(tb)):
-        module = inspect.getmodule(finfo.frame)
-        if module is None:
-            continue
-
-        if not module.__name__.startswith('reframe'):
+        relpath = os.path.relpath(finfo.filename, sys.path[0])
+        if relpath.split(os.sep)[0] != 'reframe':
             return finfo
 
     return None
 
 
 def format_exception(exc_type, exc_value, tb):
     def format_user_frame(frame):
-        return '%s:%s: %s\n%s' % (frame.filename, frame.lineno,
+        relpath = os.path.relpath(frame.filename)
+        return '%s:%s: %s\n%s' % (relpath, frame.lineno,
                                   exc_value, ''.join(frame.code_context))
 
     if exc_type is None:
@@ -238,8 +238,8 @@ def format_user_frame(frame):
         return 'OS error: %s' % exc_value
 
     frame = user_frame(tb)
-    # if isinstance(exc_value, TypeError) and frame is not None:
-    #     return 'type error: ' + format_user_frame(frame)
+    if isinstance(exc_value, TypeError) and frame is not None:
+        return 'type error: ' + format_user_frame(frame)
 
     if isinstance(exc_value, ValueError) and frame is not None:
         return 'value error: ' + format_user_frame(frame)

reframe/utility/versioning.py

@@ -1,5 +1,9 @@
+import abc
 import functools
 import sys
+import re
+
+from itertools import takewhile
 
 
 @functools.total_ordering
@@ -26,7 +30,7 @@ def __init__(self, version):
             raise ValueError('invalid version string: %s' % version) from None
 
     def _value(self):
-        return 1000*self._major + 100*self._minor + self._patch_level
+        return 10000*self._major + 100*self._minor + self._patch_level
 
     def __eq__(self, other):
         if not isinstance(other, type(self)):
@@ -61,3 +65,81 @@ def __str__(self):
             return base
 
         return base + '-dev%s' % self._dev_number
+
+
+class _ValidatorImpl:
+    """Abstract base class for the validation of version ranges."""
+    @abc.abstractmethod
+    def validate(version):
+        pass
+
+
+class _IntervalValidator(_ValidatorImpl):
+    """Class for the validation of version intervals.
+
+    This class takes an interval of versions "v1..v2" and its method
+    ``validate`` returns ``True`` if a given version string is inside
+    the interval including the endpoints.
+    """
+    def __init__(self, condition):
+        try:
+            min_version_str, max_version_str = condition.split('..')
+        except ValueError:
+            raise ValueError("invalid format of version interval: %s" %
+                             condition) from None
+
+        if min_version_str and max_version_str:
+            self._min_version = Version(min_version_str)
+            self._max_version = Version(max_version_str)
+        else:
+            raise ValueError("missing bound on version interval %s" %
+                             condition)
+
+    def validate(self, version):
+        version = Version(version)
+        return ((version >= self._min_version) and
+                (version <= self._max_version))
+
+
+class _RelationalValidator(_ValidatorImpl):
+    """Class for the validation of Boolean relations of versions.
+
+    This class takes a Boolean relation of versions with the form
+    ``<bool_operator><version>``, and its method ``validate`` returns
+    ``True`` if a given version string satisfies the relation.
+    """
+    def __init__(self, condition):
+        self._op_actions = {
+            ">":  lambda x, y: x > y,
+            ">=": lambda x, y: x >= y,
+            "<":  lambda x, y: x < y,
+            "<=": lambda x, y: x <= y,
+            "==": lambda x, y: x == y,
+            "!=": lambda x, y: x != y,
+        }
+        cond_match = re.match(r'(\W{0,2})(\S+)', condition)
+        if not cond_match:
+            raise ValueError("invalid condition: '%s'" % condition)
+
+        self._ref_version = Version(cond_match.group(2))
+        op = cond_match.group(1)
+        if not op:
+            op = '=='
+
+        if op not in self._op_actions.keys():
+            raise ValueError("invalid boolean operator: '%s'" % op)
+        else:
+            self._operator = op
+
+    def validate(self, version):
+        do_validate = self._op_actions[self._operator]
+        return do_validate(Version(version), self._ref_version)
+
+
+class VersionValidator:
+    """Class factory for the validation of version ranges."""
+    def __new__(cls, condition):
+        if '..' in condition:
+            return _IntervalValidator(condition)
+        else:
+            return _RelationalValidator(condition)

tutorial/example1.py

@@ -1,7 +1,7 @@
 import reframe as rfm
 import reframe.utility.sanity as sn
 
-
+@rfm.required_version()
 @rfm.simple_test
 class Example1Test(rfm.RegressionTest):
     def __init__(self):

unittests/resources/checks_unlisted/good.py

@@ -44,7 +44,16 @@ def __repr__(self):
         return 'AnotherBaseTest(%s, %s)' % (self.a, self.b)
 
 
+@rfm.required_version('>=2.13-dev1')
 @rfm.simple_test
 class MyTest(MyBaseTest):
     def __init__(self):
         super().__init__(10, 20)
+
+
+# We intentionally have swapped the order of the two decorators here.
+# The order should not play any role.
+@rfm.simple_test
+@rfm.required_version('<=2.12')
+class InvalidTest(MyBaseTest):
+    pass

unittests/resources/checks_unlisted/no_required_version.py

@@ -0,0 +1,7 @@
+import reframe as rfm
+
+
+@rfm.required_version()
+@rfm.simple_test
+class SomeTest(rfm.RegressionTest):
+    pass

unittests/test_loader.py

@@ -59,10 +59,7 @@ def test_load_error(self):
         self.assertRaises(OSError, self.loader.load_from_file,
                           'unittests/resources/checks/foo.py')
 
-    def test_load_invalid_sytax(self):
-        invalid_check = ('unittests/resources/checks_unlisted/'
-                         'invalid_syntax_check.py')
-        with self.assertRaises(SyntaxError) as e:
-            self.loader.load_from_file(invalid_check)
-
-        self.assertEqual(e.exception.filename, invalid_check)
+    def test_load_bad_required_version(self):
+        with self.assertRaises(ValueError):
+            self.loader.load_from_file('unittests/resources/checks_unlisted/'
+                                       'no_required_version.py')

unittests/test_versioning.py

@@ -1,6 +1,7 @@
 import unittest
 
-from reframe.utility.versioning import Version
+from reframe.frontend.loader import RegressionCheckLoader
+from reframe.utility.versioning import Version, VersionValidator
 
 
 class TestVersioning(unittest.TestCase):
@@ -27,3 +28,22 @@ def test_comparing_versions(self):
         self.assertEqual(Version('1.3-dev1'), Version('1.3.0-dev1'))
         self.assertGreater(Version('1.12.3'), Version('1.2.3'))
         self.assertGreater(Version('1.2.23'), Version('1.2.3'))
+
+    def test_version_validation(self):
+        conditions = [VersionValidator('<=1.0.0'),
+                      VersionValidator('2.0.0..2.5'),
+                      VersionValidator('3.0')]
+        self.assertTrue(any(c.validate('0.1') for c in conditions))
+        self.assertTrue(any(c.validate('2.0.0') for c in conditions))
+        self.assertTrue(any(c.validate('2.2') for c in conditions))
+        self.assertTrue(any(c.validate('2.5') for c in conditions))
+        self.assertTrue(any(c.validate('3.0') for c in conditions))
+        self.assertFalse(any(c.validate('3.1') for c in conditions))
+        self.assertRaises(ValueError, VersionValidator, '2.0.0..')
+        self.assertRaises(ValueError, VersionValidator, '..2.0.0')
+        self.assertRaises(ValueError, VersionValidator, '1.0.0..2.0.0..3.0.0')
+        self.assertRaises(ValueError, VersionValidator, '=>2.0.0')
+        self.assertRaises(ValueError, VersionValidator, '2.0.0>')
+        self.assertRaises(ValueError, VersionValidator, '2.0.0>1.0.0')
+        self.assertRaises(ValueError, VersionValidator, '=>')
+        self.assertRaises(ValueError, VersionValidator, '>1')