[feat] Refactor test registry

reframe/core/decorators.py

@@ -27,7 +27,82 @@
 from reframe.utility.versioning import VersionValidator
 
 
-def _register_test(cls, args=None):
+class TestRegistry:
+    '''Regression test registry.
+
+    The tests are stored in a dictionary where the test class is the key
+    and the constructor arguments for the different instantiations of the
+    test are stored as the dictionary value as a list of (args, kwargs)
+    tuples.
+
+    For backward compatibility reasons, the registry also contains a set of
+    tests to be skipped. The machinery related to this should be dropped with
+    the ``required_version`` decorator.
+    '''
+
+    def __init__(self):
+        self._tests = dict()
+        self._skip_tests = set()
+
+    @classmethod
+    def create(cls, test, *args, **kwargs):
+        obj = cls()
+        obj.add(test, *args, **kwargs)
+        return obj
+
+    def add(self, test, *args, **kwargs):
+        self._tests.setdefault(test, [])
+        self._tests[test].append((args, kwargs))
+
+    # FIXME: To drop with the required_version decorator
+    def skip(self, test):
+        '''Add a test to the skip set.'''
+        self._skip_tests.add(test)
+
+    def instantiate_all(self):
+        '''Instantiate all the registered tests.'''
+        ret = []
+        for test, variants in self._tests.items():
+            if test in self._skip_tests:
+                continue
+
+            for args, kwargs in variants:
+                try:
+                    ret.append(test(*args, **kwargs))
+                except SkipTestError as e:
+                    getlogger().warning(
+                        f'skipping test {test.__qualname__!r}: {e}'
+                    )
+                except Exception:
+                    exc_info = sys.exc_info()
+                    getlogger().warning(
+                        f"skipping test {test.__qualname__!r}: "
+                        f"{what(*exc_info)} "
+                        f"(rerun with '-v' for more information)"
+                    )
+                    getlogger().verbose(traceback.format_exc())
+
+        return ret
+
+    def __iter__(self):
+        '''Iterate over the registered test classes.'''
+        return iter(self._tests.keys())
+
+    def __contains__(self, test):
+        return test in self._tests
+
+
+def _register_test(cls, *args, **kwargs):
+    '''Register a test and its construction arguments into the registry.'''
+
+    mod = inspect.getmodule(cls)
+    if not hasattr(mod, '_rfm_test_registry'):
+        mod._rfm_test_registry = TestRegistry.create(cls, *args, **kwargs)
+    else:
+        mod._rfm_test_registry.add(cls, *args, **kwargs)
+
+
+def _register_parameterized_test(cls, args=None):
     '''Register the test.
 
     Register the test with _rfm_use_params=True. This additional argument flags
@@ -111,7 +186,7 @@ def simple_test(cls):
     '''
     if _validate_test(cls):
         for _ in cls.param_space:
-            _register_test(cls)
+            _register_test(cls, _rfm_use_params=True)
 
     return cls
 
@@ -153,7 +228,7 @@ def _do_register(cls):
                 )
 
             for args in inst:
-                _register_test(cls, args)
+                _register_parameterized_test(cls, args)
 
         return cls
 
@@ -210,12 +285,18 @@ def _skip_tests(cls):
         if not hasattr(mod, '__rfm_skip_tests'):
             mod.__rfm_skip_tests = set()
 
+        if not hasattr(mod, '_rfm_test_registry'):
+            mod._rfm_test_registry = TestRegistry()
+
         if not any(c.validate(osext.reframe_version()) for c in conditions):
             getlogger().warning(
                 f"skipping incompatible test '{cls.__qualname__}': not valid "
                 f"for ReFrame version {osext.reframe_version().split('-')[0]}"
             )
-            mod.__rfm_skip_tests.add(cls)
+            if cls in mod._rfm_test_registry:
+                mod._rfm_test_registry.skip(cls)
+            else:
+                mod.__rfm_skip_tests.add(cls)
 
         return cls
 

reframe/frontend/loader.py

@@ -8,7 +8,6 @@
 #
 
 import ast
-import collections.abc
 import inspect
 import os
 import sys
@@ -118,8 +117,14 @@ def recurse(self):
     def load_from_module(self, module):
         '''Load user checks from module.
 
-        This method tries to call the `_rfm_gettests()` method of the user
-        check and validates its return value.'''
+        This method tries to load the test registry from a given module and
+        instantiates all the tests in the registry. The instantiated checks
+        are validated before return.
+
+        For legacy reasons, a module might have the additional legacy registry
+        `_rfm_gettests`, which is a method that instantiates all the tests
+        registered with the deprecated `parameterized_test` decorator.
+        '''
         from reframe.core.pipeline import RegressionTest
 
         # Warn in case of old syntax
@@ -129,16 +134,18 @@ def load_from_module(self, module):
                 f'in test files: please use @reframe.simple_test decorator'
             )
 
-        if not hasattr(module, '_rfm_gettests'):
+        # FIXME: Remove the legacy_registry after dropping parameterized_test
+        registry = getattr(module, '_rfm_test_registry', None)
+        legacy_registry = getattr(module, '_rfm_gettests', None)
+        if not any((registry, legacy_registry)):
             getlogger().debug('No tests registered')
             return []
 
-        candidates = module._rfm_gettests()
-        if not isinstance(candidates, collections.abc.Sequence):
-            getlogger().warning(
-                f'Tests not registered correctly in {module.__name__!r}'
-            )
-            return []
+        candidates = registry.instantiate_all() if registry else []
+        legacy_candidates = legacy_registry() if legacy_registry else []
+
+        # Merge registries
+        candidates += legacy_candidates
 
         ret = []
         for c in candidates:

unittests/test_parameters.py

@@ -150,7 +150,7 @@ class MyTest(ExtendParams):
         pass
 
     mod = inspect.getmodule(MyTest)
-    tests = mod._rfm_gettests()
+    tests = mod._rfm_test_registry.instantiate_all()
     assert len(tests) == 8
     for test in tests:
         assert test.P0 is not None