[feat] Improve naming scheme of tests

reframe/core/decorators.py

@@ -56,9 +56,11 @@ def _instantiate_all():
                 ret.append(_instantiate(cls, args))
             except Exception:
                 frame = user_frame(*sys.exc_info())
-                msg = "skipping test due to errors: %s: " % cls.__name__
-                msg += "use `-v' for more information\n"
-                msg += "  FILE: %s:%s" % (frame.filename, frame.lineno)
+                msg  = f"skipping test due to errors: {cls.__qualname__}: "
+                msg += f"use `-v' for more information\n"
+                if frame:
+                    msg += f'  FILE: {frame.filename}:{frame.lineno}'
+
                 getlogger().warning(msg)
                 getlogger().verbose(traceback.format_exc())
 
@@ -117,13 +119,44 @@ def parameterized_test(*inst):
       This decorator does not instantiate any test.  It only registers them.
       The actual instantiation happens during the loading phase of the test.
     '''
+    def _extend_param_space(cls):
+        if not inst:
+            return
+
+        argnames = inspect.getfullargspec(cls.__init__).args[1:]
+        params = {}
+        for args in inst:
+            if isinstance(args, collections.abc.Sequence):
+                for i, v in enumerate(args):
+                    params.setdefault(argnames[i], ())
+                    params[argnames[i]] += (v,)
+            elif isinstance(args, collections.abc.Mapping):
+                for k, v in args.items():
+                    params.setdefault(k, ())
+                    params[k] += (v,)
+
+        # argvalues = zip(*inst)
+        # for name, values in zip(argnames, argvalues):
+        #     cls.param_space.params[name] = values
+
+        cls.param_space.params.update(params)
+
     def _do_register(cls):
         _validate_test(cls)
         if not cls.param_space.is_empty():
             raise ValueError(
                 f'{cls.__qualname__!r} is already a parameterized test'
             )
 
+        # NOTE: We need to mark that the test is using the old parameterized
+        # test syntax, so that any derived test does not extend its
+        # parameterization space with these parameters. This is to keep
+        # compatibility with the original behavior of this decorator, when it
+        # was not implemented using the new parameter machinery.
+        cls._rfm_compat_parameterized = True
+
+        _extend_param_space(cls)
+        cls.param_space.set_iter_fn(zip)
         for args in inst:
             _register_test(cls, args)
 

reframe/core/meta.py

@@ -18,6 +18,7 @@ class RegressionTestMeta(type):
 
     class MetaNamespace(namespaces.LocalNamespace):
         '''Custom namespace to control the cls attribute assignment.'''
+
         def __setitem__(self, key, value):
             if isinstance(value, variables.VarDirective):
                 # Insert the attribute in the variable namespace
@@ -159,7 +160,7 @@ def __call__(cls, *args, **kwargs):
         to perform specific reframe-internal actions. This gives extra control
         over the class instantiation process, allowing reframe to instantiate
         the regression test class differently if this class was registered or
-        not (e.g. when deep-copying a regression test object). These interal
+        not (e.g. when deep-copying a regression test object). These internal
         arguments must be intercepted before the object initialization, since
         these would otherwise affect the __init__ method's signature, and these
         internal mechanisms must be fully transparent to the user.

reframe/core/namespaces.py

@@ -135,11 +135,21 @@ def assert_target_cls(self, cls):
         assert isinstance(getattr(cls, self.local_namespace_name),
                           LocalNamespace)
 
+    def allow_inheritance(self, cls, base):
+        '''Return true if cls is allowed to inherit the namespace of base.
+
+        Subclasses may override that to customize the behavior.
+        '''
+        return True
+
     def inherit(self, cls):
         '''Inherit the Namespaces from the bases.'''
 
         for base in filter(lambda x: hasattr(x, self.namespace_name),
                            cls.__bases__):
+            if not self.allow_inheritance(cls, base):
+                continue
+
             assert isinstance(getattr(base, self.namespace_name), type(self))
             self.join(getattr(base, self.namespace_name), cls)
 

reframe/core/parameters.py

@@ -27,24 +27,30 @@ class TestParam:
     '''
 
     def __init__(self, values=None,
-                 inherit_params=False, filter_params=None):
+                 inherit_params=False, filter_params=None, fmtval=None):
         if values is None:
             values = []
 
-        # By default, filter out all the parameter values defined in the
-        # base classes.
         if not inherit_params:
+            # By default, filter out all the parameter values defined in the
+            # base classes.
             def filter_params(x):
                 return ()
-
-        # If inherit_params==True, inherit all the parameter values from the
-        # base classes as default behaviour.
         elif filter_params is None:
+            # If inherit_params==True, inherit all the parameter values from the
+            # base classes as default behaviour.
             def filter_params(x):
                 return x
 
+        if not callable(filter_params):
+            raise TypeError('filter_params must be a callable')
+
+        if fmtval and not callable(fmtval):
+            raise TypeError('fmtval must be a callable')
+
         self.values = tuple(values)
         self.filter_params = filter_params
+        self.format_value = fmtval
 
     def __set_name__(self, owner, name):
         self.name = name
@@ -90,7 +96,16 @@ def __init__(self, target_cls=None, target_namespace=None):
         super().__init__(target_cls, target_namespace)
 
         # Internal parameter space usage tracker
-        self.__unique_iter = iter(self)
+        self.set_iter_fn(itertools.product)
+        # self.__unique_iter = iter(self)
+
+        # Alternative value formatting functions
+        self.__format_fn = {}
+
+    def allow_inheritance(self, cls, base):
+        # Do not allow inheritance of the parameterization space in case of
+        # the old @parameterized_test decorator.
+        return not hasattr(cls, '_rfm_compat_parameterized')
 
     def join(self, other, cls):
         '''Join other parameter space into the current one.
@@ -103,6 +118,7 @@ def join(self, other, cls):
         :param other: instance of the ParamSpace class.
         :param cls: the target class.
         '''
+
         for key in other.params:
             # With multiple inheritance, a single parameter
             # could be doubly defined and lead to repeated
@@ -128,6 +144,8 @@ def extend(self, cls):
             self.params[name] = (
                 p.filter_params(self.params.get(name, ())) + p.values
             )
+            if p.format_value:
+                self.__format_fn[name] = p.format_value
 
         # If any previously declared parameter was defined in the class body
         # by directly assigning it a value, raise an error. Parameters must be
@@ -158,26 +176,34 @@ def inject(self, obj, cls=None, use_params=False):
             parameter values defined in the parameter space.
 
         '''
+
         # Set the values of the test parameters (if any)
+        injected = []
         if use_params and self.params:
             try:
                 # Consume the parameter space iterator
                 param_values = next(self.unique_iter)
                 for index, key in enumerate(self.params):
                     setattr(obj, key, param_values[index])
+                    format_fn = self.__format_fn.get(key)
+                    injected.append(
+                        (key, param_values[index], format_fn)
+                    )
 
+                obj.params_inserted(injected)
             except StopIteration as no_params:
                 raise RuntimeError(
                     f'exhausted parameter space: all possible parameter value'
                     f' combinations have been used for '
                     f'{obj.__class__.__qualname__}'
                 ) from None
-
         else:
             # Otherwise init the params as None
             for key in self.params:
                 setattr(obj, key, None)
 
+        obj.params_inserted(injected)
+
     def __iter__(self):
         '''Create a generator object to iterate over the parameter space
 
@@ -186,10 +212,17 @@ def __iter__(self):
 
         :return: generator object to iterate over the parameter space.
         '''
-        yield from itertools.product(
+        # yield from itertools.product(
+        #     *(copy.deepcopy(p) for p in self.params.values())
+        # )
+        yield from self.__iterator(
             *(copy.deepcopy(p) for p in self.params.values())
         )
 
+    def set_iter_fn(self, fn):
+        self.__iterator = fn
+        self.__unique_iter = iter(self)
+
     @property
     def params(self):
         return self._namespace