diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index 19b8f5ab91..fdd20e5e18 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -36,57 +36,168 @@ Pipeline Hooks .. autodecorator:: reframe.core.decorators.require_deps -.. _directives: -Directives ----------- +Builtins +-------- -Directives are functions that can be called directly in the body of a ReFrame regression test class. -These functions exert control over the test creation, and they allow adding and/or modifying certain attributes of the regression test. -For example, a test can be parameterized using the :func:`parameter` directive as follows: +.. versionadded:: 3.4.2 -.. code:: python +ReFrame provides built-in functions that facilitate the creation of extensible tests (i.e. a test library). +These *builtins* are intended to be used directly in the class body of the test, allowing the ReFrame internals to *pre-process* their input before the actual test creation takes place. +This provides the ReFrame internals with further control over the user's input, making the process of writing regression tests less error-prone thanks to a better error checking. +In essence, these builtins exert control over the test creation, and they allow adding and/or modifying certain attributes of the regression test. + + +.. py:function:: reframe.core.pipeline.RegressionTest.parameter(values=None, inherit_params=False, filter_params=None) + + Inserts or modifies a regression test parameter. + If a parameter with a matching name is already present in the parameter space of a parent class, the existing parameter values will be combined with those provided by this method following the inheritance behavior set by the arguments ``inherit_params`` and ``filter_params``. + Instead, if no parameter with a matching name exists in any of the parent parameter spaces, a new regression test parameter is created. + A regression test can be parametrized as follows: + + .. code:: python + + class Foo(rfm.RegressionTest): + variant = parameter(['A', 'B']) - class MyTest(rfm.RegressionTest): - parameter('variant', ['A', 'B']) - def __init__(self): if self.variant == 'A': do_this() else: do_other() -One of the most powerful features about using directives is that they store their input information at the class level. -This means if one were to extend or specialize an existing regression test, the test attribute additions and modifications made through directives in the parent class will be automatically inherited by the child test. -For instance, continuing with the example above, one could override the :func:`__init__` method in the :class:`MyTest` regression test as follows: - -.. code:: python + One of the most powerful features about these built-in functions is that they store their input information at the class level. + This means if one were to extend or specialize an existing regression test, the test attribute additions and modifications made through built-in functions in the parent class will be automatically inherited by the child test. + For instance, continuing with the example above, one could override the :func:`__init__` method in the :class:`MyTest` regression test as follows: - class MyModifiedTest(MyTest): + .. code:: python + class Bar(Foo): def __init__(self): if self.variant == 'A': override_this() else: override_other() + Note that this built-in parameter function provides an alternative method to parameterize a test to :func:`reframe.core.decorators.parameterized_test`, and the use of both approaches in the same test is currently disallowed. + The two main advantages of the built-in :func:`parameter` over the decorated approach reside in the parameter inheritance across classes and the handling of large parameter sets. + As shown in the example above, the parameters declared with the built-in :func:`parameter` are automatically carried over into derived tests through class inheritance, whereas tests using the decorated approach would have to redefine the parameters on every test. + Similarly, parameters declared through the built-in :func:`parameter` are regarded as fully independent from each other and ReFrame will automatically generate as many tests as available parameter combinations. This is a major advantage over the decorated approach, where one would have to manually expand the parameter combinations. + This is illustrated in the example below, consisting of a case with two parameters, each having two possible values. + + .. code:: python + + # Parameterized test with two parameters (p0 = ['a', 'b'] and p1 = ['x', 'y']) + @rfm.parameterized_test(['a','x'], ['a','y'], ['b','x'], ['b', 'y']) + class Foo(rfm.RegressionTest): + def __init__(self, p0, p1): + do_something(p0, p1) + + # This is easier to write with the parameter built-in. + @rfm.simple_test + class Bar(rfm.RegressionTest): + p0 = parameter(['a', 'b']) + p1 = parameter(['x', 'y']) + + def __init__(self): + do_something(self.p0, self.p1) + + + :param values: A list containing the parameter values. + If no values are passed when creating a new parameter, the parameter is considered as *declared* but not *defined* (i.e. an abstract parameter). + Instead, for an existing parameter, this depends on the parameter's inheritance behaviour and on whether any values where provided in any of the parent parameter spaces. + :param inherit_params: If :obj:`False`, no parameter values that may have been defined in any of the parent parameter spaces will be inherited. + :param filter_params: Function to filter/modify the inherited parameter values that may have been provided in any of the parent parameter spaces. + This function must accept a single argument, which will be passed as an iterable containing the inherited parameter values. + This only has an effect if used with ``inherit_params=True``. + + +.. py:function:: reframe.core.pipeline.RegressionTest.variable(*types, value=None) + + Inserts a new regression test variable. + Declaring a test variable through the :func:`variable` built-in allows for a more robust test implementation than if the variables were just defined as regular test attributes (e.g. ``self.a = 10``). + Using variables declared through the :func:`variable` built-in guarantees that these regression test variables will not be redeclared by any child class, while also ensuring that any values that may be assigned to such variables comply with its original declaration. + In essence, by using test variables, the user removes any potential test errors that might be caused by accidentally overriding a class attribute. See the example below. + + + .. code:: python + + class Foo(rfm.RegressionTest): + my_var = variable(int, value=8) + not_a_var = 4 + + def __init__(self): + print(self.my_var) # prints 8. + # self.my_var = 'override' # Error: my_var must be an int! + self.not_a_var = 'override' # However, this would work. Dangerous! + self.my_var = 10 # tests may also assign values the standard way + + The argument ``value`` in the :func:`variable` built-in sets the default value for the variable. + As mentioned above, a variable may not be declared more than once, but its default value can be updated by simply assigning it a new value directly in the class body. + + .. code:: python + + class Bar(Foo): + my_var = 4 + # my_var = 'override' # Error again! + + def __init__(self): + print(self.my_var) # prints 4. + + Here, the class :class:`Bar` inherits the variables from :class:`Foo` and can see that ``my_var`` has already been declared in the parent class. Therefore, the value of ``my_var`` is updated ensuring that the new value complies to the original variable declaration. + + These examples above assumed that a default value can be provided to the variables in the bases tests, but that might not always be the case. + For example, when writing a test library, one might want to leave some variables undefined and force the user to set these when using the test. + As shown in the example below, imposing such requirement is as simple as not passing any ``value`` to the :func:`variable` built-in, which marks the given variable as *required*. + + .. code:: python + + # Test as written in the library + class EchoBaseTest(rfm.RunOnlyRegressionTest): + what = variable(str) + + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['PrgEnv-gnu'] + self.executable = f'echo {self.what}' + self.sanity_patterns = sn.assert_found(fr'{self.what}') + + + # Test as written by the user + @rfm.simple_test + class HelloTest(EchoBaseTest): + what = 'Hello' + + + # A parametrized test with type-checking + @rfm.simple_test + class FoodTest(EchoBaseTest): + param = parameter(['Bacon', 'Eggs']) + + def __init__(self): + self.what = self.param + super().__init__() + + + Similarly to a variable with a value already assigned to it, the value of a required variable may be set either directly in the class body, on the :func:`__init__` method, or in any other hook before it is referenced. + Otherwise an error will be raised indicating that a required variable has not been set. + Conversely, a variable with a default value already assigned to it can be made required by assigning it the ``required`` keyword. -.. py:function:: reframe.core.pipeline.RegressionTest.parameter(name, values=None, inherit_params=False, filter_params=None) + .. code:: python - Inserts or modifies a regression test parameter. - If a parameter with a matching name is already present in the parameter space of a parent class, the existing parameter values will be combined with those provided by this method following the inheritance behaviour set by the arguments ``inherit_params`` and ``filter_params``. - Instead, if no parameter with a matching name exists in any of the parent parameter spaces, a new regression test parameter is created. + class MyRequiredTest(HelloTest): + what = required - :param name: The parameter name. - :param values: A list containing the parameter values. - If no values are passed when creating a new parameter, the parameter is considered as *declared* but not *defined* (i.e. an abstract parameter). - Instead, for an existing parameter, this depends on the parameter's inheritance behaviour and on whether any values where provided in any of the parent parameter spaces. - :param inherit_params: If :obj:`False`, no parameter values that may have been defined in any of the parent parameter spaces will be inherited. - :param filter_params: Function to filter/modify the inherited parameter values that may have been provided in any of the parent parameter spaces. - This function must accept a single argument, which will be passed as an iterable containing the inherited parameter values. - This only has an effect if used with ``inherit_params=True``. + Running the above test will cause the :func:`__init__` method from :class:`EchoBaseTest` to throw an error indicating that the variable ``what`` has not been set. + :param types: the supported types for the variable. + :param value: the default value assigned to the variable. If no value is provided, the variable is set as ``required``. + :param field: the field validator to be used for this variable. + If no field argument is provided, it defaults to + :class:`reframe.core.fields.TypedField`. + Note that the field validator provided by this argument must derive from + :class:`reframe.core.fields.Field`. Environments and Systems diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 13b0651a06..1b4a71aaca 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -9,29 +9,69 @@ from reframe.core.exceptions import ReframeSyntaxError +import reframe.core.namespaces as namespaces import reframe.core.parameters as parameters +import reframe.core.variables as variables class RegressionTestMeta(type): + + class MetaNamespace(namespaces.LocalNamespace): + '''Custom namespace to control the cls attribute assignment.''' + def __setitem__(self, key, value): + if isinstance(value, variables.VarDirective): + # Insert the attribute in the variable namespace + self['_rfm_local_var_space'][key] = value + elif isinstance(value, parameters.TestParam): + # Insert the attribute in the parameter namespace + self['_rfm_local_param_space'][key] = value + else: + super().__setitem__(key, value) + @classmethod - def __prepare__(cls, name, bases, **kwargs): + def __prepare__(metacls, name, bases, **kwargs): namespace = super().__prepare__(name, bases, **kwargs) # Regression test parameter space defined at the class level - local_param_space = parameters.LocalParamSpace() + local_param_space = namespaces.LocalNamespace() namespace['_rfm_local_param_space'] = local_param_space - # Directive to add a regression test parameter directly in the - # class body as: `parameter('P0', 0,1,2,3)`. - namespace['parameter'] = local_param_space.add_param + # Directive to insert a regression test parameter directly in the + # class body as: `P0 = parameter([0,1,2,3])`. + namespace['parameter'] = parameters.TestParam + + # Regression test var space defined at the class level + local_var_space = namespaces.LocalNamespace() + namespace['_rfm_local_var_space'] = local_var_space - return namespace + # Directives to add/modify a regression test variable + namespace['variable'] = variables.TestVar + namespace['required'] = variables.UndefineVar() + return metacls.MetaNamespace(namespace) + + def __new__(metacls, name, bases, namespace, **kwargs): + return super().__new__(metacls, name, bases, dict(namespace), **kwargs) def __init__(cls, name, bases, namespace, **kwargs): super().__init__(name, bases, namespace, **kwargs) - # Build the regression test parameter space - cls._rfm_param_space = parameters.ParamSpace(cls) + # Create a set with the attribute names already in use. + cls._rfm_dir = set() + for base in bases: + if hasattr(base, '_rfm_dir'): + cls._rfm_dir.update(base._rfm_dir) + + used_attribute_names = set(cls._rfm_dir) + + # Build the var space and extend the target namespace + variables.VarSpace(cls, used_attribute_names) + used_attribute_names.update(cls._rfm_var_space.vars) + + # Build the parameter space + parameters.ParamSpace(cls, used_attribute_names) + + # Update used names set with the local __dict__ + cls._rfm_dir.update(cls.__dict__) # Set up the hooks for the pipeline stages based on the _rfm_attach # attribute; all dependencies will be resolved first in the post-setup @@ -100,6 +140,27 @@ def __call__(cls, *args, **kwargs): obj.__init__(*args, **kwargs) return obj + def __getattribute__(cls, name): + ''' Attribute lookup method for the MetaNamespace. + + This metaclass implements a custom namespace, where built-in `variable` + and `parameter` types are stored in their own sub-namespaces (see + :class:`reframe.core.meta.RegressionTestMeta.MetaNamespace`). + This method will perform an attribute lookup on these sub-namespaces if + a call to the default `__getattribute__` method fails to retrieve the + requested class attribute. + ''' + try: + return super().__getattribute__(name) + except AttributeError: + try: + return cls._rfm_local_var_space[name] + except KeyError: + try: + return cls._rfm_local_param_space[name] + except KeyError: + return super().__getattr__(name) + @property def param_space(cls): # Make the parameter space available as read-only diff --git a/reframe/core/namespaces.py b/reframe/core/namespaces.py new file mode 100644 index 0000000000..13ce9dc62c --- /dev/null +++ b/reframe/core/namespaces.py @@ -0,0 +1,177 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# Base classes to manage the namespace of a regression test. +# + + +import abc + + +class LocalNamespace: + '''Local namespace of a regression test. + + Temporary storage for test attributes defined in the test class body. + This local namespace is populated during the test class body execution. + + Example: In the pseudo-code below, the local namespace of A is {P0}, + and the local namespace of B is {P1}. However, the final namespace + of A is still {P0}, and the final namespace of B is {P0, P1}. + + .. code:: python + + class A(RegressionTest): + var('P0') + + class B(A): + var('P1') + ''' + + def __init__(self, namespace=None): + self._namespace = namespace or {} + + def __getattr__(self, name): + return getattr(self._namespace, name) + + def __getitem__(self, key): + return self._namespace[key] + + def __setitem__(self, key, value): + if key not in self._namespace: + self._namespace[key] = value + else: + self._raise_namespace_clash(key) + + def __contains__(self, key): + return key in self._namespace + + def __iter__(self): + return iter(self._namespace) + + def __len__(self): + return len(self._namespace) + + def __repr__(self): + return f'{type(self).__name__}({self._namespace!r})' + + def _raise_namespace_clash(self, name): + '''Raise an error if there is a namespace clash.''' + raise ValueError( + f'{name!r} is already present in the current namespace' + ) + + +class Namespace(metaclass=abc.ABCMeta): + '''Namespace of a regression test. + + The final namespace may be built by inheriting namespaces from + the base classes, and extended with the information stored in the local + namespace of the target class. In this context, the target class is + simply the regression test class where the namespace is to be built. + + To allow for this inheritance and extension of the namespace, this + class must define the names under which the local and final namespaces + are inserted in the target classes. + + If a target class is provided, the constructor will attach the Namespace + instance into the target class with the class attribute name as defined + in ``namespace_name``. + + Eventually, the items from a Namespace are injected as attributes of + the target class instance by the :func:`inject` method, which must be + called by the target class during its instantiation process. Also, a target + class may use more that one Namespace, which raises the need for name + checking across namespaces. Thus, the :func:`__init__` method accepts the + additional argument ``illegal_names``, which is a set of class attribute + names already in use by the target class or other namespaces from this + target class. Then, after the Namespace is built, if ``illegal_names`` is + provided, a sanity check is performed, ensuring that no name clashing + will occur during the target class instantiation process. + ''' + + @property + @abc.abstractmethod + def local_namespace_name(self): + '''Name of the local namespace in the target class. + + Name under which the local namespace is stored in the + :class:`reframe.core.pipeline.RegressionTest` class. + ''' + + @property + @abc.abstractmethod + def namespace_name(self): + '''Name of the namespace in the target class. + + Name under which the namespace is stored in the + :class:`reframe.core.pipeline.RegressionTest` class. + ''' + + def __init__(self, target_cls=None, illegal_names=None): + self._namespace = {} + if target_cls: + # Assert the Namespace can be built for the target_cls + self.assert_target_cls(target_cls) + + # Inherit Namespaces from the base clases + self.inherit(target_cls) + + # Extend the Namespace with the LocalNamespace + self.extend(target_cls) + + # Sanity checkings on the resulting Namespace + self.sanity(target_cls, illegal_names) + + # Attach the Namespace to the target class + setattr(target_cls, self.namespace_name, self) + + def assert_target_cls(self, cls): + '''Assert the target class has a valid local namespace.''' + + assert hasattr(cls, self.local_namespace_name) + assert isinstance(getattr(cls, self.local_namespace_name), + LocalNamespace) + + def inherit(self, cls): + '''Inherit the Namespaces from the bases.''' + + for base in filter(lambda x: hasattr(x, self.namespace_name), + cls.__bases__): + assert isinstance(getattr(base, self.namespace_name), type(self)) + self.join(getattr(base, self.namespace_name), cls) + + @abc.abstractmethod + def join(self, other, cls): + '''Join other Namespace with the current one.''' + + @abc.abstractmethod + def extend(self, cls): + '''Extend the namespace with the local namespace.''' + + def sanity(self, cls, illegal_names=None): + '''Sanity checks post-creation of the namespace. + + By default, we make illegal to have any item in the namespace + that clashes with a member of the target class. + ''' + if illegal_names is None: + illegal_names = set(dir(cls)) + + for key in self._namespace: + if key in illegal_names: + raise ValueError( + f'{key!r} already defined in class ' + f'{cls.__qualname__!r}' + ) + + @abc.abstractmethod + def inject(self, obj, objtype=None): + '''Insert the items from the namespace as attributes of the object + ``obj``. + ''' + + def items(self): + return self._namespace.items() diff --git a/reframe/core/parameters.py b/reframe/core/parameters.py index 07a508e3be..bbb7643fd6 100644 --- a/reframe/core/parameters.py +++ b/reframe/core/parameters.py @@ -1,4 +1,4 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause @@ -7,13 +7,14 @@ # Functionality to build extensible parameterized tests. # +import copy import functools import itertools -from reframe.core.exceptions import ReframeSyntaxError +import reframe.core.namespaces as namespaces -class _TestParameter: +class TestParam: '''Regression test paramter class. Stores the attributes of a regression test parameter as defined directly @@ -21,9 +22,11 @@ class _TestParameter: values, and inheritance behaviour. This class should be thought of as a temporary storage for these parameter attributes, before the full final parameter space is built. + + :meta private: ''' - def __init__(self, name, values=None, + def __init__(self, values=None, inherit_params=False, filter_params=None): if values is None: values = [] @@ -40,103 +43,33 @@ def filter_params(x): def filter_params(x): return x - self.name = name self.values = tuple(values) self.filter_params = filter_params - -class LocalParamSpace: - '''Local parameter space of a regression test. - - Stores all the regression test parameters defined in the test class body. - In the context of this class, a regression test parameter is an instance - of the class _TestParameter. This local parameter space is populated - during the test class body execution through the add_param method, and the - different parameters are stored under the _params attribute. This class - should be thought of as a temporary storage for this local parameter space, - before the full final parameter space is built. - - Example: In the pseudo-code below, the local parameter space of A is {P0}, - and the local parameter space of B is {P1}. However, the final parameter - space of A is still {P0}, and the final parameter space of B is {P0, P1}. - - .. code:: python - - class A(RegressionTest): - -> define parameter P0 with value X. - - class B(A): - -> define parameter P1 with value Y. - ''' - - def __init__(self): - self._params = {} - - def __getattr__(self, name): - # Delegate any unknown attribute access to the actual parameter space - return getattr(self._params, name) - - def __setitem__(self, name, value): - if name not in self._params: - self._params[name] = value - else: - raise ValueError( - f'parameter {name!r} already defined in this class' - ) - - def add_param(self, name, values=None, **kwargs): - '''Insert or modify a regression test parameter. - - This method must be called directly in the class body. For each - regression test class definition, this function may only be called - once per parameter. Calling this method during or after the class - instantiation has an undefined behavior. - - .. seealso:: - - :ref:`directives` - - ''' - self[name] = _TestParameter(name, values, **kwargs) - - @property - def params(self): - return self._params - - def items(self): - return self._params.items() + def __set_name__(self, owner, name): + self.name = name -class ParamSpace: +class ParamSpace(namespaces.Namespace): ''' Regression test parameter space Host class for the parameter space of a regresion test. The parameter - space is stored as a dictionary (self._params), where the keys are the + space is stored as a dictionary (self.params), where the keys are the parameter names and the values are tuples with all the available values for each parameter. The __init__ method in this class takes an optional argument (target_class), which is the regression test class where the - parameter space is to be built. If this target class is provided, the - __init__ method performs three main steps. These are (in order of exec) - the inheritance of the parameter spaces from the direct parent classes, - the extension of the inherited parameter space with the local parameter - space (this must be an instance of - :class `reframe.core.parameters.LocalParamSpace`), and lastly, a check to - ensure that none of the parameter names clashes with any of the class - attributes existing in the target class. If no target class is provided, - the parameter space is initialized as empty. After the parameter space is - set, a parameter space iterator is created under self.__unique_iter, which - acts as an internal control variable that tracks the usage of this - parameter space. This iterator walks through all possible parameter - combinations and cannot be restored after reaching exhaustion. This unique - iterator is made available as read-only through cls.unique_iterator and it - may be used by an external class to track the usage of the parameter space - (i.e. the - :class `reframe.core.pipeline.RegressionTest` can use this unique iterator - to ensure that each point of the parameter space has only been instantiated - once). The length of this iterator matches the value returned by the member - function __len__. + parameter space is to e inserted as the ``_rfm_param_space`` class + attribute. If no target class is provided, the parameter space is + initialized as empty. After the parameter space is set, a parameter space + iterator is created under self.__unique_iter, which acts as an internal + control variable that tracks the usage of this parameter space. This + iterator walks through all possible parameter combinations and cannot be + restored after reaching exhaustion. The length of this iterator matches + the value returned by the member function __len__. :param target_cls: the class where the full parameter space is to be built. + :param target_namespace: a reference namespace to ensure that no name + clashes occur (see :class:`reframe.core.namespaces.Namespace`). .. note:: The __init__ method is aware of the implementation details of the @@ -145,76 +78,121 @@ class ParamSpace: the target class. ''' - def __init__(self, target_cls=None): - self._params = {} - - # If a target class is provided, build the param space for it - if target_cls: - assert hasattr(target_cls, '_rfm_local_param_space') - assert isinstance(target_cls._rfm_local_param_space, - LocalParamSpace) - - # Inherit the parameter spaces from the direct parent classes - for base in filter(lambda x: hasattr(x, 'param_space'), - target_cls.__bases__): - self.join(base._rfm_param_space) + @property + def local_namespace_name(self): + return '_rfm_local_param_space' - # Extend the parameter space with the local parameter space - for name, p in target_cls._rfm_local_param_space.items(): - self._params[name] = ( - p.filter_params(self._params.get(name, ())) + p.values - ) + @property + def namespace_name(self): + return '_rfm_param_space' - # Make sure that none of the parameters clashes with the target - # class namespace - target_namespace = set(dir(target_cls)) - for key in self._params: - if key in target_namespace: - raise ReframeSyntaxError( - f'parameter {key!r} clashes with other variables' - f' present in the namespace from class ' - f'{target_cls.__qualname__!r}' - ) + def __init__(self, target_cls=None, target_namespace=None): + super().__init__(target_cls, target_namespace) # Internal parameter space usage tracker self.__unique_iter = iter(self) - def join(self, other): - '''Join two parameter spaces into one + def join(self, other, cls): + '''Join other parameter space into the current one. Join two different parameter spaces into a single one. Both parameter spaces must be an instance ot the ParamSpace class. This method will raise an error if a parameter is defined in the two parameter spaces to be merged. - :param other: instance of the ParamSpace class + :param other: instance of the ParamSpace class. + :param cls: the target class. ''' for key in other.params: # With multiple inheritance, a single parameter # could be doubly defined and lead to repeated # values - if (key in self._params and - self._params[key] != () and + if (key in self.params and + self.params[key] != () and other.params[key] != ()): - raise ReframeSyntaxError(f'parameter space conflict: ' - f'parameter {key!r} already defined ' - f'in {b.__qualname__!r}') + raise ValueError( + f'parameter space conflict: ' + f'parameter {key!r} is defined in more than ' + f'one base class of class {cls.__qualname__!r}' + ) + + self.params[key] = ( + other.params.get(key, ()) + self.params.get(key, ()) + ) + + def extend(self, cls): + '''Extend the parameter space with the local parameter space.''' - self._params[key] = ( - other.params.get(key, ()) + self._params.get(key, ()) + for name, p in cls._rfm_local_param_space.items(): + self.params[name] = ( + p.filter_params(self.params.get(name, ())) + p.values ) + # If any previously declared parameter was defined in the class body + # by directly assigning it a value, raise an error. Parameters must be + # changed using the `x = parameter([...])` syntax. + for key, values in cls.__dict__.items(): + if key in self.params: + raise ValueError( + f'parameter {key!r} must be modified through the built-in ' + f'parameter type' + ) + + def inject(self, obj, cls=None, use_params=False): + '''Insert the params in the regression test. + + Create and initialize the regression test parameters as object + attributes. The values assigned to these parameters exclusively depend + on the use_params argument. If this is set to True, the current object + uses the parameter space iterator (see + :class:`reframe.core.pipeline.RegressionTest` and consumes a set of + parameter values (i.e. a point in the parameter space). Contrarily, if + use_params is False, the regression test parameters are initialized as + None. + + :param obj: The test object. + :param cls: The test class. + :param use_param: bool that dictates whether an instance of the + :class:`reframe.core.pipeline.RegressionTest` is to use the + parameter values defined in the parameter space. + + ''' + # Set the values of the test parameters (if any) + if use_params and self.params: + try: + # Consume the parameter space iterator + param_values = next(self.unique_iter) + for index, key in enumerate(self.params): + setattr(obj, key, param_values[index]) + + except StopIteration as no_params: + raise RuntimeError( + f'exhausted parameter space: all possible parameter value' + f' combinations have been used for ' + f'{obj.__class__.__qualname__}' + ) from None + + else: + # Otherwise init the params as None + for key in self.params: + setattr(obj, key, None) + def __iter__(self): '''Create a generator object to iterate over the parameter space + The parameters must be deep-copied to prevent an instance from + modifying the class parameter space. + :return: generator object to iterate over the parameter space. ''' - yield from itertools.product(*(p for p in self._params.values())) + yield from itertools.product( + *(copy.deepcopy(p) for p in self.params.values()) + ) @property def params(self): - return self._params + return self._namespace @property def unique_iter(self): @@ -233,17 +211,18 @@ def __len__(self): the parameter space), the returned parameter space length is 0. :return: length of the parameter space + ''' - if not self._params: + if not self.params: return 1 return functools.reduce( lambda x, y: x*y, - (len(p) for p in self._params.values()) + (len(p) for p in self.params.values()) ) def __getitem__(self, key): - return self._params.get(key, ()) + return self.params.get(key, ()) def is_empty(self): - return self._params == {} + return self.params == {} diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index b3f62d6e77..1b1d644b65 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -9,7 +9,7 @@ __all__ = [ 'CompileOnlyRegressionTest', 'RegressionTest', 'RunOnlyRegressionTest', - 'DEPEND_BY_ENV', 'DEPEND_EXACT', 'DEPEND_FULLY', 'final' + 'DEPEND_BY_ENV', 'DEPEND_EXACT', 'DEPEND_FULLY', 'final', 'RegressionMixin' ] @@ -126,6 +126,20 @@ def _wrapped(*args, **kwargs): return _wrapped +class RegressionMixin(metaclass=RegressionTestMeta): + '''Base mixin class for regression tests. + + Multiple inheritance from more than one + :class:`RegressionTest` class is not allowed in ReFrame. Hence, mixin + classes provide the flexibility to bundle reusable test add-ons, leveraging + the metaclass magic implemented in + :class:`RegressionTestMeta`. Using this metaclass allows mixin classes to + use powerful ReFrame features, such as hooks, parameters or variables. + + .. versionadded:: 3.4.2 + ''' + + class RegressionTest(jsonext.JSONSerializable, metaclass=RegressionTestMeta): '''Base class for regression tests. @@ -133,6 +147,10 @@ class RegressionTest(jsonext.JSONSerializable, metaclass=RegressionTestMeta): This class provides the implementation of the pipeline phases that the regression test goes through during its lifetime. + .. warning:: + .. versionchanged:: 3.4.2 + Multiple inheritance with a shared common ancestor is not allowed. + .. note:: .. versionchanged:: 2.19 Base constructor takes no arguments. @@ -163,7 +181,7 @@ def pipeline_hooks(cls): #: The name of the test. #: #: :type: string that can contain any character except ``/`` - name = fields.TypedField(typ.Str[r'[^\/]+']) + name = variable(typ.Str[r'[^\/]+']) #: List of programming environments supported by this test. #: @@ -171,7 +189,7 @@ def pipeline_hooks(cls): #: by this test. #: #: :type: :class:`List[str]` - #: :default: ``[]`` + #: :default: ``None`` #: #: .. note:: #: .. versionchanged:: 2.12 @@ -180,7 +198,10 @@ def pipeline_hooks(cls): #: .. versionchanged:: 2.17 #: Support for wildcards is dropped. #: - valid_prog_environs = fields.TypedField(typ.List[str], type(None)) + #: .. versionchanged:: 3.3 + #: Default value changed from ``[]`` to ``None``. + #: + valid_prog_environgs = variable(typ.List[str], type(None), value=None) #: List of systems supported by this test. #: The general syntax for systems is ``[:]``. @@ -188,14 +209,18 @@ def pipeline_hooks(cls): #: ``*`` is an alias of ``*:*`` #: #: :type: :class:`List[str]` - #: :default: ``[]`` - valid_systems = fields.TypedField(typ.List[str], type(None)) + #: :default: ``None`` + #: + #: .. versionchanged:: 3.3 + #: Default value changed from ``[]`` to ``None``. + #: + valid_systems = variable(typ.List[str], type(None), value=None) #: A detailed description of the test. #: #: :type: :class:`str` #: :default: ``self.name`` - descr = fields.TypedField(str) + descr = variable(str) #: The path to the source file or source directory of the test. #: @@ -213,7 +238,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`str` #: :default: ``''`` - sourcepath = fields.TypedField(str) + sourcepath = variable(str, value='') #: The directory containing the test's resources. #: @@ -243,7 +268,7 @@ def pipeline_hooks(cls): #: .. versionchanged:: 3.0 #: Default value is now conditionally set to either ``'src'`` or #: :class:`None`. - sourcesdir = fields.TypedField(str, type(None)) + sourcesdir = variable(str, type(None), value='src') #: .. versionadded:: 2.14 #: @@ -260,7 +285,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`str` or :class:`reframe.core.buildsystems.BuildSystem`. #: :default: :class:`None`. - build_system = BuildSystemField(type(None)) + build_system = variable(type(None), field=BuildSystemField, value=None) #: .. versionadded:: 3.0 #: @@ -272,7 +297,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - prebuild_cmds = fields.TypedField(typ.List[str]) + prebuild_cmds = variable(typ.List[str], value=[]) #: .. versionadded:: 3.0 #: @@ -284,19 +309,19 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - postbuild_cmds = fields.TypedField(typ.List[str]) + postbuild_cmds = variable(typ.List[str], value=[]) #: The name of the executable to be launched during the run phase. #: #: :type: :class:`str` #: :default: ``os.path.join('.', self.name)`` - executable = fields.TypedField(str) + executable = variable(str) #: List of options to be passed to the :attr:`executable`. #: #: :type: :class:`List[str]` #: :default: ``[]`` - executable_opts = fields.TypedField(typ.List[str]) + executable_opts = variable(typ.List[str], value=[]) #: .. versionadded:: 2.20 #: @@ -320,7 +345,8 @@ def pipeline_hooks(cls): #: :type: :class:`str` or #: :class:`reframe.core.containers.ContainerPlatform`. #: :default: :class:`None`. - container_platform = ContainerPlatformField(type(None)) + container_platform = variable(type(None), + field=ContainerPlatformField, value=None) #: .. versionadded:: 3.0 #: @@ -332,7 +358,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - prerun_cmds = fields.TypedField(typ.List[str]) + prerun_cmds = variable(typ.List[str], value=[]) #: .. versionadded:: 3.0 #: @@ -343,7 +369,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - postrun_cmds = fields.TypedField(typ.List[str]) + postrun_cmds = variable(typ.List[str], value=[]) #: List of files to be kept after the test finishes. #: @@ -363,7 +389,7 @@ def pipeline_hooks(cls): #: .. versionchanged:: 3.3 #: This field accepts now also file glob patterns. #: - keep_files = fields.TypedField(typ.List[str]) + keep_files = variable(typ.List[str], value=[]) #: List of files or directories (relative to the :attr:`sourcesdir`) that #: will be symlinked in the stage directory and not copied. @@ -373,7 +399,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - readonly_files = fields.TypedField(typ.List[str]) + readonly_files = variable(typ.List[str], value=[]) #: Set of tags associated with this test. #: @@ -381,7 +407,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`Set[str]` #: :default: an empty set - tags = fields.TypedField(typ.Set[str]) + tags = variable(typ.Set[str], value=set()) #: List of people responsible for this test. #: @@ -389,7 +415,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - maintainers = fields.TypedField(typ.List[str]) + maintainers = variable(typ.List[str], value=[]) #: Mark this test as a strict performance test. #: @@ -399,7 +425,7 @@ def pipeline_hooks(cls): #: #: :type: boolean #: :default: :class:`True` - strict_check = fields.TypedField(bool) + strict_check = variable(bool, value=True) #: Number of tasks required by this test. #: @@ -425,7 +451,7 @@ def pipeline_hooks(cls): #: #: .. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` #: .. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes - num_tasks = fields.TypedField(int) + num_tasks = variable(int, value=1) #: Number of tasks per node required by this test. #: @@ -433,7 +459,7 @@ def pipeline_hooks(cls): #: #: :type: integral or :class:`None` #: :default: :class:`None` - num_tasks_per_node = fields.TypedField(int, type(None)) + num_tasks_per_node = variable(int, type(None), value=None) #: Number of GPUs per node required by this test. #: This attribute is translated internally to the ``_rfm_gpu`` resource. @@ -442,7 +468,7 @@ def pipeline_hooks(cls): #: #: :type: integral #: :default: ``0`` - num_gpus_per_node = fields.TypedField(int) + num_gpus_per_node = variable(int, value=0) #: Number of CPUs per task required by this test. #: @@ -450,7 +476,7 @@ def pipeline_hooks(cls): #: #: :type: integral or :class:`None` #: :default: :class:`None` - num_cpus_per_task = fields.TypedField(int, type(None)) + num_cpus_per_task = variable(int, type(None), value=None) #: Number of tasks per core required by this test. #: @@ -458,7 +484,7 @@ def pipeline_hooks(cls): #: #: :type: integral or :class:`None` #: :default: :class:`None` - num_tasks_per_core = fields.TypedField(int, type(None)) + num_tasks_per_core = variable(int, type(None), value=None) #: Number of tasks per socket required by this test. #: @@ -466,7 +492,7 @@ def pipeline_hooks(cls): #: #: :type: integral or :class:`None` #: :default: :class:`None` - num_tasks_per_socket = fields.TypedField(int, type(None)) + num_tasks_per_socket = variable(int, type(None), value=None) #: Specify whether this tests needs simultaneous multithreading enabled. #: @@ -474,7 +500,7 @@ def pipeline_hooks(cls): #: #: :type: boolean or :class:`None` #: :default: :class:`None` - use_multithreading = fields.TypedField(bool, type(None)) + use_multithreading = variable(bool, type(None), value=None) #: .. versionadded:: 3.0 #: @@ -484,19 +510,20 @@ def pipeline_hooks(cls): #: #: :type: :class:`str` or :class:`datetime.timedelta` #: :default: :class:`None` - max_pending_time = fields.TimerField(type(None)) + max_pending_time = variable( + type(None), field=fields.TimerField, value=None) #: Specify whether this test needs exclusive access to nodes. #: #: :type: boolean #: :default: :class:`False` - exclusive_access = fields.TypedField(bool) + exclusive_access = variable(bool, value=False) #: Always execute this test locally. #: #: :type: boolean #: :default: :class:`False` - local = fields.TypedField(bool) + local = variable(bool, value=False) #: The set of reference values for this test. #: @@ -528,9 +555,8 @@ def pipeline_hooks(cls): #: .. versionchanged:: 3.0 #: The measurement unit is required. The user should explicitly #: specify :class:`None` if no unit is available. - reference = fields.ScopedDictField( - typ.Tuple[object, object, object, object] - ) + reference = variable(typ.Tuple[object, object, object, object], + field=fields.ScopedDictField, value={}) # FIXME: There is not way currently to express tuples of `float`s or # `None`s, so we just use the very generic `object` @@ -556,7 +582,7 @@ def pipeline_hooks(cls): #: :: #: #: self.sanity_patterns = sn.assert_found(r'.*', self.stdout) - sanity_patterns = fields.TypedField(_DeferredExpression, type(None)) + sanity_patterns = variable(_DeferredExpression, type(None), value=None) #: Patterns for verifying the performance of this test. #: @@ -570,9 +596,8 @@ def pipeline_hooks(cls): #: `) as values. #: :class:`None` is also allowed. #: :default: :class:`None` - perf_patterns = fields.TypedField( - typ.Dict[str, _DeferredExpression], type(None) - ) + perf_patterns = variable(typ.Dict[str, _DeferredExpression], + type(None), value=None) #: List of modules to be loaded before running this test. #: @@ -580,7 +605,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`List[str]` #: :default: ``[]`` - modules = fields.TypedField(typ.List[str]) + modules = variable(typ.List[str], value=[]) #: Environment variables to be set before running this test. #: @@ -588,7 +613,7 @@ def pipeline_hooks(cls): #: #: :type: :class:`Dict[str, str]` #: :default: ``{}`` - variables = fields.TypedField(typ.Dict[str, str]) + variables = variable(typ.Dict[str, str], value={}) #: Time limit for this test. #: @@ -612,7 +637,7 @@ def pipeline_hooks(cls): #: - The old syntax using a ``(h, m, s)`` tuple is dropped. #: - Support of `timedelta` objects is dropped. #: - Number values are now accepted. - time_limit = fields.TimerField(type(None)) + time_limit = variable(type(None), field=fields.TimerField, value='10m') #: .. versionadded:: 2.8 #: @@ -680,9 +705,7 @@ def pipeline_hooks(cls): #: .. versionchanged:: 2.9 #: A new more powerful syntax was introduced #: that allows also custom job script directive prefixes. - extra_resources = fields.TypedField( - typ.Dict[str, typ.Dict[str, object]] - ) + extra_resources = variable(typ.Dict[str, typ.Dict[str, object]], value={}) #: .. versionadded:: 3.3 #: @@ -697,13 +720,14 @@ def pipeline_hooks(cls): #: appropriate sanity check. #: #: :type: boolean : :default: :class:`True` - build_locally = fields.TypedField(bool) + build_locally = variable(bool, value=True) def __new__(cls, *args, _rfm_use_params=False, **kwargs): obj = super().__new__(cls) - # Set the test parameters in the object - cls._init_params(obj, _rfm_use_params) + # Insert the var & param spaces + cls._rfm_var_space.inject(obj, cls) + cls._rfm_param_space.inject(obj, cls, _rfm_use_params) # Create a test name from the class name and the constructor's # arguments @@ -736,39 +760,23 @@ def __new__(cls, *args, _rfm_use_params=False, **kwargs): def __init__(self): pass - @classmethod - def _init_params(cls, obj, use_params=False): - '''Attach the test parameters as class attributes. - - Create and initialize the regression test parameters as object - attributes. The values assigned to these parameters exclusively depend - on the use_params argument. If this is set to True, the current object - uses the parameter space iterator (see - :class `reframe.core.pipeline.RegressionTest` and consumes a set of - parameter values (i.e. a point in the parameter space). Contrarily, if - use_params is False, the regression test parameters are initialized as - None. - - :param use_param: bool that dictates whether an instance of the - :class `reframe.core.pipeline.RegressionTest` is to use the - parameter values defined in the parameter space. - - :meta private: - ''' - # Set the values of the test parameters (if any) - if use_params and cls._rfm_param_space.params: - # Consume the parameter space iterator - param_values = next(cls._rfm_param_space.unique_iter) - for index, key in enumerate(cls._rfm_param_space.params): - setattr(obj, key, param_values[index]) - else: - # Otherwise init the params as None - for key in cls._rfm_param_space.params: - setattr(obj, key, None) + def __getattribute__(self, name): + try: + return super().__getattribute__(name) + except AttributeError: + # Intercept the AttributeError if the name corresponds to a + # required variable. + if (name in self._rfm_var_space.vars and + not self._rfm_var_space.vars[name].is_defined()): + raise AttributeError( + f'required variable {name!r} has not been set' + ) from None + else: + super().__getattr__(name) def _append_parameters_to_name(self): if self._rfm_param_space.params: - return '_' + '_'.join([str(self.__dict__[key]) + return '_' + '_'.join([util.toalphanum(str(self.__dict__[key])) for key in self._rfm_param_space.params]) else: return '' @@ -790,68 +798,20 @@ def _rfm_init(self, name=None, prefix=None): self.name = name self.descr = self.name - self.valid_prog_environs = None - self.valid_systems = None - self.sourcepath = '' - self.prebuild_cmds = [] - self.postbuild_cmds = [] self.executable = os.path.join('.', self.name) - self.executable_opts = [] - self.prerun_cmds = [] - self.postrun_cmds = [] - self.keep_files = [] - self.readonly_files = [] - self.tags = set() - self.maintainers = [] self._perfvalues = {} - self.container_platform = None - - # Strict performance check, if applicable - self.strict_check = True - - # Default is a single node check - self.num_tasks = 1 - self.num_tasks_per_node = None - self.num_gpus_per_node = 0 - self.num_cpus_per_task = None - self.num_tasks_per_core = None - self.num_tasks_per_socket = None - self.use_multithreading = None - self.exclusive_access = False - self.max_pending_time = None - - # True only if check is to be run locally - self.local = False - self.build_locally = True # Static directories of the regression check self._prefix = os.path.abspath(prefix) - if os.path.isdir(os.path.join(self._prefix, 'src')): - self.sourcesdir = 'src' - else: + if not os.path.isdir(os.path.join(self._prefix, self.sourcesdir)): self.sourcesdir = None - # Output patterns - self.sanity_patterns = None - - # Performance patterns: None -> no performance checking - self.perf_patterns = None - self.reference = {} - - # Environment setup - self.modules = [] - self.variables = {} - - # Time limit for the check - self.time_limit = '10m' - # Runtime information of the test self._current_partition = None self._current_environ = None # Associated job self._job = None - self.extra_resources = {} # Dynamic paths of the regression check; will be set in setup() self._stagedir = None @@ -862,7 +822,6 @@ def _rfm_init(self, name=None, prefix=None): # Compilation process output self._build_job = None self._compile_proc = None - self.build_system = None # Performance logging self._perf_logger = logging.null_logger diff --git a/reframe/core/variables.py b/reframe/core/variables.py new file mode 100644 index 0000000000..ef1fce3603 --- /dev/null +++ b/reframe/core/variables.py @@ -0,0 +1,216 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# Functionality to build extensible variable spaces into ReFrame tests. +# + +import copy + +import reframe.core.namespaces as namespaces +import reframe.core.fields as fields + + +class _UndefinedType: + '''Custom type to flag a variable as undefined.''' + __slots__ = () + + +_Undefined = _UndefinedType() + + +class VarDirective: + '''Base class for the variable directives.''' + + +class TestVar(VarDirective): + '''Regression test variable class. + + Stores the attributes of a variable when defined directly in the class + body. Instances of this class are injected into the regression test + during class instantiation. + + :meta private: + ''' + + def __init__(self, *args, **kwargs): + self.field_type = kwargs.pop('field', fields.TypedField) + self._default_value = kwargs.pop('value', _Undefined) + + if not issubclass(self.field_type, fields.Field): + raise ValueError( + f'field {self.field_type!r} is not derived from ' + f'{fields.Field.__qualname__}' + ) + + self.args = args + self.kwargs = kwargs + + def is_defined(self): + return self._default_value is not _Undefined + + def undefine(self): + self._default_value = _Undefined + + def define(self, value): + self._default_value = value + + def __set_name__(self, owner, name): + self.name = name + + @property + def default_value(self): + # Variables must be returned by-value to prevent an instance from + # modifying the class variable space. + return copy.deepcopy(self._default_value) + + +class UndefineVar(VarDirective): + def __init__(self): + self.default_value = _Undefined + + +class VarSpace(namespaces.Namespace): + '''Variable space of a regression test. + + Store the variables of a regression test. This variable space is stored + in the regression test class under the class attribute ``_rfm_var_space``. + A target class can be provided to the + :func:`__init__` method, which is the regression test where the + VarSpace is to be built. During this call to + :func:`__init__`, the VarSpace inherits all the VarSpace from the base + classes of the target class. After this, the VarSpace is extended with + the information from the local variable space, which is stored under the + target class' attribute ``_rfm_local_var_space``. If no target class is + provided, the VarSpace is simply initialized as empty. + ''' + + @property + def local_namespace_name(self): + return '_rfm_local_var_space' + + @property + def namespace_name(self): + return '_rfm_var_space' + + def __init__(self, target_cls=None, illegal_names=None): + # Set to register the variables already injected in the class + self._injected_vars = set() + super().__init__(target_cls, illegal_names) + + def join(self, other, cls): + '''Join an existing VarSpace into the current one. + + :param other: instance of the VarSpace class. + :param cls: the target class. + ''' + for key, var in other.items(): + + # Make doubly declared vars illegal. Note that this will be + # triggered when inheriting from multiple RegressionTest classes. + if key in self.vars: + raise ValueError( + f'variable {key!r} is declared in more than one of the ' + f'parent classes of class {cls.__qualname__!r}' + ) + + self.vars[key] = var + + # Carry over the set of injected variables + self._injected_vars.update(other._injected_vars) + + def extend(self, cls): + '''Extend the VarSpace with the content in the LocalVarSpace. + + Merge the VarSpace inherited from the base classes with the + LocalVarSpace. Note that the LocalVarSpace can also contain + define and undefine actions on existing vars. Thus, since it + does not make sense to define and undefine a var in the same + class, the order on which the define and undefine functions + are called is not preserved. In fact, applying more than one + of these actions on the same var for the same local var space + is disallowed. + ''' + local_varspace = getattr(cls, self.local_namespace_name) + for key, var in local_varspace.items(): + if isinstance(var, TestVar): + # Disable redeclaring a variable + if key in self.vars: + raise ValueError( + f'cannot redeclare the variable {key!r}' + ) + + # Add a new var + self.vars[key] = var + elif isinstance(var, VarDirective): + # Modify the value of a previously declared var. + # If var is an instance of UndefineVar, we set its default + # value to _Undefined. Alternatively, the value is just updated + # with the user's input. + self._check_var_is_declared(key) + self.vars[key].define(var.default_value) + + # If any previously declared variable was defined in the class body + # by directly assigning it a value, retrieve this value from the class + # namespace and update it into the variable space. + _assigned_vars = set() + for key, value in cls.__dict__.items(): + if key in local_varspace: + raise ValueError( + f'cannot specify more than one action on variable ' + f'{key!r} in the same class' + ) + elif key in self.vars: + self.vars[key].define(value) + _assigned_vars.add(key) + + # Delete the vars from the class __dict__. + for key in _assigned_vars: + delattr(cls, key) + + def _check_var_is_declared(self, key): + if key not in self.vars: + raise ValueError( + f'variable {key!r} has not been declared' + ) + + def sanity(self, cls, illegal_names=None): + '''Sanity checks post-creation of the var namespace. + + By default, we make illegal to have any item in the namespace + that clashes with a member of the target class unless this member + was injected by this namespace. + ''' + if illegal_names is None: + illegal_names = set(dir(cls)) + + for key in self._namespace: + if key in illegal_names and key not in self._injected_vars: + raise ValueError( + f'{key!r} already defined in class ' + f'{cls.__qualname__!r}' + ) + + def inject(self, obj, cls): + '''Insert the vars in the regression test. + + :param obj: The test object. + :param cls: The test class. + ''' + + for name, var in self.items(): + setattr(cls, name, var.field_type(*var.args, **var.kwargs)) + getattr(cls, name).__set_name__(obj, name) + + # If the var is defined, set its value + if var.is_defined(): + setattr(obj, name, var.default_value) + + # Track the variables that have been injected. + self._injected_vars.add(name) + + @property + def vars(self): + return self._namespace diff --git a/unittests/test_parameters.py b/unittests/test_parameters.py index 4505603e5d..4c13386d8f 100644 --- a/unittests/test_parameters.py +++ b/unittests/test_parameters.py @@ -1,4 +1,4 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,7 @@ import pytest import inspect + import reframe as rfm @@ -15,17 +16,17 @@ class NoParams(rfm.RunOnlyRegressionTest): class TwoParams(NoParams): - parameter('P0', ['a']) - parameter('P1', ['b']) + P0 = parameter(['a']) + P1 = parameter(['b']) class Abstract(TwoParams): - parameter('P0') + P0 = parameter() class ExtendParams(TwoParams): - parameter('P1', ['c', 'd', 'e'], inherit_params=True) - parameter('P2', ['f', 'g']) + P1 = parameter(['c', 'd', 'e'], inherit_params=True) + P2 = parameter(['f', 'g']) def test_param_space_is_empty(): @@ -53,7 +54,7 @@ class MyTest(Abstract): def test_param_override(): class MyTest(TwoParams): - parameter('P1', ['-']) + P1 = parameter(['-']) assert MyTest.param_space['P0'] == ('a',) assert MyTest.param_space['P1'] == ('-',) @@ -61,7 +62,7 @@ class MyTest(TwoParams): def test_param_inheritance(): class MyTest(TwoParams): - parameter('P1', ['c'], inherit_params=True) + P1 = parameter(['c'], inherit_params=True) assert MyTest.param_space['P0'] == ('a',) assert MyTest.param_space['P1'] == ('b', 'c',) @@ -69,7 +70,7 @@ class MyTest(TwoParams): def test_filter_params(): class MyTest(ExtendParams): - parameter('P1', inherit_params=True, filter_params=lambda x: x[2:]) + P1 = parameter(inherit_params=True, filter_params=lambda x: x[2:]) assert MyTest.param_space['P0'] == ('a',) assert MyTest.param_space['P1'] == ('d', 'e',) @@ -137,7 +138,7 @@ class MyTest(ExtendParams): assert test.P1 is None assert test.P2 is None - with pytest.raises(StopIteration): + with pytest.raises(RuntimeError): test = MyTest(_rfm_use_params=True) @@ -168,3 +169,62 @@ def test_parameterized_test_is_incompatible(): class MyTest(TwoParams): def __init__(self, var): pass + + +def test_param_space_clash(): + class Spam(rfm.RegressionMixin): + P0 = parameter([1]) + + class Ham(rfm.RegressionMixin): + P0 = parameter([2]) + + with pytest.raises(ValueError): + class Eggs(Spam, Ham): + '''Trigger error from param name clashing.''' + + +def test_namespace_clash(): + class Spam(rfm.RegressionTest): + foo = variable(int, 1) + + with pytest.raises(ValueError): + class Ham(Spam): + foo = parameter([1]) + + +def test_double_declare(): + with pytest.raises(ValueError): + class MyTest(rfm.RegressionTest): + P0 = parameter([1, 2, 3]) + P0 = parameter() + + +def test_overwrite_param(): + with pytest.raises(ValueError): + class MyTest(TwoParams): + P0 = [1, 2, 3] + + +def test_param_deepcopy(): + '''Test that there is no cross-class pollution. + + Each instance must deal with its own copies of the parameters. + ''' + class MyParam: + def __init__(self, val): + self.val = val + + class Base(rfm.RegressionTest): + p0 = parameter([MyParam(1), MyParam(2)]) + + class Foo(Base): + def __init__(self): + self.p0.val = -20 + + class Bar(Base): + pass + + assert Foo(_rfm_use_params=True).p0.val == -20 + assert Foo(_rfm_use_params=True).p0.val == -20 + assert Bar(_rfm_use_params=True).p0.val == 1 + assert Bar(_rfm_use_params=True).p0.val == 2 diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index bdd0611244..3d4af3cfcc 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -6,6 +6,7 @@ import os import pytest import re +import sys import reframe as rfm import reframe.core.runtime as rt @@ -14,9 +15,6 @@ import unittests.fixtures as fixtures from reframe.core.exceptions import (BuildError, PipelineError, ReframeError, PerformanceError, SanityError) -from reframe.frontend.loader import RegressionCheckLoader -from unittests.resources.checks.hellocheck import HelloTest -from unittests.resources.checks.pinnedcheck import PinnedTest def _run(test, partition, prgenv): @@ -30,9 +28,30 @@ def _run(test, partition, prgenv): test.cleanup(remove_files=True) -def load_test(testfile): - loader = RegressionCheckLoader(['unittests/resources/checks']) - return loader.load_from_file(testfile) +@pytest.fixture +def HelloTest(): + from unittests.resources.checks.hellocheck import HelloTest + yield HelloTest + del sys.modules['unittests.resources.checks.hellocheck'] + + +@pytest.fixture +def hellotest(HelloTest): + yield HelloTest() + + +@pytest.fixture +def hellomaketest(): + from unittests.resources.checks.hellocheck_make import HelloMakeTest + yield HelloMakeTest + del sys.modules['unittests.resources.checks.hellocheck_make'] + + +@pytest.fixture +def pinnedtest(): + from unittests.resources.checks.pinnedcheck import PinnedTest + yield PinnedTest + del sys.modules['unittests.resources.checks.pinnedcheck'] @pytest.fixture @@ -64,11 +83,6 @@ def user_system(temp_runtime): yield generic_system -@pytest.fixture -def hellotest(): - yield load_test('unittests/resources/checks/hellocheck.py')[0] - - @pytest.fixture def local_exec_ctx(generic_system): partition = fixtures.partition_by_name('default') @@ -160,9 +174,8 @@ def test_hellocheck(hellotest, remote_exec_ctx): _run(hellotest, *remote_exec_ctx) -def test_hellocheck_make(remote_exec_ctx): - test = load_test('unittests/resources/checks/hellocheck_make.py')[0] - _run(test, *remote_exec_ctx) +def test_hellocheck_make(hellomaketest, remote_exec_ctx): + _run(hellomaketest(), *remote_exec_ctx) def test_hellocheck_local(hellotest, local_exec_ctx): @@ -269,8 +282,8 @@ def __init__(self): _run(MyTest(), *local_exec_ctx) -def test_pinned_test(local_exec_ctx): - class MyTest(PinnedTest): +def test_pinned_test(pinnedtest, local_exec_ctx): + class MyTest(pinnedtest): pass pinned = MyTest() @@ -450,7 +463,7 @@ def __init__(self): test.compile_wait() -def test_extra_resources(testsys_system): +def test_extra_resources(HelloTest, testsys_system): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -479,7 +492,7 @@ def set_resources(self): assert expected_job_options == set(test.job.options) -def test_setup_hooks(local_exec_ctx): +def test_setup_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -503,7 +516,7 @@ def postfoo(self): assert test.count == 2 -def test_compile_hooks(local_exec_ctx): +def test_compile_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -528,7 +541,7 @@ def check_executable(self): assert test.count == 1 -def test_run_hooks(local_exec_ctx): +def test_run_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -550,7 +563,7 @@ def check_executable(self): _run(MyTest(), *local_exec_ctx) -def test_multiple_hooks(local_exec_ctx): +def test_multiple_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -576,7 +589,7 @@ def z(self): assert test.var == 3 -def test_stacked_hooks(local_exec_ctx): +def test_stacked_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): @@ -596,7 +609,13 @@ def x(self): assert test.var == 3 -def test_inherited_hooks(local_exec_ctx): +def test_multiple_inheritance(HelloTest): + with pytest.raises(ValueError): + class MyTest(rfm.RunOnlyRegressionTest, HelloTest): + pass + + +def test_inherited_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class BaseTest(HelloTest): def __init__(self): @@ -609,7 +628,7 @@ def __init__(self): def x(self): self.var += 1 - class C(rfm.RegressionTest): + class C(rfm.RegressionMixin): @rfm.run_before('run') def y(self): self.foo = 1 @@ -632,7 +651,7 @@ class MyTest(DerivedTest): } -def test_overriden_hooks(local_exec_ctx): +def test_overriden_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class BaseTest(HelloTest): def __init__(self): @@ -666,7 +685,7 @@ def y(self): assert test.foo == 10 -def test_disabled_hooks(local_exec_ctx): +def test_disabled_hooks(HelloTest, local_exec_ctx): @fixtures.custom_prefix('unittests/resources/checks') class BaseTest(HelloTest): def __init__(self): @@ -696,7 +715,7 @@ def x(self): assert test.foo == 0 -def test_require_deps(local_exec_ctx): +def test_require_deps(HelloTest, local_exec_ctx): import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors diff --git a/unittests/test_variables.py b/unittests/test_variables.py new file mode 100644 index 0000000000..e678824c23 --- /dev/null +++ b/unittests/test_variables.py @@ -0,0 +1,182 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + + +import pytest + + +import reframe as rfm +from reframe.core.fields import Field + + +@pytest.fixture +def NoVarsTest(): + '''Variables are injected as descriptors in the classes. + + Thus, fixtures are needed to provide a fresh class to each test. + ''' + class NoVarsTest(rfm.RegressionTest): + pass + + yield NoVarsTest + + +@pytest.fixture +def OneVarTest(NoVarsTest): + class OneVarTest(NoVarsTest): + foo = variable(int, value=10) + + yield OneVarTest + + +def test_custom_variable(OneVarTest): + assert hasattr(OneVarTest, 'foo') + assert not isinstance(OneVarTest.foo, Field) + inst = OneVarTest() + assert hasattr(OneVarTest, 'foo') + assert isinstance(OneVarTest.foo, Field) + assert hasattr(inst, 'foo') + assert inst.foo == 10 + + +def test_redeclare_builtin_var_clash(NoVarsTest): + with pytest.raises(ValueError): + class MyTest(NoVarsTest): + name = variable(str) + + +def test_name_clash_builtin_property(NoVarsTest): + with pytest.raises(ValueError): + class MyTest(NoVarsTest): + current_environ = variable(str) + + +def test_redeclare_var_clash(OneVarTest): + with pytest.raises(ValueError): + class MyTest(OneVarTest): + foo = variable(str) + + +def test_inheritance_clash(NoVarsTest): + class MyMixin(rfm.RegressionMixin): + name = variable(str) + + with pytest.raises(ValueError): + class MyTest(NoVarsTest, MyMixin): + '''Trigger error from inheritance clash.''' + + +def test_instantiate_and_inherit(OneVarTest): + '''Instantiation will inject the vars as class attributes. + + Ensure that inheriting from this class after the instantiation does not + raise a namespace clash with the vars. + ''' + inst = OneVarTest() + + class MyTest(OneVarTest): + pass + + +def test_var_space_clash(): + class Spam(rfm.RegressionMixin): + v0 = variable(int, value=1) + + class Ham(rfm.RegressionMixin): + v0 = variable(int, value=2) + + with pytest.raises(ValueError): + class Eggs(Spam, Ham): + '''Trigger error from var name clashing.''' + + +def test_double_declare(): + with pytest.raises(ValueError): + class MyTest(rfm.RegressionTest): + v0 = variable(int, value=1) + v0 = variable(float, value=0.5) + + +def test_double_action_on_variable(): + with pytest.raises(ValueError): + class MyTest(rfm.RegressionTest): + v0 = variable(int, value=2) + v0 = 2 + + +def test_set_var(OneVarTest): + class MyTest(OneVarTest): + foo = 4 + + inst = MyTest() + assert hasattr(OneVarTest, 'foo') + assert not isinstance(OneVarTest.foo, Field) + assert hasattr(MyTest, 'foo') + assert isinstance(MyTest.foo, Field) + assert hasattr(inst, 'foo') + assert inst.foo == 4 + + +def test_var_type(OneVarTest): + class MyTest(OneVarTest): + foo = 'bananas' + + with pytest.raises(TypeError): + inst = MyTest() + + +def test_require_var(OneVarTest): + class MyTest(OneVarTest): + foo = required + + def __init__(self): + print(self.foo) + + with pytest.raises(AttributeError): + inst = MyTest() + + +def test_required_var_not_present(OneVarTest): + class MyTest(OneVarTest): + foo = required + + def __init__(self): + pass + + mytest = MyTest() + + +def test_require_undeclared_variable(NoVarsTest): + with pytest.raises(ValueError): + class MyTest(NoVarsTest): + foo = required + + +def test_invalid_field(): + class Foo: + '''An invalid descriptor''' + + with pytest.raises(ValueError): + class MyTest(rfm.RegressionTest): + a = variable(int, value=4, field=Foo) + + +def test_var_deepcopy(): + '''Test that there is no cross-class pollution. + + Each instance must have its own copies of each variable. + ''' + class Base(rfm.RegressionTest): + my_var = variable(list, value=[1, 2]) + + class Foo(Base): + def __init__(self): + self.my_var += [3] + + class Bar(Base): + pass + + assert Foo().my_var == [1, 2, 3] + assert Bar().my_var == [1, 2]