[feat] Allow setting test variables from the command line

docs/manpage.rst

@@ -386,6 +386,60 @@ Options controlling ReFrame execution
    .. versionchanged:: 3.6.1
       Multiple report files are now accepted.
 
+.. option:: -S, --setvar=[TEST.]VAR=VAL
+
+   Set variable ``VAR`` in all tests or optionally only in test ``TEST`` to ``VAL``.
+
+   Multiple variables can be set at the same time by passing this option multiple times.
+   This option *cannot* change arbitrary test attributes, but only test variables declared with the :attr:`~reframe.core.pipeline.RegressionMixin.variable` built-in.
+   If an attempt is made to change an inexistent variable or a test parameter, a warning will be issued.
+
+   ReFrame will try to convert ``VAL`` to the type of the variable.
+   If it does not succeed, a warning will be issued and the variable will not be set.
+   ``VAL`` can take the special value ``@none`` to denote that the variable must be set to :obj:`None`.
+
+   Sequence and mapping types can also be set from the command line by using the following syntax:
+
+   - Sequence types: ``-S seqvar=1,2,3,4``
+   - Mapping types: ``-S mapvar=a:1,b:2,c:3``
+
+   Conversions to arbitrary objects are also supported.
+   See :class:`~reframe.utility.typecheck.ConvertibleType` for more details.
+
+
+   The optional ``TEST.`` prefix refers to the test class name, *not* the test name.
+
+   Variable assignments passed from the command line happen *before* the test is instantiated and is the exact equivalent of assigning a new value to the variable *at the end* of the test class body.
+   This has a number of implications that users of this feature should be aware of:
+
+   - In the following test, :attr:`num_tasks` will have always the value ``1`` regardless of any command-line assignment of the variable :attr:`foo`:
+
+   .. code-block:: python
+
+      @rfm.simple_test
+      class my_test(rfm.RegressionTest):
+          foo = variable(int, value=1)
+          num_tasks = foo
+
+   - If the variable is set in any pipeline hook, the command line assignment will have an effect until the variable assignment in the pipeline hook is reached.
+     The variable will be then overwritten.
+   - The `test filtering <#test-filtering>`__ happens *after* a test is instantiated, so the only way to scope a variable assignment is to prefix it with the test class name.
+     However, this has some positive side effects:
+
+     - Passing ``-S valid_systems='*'`` and ``-S valid_prog_environs='*'`` is the equivalent of passing the :option:`--skip-system-check` and :option:`--skip-prgenv-check` options.
+     - Users could alter the behavior of tests based on tag values that they pass from the command line, by changing the behavior of a test in a post-init hook based on the value of the :attr:`~reframe.core.pipeline.RegressionTest.tags` attribute.
+     - Users could force a test with required variables to run if they set these variables from the command line.
+       For example, the following test could only be run if invoked with ``-S num_tasks=<NUM>``:
+
+     .. code-block:: python
+
+        @rfm.simple_test
+        class my_test(rfm.RegressionTest):
+            num_tasks = required
+
+   .. versionadded:: 3.8.0
+
+
 ----------------------------------
 Options controlling job submission
 ----------------------------------

reframe/core/fields.py

@@ -15,6 +15,26 @@
 from reframe.utility import ScopedDict
 
 
+class _Convertible:
+    '''Wrapper for values that allowed to be converted implicitly'''
+
+    __slots__ = ('value')
+
+    def __init__(self, value):
+        self.value = value
+
+
+def make_convertible(value):
+    return _Convertible(value)
+
+
+def remove_convertible(value):
+    if isinstance(value, _Convertible):
+        return value.value
+    else:
+        return value
+
+
 class Field:
     '''Base class for attribute validators.'''
 
@@ -34,7 +54,7 @@ def __get__(self, obj, objtype):
                                  (objtype.__name__, self._name)) from None
 
     def __set__(self, obj, value):
-        obj.__dict__[self._name] = value
+        obj.__dict__[self._name] = remove_convertible(value)
 
 
 class TypedField(Field):
@@ -46,6 +66,10 @@ def __init__(self, main_type, *other_types):
             raise TypeError('{0} is not a sequence of types'.
                             format(self._types))
 
+    @property
+    def valid_types(self):
+        return self._types
+
     def _check_type(self, value):
         if not any(isinstance(value, t) for t in self._types):
             typedescr = '|'.join(t.__name__ for t in self._types)
@@ -54,8 +78,33 @@ def _check_type(self, value):
                 (self._name, value, typedescr))
 
     def __set__(self, obj, value):
-        self._check_type(value)
-        super().__set__(obj, value)
+        try:
+            self._check_type(value)
+        except TypeError:
+            raw_value = remove_convertible(value)
+            if raw_value is value:
+                # value was not convertible; reraise
+                raise
+
+            # Try to convert value to any of the supported types
+            value = raw_value
+            for t in self._types:
+                try:
+                    value = t(value)
+                except TypeError:
+                    continue
+                else:
+                    super().__set__(obj, value)
+                    return
+
+            # Conversion failed
+            raise TypeError(
+                f'failed to set field {self._name!r}: '
+                f'could not convert to any of the supported types: '
+                f'{self._types}'
+            )
+        else:
+            super().__set__(obj, value)
 
 
 class ConstantField(Field):
@@ -88,6 +137,7 @@ def __init__(self, *other_types):
         super().__init__(str, int, float, *other_types)
 
     def __set__(self, obj, value):
+        value = remove_convertible(value)
         self._check_type(value)
         if isinstance(value, str):
             time_match = re.match(r'^((?P<days>\d+)d)?'
@@ -119,6 +169,7 @@ def __init__(self, valuetype, *other_types):
                          ScopedDict, *other_types)
 
     def __set__(self, obj, value):
+        value = remove_convertible(value)
         self._check_type(value)
         if not isinstance(value, ScopedDict):
             value = ScopedDict(value) if value is not None else value

reframe/core/meta.py

@@ -473,6 +473,41 @@ def __getattr__(cls, name):
             f'class {cls.__qualname__!r} has no attribute {name!r}'
         ) from None
 
+    def setvar(cls, name, value):
+        '''Set the value of a variable.
+
+        :param name: The name of the variable.
+        :param value: The value of the variable.
+
+        :returns: :class:`True` if the variable was set.
+            A variable will *not* be set, if it does not exist or when an
+            attempt is made to set it with its underlying descriptor.
+            This happens during the variable injection time and it should be
+            delegated to the class' :func:`__setattr__` method.
+
+        :raises ReframeSyntaxError: If an attempt is made to override a
+            variable with a descriptor other than its underlying one.
+
+        '''
+
+        try:
+            var_space = super().__getattribute__('_rfm_var_space')
+            if name in var_space:
+                if not hasattr(value, '__get__'):
+                    var_space[name].define(value)
+                    return True
+                elif var_space[name].field is not value:
+                    desc = '.'.join([cls.__qualname__, name])
+                    raise ReframeSyntaxError(
+                        f'cannot override variable descriptor {desc!r}'
+                    )
+                else:
+                    # Variable is being injected
+                    return False
+        except AttributeError:
+            '''Catch early access attempt to the variable space.'''
+            return False
+
     def __setattr__(cls, name, value):
         '''Handle the special treatment required for variables and parameters.
 
@@ -489,31 +524,20 @@ class attribute. This behavior does not apply when the assigned value
         is not allowed. This would break the parameter space internals.
         '''
 
-        # Set the value of a variable (except when the value is a descriptor).
-        try:
-            var_space = super().__getattribute__('_rfm_var_space')
-            if name in var_space:
-                if not hasattr(value, '__get__'):
-                    var_space[name].define(value)
-                    return
-                elif not var_space[name].field is value:
-                    desc = '.'.join([cls.__qualname__, name])
-                    raise ReframeSyntaxError(
-                        f'cannot override variable descriptor {desc!r}'
-                    )
-
-        except AttributeError:
-            pass
+        # Try to treat `name` as variable
+        if cls.setvar(name, value):
+            return
 
-        # Catch attempts to override a test parameter
+        # Try to treat `name` as a parameter
         try:
+            # Catch attempts to override a test parameter
             param_space = super().__getattribute__('_rfm_param_space')
             if name in param_space.params:
                 raise ReframeSyntaxError(f'cannot override parameter {name!r}')
-
         except AttributeError:
-            pass
+            '''Catch early access attempt to the parameter space.'''
 
+        # Treat `name` as normal class attribute
         super().__setattr__(name, value)
 
     @property

reframe/core/pipeline.py

@@ -213,6 +213,12 @@ def pipeline_hooks(cls):
     #: The name of the test.
     #:
     #: :type: string that can contain any character except ``/``
+    #: :default: For non-parameterised tests, the default name is the test
+    #:   class name. For parameterised tests, the default name is constructed
+    #:   by concatenating the test class name and the string representations
+    #:   of every test parameter: ``TestClassName_<param1>_<param2>``.
+    #:   Any non-alphanumeric value in a parameter's representation is
+    #:   converted to ``_``.
     name = variable(typ.Str[r'[^\/]+'])
 
     #: List of programming environments supported by this test.

reframe/core/variables.py

@@ -448,7 +448,6 @@ def join(self, other, cls):
         :param cls: the target class.
         '''
         for key, var in other.items():
-
             # Make doubly declared vars illegal. Note that this will be
             # triggered when inheriting from multiple RegressionTest classes.
             if key in self.vars:

reframe/frontend/cli.py

@@ -329,6 +329,12 @@ def main():
         '--skip-prgenv-check', action='store_true',
         help='Skip programming environment check'
     )
+    run_options.add_argument(
+        '-S', '--setvar', action='append', metavar='[TEST.]VAR=VAL',
+        dest='vars', default=[],
+        help=('Set test variable VAR to VAL in all tests '
+              'or optionally in TEST only')
+    )
     run_options.add_argument(
         '--exec-policy', metavar='POLICY', action='store',
         choices=['async', 'serial'], default='async',
@@ -711,10 +717,21 @@ def main():
         )
         check_search_path = site_config.get('general/0/check_search_path')
 
-    loader = RegressionCheckLoader(
-        load_path=check_search_path,
-        recurse=check_search_recursive
-    )
+    # Collect any variables set from the command line
+    external_vars = {}
+    for expr in options.vars:
+        try:
+            lhs, rhs = expr.split('=', maxsplit=1)
+        except ValueError:
+            printer.warning(
+                f'invalid test variable assignment: {expr!r}; skipping'
+            )
+        else:
+            external_vars[lhs] = rhs
+
+    loader = RegressionCheckLoader(check_search_path,
+                                   check_search_recursive,
+                                   external_vars)
 
     def print_infoline(param, value):
         param = param + ':'