diff --git a/cscs-checks/system/jobreport/gpu_report.py b/cscs-checks/system/jobreport/gpu_report.py index 4a7365e816..5967ff59eb 100644 --- a/cscs-checks/system/jobreport/gpu_report.py +++ b/cscs-checks/system/jobreport/gpu_report.py @@ -43,9 +43,9 @@ def set_launcher_opts(self): ''' self.job.launcher.options = ['-u'] - @run_before('sanity') + @sanity_function def set_sanity_patterns(self): - '''Set sanity patterns and wait for the jobreport. + '''Extend sanity and wait for the jobreport. If a large number of nodes is used, the final jobreport output happens much later after job has already completed (this could be up to 25s). @@ -54,16 +54,16 @@ def set_sanity_patterns(self): sanity function does not succeed. ''' - super().set_sanity_patterns() - self.sanity_patterns = sn.all([ - self.sanity_patterns, self.gpu_usage_sanity() - ]) try: sn.evaluate(self.gpu_usage_sanity()) except SanityError: time.sleep(25) - @sn.sanity_function + return sn.all([ + self.count_successful_burns(), self.gpu_usage_sanity() + ]) + + @deferrable def gpu_usage_sanity(self): '''Verify that the jobreport output has sensible numbers. diff --git a/docs/deferrable_functions_reference.rst b/docs/deferrable_functions_reference.rst new file mode 100644 index 0000000000..996820b0fe --- /dev/null +++ b/docs/deferrable_functions_reference.rst @@ -0,0 +1,79 @@ + .. _deferrable-functions: + +============================== +Deferrable Functions Reference +============================== + +*Deferrable functions* are the functions whose execution may be postponed to a later time after they are called. +The key characteristic of these functions is that they store their arguments when they are called, and the execution itself does not occur until the function is evaluated either explicitly or implicitly. + +Explicit evaluation of deferrable functions +------------------------------------------- + +Deferrable functions may be evaluated at any time by calling :func:`evaluate` on their return value or by passing the deferred function itself to the :func:`~reframe.utility.sanity.evaluate()` free function. + +Implicit evaluation of deferrable functions +------------------------------------------- + +Deferrable functions may also be evaluated implicitly in the following situations: + +- When you try to get their truthy value by either explicitly or implicitly calling :func:`bool ` on their return value. + This implies that when you include the result of a deferrable function in an :keyword:`if` statement or when you apply the :keyword:`and`, :keyword:`or` or :keyword:`not` operators, this will trigger their immediate evaluation. + +- When you try to iterate over their result. + This implies that including the result of a deferrable function in a :keyword:`for` statement will trigger its evaluation immediately. + +- When you try to explicitly or implicitly get its string representation by calling :func:`str ` on its result. + This implies that printing the return value of a deferrable function will automatically trigger its evaluation. + + +Categories of deferrable functions +---------------------------------- + +Currently ReFrame provides three broad categories of deferrable functions: + +1. Deferrable replacements of certain Python built-in functions. + These functions simply delegate their execution to the actual built-ins. +2. Assertion functions. + These functions are used to assert certain conditions and they either return :class:`True` or raise :class:`~reframe.core.exceptions.SanityError` with a message describing the error. + Users may provide their own formatted messages through the ``msg`` argument. + For example, in the following call to :func:`assert_eq` the ``{0}`` and ``{1}`` placeholders will obtain the actual arguments passed to the assertion function. + + .. code:: python + + assert_eq(a, 1, msg="{0} is not equal to {1}") + + If in the user provided message more placeholders are used than the arguments of the assert function (except the ``msg`` argument), no argument substitution will be performed in the user message. +3. Utility functions. + They include, but are not limited to, functions to iterate over regex matches in a file, extracting and converting values from regex matches, computing statistical information on series of data etc. + + +Users can write their own deferrable functions as well. +The page ":doc:`deferrables`" explains in detail how deferrable functions work and how users can write their own. + + +.. py:decorator:: reframe.utility.sanity.deferrable(func) + + Deferrable decorator. + + Converts the decorated free function into a deferrable function. + + .. code:: python + + import reframe.utility.sanity as sn + + @sn.deferrable + def myfunc(*args): + do_sth() + + +.. py:decorator:: reframe.utility.sanity.sanity_function(func) + + Please use the :func:`reframe.core.pipeline.RegressionMixin.deferrable` decorator when possible. Alternatively, please use the :func:`reframe.utility.sanity.deferrable` decorator instead. + + .. warning:: Not to be mistaken with :func:`~reframe.core.pipeline.RegressionMixin.sanity_function` built-in. + .. deprecated:: 3.8.0 + + +.. automodule:: reframe.utility.sanity + :members: diff --git a/docs/deferrables.rst b/docs/deferrables.rst index 7b77030b11..be78c33265 100644 --- a/docs/deferrables.rst +++ b/docs/deferrables.rst @@ -1,23 +1,24 @@ -=============================================== -Understanding the Mechanism of Sanity Functions -=============================================== +=================================================== +Understanding the Mechanism of Deferrable Functions +=================================================== -This section describes the mechanism behind the sanity functions that are used for the sanity and performance checking. -Generally, writing a new sanity function is as straightforward as decorating a simple Python function with the :func:`reframe.utility.sanity.sanity_function` decorator. -However, it is important to understand how and when a deferrable function is evaluated, especially if your function takes as arguments the results of other deferrable functions. +This section describes the mechanism behind deferrable functions, which in ReFrame, they are used for sanity and performance checking. +Generally, writing a new sanity function in a :class:`~reframe.core.pipeline.RegressionTest` is as straightforward as decorating a simple member function with the built-in :func:`~reframe.core.pipeline.RegressionMixin.sanity_function` decorator. +Behind the scenes, this decorator will convert the Python function into a deferrable function and schedule its evaluation for the sanity stage of the test. +However, when dealing with more complex scenarios such as a deferrable function taking as an argument the results from other deferrable functions, it is crucial to understand how a deferrable function differs from a regular Python function, and when is it actually evaluated. What Is a Deferrable Function? ------------------------------ A deferrable function is a function whose a evaluation is deferred to a later point in time. -You can define any function as deferrable by wrapping it with the :func:`reframe.utility.sanity.sanity_function` decorator before its definition. +You can define any function as deferrable by wrapping it with the :func:`~reframe.core.pipeline.RegressionMixin.deferrable` when decorating a member function of a class derived from :class:`~reframe.core.pipeline.RegressionMixin`, or alternatively, the :func:`reframe.utility.sanity.deferrable` decorator can be used for any other function. The example below demonstrates a simple scenario: .. code-block:: python import reframe.utility.sanity as sn - @sn.sanity_function + @sn.deferrable def foo(): print('hello') @@ -47,11 +48,11 @@ Deferrable functions may also be combined as we do with normal functions. Let's import reframe.utility.sanity as sn - @sn.sanity_function + @sn.deferrable def foo(arg): print(arg) - @sn.sanity_function + @sn.deferrable def greetings(): return 'hello' @@ -85,7 +86,7 @@ To demonstrate more clearly how the deferred evaluation of a function works, let .. code-block:: python - @sn.sanity_function + @sn.deferrable def size3(iterable): return len(iterable) == 3 @@ -160,7 +161,7 @@ Here is why: .. code-block:: pycon - >>> @sn.sanity_function + >>> @sn.deferrable ... def size(iterable): ... return len(iterable) ... @@ -184,7 +185,7 @@ If you want to defer the execution of such operators, you should use the corresp In summary deferrable functions have the following characteristics: -* You can make any function deferrable by wrapping it with the :func:`reframe.utility.sanity.sanity_function` decorator. +* You can make any function deferrable by wrapping it with the :func:`~reframe.utility.sanity.deferrable` decorator. * When you call a deferrable function, its body is not executed but its arguments are *captured* and an object representing the deferred function is returned. * You can execute the body of a deferrable function at any later point by calling :func:`evaluate ` on the deferred expression object that it has been returned by the call to the deferred function. * Deferred functions can accept other deferred expressions as arguments and may also return a deferred expression. @@ -194,7 +195,7 @@ In summary deferrable functions have the following characteristics: How a Deferred Expression Is Evaluated? --------------------------------------- -As discussed before, you can create a new deferred expression by calling a function whose definition is decorated by the ``@sanity_function`` or ``@deferrable`` decorator or by including an already deferred expression in any sort of arithmetic operation. +As discussed before, you can create a new deferred expression by calling a function whose definition is decorated by the ``@deferrable`` decorator or by including an already deferred expression in any sort of arithmetic operation. When you call :func:`evaluate ` on a deferred expression, you trigger the evaluation of the whole subexpression tree. Here is how the evaluation process evolves: @@ -209,15 +210,15 @@ Here is an example where we define two deferrable variations of the builtins :fu .. code-block:: python - @sn.sanity_function + @sn.deferrable def dsum(iterable): return sum(iterable) - @sn.sanity_function + @sn.deferrable def dlen(iterable): return len(iterable) - @sn.sanity_function + @sn.deferrable def avg(iterable): return dsum(iterable) / dlen(iterable) @@ -285,7 +286,7 @@ Although you can trigger the evaluation of a deferred expression at any time by .. code-block:: pycon - >>> @sn.sanity_function + >>> @sn.deferrable ... def getlist(iterable): ... ret = list(iterable) ... ret += [1, 2, 3] @@ -336,12 +337,12 @@ The following example demonstrates two different ways writing a deferrable funct import reframe.utility.sanity as sn - @sn.sanity_function + @sn.deferrable def check_avg_with_deferrables(iterable): avg = sn.sum(iterable) / sn.len(iterable) return -1 if avg > 2 else 1 - @sn.sanity_function + @sn.deferrable def check_avg_without_deferrables(iterable): avg = sum(iterable) / len(iterable) return -1 if avg > 2 else 1 @@ -360,14 +361,14 @@ In the version with the deferrables, ``avg`` is a deferred expression but it is Generally, inside a sanity function, it is a preferable to use the non-deferrable version of a function, if that exists, since you avoid the extra overhead and bookkeeping of the deferring mechanism. -Deferrable Sanity Functions ---------------------------- +Ready to Go Deferrable Functions +-------------------------------- -Normally, you will not have to implement your own sanity functions, since ReFrame provides already a variety of them. -You can find the complete list of provided sanity functions `here `__. +Normally, you will not have to implement your own deferrable functions, since ReFrame provides already a variety of them. +You can find the complete list of provided sanity functions in :ref:`deferrable-functions`. -Similarities and Differences with Generators -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Deferrable functions vs Generators +---------------------------------- Python allows you to create functions that will be evaluated lazily. These are called `generator functions `__. @@ -390,7 +391,7 @@ Differences .. code-block:: pycon - >>> @sn.sanity_function + >>> @sn.deferrable ... def dsize(iterable): ... print(len(iterable)) ... return len(iterable) diff --git a/docs/programming_apis.rst b/docs/programming_apis.rst index 85fa9c2c80..c204883fde 100644 --- a/docs/programming_apis.rst +++ b/docs/programming_apis.rst @@ -6,6 +6,6 @@ Programming APIs :maxdepth: 3 regression_test_api - sanity_functions_reference + deferrable_functions_reference utility_functions_reference exceptions diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index 6bb27acb45..52d31af751 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -262,13 +262,36 @@ The framework will then continue with other activities and it will execute the p Built-in functions ------------------ +.. py:decorator:: RegressionMixin.sanity_function(func) + + Decorate a member function as the sanity function of the test. + + This decorator will convert the decorated method into a :func:`~RegressionMixin.deferrable` and mark it to be executed during the test's sanity stage. + When this decorator is used, manually assigning a value to :attr:`~RegressionTest.sanity_patterns` in the test is not allowed. + + Decorated functions may be overridden by derived classes, and derived classes may also decorate a different method as the test's sanity function. + Decorating multiple member functions in the same class is not allowed. + However, a :class:`RegressionTest` may inherit from multiple :class:`RegressionMixin` classes with their own sanity functions. + In this case, the derived class will follow Python's `MRO `_ to find a suitable sanity function. + + .. versionadded:: 3.7.0 + +.. py:decorator:: RegressionMixin.deferrable(func) + + Converts the decorated method into a deferrable function. + + See :ref:`deferrable-functions` for further information on deferrable functions. + + .. versionadded:: 3.7.0 + .. py:function:: RegressionMixin.bind(func, name=None) Bind a free function to a regression test. + By default, the function is bound with the same name as the free function. However, the function can be bound using a different name with the ``name`` argument. - :param fn: external function to be bound to a class. + :param func: external function to be bound to a class. :param name: bind the function under a different name. .. versionadded:: 3.6.2 diff --git a/docs/sanity_functions_reference.rst b/docs/sanity_functions_reference.rst deleted file mode 100644 index 05ef3e37a1..0000000000 --- a/docs/sanity_functions_reference.rst +++ /dev/null @@ -1,70 +0,0 @@ -========================== -Sanity Functions Reference -========================== - -*Sanity functions* are the functions used with the :attr:`sanity_patterns ` and :attr:`perf_patterns `. -The key characteristic of these functions is that they are not executed the time they are called. -Instead they are evaluated at a later point by the framework (inside the :func:`check_sanity ` and :func:`check_performance ` methods). -Any sanity function may be evaluated either explicitly or implicitly. - - -Explicit evaluation of sanity functions ---------------------------------------- - -Sanity functions may be evaluated at any time by calling :func:`evaluate` on their return value or by passing the result of a sanity function to the :func:`reframe.utility.sanity.evaluate()` free function. - - -Implicit evaluation of sanity functions ---------------------------------------- - -Sanity functions may also be evaluated implicitly in the following situations: - -- When you try to get their truthy value by either explicitly or implicitly calling :func:`bool ` on their return value. - This implies that when you include the result of a sanity function in an :keyword:`if` statement or when you apply the :keyword:`and`, :keyword:`or` or :keyword:`not` operators, this will trigger their immediate evaluation. -- When you try to iterate over their result. - This implies that including the result of a sanity function in a :keyword:`for` statement will trigger its evaluation immediately. -- When you try to explicitly or implicitly get its string representation by calling :func:`str ` on its result. - This implies that printing the return value of a sanity function will automatically trigger its evaluation. - - -Categories of sanity functions ------------------------------- - -Currently ReFrame provides three broad categories of sanity functions: - -1. Deferrable replacements of certain Python built-in functions. - These functions simply delegate their execution to the actual built-ins. -2. Assertion functions. - These functions are used to assert certain conditions and they either return :class:`True` or raise :class:`reframe.core.exceptions.SanityError` with a message describing the error. - Users may provide their own formatted messages through the ``msg`` argument. - For example, in the following call to :func:`assert_eq` the ``{0}`` and ``{1}`` placeholders will obtain the actual arguments passed to the assertion function. - - .. code:: python - - assert_eq(a, 1, msg="{0} is not equal to {1}") - - If in the user provided message more placeholders are used than the arguments of the assert function (except the ``msg`` argument), no argument substitution will be performed in the user message. -3. Utility functions. - The are functions that you will normally use when defining :attr:`sanity_patterns ` and :attr:`perf_patterns `. - They include, but are not limited to, functions to iterate over regex matches in a file, extracting and converting values from regex matches, computing statistical information on series of data etc. - - -Users can write their own sanity functions as well. -The page ":doc:`deferrables`" explains in detail how sanity functions work and how users can write their own. - - -.. py:decorator:: sanity_function - - Sanity function decorator. - - The evaluation of the decorated function will be deferred and it will become suitable for use in the sanity and performance patterns of a regression test. - - .. code:: python - - @sanity_function - def myfunc(*args): - do_sth() - - -.. automodule:: reframe.utility.sanity - :members: diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index 71921ff412..b0cb6a698e 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -575,7 +575,7 @@ They will not be executed where they appear, but rather at the sanity checking p ReFrame provides lazily evaluated counterparts for most of the builtin Python functions, such the :func:`len` function here. Also whole expressions can be lazily evaluated if one of the operands is deferred, as is the case in this example with the assignment to ``num_messages``. This makes the sanity checking mechanism quite powerful and straightforward to reason about, without having to rely on complex pattern matching techniques. -:doc:`sanity_functions_reference` provides a complete reference of the sanity functions provided by ReFrame, but users can also define their own, as described in :doc:`deferrables`. +:doc:`deferrable_functions_reference` provides a complete reference of the sanity functions provided by ReFrame, but users can also define their own, as described in :doc:`deferrables`. Let's run this version of the test now and see if it fails: diff --git a/docs/tutorial_tips_tricks.rst b/docs/tutorial_tips_tricks.rst index bc0cffd7b2..1dee279aa5 100644 --- a/docs/tutorial_tips_tricks.rst +++ b/docs/tutorial_tips_tricks.rst @@ -107,7 +107,7 @@ As suggested by the warning message, passing :option:`-v` will give you the stac Debugging deferred expressions ============================== -Although deferred expression that are used in :attr:`sanity_patterns` and :attr:`perf_patterns` behave similarly to normal Python expressions, you need to understand their `implicit evaluation rules `__. +Although deferred expression that are used in :attr:`sanity_patterns` and :attr:`perf_patterns` behave similarly to normal Python expressions, you need to understand their `implicit evaluation rules `__. One of the rules is that :func:`str` triggers the implicit evaluation, so trying to use the standard :func:`print` function with a deferred expression, you might get unexpected results if that expression is not yet to be evaluated. For this reason, ReFrame offers a sanity function counterpart of :func:`print`, which allows you to safely print deferred expressions. diff --git a/hpctestlib/microbenchmarks/gpu/dgemm/__init__.py b/hpctestlib/microbenchmarks/gpu/dgemm/__init__.py index 78daf1316d..46e00f5177 100644 --- a/hpctestlib/microbenchmarks/gpu/dgemm/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/dgemm/__init__.py @@ -74,10 +74,13 @@ def set_gpu_build(self): else: raise ValueError('unknown gpu_build option') - @run_before('sanity') - def set_sanity_patterns(self): - '''Set the sanity patterns.''' - self.sanity_patterns = self.assert_num_gpus() + @sanity_function + def assert_num_gpus(self): + '''Assert that that all tasks passed.''' + + return sn.assert_eq( + sn.count(sn.findall(r'^\s*\[[^\]]*\]\s*Test passed', self.stdout)), + sn.getattr(self.job, 'num_tasks')) @run_before('performance') def set_perf_patterns(self): @@ -88,11 +91,3 @@ def set_perf_patterns(self): r'^\s*\[[^\]]*\]\s*GPU\s*\d+: (?P\S+) TF/s', self.stdout, 'fp', float)) } - - @sn.sanity_function - def assert_num_gpus(self): - '''Assert that that all tasks passed.''' - - return sn.assert_eq( - sn.count(sn.findall(r'^\s*\[[^\]]*\]\s*Test passed', self.stdout)), - sn.getattr(self.job, 'num_tasks')) diff --git a/hpctestlib/microbenchmarks/gpu/gpu_burn/__init__.py b/hpctestlib/microbenchmarks/gpu/gpu_burn/__init__.py index ccf8124a76..6732ffa012 100644 --- a/hpctestlib/microbenchmarks/gpu/gpu_burn/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/gpu_burn/__init__.py @@ -83,7 +83,7 @@ def set_gpu_build(self): raise ValueError('unknown gpu_build option') @property - @sn.sanity_function + @deferrable def num_tasks_assigned(self): '''Total number of times the gpu burn will run. @@ -95,11 +95,11 @@ def num_tasks_assigned(self): return self.job.num_tasks * self.num_gpus_per_node - @run_before('sanity') - def set_sanity_patterns(self): + @sanity_function + def count_successful_burns(self): '''Set the sanity patterns to count the number of successful burns.''' - self.sanity_patterns = sn.assert_eq(sn.count(sn.findall( + return sn.assert_eq(sn.count(sn.findall( r'^\s*\[[^\]]*\]\s*GPU\s*\d+\(OK\)', self.stdout) ), self.num_tasks_assigned) diff --git a/hpctestlib/microbenchmarks/gpu/kernel_latency/__init__.py b/hpctestlib/microbenchmarks/gpu/kernel_latency/__init__.py index 83c31f599d..4804ddc1a3 100644 --- a/hpctestlib/microbenchmarks/gpu/kernel_latency/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/kernel_latency/__init__.py @@ -93,21 +93,7 @@ def set_gpu_build(self): else: raise ValueError('unknown gpu_build option') - @run_before('sanity') - def set_sanity_patterns(self): - '''Set sanity function''' - self.sanity_patterns = self.assert_count_gpus() - - @run_before('performance') - def set_perf_patterns(self): - '''Set performance patterns.''' - self.perf_patterns = { - 'latency': sn.max(sn.extractall( - r'\[\S+\] \[gpu \d+\] Kernel launch latency: ' - r'(?P\S+) us', self.stdout, 'latency', float)) - } - - @sn.sanity_function + @sanity_function def assert_count_gpus(self): '''Assert GPU count is consistent.''' return sn.all([ @@ -126,3 +112,12 @@ def assert_count_gpus(self): self.job.num_tasks * self.num_gpus_per_node ) ]) + + @run_before('performance') + def set_perf_patterns(self): + '''Set performance patterns.''' + self.perf_patterns = { + 'latency': sn.max(sn.extractall( + r'\[\S+\] \[gpu \d+\] Kernel launch latency: ' + r'(?P\S+) us', self.stdout, 'latency', float)) + } diff --git a/hpctestlib/microbenchmarks/gpu/memory_bandwidth/__init__.py b/hpctestlib/microbenchmarks/gpu/memory_bandwidth/__init__.py index acd52aabb5..4198f8f399 100644 --- a/hpctestlib/microbenchmarks/gpu/memory_bandwidth/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/memory_bandwidth/__init__.py @@ -84,32 +84,29 @@ def set_exec_opts(self): f'--copies {self.num_copies}', ] - @run_before('sanity') - def set_sanity_patterns(self): - self.sanity_patterns = self.do_sanity_check() - - @sn.sanity_function - def do_sanity_check(self): + @sanity_function + def assert_successful_completion(self): '''Check that all nodes completed successfully.''' node_names = set(sn.extractall( r'^\s*\[([^\]]*)\]\s*Found %s device\(s\).' % self.num_gpus_per_node, self.stdout, 1 )) - sn.evaluate(sn.assert_eq( + req_nodes = sn.assert_eq( self.job.num_tasks, len(node_names), msg='requested {0} node(s), got {1} (nodelist: %s)' % - ','.join(sorted(node_names)))) + ','.join(sorted(node_names))) good_nodes = set(sn.extractall( r'^\s*\[([^\]]*)\]\s*Test Result\s*=\s*PASS', self.stdout, 1 )) - sn.evaluate(sn.assert_eq( + failed_nodes = sn.assert_eq( node_names, good_nodes, msg='check failed on the following node(s): %s' % - ','.join(sorted(node_names - good_nodes))) + ','.join(sorted(node_names - good_nodes)) ) - return True + + return sn.all([req_nodes, failed_nodes]) class GpuBandwidth(GpuBandwidthBase): diff --git a/hpctestlib/microbenchmarks/gpu/pointer_chase/__init__.py b/hpctestlib/microbenchmarks/gpu/pointer_chase/__init__.py index 30486904d2..5275a31abc 100644 --- a/hpctestlib/microbenchmarks/gpu/pointer_chase/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/pointer_chase/__init__.py @@ -66,11 +66,11 @@ def set_gpu_build(self): else: raise ValueError('unknown gpu_build option') - @run_before('sanity') - def set_sanity(self): + @sanity_function + def assert_exec_present(self): '''Assert that the executable is present.''' - self.sanity_patterns = sn.assert_found(r'pChase.x', self.stdout) + return sn.assert_found(r'pChase.x', self.stdout) class RunGpuPchaseBase(rfm.RunOnlyRegressionTest, pin_prefix=True): @@ -124,12 +124,8 @@ def set_exec_opts(self): f'--num-jumps {self.num_node_jumps}' ] - @run_before('sanity') - def set_sanity(self): - self.sanity_patterns = self.do_sanity_check() - - @sn.sanity_function - def do_sanity_check(self): + @sanity_function + def assert_correct_num_gpus_per_node(self): '''Check that every node has the right number of GPUs.''' my_nodes = set(sn.extractall( @@ -173,7 +169,7 @@ class RunGpuPchaseD2D(RunGpuPchaseBase): executable_opts = ['--multi-gpu'] - @sn.sanity_function + @deferrable def average_D2D_latency(self): '''Extract the average D2D latency. diff --git a/hpctestlib/microbenchmarks/gpu/shmem/__init__.py b/hpctestlib/microbenchmarks/gpu/shmem/__init__.py index 1f1f64ed8b..e647f5b72e 100644 --- a/hpctestlib/microbenchmarks/gpu/shmem/__init__.py +++ b/hpctestlib/microbenchmarks/gpu/shmem/__init__.py @@ -74,10 +74,14 @@ def set_gpu_build(self): else: raise ValueError('unknown gpu_build option') - @run_before('sanity') - def set_sanity_patterns(self): - '''Set the ``sanity_patterns`` variable.''' - self.sanity_patterns = self.assert_count_gpus() + @sanity_function + def assert_count_gpus(self): + '''Count the number of GPUs testd is correct.''' + + return sn.assert_eq( + sn.count(sn.findall(r'Bandwidth', self.stdout)), + self.job.num_tasks * 2 * self.num_gpus_per_node + ) @run_before('performance') def set_perf_patterns(self): @@ -90,12 +94,3 @@ def set_perf_patterns(self): self.stdout, 'bw', float )) } - - @sn.sanity_function - def assert_count_gpus(self): - '''Count the number of GPUs testd is correct.''' - - return sn.assert_eq( - sn.count(sn.findall(r'Bandwidth', self.stdout)), - self.job.num_tasks * 2 * self.num_gpus_per_node - ) diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 309f43db76..089986c804 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -16,6 +16,7 @@ import reframe.core.hooks as hooks from reframe.core.exceptions import ReframeSyntaxError +from reframe.core.deferrable import deferrable _USER_PIPELINE_STAGES = ( @@ -229,6 +230,21 @@ def run_after(stage): namespace['run_after'] = run_after namespace['require_deps'] = hooks.require_deps + + # Machinery to add a sanity function + def sanity_function(fn): + '''Mark a function as the test's sanity function. + + Decorated functions must be unary and they will be converted into + deferred expressions. + ''' + + _def_fn = deferrable(fn) + setattr(_def_fn, '_rfm_sanity_fn', True) + return _def_fn + + namespace['sanity_function'] = sanity_function + namespace['deferrable'] = deferrable return metacls.MetaNamespace(namespace) def __new__(metacls, name, bases, namespace, **kwargs): @@ -243,7 +259,7 @@ class was created or even at the instance level (e.g. doing blacklist = [ 'parameter', 'variable', 'bind', 'run_before', 'run_after', - 'require_deps', 'required' + 'require_deps', 'required', 'deferrable', 'sanity_function' ] for b in blacklist: namespace.pop(b, None) @@ -255,9 +271,8 @@ def __init__(cls, name, bases, namespace, **kwargs): # Create a set with the attribute names already in use. cls._rfm_dir = set() - for base in bases: - if hasattr(base, '_rfm_dir'): - cls._rfm_dir.update(base._rfm_dir) + for base in (b for b in bases if hasattr(b, '_rfm_dir')): + cls._rfm_dir.update(base._rfm_dir) used_attribute_names = set(cls._rfm_dir) @@ -275,11 +290,37 @@ def __init__(cls, name, bases, namespace, **kwargs): # attribute; all dependencies will be resolved first in the post-setup # phase if not assigned elsewhere hook_reg = hooks.HookRegistry.create(namespace) - for b in bases: - if hasattr(b, '_rfm_pipeline_hooks'): - hook_reg.update(getattr(b, '_rfm_pipeline_hooks')) + for base in (b for b in bases if hasattr(b, '_rfm_pipeline_hooks')): + hook_reg.update(getattr(base, '_rfm_pipeline_hooks')) cls._rfm_pipeline_hooks = hook_reg + + # Gather all the locally defined sanity functions based on the + # _rfm_sanity_fn attribute. + + sn_fn = [v for v in namespace.values() if hasattr(v, '_rfm_sanity_fn')] + if sn_fn: + cls._rfm_sanity = sn_fn[0] + if len(sn_fn) > 1: + raise ReframeSyntaxError( + f'{cls.__qualname__!r} defines more than one sanity ' + 'function in the class body.' + ) + + else: + # Search the bases if no local sanity functions exist. + for base in (b for b in bases if hasattr(b, '_rfm_sanity')): + cls._rfm_sanity = getattr(base, '_rfm_sanity') + if cls._rfm_sanity.__name__ in namespace: + raise ReframeSyntaxError( + f'{cls.__qualname__!r} overrides the candidate ' + f'sanity function ' + f'{cls._rfm_sanity.__qualname__!r} without ' + f'defining an alternative' + ) + + break + cls._final_methods = {v.__name__ for v in namespace.values() if hasattr(v, '_rfm_final')} @@ -287,14 +328,12 @@ def __init__(cls, name, bases, namespace, **kwargs): cls._final_methods.update(*(b._final_methods for b in bases if hasattr(b, '_final_methods'))) - if hasattr(cls, '_rfm_special_test') and cls._rfm_special_test: + if getattr(cls, '_rfm_special_test', None): return + bases_w_final = [b for b in bases if hasattr(b, '_final_methods')] for v in namespace.values(): - for b in bases: - if not hasattr(b, '_final_methods'): - continue - + for b in bases_w_final: if callable(v) and v.__name__ in b._final_methods: msg = (f"'{cls.__qualname__}.{v.__name__}' attempts to " f"override final method " @@ -333,7 +372,6 @@ def __getattr__(cls, name): method will perform an attribute lookup on these sub-namespaces if a call to the default :func:`__getattribute__` method fails to retrieve the requested class attribute. - ''' try: diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index aeea2ce53a..fe92d0a5bb 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -38,7 +38,8 @@ from reframe.core.deferrable import _DeferredExpression from reframe.core.exceptions import (BuildError, DependencyError, PerformanceError, PipelineError, - SanityError, SkipTestError) + SanityError, SkipTestError, + ReframeSyntaxError) from reframe.core.meta import RegressionTestMeta from reframe.core.schedulers import Job from reframe.core.warnings import user_deprecation_warning @@ -551,10 +552,11 @@ def pipeline_hooks(cls): #: Refer to the :doc:`ReFrame Tutorials ` for concrete usage #: examples. #: - #: If not set a sanity error will be raised during sanity checking. + #: If not set, a sanity error may be raised during sanity checking if no + #: other sanity checking functions already exist. #: #: :type: A deferrable expression (i.e., the result of a :doc:`sanity - #: function `) + #: function `) #: :default: :class:`required` #: #: .. note:: @@ -582,7 +584,7 @@ def pipeline_hooks(cls): #: #: :type: A dictionary with keys of type :class:`str` and deferrable #: expressions (i.e., the result of a :doc:`sanity function - #: `) as values. + #: `) as values. #: :class:`None` is also allowed. #: :default: :class:`None` perf_patterns = variable(typ.Dict[str, _DeferredExpression], @@ -955,7 +957,7 @@ def outputdir(self): return self._outputdir @property - @sn.sanity_function + @deferrable def stdout(self): '''The name of the file containing the standard output of the test. @@ -970,7 +972,7 @@ def stdout(self): return self.job.stdout if self.job else None @property - @sn.sanity_function + @deferrable def stderr(self): '''The name of the file containing the standard error of the test. @@ -989,12 +991,12 @@ def build_job(self): return self._build_job @property - @sn.sanity_function + @deferrable def build_stdout(self): return self.build_job.stdout if self.build_job else None @property - @sn.sanity_function + @deferrable def build_stderr(self): return self.build_job.stderr if self.build_job else None @@ -1492,6 +1494,8 @@ def check_sanity(self): '''The sanity checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the sanity check fails. + :raises reframe.core.exceptions.ReframeSyntaxError: If the sanity + function cannot be resolved due to ambiguous syntax. .. warning:: @@ -1507,6 +1511,19 @@ def check_sanity(self): more details. ''' + + if hasattr(self, '_rfm_sanity'): + # Using more than one type of syntax to set the sanity patterns is + # not allowed. + if hasattr(self, 'sanity_patterns'): + raise ReframeSyntaxError( + f"assigning a sanity function to the 'sanity_patterns' " + f"variable conflicts with using the 'sanity_function' " + f"decorator (class {self.__class__.__qualname__})" + ) + + self.sanity_patterns = self._rfm_sanity() + if rt.runtime().get_option('general/0/trap_job_errors'): sanity_patterns = [ sn.assert_eq(self.job.exitcode, 0, @@ -1948,12 +1965,12 @@ def setup(self, partition, environ, **job_opts): **job_opts) @property - @sn.sanity_function + @deferrable def stdout(self): return self.build_job.stdout if self.build_job else None @property - @sn.sanity_function + @deferrable def stderr(self): return self.build_job.stderr if self.build_job else None diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index 0e3c6e16cc..e6047bcfa2 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -13,6 +13,7 @@ import sys import reframe.utility as util +import reframe.core.warnings as warn from reframe.core.deferrable import deferrable, _DeferredExpression from reframe.core.exceptions import SanityError @@ -40,7 +41,13 @@ def _open(filename, *args, **kwargs): # Create an alias decorator -sanity_function = deferrable +def sanity_function(func): + warn.user_deprecation_warning( + 'using the @sn.sanity_function decorator from the sn module is ' + 'deprecated; please use the built-in decorator @deferrable instead.', + from_version='3.8.0' + ) + return deferrable(func) # Deferrable versions of selected builtins diff --git a/unittests/test_meta.py b/unittests/test_meta.py index f8244b2cc2..20b58645f8 100644 --- a/unittests/test_meta.py +++ b/unittests/test_meta.py @@ -3,23 +3,38 @@ # # SPDX-License-Identifier: BSD-3-Clause -import reframe as rfm +import pytest + import reframe.core.meta as meta +import reframe.core.deferrable as deferrable + +from reframe.core.exceptions import ReframeSyntaxError + + +@pytest.fixture +def MyMeta(): + '''Utility fixture just for convenience.''' + class Foo(metaclass=meta.RegressionTestMeta): + pass + + yield Foo -def test_directives(): +def test_directives(MyMeta): '''Test that directives are not available as instance attributes.''' def ext_fn(x): pass - class MyTest(rfm.RegressionTest): + class MyTest(MyMeta): p = parameter() v = variable(int) bind(ext_fn, name='ext') run_before('run')(ext) run_after('run')(ext) require_deps(ext) + deferrable(ext) + sanity_function(ext) v = required def __init__(self): @@ -29,18 +44,20 @@ def __init__(self): assert not hasattr(self, 'run_before') assert not hasattr(self, 'run_after') assert not hasattr(self, 'require_deps') + assert not hasattr(self, 'deferrable') + assert not hasattr(self, 'sanity_function') assert not hasattr(self, 'required') MyTest() -def test_bind_directive(): +def test_bind_directive(MyMeta): def ext_fn(x): return x ext_fn._rfm_foo = True - class MyTest(rfm.RegressionTest): + class MyTest(MyMeta): bind(ext_fn) bind(ext_fn, name='ext') @@ -69,3 +86,61 @@ def __init__(self): # Test __call__ assert MyTest.ext_fn(2) == 2 assert MyTest.ext(2) == 2 + + +def test_sanity_function_decorator(MyMeta): + class Foo(MyMeta): + @sanity_function + def my_sanity(self): + return True + + assert hasattr(Foo, '_rfm_sanity') + assert Foo._rfm_sanity.__name__ == 'my_sanity' + assert type(Foo._rfm_sanity()) is deferrable._DeferredExpression + + # Test override sanity + class Bar(Foo): + @sanity_function + def extended_sanity(self): + return self.my_sanity() + + assert hasattr(Bar, '_rfm_sanity') + assert Bar._rfm_sanity.__name__ == 'extended_sanity' + assert type(Bar._rfm_sanity()) is deferrable._DeferredExpression + + # Test bases lookup + class Baz(MyMeta): + pass + + class MyTest(Baz, Foo): + pass + + assert hasattr(MyTest, '_rfm_sanity') + assert MyTest._rfm_sanity.__name__ == 'my_sanity' + assert type(MyTest._rfm_sanity()) is deferrable._DeferredExpression + + # Test incomplete sanity override + with pytest.raises(ReframeSyntaxError): + class MyWrongTest(Foo): + def my_sanity(self): + pass + + # Test error when double-declaring @sanity_function in the same class + with pytest.raises(ReframeSyntaxError): + class MyWrongTest(MyMeta): + @sanity_function + def sn_fn_a(self): + pass + + @sanity_function + def sn_fn_b(self): + pass + + +def test_deferrable_decorator(MyMeta): + class MyTest(MyMeta): + @deferrable + def my_deferrable(self): + pass + + assert type(MyTest.my_deferrable()) is deferrable._DeferredExpression diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index c61a597e7d..5bcdd385dc 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -16,7 +16,8 @@ from reframe.core.containers import _STAGEDIR_MOUNT from reframe.core.exceptions import (BuildError, PipelineError, ReframeError, - PerformanceError, SanityError) + PerformanceError, SanityError, + ReframeSyntaxError) def _run(test, partition, prgenv): @@ -249,6 +250,29 @@ def set_sanity(self): _run(MyTest(), *local_exec_ctx) +def test_run_only_decorated_sanity(local_exec_ctx): + @test_util.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RunOnlyRegressionTest): + executable = './hello.sh' + executable_opts = ['Hello, World!'] + local = True + valid_prog_environs = ['*'] + valid_systems = ['*'] + + @sanity_function + def set_sanity(self): + return sn.assert_found(r'Hello, World\!', self.stdout) + + _run(MyTest(), *local_exec_ctx) + + class MyOtherTest(MyTest): + '''Test both syntaxes are incompatible.''' + sanity_patterns = sn.assert_true(1) + + with pytest.raises(ReframeSyntaxError): + _run(MyOtherTest(), *local_exec_ctx) + + def test_run_only_no_srcdir(local_exec_ctx): @test_util.custom_prefix('foo/bar/') class MyTest(rfm.RunOnlyRegressionTest):