diff --git a/docs/index.rst b/docs/index.rst index 5969b7dcf7..d348901eaf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,7 +46,7 @@ Publications .. toctree:: - :caption: Table of Contents: + :caption: Table of Contents :hidden: Getting Started @@ -58,6 +58,7 @@ Publications Understanding The Mechanism Of Sanity Functions Running ReFrame Use cases + Migrating to ReFrame 3 About ReFrame Reference Guide Sanity Functions Reference diff --git a/docs/migration_2_to_3.rst b/docs/migration_2_to_3.rst new file mode 100644 index 0000000000..c14488bb0a --- /dev/null +++ b/docs/migration_2_to_3.rst @@ -0,0 +1,78 @@ +====================== +Migrating to ReFrame 3 +====================== + + +Updating your tests +------------------- + + +ReFrame 2.20 introduced a new powerful mechanism for attaching arbitrary functions hooks at the different pipeline stages. +This mechanism provides an easy way to configure and extend the functionality of a test, eliminating essentially the need to override pipeline stages for this purpose. +ReFrame 3.0 deprecates the old practice for overriding pipeline stage methods in favor of using pipeline hooks. +In the old syntax, it was quite common to override the ``setup()`` method, in order to configure your test based on the current programming environment or the current system partition. +The following is a typical example of that: + + +.. code:: python + + def setup(self, partition, environ, **job_opts): + if environ.name == 'gnu': + self.build_system.cflags = ['-fopenmp'] + elif environ.name == 'intel': + self.build_system.cflags = ['-qopenmp'] + + super().setup(partition, environ, **job_opts) + + +Alternatively, this example could have been written as follows: + +.. code:: python + + def setup(self, partition, environ, **job_opts): + super().setup(partition, environ, **job_opts) + if self.current_environ.name == 'gnu': + self.build_system.cflags = ['-fopenmp'] + elif self.current_environ.name == 'intel': + self.build_system.cflags = ['-qopenmp'] + + +This syntax now issues a deprecation warning. +Rewriting this using pipeline hooks is quite straightforward and leads to nicer and more intuitive code: + +.. code:: python + + @rfm.run_before('compile') + def setflags(self): + if self.current_environ.name == 'gnu': + self.build_system.cflags = ['-fopenmp'] + elif self.current_environ.name == 'intel': + self.build_system.cflags = ['-qopenmp'] + + +You could equally attach this function to run after the "setup" phase with ``@rfm.run_after('setup')``, as in the original example, but attaching it to the "compile" phase makes more sense. +However, you can't attach this function *before* the "setup" phase, because the ``current_environ`` will not be available and it will be still ``None``. + + +Force override a pipeline method +================================ + +Although pipeline hooks should be able to cover almost all the cases for writing tests in ReFrame, there might be corner cases that you need to override one of the pipeline methods, e.g., because you want to implement a stage differently. +In this case, all you have to do is mark your test class as "special", and ReFrame will not issue any deprecation warning if you override pipeline stage methods: + +.. code:: python + + class MyExtendedTest(rfm.RegressionTest, special=True): + def setup(self, partition, environ, **job_opts): + # do your custom stuff + super().setup(partition, environ, **job_opts) + + +If you try to override the ``setup()`` method in any of the subclasses of ``MyExtendedTest``, you will still get a deprecation warning, which a desired behavior since the subclasses should be normal tests. + + +Suppressing deprecation warnings +================================ + +You can suppress any deprecation warning issued by ReFrame by passing the ``--no-deprecation-warnings`` flag. + diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 26de482a98..9ecd0971ca 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -461,12 +461,16 @@ Notice that in order to redefine a hook, you need not only redefine the method i Otherwise, the base class hook will be executed. -.. note:: - You may still configure your test per programming environment and per system partition by overriding the :func:`setup ` method, as in ReFrame versions prior to 2.20, but this is now discouraged since it is more error prone, as you have to memorize the signature of the pipeline methods that you override and also remember to call ``super()``. +.. warning:: + Configuring your test per programming environment and per system partition by overriding the :func:`setup() ` method is deprecated. + Please refer to the `Migrate to ReFrame 3 `__ guide for more details. + .. versionchanged:: 3.0 .. warning:: - Setting the compiler flags in the programming environment has been dropped completely in version 2.17. + Support for setting the compiler flags in the programming environment has been dropped completely. + + .. versionchanged:: 2.17 An alternative implementation using dictionaries diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 8476434d7e..d20d1f30cc 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -7,6 +7,8 @@ # Met-class for creating regression tests. # +from reframe.core.exceptions import user_deprecation_warning + class RegressionTestMeta(type): def __init__(cls, name, bases, namespace, **kwargs): @@ -33,3 +35,22 @@ def __init__(cls, name, bases, namespace, **kwargs): hooks['post_setup'] = fn_with_deps + hooks.get('post_setup', []) cls._rfm_pipeline_hooks = hooks + + cls._final_methods = {v.__name__ for v in namespace.values() + if hasattr(v, '_rfm_final')} + + # Add the final functions from its parents + cls._final_methods.update(*(b._final_methods for b in bases + if hasattr(b, '_final_methods'))) + + if hasattr(cls, '_rfm_special_test') and cls._rfm_special_test: + return + + for v in namespace.values(): + for b in bases: + if callable(v) and v.__name__ in b._final_methods: + msg = (f"'{cls.__qualname__}.{v.__name__}' attempts to " + f"override final method " + f"'{b.__qualname__}.{v.__name__}'; " + f"consider using the reframe hooks instead") + user_deprecation_warning(msg) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 858bae8aac..ed1f1e4bbb 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -7,7 +7,7 @@ # Basic functionality for regression tests # -__all__ = ['RegressionTest', +__all__ = ['final', 'RegressionTest', 'RunOnlyRegressionTest', 'CompileOnlyRegressionTest', 'DEPEND_EXACT', 'DEPEND_BY_ENV', 'DEPEND_FULLY'] @@ -107,6 +107,16 @@ def _fn(obj, *args, **kwargs): return _deco +def final(fn): + fn._rfm_final = True + + @functools.wraps(fn) + def _wrapped(*args, **kwargs): + return fn(*args, **kwargs) + + return _wrapped + + class RegressionTest(metaclass=RegressionTestMeta): '''Base class for regression tests. @@ -701,6 +711,11 @@ def __new__(cls, *args, **kwargs): def __init__(self): pass + @classmethod + def __init_subclass__(cls, *, special=False, **kwargs): + super().__init_subclass__(**kwargs) + cls._rfm_special_test = special + def _rfm_init(self, name=None, prefix=None): if name is not None: self.name = name @@ -1024,6 +1039,7 @@ def _setup_perf_logging(self): self._perf_logger = logging.getperflogger(self) @_run_hooks() + @final def setup(self, partition, environ, **job_opts): '''The setup phase of the regression test pipeline. @@ -1033,6 +1049,15 @@ def setup(self, partition, environ, **job_opts): When overriding this method users should always pass through ``job_opts`` to the base class method. :raises reframe.core.exceptions.ReframeError: In case of errors. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' self._current_partition = partition self._current_environ = environ @@ -1056,10 +1081,20 @@ def _clone_to_stagedir(self, url): os_ext.git_clone(self.sourcesdir, self._stagedir) @_run_hooks('pre_compile') + @final def compile(self): '''The compilation phase of the regression test pipeline. :raises reframe.core.exceptions.ReframeError: In case of errors. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' if not self._current_environ: raise PipelineError('no programming environment set') @@ -1144,10 +1179,20 @@ def compile(self): self._build_job.submit() @_run_hooks('post_compile') + @final def compile_wait(self): '''Wait for compilation phase to finish. .. versionadded:: 2.13 + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' self._build_job.wait() self.logger.debug('compilation finished') @@ -1157,11 +1202,21 @@ def compile_wait(self): raise BuildError(self._build_job.stdout, self._build_job.stderr) @_run_hooks('pre_run') + @final def run(self): '''The run phase of the regression test pipeline. This call is non-blocking. It simply submits the job associated with this test and returns. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' if not self.current_system or not self._current_partition: raise PipelineError('no system or system partition is set') @@ -1245,6 +1300,7 @@ def run(self): if self.job.sched_flex_alloc_nodes: self.num_tasks = self.job.num_tasks + @final def poll(self): '''Poll the test's state. @@ -1254,6 +1310,15 @@ def poll(self): If no job descriptor is yet associated with this test, :class:`True` is returned. :raises reframe.core.exceptions.ReframeError: In case of errors. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' if not self._job: return True @@ -1261,19 +1326,31 @@ def poll(self): return self._job.finished() @_run_hooks('post_run') + @final def wait(self): '''Wait for this test to finish. :raises reframe.core.exceptions.ReframeError: In case of errors. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' self._job.wait() self.logger.debug('spawned job finished') @_run_hooks() + @final def sanity(self): self.check_sanity() @_run_hooks() + @final def performance(self): try: self.check_performance() @@ -1281,10 +1358,20 @@ def performance(self): if self.strict_check: raise + @final def check_sanity(self): '''The sanity checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the sanity check fails. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' if self.sanity_patterns is None: raise SanityError('sanity_patterns not set') @@ -1294,11 +1381,21 @@ def check_sanity(self): if not success: raise SanityError() + @final def check_performance(self): '''The performance checking phase of the regression test pipeline. :raises reframe.core.exceptions.SanityError: If the performance check fails. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' if self.perf_patterns is None: return @@ -1398,11 +1495,21 @@ def _copy_to_outputdir(self): shutil.copytree(f, os.path.join(self.outputdir, f_orig)) @_run_hooks() + @final def cleanup(self, remove_files=False): '''The cleanup phase of the regression test pipeline. :arg remove_files: If :class:`True`, the stage directory associated with this test will be removed. + + .. warning:: + + You may not override this method directly unless you are in special + test. See `here + `__ for + more details. + + .. versionchanged:: 3.0 ''' aliased = os.path.samefile(self._stagedir, self._outputdir) if aliased: @@ -1502,7 +1609,7 @@ def __str__(self): self.name, self.prefix) -class RunOnlyRegressionTest(RegressionTest): +class RunOnlyRegressionTest(RegressionTest, special=True): '''Base class for run-only regression tests. This class is also directly available under the top-level :mod:`reframe` @@ -1538,7 +1645,7 @@ def run(self): super().run.__wrapped__(self) -class CompileOnlyRegressionTest(RegressionTest): +class CompileOnlyRegressionTest(RegressionTest, special=True): '''Base class for compile-only regression tests. These tests are by default local and will skip the run phase of the diff --git a/unittests/resources/checks/frontend_checks.py b/unittests/resources/checks/frontend_checks.py index 36071bbc55..6b7a30c013 100644 --- a/unittests/resources/checks/frontend_checks.py +++ b/unittests/resources/checks/frontend_checks.py @@ -28,11 +28,10 @@ def __init__(self): self.valid_systems = ['*'] self.valid_prog_environs = ['*'] - def setup(self, system, environ, **job_opts): - super().setup(system, environ, **job_opts) + @rfm.run_after('setup') + def raise_error(self): raise ReframeError('Setup failure') - @rfm.simple_test class BadSetupCheckEarly(BaseFrontendCheck): def __init__(self): @@ -41,7 +40,8 @@ def __init__(self): self.valid_prog_environs = ['*'] self.local = False - def setup(self, system, environ, **job_opts): + @rfm.run_before('setup') + def raise_error_early(self): raise ReframeError('Setup failure') @@ -98,7 +98,7 @@ def check_performance(self): raise PerformanceError('performance failure') -class KeyboardInterruptCheck(BaseFrontendCheck): +class KeyboardInterruptCheck(BaseFrontendCheck, special=True): '''Simulate keyboard interrupt during test's execution.''' def __init__(self, phase='wait'): @@ -108,12 +108,11 @@ def __init__(self, phase='wait'): self.valid_prog_environs = ['*'] self.phase = phase - def setup(self, system, environ, **job_opts): + @rfm.run_before('setup') + def raise_before_setup(self): if self.phase == 'setup': raise KeyboardInterrupt - super().setup(system, environ, **job_opts) - def wait(self): # We do our nasty stuff in wait() to make things more complicated if self.phase == 'wait': @@ -122,7 +121,7 @@ def wait(self): super().wait() -class SystemExitCheck(BaseFrontendCheck): +class SystemExitCheck(BaseFrontendCheck, special=True): '''Simulate system exit from within a check.''' def __init__(self): @@ -173,14 +172,14 @@ def __init__(self, sleep_time): SleepCheck._next_id += 1 -class SleepCheckPollFail(SleepCheck): +class SleepCheckPollFail(SleepCheck, special=True): '''Emulate a test failing in the polling phase.''' def poll(self): raise ValueError -class SleepCheckPollFailLate(SleepCheck): +class SleepCheckPollFailLate(SleepCheck, special=True): '''Emulate a test failing in the polling phase after the test has finished.''' diff --git a/unittests/resources/checks_unlisted/kbd_interrupt.py b/unittests/resources/checks_unlisted/kbd_interrupt.py index 84676f9e0b..0010e8d777 100644 --- a/unittests/resources/checks_unlisted/kbd_interrupt.py +++ b/unittests/resources/checks_unlisted/kbd_interrupt.py @@ -22,5 +22,6 @@ def __init__(self): self.valid_prog_environs = ['*'] self.tags = {self.name} - def setup(self, system, environ, **job_opts): + @rfm.run_before('setup') + def raise_keyboard_interrupt(self): raise KeyboardInterrupt diff --git a/unittests/test_loader.py b/unittests/test_loader.py index 8d0a20b3af..b8e63fea44 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -7,7 +7,9 @@ import pytest import unittest +import reframe as rfm from reframe.core.exceptions import (ConfigError, NameConflictError, + ReframeDeprecationWarning, RegressionTestLoadError) from reframe.core.systems import System from reframe.frontend.loader import RegressionCheckLoader @@ -71,3 +73,98 @@ def test_load_bad_init(self): tests = self.loader.load_from_file( 'unittests/resources/checks_unlisted/bad_init_check.py') assert 0 == len(tests) + + def test_special_test(self): + with pytest.warns(ReframeDeprecationWarning): + @rfm.simple_test + class TestDeprecated(rfm.RegressionTest): + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + with pytest.warns(ReframeDeprecationWarning): + @rfm.simple_test + class TestDeprecatedRunOnly(rfm.RunOnlyRegressionTest): + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + with pytest.warns(ReframeDeprecationWarning): + @rfm.simple_test + class TestDeprecatedCompileOnly(rfm.CompileOnlyRegressionTest): + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + with pytest.warns(ReframeDeprecationWarning): + @rfm.simple_test + class TestDeprecatedCompileOnlyDerived(TestDeprecatedCompileOnly): + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + with pytest.warns(None) as warnings: + @rfm.simple_test + class TestSimple(rfm.RegressionTest): + def __init__(self): + pass + + @rfm.simple_test + class TestSpecial(rfm.RegressionTest, special=True): + def __init__(self): + pass + + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + @rfm.simple_test + class TestSpecialRunOnly(rfm.RunOnlyRegressionTest, + special=True): + def __init__(self): + pass + + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + def run(self): + super().run() + + @rfm.simple_test + class TestSpecialCompileOnly(rfm.CompileOnlyRegressionTest, + special=True): + def __init__(self): + pass + + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + def run(self): + super().run() + + assert not any(isinstance(w.message, ReframeDeprecationWarning) + for w in warnings) + + with pytest.warns(ReframeDeprecationWarning) as warnings: + @rfm.simple_test + class TestSpecialDerived(TestSpecial): + def __init__(self): + pass + + def setup(self, partition, environ, **job_opts): + super().setup(system, environ, **job_opts) + + def run(self): + super().run() + + assert len(warnings) == 2 + + @rfm.simple_test + class TestFinal(rfm.RegressionTest): + def __init__(self): + pass + + @rfm.final + def my_new_final(seld): + pass + + with pytest.warns(ReframeDeprecationWarning): + @rfm.simple_test + class TestFinalDerived(TestFinal): + def my_new_final(self, a, b): + pass