diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 61c1a0d6f1..839c5d44de 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -4,11 +4,11 @@ scriptname=`basename $0` CI_FOLDER="" CI_GENERIC=0 CI_TUTORIAL=0 +CI_EXITCODE=0 TERM="${TERM:-xterm}" PROFILE="" MODULEUSE="" -CI_EXITCODE=0 # # This function prints the script usage form @@ -41,41 +41,29 @@ checked_exec() run_tutorial_checks() { - cmd="./bin/reframe --exec-policy=async --save-log-files -r -t tutorial $@" + cmd="./bin/reframe -C tutorial/config/settings.py --exec-policy=async \ +--save-log-files -r -t tutorial $@" echo "Running tutorial checks with \`$cmd'" checked_exec $cmd } run_user_checks() { - cmd="./bin/reframe --exec-policy=async --save-log-files -r -t production $@" + cmd="./bin/reframe -C config/cscs.py --exec-policy=async --save-log-files \ +-r -t production $@" echo "Running user checks with \`$cmd'" checked_exec $cmd } run_serial_user_checks() { - cmd="./bin/reframe --exec-policy=serial --save-log-files -r -t production-serial $@" + cmd="./bin/reframe -C config/cscs.py --exec-policy=serial --save-log-files \ +-r -t production-serial $@" echo "Running user checks with \`$cmd'" checked_exec $cmd } -save_settings() -{ - tempfile=$(mktemp) - cp reframe/settings.py $tempfile - echo $tempfile -} - -restore_settings() -{ - saved=$1 - cp $saved reframe/settings.py - /bin/rm $saved -} - - ### Main script ### shortopts="h,g,t,f:,i:,l:,m:" @@ -167,9 +155,6 @@ if [ $CI_GENERIC -eq 1 ]; then grep -- '--- Logging error ---' elif [ $CI_TUTORIAL -eq 1 ]; then # Run tutorial checks - settings_orig=$(save_settings) - cp tutorial/config/settings.py reframe/settings.py - # Find modified or added tutorial checks tutorialchecks=( $(git log --name-status --oneline --no-merges -1 | \ awk '/^[AM]/ { print $2 } /^R0[0-9][0-9]/ { print $3 }' | \ @@ -190,17 +175,13 @@ elif [ $CI_TUTORIAL -eq 1 ]; then run_tutorial_checks ${tutorialchecks_path} ${invocations[i]} done fi - - restore_settings $settings_orig else # Performing the unittests echo "==================" echo "Running unit tests" echo "==================" - settings_orig=$(save_settings) - cp config/cscs.py reframe/settings.py - checked_exec ./test_reframe.py + checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py # Find modified or added user checks userchecks=( $(git log --name-status --oneline --no-merges -1 | \ @@ -226,7 +207,5 @@ else run_serial_user_checks ${userchecks_path} ${invocations[i]} done fi - - restore_settings $settings_orig fi exit $CI_EXITCODE diff --git a/cscs-checks/prgenv/environ_check.py b/cscs-checks/prgenv/environ_check.py index b0cdee9c96..a94631f950 100644 --- a/cscs-checks/prgenv/environ_check.py +++ b/cscs-checks/prgenv/environ_check.py @@ -1,7 +1,7 @@ import os -from reframe.core.modules import get_modules_system from reframe.core.pipeline import RunOnlyRegressionTest +from reframe.core.runtime import runtime class DefaultPrgEnvCheck(RunOnlyRegressionTest): @@ -31,7 +31,7 @@ def wait(self): pass def check_sanity(self): - return get_modules_system().is_module_loaded('PrgEnv-cray') + return runtime().modules_system.is_module_loaded('PrgEnv-cray') def cleanup(self, remove_files=False, unload_env=True): pass @@ -53,7 +53,8 @@ def __init__(self, **kwargs): self.tags = {'production'} def check_sanity(self): - return get_modules_system().is_module_loaded(self.current_environ.name) + return runtime().modules_system.is_module_loaded( + self.current_environ.name) def _get_checks(**kwargs): diff --git a/docs/advanced.rst b/docs/advanced.rst index 7acc0f8909..41d9d1e168 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -43,7 +43,7 @@ The contents of this regression test are the following (``tutorial/advanced/adva The important bit here is the ``compile()`` method. .. literalinclude:: ../tutorial/advanced/advanced_example1.py - :lines: 21-23 + :lines: 18-20 :dedent: 4 As in the simple single source file examples we showed in the `tutorial `__, we use the current programming environment's flags for modifying the compilation. @@ -162,7 +162,7 @@ This ensures that the environment of the test is also set correctly at runtime. Finally, as already mentioned `previously <#leveraging-makefiles>`__, since the ``Makefile`` name is not one of the standard ones, it has to be passed as an argument to the :func:`compile ` method of the base :class:`RegressionTest ` class as follows: .. literalinclude:: ../tutorial/advanced/advanced_example4.py - :lines: 24 + :lines: 21 :dedent: 8 Setting a Time Limit for Regression Tests @@ -175,18 +175,17 @@ The following example (``tutorial/advanced/advanced_example5.py``) demonstrates The important bit here is the following line that sets the time limit for the test to one minute: - .. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 16 + :lines: 13 :dedent: 8 The :attr:`time_limit ` attribute is a three-tuple in the form ``(HOURS, MINUTES, SECONDS)``. Time limits are implemented for all the scheduler backends. -The sanity condition for this test verifies that associated job has been canceled due to the time limit. +The sanity condition for this test verifies that associated job has been canceled due to the time limit (note that this message is SLURM-specific). .. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 19-20 + :lines: 16-17 :dedent: 8 Applying a sanity function iteratively @@ -212,7 +211,7 @@ The contents of the ReFrame regression test contained in ``advanced_example6.py` First the random numbers are extracted through the :func:`extractall ` function as follows: .. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 16-17 + :lines: 14-15 :dedent: 8 The ``numbers`` variable is a deferred iterable, which upon evaluation will return all the extracted numbers. @@ -229,7 +228,7 @@ Note that the ``and`` operator is not deferrable and will trigger the evaluation The full syntax for the :attr:`sanity_patterns` is the following: .. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 18-20 + :lines: 16-18 :dedent: 8 Customizing the Generated Job Script @@ -286,4 +285,72 @@ The parallel launch itself consists of three parts: #. the regression test executable as specified in the :attr:`executable ` attribute and #. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. -A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against inside it. +A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. + + +Working with parameterized tests +-------------------------------- + +.. versionadded:: 2.13 + +We have seen already in the `basic tutorial `__ how we could better organize the tests so as to avoid code duplication by using test class hierarchies. +An alternative technique, which could also be used in parallel with the class hierarchies, is to use `parameterized tests`. +The following is a test that takes a ``variant`` parameter, which controls which variant of the code will be used. +Depending on that value, the test is set up differently: + +.. literalinclude:: ../tutorial/advanced/advanced_example8.py + +If you have already gone through the `tutorial `__, this test can be easily understood. +The new bit here is the ``@parameterized_test`` decorator of the ``MatrixVectorTest`` class. +This decorator takes as argument an iterable of either sequence (i.e., lists, tuples etc.) or mapping types (i.e., dictionaries). +Each of this iterable's elements corresponds to the arguments that will be used to instantiate the decorated test each time. +In the example shown, the test will instantiated twice, once passing ``variant`` as ``MPI`` and a second time with ``variant`` passed as ``OpenMP``. +The framework will try to generate unique names for the generated tests by stringifying the arguments passed to the test's constructor: + + +.. code-block:: none + + Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/advanced/advanced_example8.py -l + Reframe version: 2.13-dev0 + Launched by user: XXX + Launched on host: daint101 + Reframe paths + ============= + Check prefix : + Check search path : 'tutorial/advanced/advanced_example8.py' + Stage dir prefix : current/working/dir/reframe/stage/ + Output dir prefix : current/working/dir/reframe/output/ + Logging dir : current/working/dir/reframe/logs + List of matched checks + ====================== + * MatrixVectorTest_MPI (Matrix-vector multiplication test (MPI)) + tags: [tutorial], maintainers: [you-can-type-your-email-here] + * MatrixVectorTest_OpenMP (Matrix-vector multiplication test (OpenMP)) + tags: [tutorial], maintainers: [you-can-type-your-email-here] + Found 2 check(s). + + +There are a couple of different ways that we could have used the ``@parameterized_test`` decorator. +One is to use dictionaries for specifying the instantiations of our test class. +The dictionaries will be converted to keyword arguments and passed to the constructor of the test class: + +.. code-block:: python + + @rfm.parameterized_test([{'variant': 'MPI'}, {'variant': 'OpenMP'}]) + + +Another way, which is quite useful if you want to generate lots of different tests at the same time, is to use either `list comprehensions `__ or `generator expressions `__ for specifying the different test instantiations: + +.. code-block:: python + + @rfm.parameterized_test((variant,) for variant in ['MPI', 'OpenMP']) + + +.. note:: + In versions of the framework prior to 2.13, this could be achieved by explicitly instantiating your tests inside the ``_get_checks()`` method. + + +.. tip:: + + Combining parameterized tests and test class hierarchies can offer you a very flexible way for generating multiple related tests at once keeping at the same time the maintenance cost low. + We use this technique extensively in our tests. diff --git a/docs/reference.rst b/docs/reference.rst index 965525f7c8..e181154921 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,10 +1,62 @@ +=============== Reference Guide ---------------- +=============== + +This page provides a reference guide of the ReFrame API for writing regression tests covering all the relevant details. +Internal data structures and APIs are covered only to the extent that might be helpful to the final user of the framework. + + + +Regression test classes and related utilities +--------------------------------------------- + +.. class:: reframe.RegressionTest(name=None, prefix=None) + + This is an alias of :class:`reframe.core.pipeline.RegressionTest`. + + .. versionadded:: 2.13 + + +.. class:: reframe.RunOnlyRegressionTest(*args, **kwargs) + + This is an alias of :class:`reframe.core.pipeline.RunOnlyRegressionTest`. + + .. versionadded:: 2.13 + + +.. class:: reframe.CompileOnlyRegressionTest(*args, **kwargs) + + This is an alias of :class:`reframe.core.pipeline.CompileOnlyRegressionTest`. + + .. versionadded:: 2.13 + + +.. py:decorator:: reframe.simple_test + + This is an alias of :func:`reframe.core.decorators.simple_test`. + + .. versionadded:: 2.13 + + +.. py:decorator:: reframe.parameterized_test(inst=[]) + + This is an alias of :func:`reframe.core.decorators.parameterized_test`. + + .. versionadded:: 2.13 + + +.. automodule:: reframe.core.decorators + :members: + :show-inheritance: .. automodule:: reframe.core.pipeline :members: :show-inheritance: + +Environments and Systems +------------------------ + .. automodule:: reframe.core.environments :members: :show-inheritance: @@ -13,6 +65,10 @@ Reference Guide :members: :show-inheritance: + +Job schedulers and parallel launchers +------------------------------------- + .. autoclass:: reframe.core.schedulers.Job :members: :show-inheritance: @@ -21,6 +77,22 @@ Reference Guide :members: :show-inheritance: + .. automodule:: reframe.core.launchers.registry :members: :show-inheritance: + + +Runtime services +---------------- + +.. automodule:: reframe.core.runtime + :members: + :exclude-members: temp_runtime, switch_runtime + :show-inheritance: + + +Modules System API +------------------ + +.. autoclass:: reframe.core.modules.ModulesSystem diff --git a/docs/tutorial.rst b/docs/tutorial.rst index fa761ee6a1..8cbeeaf718 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -32,33 +32,46 @@ Here is the full code for this test: .. literalinclude:: ../tutorial/example1.py A regression test written in ReFrame is essentially a Python class that must eventually derive from :class:`RegressionTest `. -In order to make the test available to the framework, every file defining regression tests must define the special function ``_get_checks()``, which should return a list of instantiated regression tests. -This method will be called by the framework upon loading your file, in order to retrieve the regression tests defined. -The framework will pass some special arguments to the ``_get_checks()`` function through the ``kwargs`` parameter, which are needed for the correct initialization of the regression test. +To make a test visible to the framework, you must decorate your final test class with one of the following decorators: -Now let's move on to the actual definition of the ``SerialTest`` here: +* ``@simple_test``: for registering a single parameterless instantiation of your test. +* ``@parameterized_test``: for registering multiple instantiations of your test. + +Let's see in more detail how the ``Example1Test`` is defined: .. literalinclude:: ../tutorial/example1.py - :lines: 7-10 + :lines: 5-8 The ``__init__()`` method is the constructor of your test. It is usually the only method you need to implement for your tests, especially if you don't want to customize any of the regression test pipeline stages. -The first statement in the ``SerialTest`` constructor calls the constructor of the base class, passing as arguments the name of the regression test (``example1_check`` here), the path to the test directory and any other arguments passed to the ``SerialTest``'s constructor. -You can consider these first three lines and especially the way you should call the constructor of the base class, as boilerplate code. -As you will see, it remains the same across all our examples, except, of course, for the check name. +The first statement in the ``Example1Test`` constructor calls the constructor of the base class. +This is essential for properly initializing your test. +When your test is instantiated, the framework assigns a default name to it. +This name is essentially a concatenation of the fully qualified name of the class and string representations of the constructor arguments, with any non-alphanumeric characters converted to underscores. +In this example, the auto-generated test name is simply ``Example1Test``. +You may change the name of the test later in the constructor by setting the :attr:`name ` attribute. + +.. note:: + ReFrame requires that the names of all the tests it loads are unique. + In case of name clashes, it will refuse to load the conflicting test. + + .. versionadded:: 2.12 The next line sets a more detailed description of the test: .. literalinclude:: ../tutorial/example1.py - :lines: 11 + :lines: 9 :dedent: 8 -This is optional and it defaults to the regression test's name, if not specified. +This is optional and it defaults to the auto-generated test's name, if not specified. + +.. note:: + If you explicitly set only the name of the test, the description will not be automatically updated and will still keep its default value. The next two lines specify the systems and the programming environments that this test is valid for: .. literalinclude:: ../tutorial/example1.py - :lines: 12-13 + :lines: 10-11 :dedent: 8 Both of these variables accept a list of system names or environment names, respectively. @@ -74,7 +87,7 @@ If only a system name (without a partition) is specified in the :attr:`self.vali The next line specifies the source file that needs to be compiled: .. literalinclude:: ../tutorial/example1.py - :lines: 14 + :lines: 12 :dedent: 8 ReFrame expects any source files, or generally resources, of the test to be inside an ``src/`` directory, which is at the same level as the regression test file. @@ -96,7 +109,7 @@ A user can associate compilers with programming environments in the ReFrame's `s The next line in our first regression test specifies a list of options to be used for running the generated executable (the matrix dimension and the number of iterations in this particular example): .. literalinclude:: ../tutorial/example1.py - :lines: 15 + :lines: 13 :dedent: 8 Notice that you do not need to specify the executable name. @@ -106,7 +119,7 @@ We will see in the `"Customizing Further A ReFrame Regression Test" ` accepts any filename, which will be resolved against the stage directory of the test. You can also use the :attr:`stdout ` and :attr:`stderr ` attributes to reference the standard output and standard error, respectively. -.. note:: You need not to care about handling exceptions, and error handling in general, inside your test. +.. tip:: You need not to care about handling exceptions, and error handling in general, inside your test. The framework will automatically abort the execution of the test, report the error and continue with the next test case. The last two lines of the regression test are optional, but serve a good role in a production environment: .. literalinclude:: ../tutorial/example1.py - :lines: 18-19 + :lines: 16-17 :dedent: 8 In the :attr:`maintainers ` attribute you may store a list of people responsible for the maintenance of this test. @@ -149,54 +162,56 @@ Here we will only show you how to run a specific tutorial test: .. code-block:: bash - ./bin/reframe -c tutorial/ -n example1_check -r + ./bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r If everything is configured correctly for your system, you should get an output similar to the following: .. code-block:: none - Reframe version: 2.7 - Launched by user: - Launched on host: daint104 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial/' - Stage dir prefix : /stage/ - Output dir prefix : /output/ - Logging dir : /logs - [==========] Running 1 check(s) - [==========] Started on Fri Oct 20 15:11:38 2017 - - [----------] started processing example1_check (Simple matrix-vector multiplication example) - [ RUN ] example1_check on daint:mc using PrgEnv-cray - [ OK ] example1_check on daint:mc using PrgEnv-cray - [ RUN ] example1_check on daint:mc using PrgEnv-gnu - [ OK ] example1_check on daint:mc using PrgEnv-gnu - [ RUN ] example1_check on daint:mc using PrgEnv-intel - [ OK ] example1_check on daint:mc using PrgEnv-intel - [ RUN ] example1_check on daint:mc using PrgEnv-pgi - [ OK ] example1_check on daint:mc using PrgEnv-pgi - [ RUN ] example1_check on daint:login using PrgEnv-cray - [ OK ] example1_check on daint:login using PrgEnv-cray - [ RUN ] example1_check on daint:login using PrgEnv-gnu - [ OK ] example1_check on daint:login using PrgEnv-gnu - [ RUN ] example1_check on daint:login using PrgEnv-intel - [ OK ] example1_check on daint:login using PrgEnv-intel - [ RUN ] example1_check on daint:login using PrgEnv-pgi - [ OK ] example1_check on daint:login using PrgEnv-pgi - [ RUN ] example1_check on daint:gpu using PrgEnv-cray - [ OK ] example1_check on daint:gpu using PrgEnv-cray - [ RUN ] example1_check on daint:gpu using PrgEnv-gnu - [ OK ] example1_check on daint:gpu using PrgEnv-gnu - [ RUN ] example1_check on daint:gpu using PrgEnv-intel - [ OK ] example1_check on daint:gpu using PrgEnv-intel - [ RUN ] example1_check on daint:gpu using PrgEnv-pgi - [ OK ] example1_check on daint:gpu using PrgEnv-pgi - [----------] finished processing example1_check (Simple matrix-vector multiplication example) - - [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Fri Oct 20 15:15:25 2017 + Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r + Reframe version: 2.13-dev0 + Launched by user: XXX + Launched on host: daint104 + Reframe paths + ============= + Check prefix : + Check search path : 'tutorial/example1.py' + Stage dir prefix : /current/working/dir/stage/ + Output dir prefix : /current/working/dir/output/ + Logging dir : /current/working/dir/logs + [==========] Running 1 check(s) + [==========] Started on Fri May 18 13:19:12 2018 + + [----------] started processing Example1Test (Simple matrix-vector multiplication example) + [ RUN ] Example1Test on daint:login using PrgEnv-cray + [ OK ] Example1Test on daint:login using PrgEnv-cray + [ RUN ] Example1Test on daint:login using PrgEnv-gnu + [ OK ] Example1Test on daint:login using PrgEnv-gnu + [ RUN ] Example1Test on daint:login using PrgEnv-intel + [ OK ] Example1Test on daint:login using PrgEnv-intel + [ RUN ] Example1Test on daint:login using PrgEnv-pgi + [ OK ] Example1Test on daint:login using PrgEnv-pgi + [ RUN ] Example1Test on daint:gpu using PrgEnv-cray + [ OK ] Example1Test on daint:gpu using PrgEnv-cray + [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu + [ OK ] Example1Test on daint:gpu using PrgEnv-gnu + [ RUN ] Example1Test on daint:gpu using PrgEnv-intel + [ OK ] Example1Test on daint:gpu using PrgEnv-intel + [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi + [ OK ] Example1Test on daint:gpu using PrgEnv-pgi + [ RUN ] Example1Test on daint:mc using PrgEnv-cray + [ OK ] Example1Test on daint:mc using PrgEnv-cray + [ RUN ] Example1Test on daint:mc using PrgEnv-gnu + [ OK ] Example1Test on daint:mc using PrgEnv-gnu + [ RUN ] Example1Test on daint:mc using PrgEnv-intel + [ OK ] Example1Test on daint:mc using PrgEnv-intel + [ RUN ] Example1Test on daint:mc using PrgEnv-pgi + [ OK ] Example1Test on daint:mc using PrgEnv-pgi + [----------] finished processing Example1Test (Simple matrix-vector multiplication example) + + [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Fri May 18 13:20:17 2018 + Notice how our regression test is run on every partition of the configured system and for every programming environment. @@ -209,7 +224,7 @@ In this example, we write a regression test to compile and run the OpenMP versio The full code of this test follows: .. literalinclude:: ../tutorial/example2.py - :lines: 1-36 + :lines: 1-34 This example introduces two new concepts: @@ -234,7 +249,7 @@ This variable is available to regression tests after the setup phase. Before it Let's have a closer look at the ``compile()`` method: .. literalinclude:: ../tutorial/example2.py - :lines: 25-36 + :lines: 23-34 :dedent: 4 We first take the name of the current programming environment (``self.current_environ.name``) and we check it against the set of the known programming environments. @@ -251,11 +266,11 @@ The advantage of this implementation is that you move the different compilation The ``compile()`` method is now very simple: it gets the correct compilation flags from the ``prgenv_flags`` dictionary and applies them to the current programming environment. -.. note:: A regression test is like any other Python class, so you can freely define your own attributes. - If you accidentally try to write on a reserved :class:`RegressionTest ` attribute that is not writeable, ReFrame will prevent this and it will throw an error. - .. literalinclude:: ../tutorial/example2.py - :lines: 1-6,39-66 + :lines: 1-4,37-64 + +.. tip:: A regression test is like any other Python class, so you can freely define your own attributes. + If you accidentally try to write on a reserved :class:`RegressionTest ` attribute that is not writeable, ReFrame will prevent this and it will throw an error. Running on Multiple Nodes ------------------------- @@ -275,7 +290,7 @@ Let's take the changes step-by-step: First we need to specify for which partitions this test is meaningful by setting the :attr:`valid_systems ` attribute: .. literalinclude:: ../tutorial/example3.py - :lines: 12 + :lines: 10 :dedent: 8 We only specify the partitions that are configured with a job scheduler. @@ -285,7 +300,7 @@ So we remove this partition from the list of the supported systems. The most important addition to this check are the variables controlling the distributed execution: .. literalinclude:: ../tutorial/example3.py - :lines: 25-27 + :lines: 23-25 :dedent: 8 By setting these variables, we specify that this test should run with 8 MPI tasks in total, using two tasks per node. @@ -323,7 +338,7 @@ Let's start with the OpenACC regression test: The things to notice in this test are the restricted list of system partitions and programming environments that this test supports and the use of the :attr:`modules ` variable: .. literalinclude:: ../tutorial/example4.py - :lines: 16 + :lines: 14 :dedent: 8 The :attr:`modules ` variable takes a list of modules that should be loaded during the setup phase of the test. @@ -332,7 +347,7 @@ In this particular test, we need to load the ``craype-accel-nvidia60`` module, w It is also important to note that in GPU-enabled tests the number of GPUs for each node have to be specified by setting the corresponding variable :attr:`num_gpus_per_node `, as follows: .. literalinclude:: ../tutorial/example4.py - :lines: 17 + :lines: 15 :dedent: 8 The regression test for the CUDA code is slightly simpler: @@ -357,7 +372,7 @@ Let's go over it line-by-line. The first thing we do is to extract the norm printed in the standard output. .. literalinclude:: ../tutorial/example6.py - :lines: 21-23 + :lines: 19-21 :dedent: 8 The :func:`extractsingle ` sanity function extracts some information from a single occurrence (by default the first) of a pattern in a filename. @@ -379,13 +394,13 @@ Notice that we replaced the ``'norm'`` argument with ``1``, which is the capturi A useful counterpart of :func:`extractsingle ` is the :func:`extractall ` function, which instead of a single occurrence, returns a list of all the occurrences found. For a more detailed description of this and other sanity functions, please refer to the `sanity function reference `__. -The next couple of lines is the actual sanity check: +The next four lines is the actual sanity check: .. literalinclude:: ../tutorial/example6.py - :lines: 24-28 + :lines: 22-26 :dedent: 8 -This expression combines two conditions that need to true, in order for the sanity check to succeed: +This expression combines two conditions that need to be true, in order for the sanity check to succeed: 1. Find in standard output the same line we were looking for already in the first example. 2. Verify that the printed norm does not deviate significantly from the expected value. @@ -423,7 +438,7 @@ The are two new variables set in this test that basically enable the performance Let's have a closer look at each of them: .. literalinclude:: ../tutorial/example7.py - :lines: 20-23 + :lines: 18-21 :dedent: 8 The :attr:`perf_patterns ` attribute is a dictionary, whose keys are *performance variables* (i.e., arbitrary names assigned to the performance values we are looking for), and its values are *sanity expressions* that specify how to obtain these performance values from the output. @@ -435,7 +450,7 @@ When the framework obtains a performance value from the output of the test it se Let's go over the :attr:`reference ` dictionary of our example and explain its syntax in more detail: .. literalinclude:: ../tutorial/example7.py - :lines: 24-28 + :lines: 22-26 :dedent: 8 This is a special type of dictionary that we call ``scoped dictionary``, because it defines scopes for its keys. @@ -463,13 +478,11 @@ Here is the final example code that combines all the tests discussed before: This test abstracts away the common functionality found in almost all of our tutorial tests (executable options, sanity checking, etc.) to a base class, from which all the concrete regression tests derive. Each test then redefines only the parts that are specific to it. -The ``_get_checks()`` now instantiates all the interesting tests and returns them as a list to the framework. -The total line count of this refactored example is less than half of that of the individual tutorial tests. -Notice how the base class for all tutorial regression tests specify additional parameters to its constructor, so that the concrete subclasses can initialize it based on their needs. +Notice also that only the actual tests, i.e., the derived classes, are made visible to the framework through the ``@simple_test`` decorator. +Decorating the base class has now meaning, because it does not correspond to an actual test. -Another interesting technique, not demonstrated here, is to create regression test factories that will create different regression tests based on specific arguments they take in their constructor. - -We use such techniques extensively in the regression tests for our production systems, in order to facilitate their maintenance. +The total line count of this refactored example is less than half of that of the individual tutorial tests. +Another interesting thing to note here is the base class accepting additional additional parameters to its constructor, so that the concrete subclasses can initialize it based on their needs. Summary ------- diff --git a/reframe/__init__.py b/reframe/__init__.py index de6edffaac..07eeaf0420 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -11,3 +11,8 @@ sys.stderr.write('Unsupported Python version: ' 'Python >= %d.%d.%d is required\n' % _required_pyver) sys.exit(1) + + +# Import important names for user tests +from reframe.core.pipeline import * +from reframe.core.decorators import * diff --git a/reframe/frontend/config.py b/reframe/core/config.py similarity index 81% rename from reframe/frontend/config.py rename to reframe/core/config.py index 3a1dbd1220..e0e4e701a5 100644 --- a/reframe/frontend/config.py +++ b/reframe/core/config.py @@ -2,44 +2,40 @@ import collections.abc import reframe.core.debug as debug -import reframe.utility.os_ext as os_ext -from reframe.core.environments import Environment -from reframe.core.exceptions import (ConfigError, ReframeError, - ReframeFatalError) -from reframe.core.fields import ScopedDictField -from reframe.core.launchers.registry import getlauncher -from reframe.core.schedulers.registry import getscheduler -from reframe.core.systems import System, SystemPartition -from reframe.utility import ScopedDict, import_module_from_file +import reframe.core.fields as fields +import reframe.utility as util +from reframe.core.exceptions import ConfigError, ReframeError, ReframeFatalError + _settings = None -def load_from_file(filename): +def load_settings_from_file(filename): global _settings try: - _settings = import_module_from_file(filename).settings + _settings = util.import_module_from_file(filename).settings + return _settings except Exception as e: raise ConfigError( "could not load configuration file `%s'" % filename) from e - return _settings - def settings(): if _settings is None: - raise ReframeFatalError('No configuration loaded') + raise ReframeFatalError('ReFrame is not configured') return _settings class SiteConfiguration: """Holds the configuration of systems and environments""" - _modes = ScopedDictField('_modes', (list, str)) + _modes = fields.ScopedDictField('_modes', (list, str)) - def __init__(self): + def __init__(self, dict_config=None): self._systems = {} self._modes = {} + if dict_config is not None: + self.load_from_dict(dict_config) def __repr__(self): return debug.repr(self) @@ -54,6 +50,9 @@ def modes(self): def get_schedsystem_config(self, descr): # Handle the special shortcuts first + from reframe.core.launchers.registry import getlauncher + from reframe.core.schedulers.registry import getscheduler + if descr == 'nativeslurm': return getscheduler('slurm'), getlauncher('srun') @@ -72,6 +71,11 @@ def load_from_dict(self, site_config): if not isinstance(site_config, collections.abc.Mapping): raise TypeError('site configuration is not a dict') + # We do all the necessary imports here and not on the top, because we + # want to remove import time dependencies + import reframe.core.environments as m_env + from reframe.core.systems import System, SystemPartition + sysconfig = site_config.get('systems', None) envconfig = site_config.get('environments', None) modes = site_config.get('modes', {}) @@ -84,13 +88,13 @@ def load_from_dict(self, site_config): # Convert envconfig to a ScopedDict try: - envconfig = ScopedDict(envconfig) + envconfig = fields.ScopedDict(envconfig) except TypeError: raise TypeError('environments configuration ' 'is not a scoped dictionary') from None # Convert modes to a `ScopedDict`; note that `modes` will implicitly - # converted to a scoped dict here, since `self._models` is a + # converted to a scoped dict here, since `self._modes` is a # `ScopedDictField`. try: self._modes = modes @@ -111,8 +115,6 @@ def create_env(system, partition, name): raise TypeError("config for `%s' is not a dictionary" % name) try: - import reframe.core.environments as m_env - envtype = m_env.__dict__[config['type']] return envtype(name, **config) except KeyError: @@ -176,7 +178,7 @@ def create_env(system, partition, name): part_scheduler, part_launcher = self.get_schedsystem_config( partconfig.get('scheduler', 'local+local') ) - part_local_env = Environment( + part_local_env = m_env.Environment( name='__rfm_env_%s' % part_name, modules=partconfig.get('modules', []), variables=partconfig.get('variables', {}) @@ -199,25 +201,3 @@ def create_env(system, partition, name): max_jobs=part_max_jobs)) self._systems[sys_name] = system - - -def autodetect_system(site_config): - """Auto-detect system""" - import re - import socket - - # Try to detect directly the cluster name from /etc/xthostname (Cray - # specific) - try: - hostname = os_ext.run_command('cat /etc/xthostname', check=True).stdout - except ReframeError: - # Try to figure it out with the standard method - hostname = socket.gethostname() - - # Go through the supported systems and try to match the hostname - for system in site_config.systems.values(): - for hostname_patt in system.hostnames: - if re.match(hostname_patt, hostname): - return system - - return None diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py new file mode 100644 index 0000000000..ba39cf89e6 --- /dev/null +++ b/reframe/core/decorators.py @@ -0,0 +1,85 @@ +# +# Decorators for registering tests with the framework +# + +__all__ = ['parameterized_test', 'simple_test'] + + +import collections +import inspect + +from reframe.core.exceptions import ReframeSyntaxError +from reframe.core.pipeline import RegressionTest + + +def _register_test(cls, args=None): + def _instantiate(): + ret = [] + for cls, args in mod.__rfm_test_registry: + if isinstance(args, collections.Sequence): + ret.append(cls(*args)) + elif isinstance(args, collections.Mapping): + ret.append(cls(**args)) + elif args is None: + ret.append(cls()) + + return ret + + mod = inspect.getmodule(cls) + if not hasattr(mod, '_rfm_gettests'): + mod._rfm_gettests = _instantiate + + try: + mod.__rfm_test_registry.append((cls, args)) + except AttributeError: + mod.__rfm_test_registry = [(cls, args)] + + +def _validate_test(cls): + if not issubclass(cls, RegressionTest): + raise ReframeSyntaxError('the decorated class must be a ' + 'subclass of RegressionTest') + + +def simple_test(cls): + """Class decorator for registering parameterless tests with ReFrame. + + The decorated class must derive from + :class:`reframe.core.pipeline.RegressionTest`. This decorator is also + available directly under the :mod:`reframe` module. + + .. versionadded:: 2.13 + + """ + + _validate_test(cls) + _register_test(cls) + return cls + + +def parameterized_test(inst=[]): + """Class decorator for registering multiple instantiations of a test class. + + The decorated class must derive from + :class:`reframe.core.pipeline.RegressionTest`. This decorator is also + available directly under the :mod:`reframe` module. + + :arg inst: An iterable of the argument lists of the difference + instantiations. Instantiation arguments may also be passed as + keyword dictionaries. + + .. versionadded:: 2.13 + + .. note:: + + This decorator does not instantiate any test. It only registers them. + The actual instantiation happens during the loading phase of the test. + """ + def _do_register(cls): + _validate_test(cls) + for args in inst: + _register_test(cls, args) + + return cls + + return _do_register diff --git a/reframe/core/environments.py b/reframe/core/environments.py index 21f5288eec..c3cc20ae60 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -6,7 +6,7 @@ import reframe.utility.os_ext as os_ext from reframe.core.exceptions import (EnvironError, SpawnedProcessError, CompilationError) -from reframe.core.modules import get_modules_system +from reframe.core.runtime import runtime class Environment: @@ -16,7 +16,7 @@ class Environment: to be set when this environment is loaded by the framework. Users may not create or modify directly environments. """ - name = fields.NonWhitespaceField('name') + name = fields.StringPatternField('name', '(\w|-)+') modules = fields.TypedListField('modules', str) variables = fields.TypedDictField('variables', str, str) @@ -74,16 +74,17 @@ def set_variable(self, name, value): def load(self): # conflicted module list must be filled at the time of load + rt = runtime() for m in self._modules: - if get_modules_system().is_module_loaded(m): + if rt.modules_system.is_module_loaded(m): self._preloaded.add(m) - self._conflicted += get_modules_system().load_module(m, force=True) + self._conflicted += rt.modules_system.load_module(m, force=True) for conflict in self._conflicted: - stmts = get_modules_system().emit_unload_commands(conflict) + stmts = rt.modules_system.emit_unload_commands(conflict) self._load_stmts += stmts - self._load_stmts += get_modules_system().emit_load_commands(m) + self._load_stmts += rt.modules_system.emit_load_commands(m) for k, v in self._variables.items(): if k in os.environ: @@ -106,11 +107,11 @@ def unload(self): # Unload modules in reverse order for m in reversed(self._modules): if m not in self._preloaded: - get_modules_system().unload_module(m) + runtime().modules_system.unload_module(m) # Reload the conflicted packages, previously removed for m in self._conflicted: - get_modules_system().load_module(m) + runtime().modules_system.load_module(m) self._loaded = False @@ -157,7 +158,7 @@ def swap_environments(src, dst): class EnvironmentSnapshot(Environment): def __init__(self, name='env_snapshot'): self._name = name - self._modules = get_modules_system().loaded_modules() + self._modules = runtime().modules_system.loaded_modules() self._variables = dict(os.environ) self._conflicted = [] diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index 372f14e9b6..e1eaffc95a 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -36,11 +36,15 @@ def __str__(self): return ret -class TestSuiteError(ReframeError): - """Raised when the regression test suite does not meet certain criteria.""" +class ReframeSyntaxError(ReframeError): + """Raised when the syntax of regression tests is not correct.""" -class NameConflictError(TestSuiteError): +class TestLoadError(ReframeError): + """Raised when the regression test cannot be loaded.""" + + +class NameConflictError(TestLoadError): """Raised when there is a name clash in the test suite.""" @@ -58,6 +62,14 @@ class ConfigError(ReframeError): """Raised when a configuration error occurs.""" +class UnknownSystemError(ConfigError): + """Raised when the host system cannot be identified.""" + + +class SystemAutodetectionError(UnknownSystemError): + """Raised when the host system cannot be auto-detected""" + + class EnvironError(ReframeError): """Raised when an error related to an environment occurs.""" @@ -226,8 +238,8 @@ def format_user_frame(frame): return 'OS error: %s' % exc_value frame = user_frame(tb) - if isinstance(exc_value, TypeError) and frame is not None: - return 'type error: ' + format_user_frame(frame) + # if isinstance(exc_value, TypeError) and frame is not None: + # return 'type error: ' + format_user_frame(frame) if isinstance(exc_value, ValueError) and frame is not None: return 'value error: ' + format_user_frame(frame) diff --git a/reframe/core/fields.py b/reframe/core/fields.py index 54190a82fe..41b72bd21b 100644 --- a/reframe/core/fields.py +++ b/reframe/core/fields.py @@ -2,9 +2,10 @@ # Useful descriptors for advanced operations on fields # -import collections.abc +import collections import copy import numbers +import os import re from reframe.core.exceptions import user_deprecation_warning @@ -80,9 +81,12 @@ def __init__(self, fieldname, fieldtype, allow_none=False): self._fieldtype = fieldtype self._allow_none = allow_none + def _check_type(self, value): + return ((self._allow_none and value is None) or + isinstance(value, self._fieldtype)) + def __set__(self, obj, value): - if ((value is not None or not self._allow_none) and - not isinstance(value, self._fieldtype)): + if not self._check_type(value): raise TypeError('attempt to set a field of different type. ' 'Required: %s, Found: %s' % (self._fieldtype.__name__, type(value).__name__)) @@ -237,52 +241,33 @@ def _check_type(self, value, typespec): return True -class AlphanumericField(TypedField): - """Stores an alphanumeric string ([A-Za-z0-9_])""" +class StringField(TypedField): + """Stores a standard string object""" def __init__(self, fieldname, allow_none=False): super().__init__(fieldname, str, allow_none) - def __set__(self, obj, value): - if value is not None: - if not isinstance(value, str): - raise TypeError('attempt to set an alphanumeric field ' - 'with a non-string value') - - # Check if the string is properly formatted - if not re.fullmatch('\w+', value, re.ASCII): - raise ValueError('Attempt to set an alphanumeric field ' - 'with a non-alphanumeric value') - - super().__set__(obj, value) +class StringPatternField(StringField): + """Stores a string that must follow a specific pattern""" -class NonWhitespaceField(TypedField): - """Stores a string without any whitespace""" - - def __init__(self, fieldname, allow_none=False): - super().__init__(fieldname, str, allow_none) + def __init__(self, fieldname, pattern, allow_none=False): + super().__init__(fieldname, allow_none) + self._pattern = pattern def __set__(self, obj, value): - if value is not None: - if not isinstance(value, str): - raise TypeError('Attempt to set a string field ' - 'with a non-string value') + if not self._check_type(value): + raise TypeError('a string type is required') - if not re.fullmatch('\S+', value, re.ASCII): - raise ValueError('Attempt to set a non-whitespace field ' - 'with a string containing whitespace') + if (value is not None and + not re.fullmatch(self._pattern, value, re.ASCII)): + raise ValueError( + 'cannot validate string "%s" against pattern: "%s"' % + (value, self._pattern)) super().__set__(obj, value) -class StringField(TypedField): - """Stores a standard string object""" - - def __init__(self, fieldname, allow_none=False): - super().__init__(fieldname, str, allow_none) - - class IntegerField(TypedField): """Stores an integer object""" @@ -376,6 +361,24 @@ def __set__(self, obj, value): Field.__set__(self, obj, value) +class AbsolutePathField(StringField): + """A string field that stores an absolute path. + + Any string assigned to such a field, will be converted to an absolute path. + """ + + def __set__(self, obj, value): + if not self._check_type(value): + raise TypeError('attempt to set a path field with a ' + 'non string value: %s' % value) + + if value is not None: + value = os.path.abspath(value) + + # Call Field's __set__() method, type checking is already performed + Field.__set__(self, obj, value) + + class ScopedDictField(AggregateTypeField): """Stores a ScopedDict with a specific type diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index 157d9bd372..576e7096d0 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -30,15 +30,15 @@ def __init__(self, options=[]): @abc.abstractmethod def command(self, job): - # The launcher command. - # - # :arg job: A :class:`reframe.core.schedulers.Job` that may be used by - # this launcher to properly emit its options. - # Subclasses may override this method and emit options according the - # num of tasks associated to the job etc. - # :returns: a list of command line arguments (including the launcher - # executable). - pass + """The launcher command. + + :arg job: A :class:`reframe.core.schedulers.Job` that will be used by + this launcher to properly emit its options. + Subclasses may override this method and emit options according the + number of tasks associated to the job etc. + :returns: a list of command line arguments (including the launcher + executable). + """ def emit_run_command(self, job, builder): return builder.verbatim( diff --git a/reframe/core/launchers/registry.py b/reframe/core/launchers/registry.py index 2216cf8ada..fd8df4dd07 100644 --- a/reframe/core/launchers/registry.py +++ b/reframe/core/launchers/registry.py @@ -1,5 +1,4 @@ import reframe.core.fields as fields - from reframe.core.exceptions import ConfigError @@ -10,18 +9,20 @@ def register_launcher(name, local=False): """Class decorator for registering new job launchers. + .. caution:: + This decorator is only relevant to developers of new job launchers. + + .. note:: + .. versionadded:: 2.8 + :arg name: The registration name of this launcher :arg local: :class:`True` if launcher may only submit local jobs, :class:`False` otherwise. :raises ValueError: if a job launcher is already registered with the same name. - - .. note:: - .. versionadded:: 2.8 - - This method is only relevant to developers of new job launchers. - """ + + # See reference.rst for documentation def _register_launcher(cls): if name in _LAUNCHERS: raise ValueError("a job launcher is already " @@ -60,15 +61,14 @@ def setup(self, partition, environ, **job_opts): You have to instantiate it explicitly before assigning it to the :attr:`launcher` attribute of the job. + .. note:: + .. versionadded:: 2.8 + :arg name: The name of the launcher to retrieve. :returns: The class of the launcher requested, which is a subclass of :class:`reframe.core.launchers.JobLauncher`. :raises reframe.core.exceptions.ConfigError: if no launcher is registered with that name. - - .. note:: - .. versionadded:: 2.8 - """ try: return _LAUNCHERS[name] diff --git a/reframe/core/modules.py b/reframe/core/modules.py index db6ed098b4..35460e1695 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -71,11 +71,28 @@ def __str__(self): class ModulesSystem: - """Implements the frontend of the module systems.""" + """A modules system abstraction inside ReFrame. + + This class interfaces between the framework internals and the actual + modules systems implementation. + """ module_map = fields.AggregateTypeField('module_map', (dict, (str, (list, str)))) + @classmethod + def create(cls, modules_kind=None): + if modules_kind is None: + return ModulesSystem(NoModImpl()) + elif modules_kind == 'tmod': + return ModulesSystem(TModImpl()) + elif modules_kind == 'tmod4': + return ModulesSystem(TMod4Impl()) + elif modules_kind == 'lmod': + return ModulesSystem(LModImpl()) + else: + raise ConfigError('unknown module system: %s' % modules_kind) + def __init__(self, backend): self._backend = backend self.module_map = {} @@ -84,7 +101,8 @@ def resolve_module(self, name): """Resolve module ``name`` in the registered module map. :returns: the list of real modules names pointed to by ``name``. - :raises: :attr:`ConfigError` if the mapping contains a cycle. + :raises: :class:`reframe.core.exceptions.ConfigError` if the mapping + contains a cycle. """ ret = [] visited = set() @@ -130,7 +148,7 @@ def loaded_modules(self): return [str(m) for m in self._backend.loaded_modules()] def conflicted_modules(self, name): - """Return the list of conflicted modules of the module ``name``. + """Return the list of the modules conflicting with module ``name``. If module ``name`` resolves to multiple real modules, then the returned list will be the concatenation of the conflict lists of all the real @@ -148,12 +166,11 @@ def _conflicted_modules(self, name): return [str(m) for m in self._backend.conflicted_modules(Module(name))] def load_module(self, name, force=False): - """Load the module `name'. + """Load the module ``name``. - If ``force`` is set, forces the loading, - unloading first any conflicting modules currently loaded. - If module ``name`` refers to to multiple real modules, all of the - target modules will be loaded. + If ``force`` is set, forces the loading, unloading first any + conflicting modules currently loaded. If module ``name`` refers to + multiple real modules, all of the target modules will be loaded. Returns the list of unloaded modules as strings. """ @@ -198,7 +215,7 @@ def is_module_loaded(self, name): """Check if module ``name`` is loaded. If module ``name`` refers to multiple real modules, this method will - return True only if all the referees are loaded. + return :class:`True` only if all the referees are loaded. """ return all(self._is_module_loaded(m) for m in self.resolve_module(name)) @@ -206,7 +223,12 @@ def _is_module_loaded(self, name): return self._backend.is_module_loaded(Module(name)) def load_mapping(self, mapping): - """Updates the internal module mapping with a single mapping""" + """Update the internal module mappings using a single mapping. + + :arg mapping: a string specifying the module mapping. + Example syntax: ``'m0: m1 m2'``. + + """ key, *rest = mapping.split(':') if len(rest) != 1: raise ConfigError('invalid mapping syntax: %s' % mapping) @@ -222,7 +244,7 @@ def load_mapping(self, mapping): self.module_map[key] = list(OrderedDict.fromkeys(values)) def load_mapping_from_file(self, filename): - """Update the internal module mapping from mappings in a file.""" + """Update the internal module mappings from mappings read from file.""" with open(filename) as fp: for lineno, line in enumerate(fp, start=1): line = line.strip().split('#')[0] @@ -262,12 +284,13 @@ def searchpath_remove(self, *dirs): return self._backend.searchpath_remove(*dirs) def emit_load_commands(self, name): - """Return the appropriate shell command for loading module.""" + """Return the appropriate shell command for loading module ``name``.""" return [self._backend.emit_load_instr(Module(name)) for name in self.resolve_module(name)] def emit_unload_commands(self, name): - """Return the appropriate shell command for unloading module.""" + """Return the appropriate shell command for unloading module + ``name``.""" return [self._backend.emit_unload_instr(Module(name)) for name in reversed(self.resolve_module(name))] @@ -294,7 +317,7 @@ def conflicted_modules(self, module): @abc.abstractmethod def load_module(self, module): - """Load the module `name'. + """Load the module ``name``. If ``force`` is set, forces the loading, unloading first any conflicting modules currently loaded. @@ -598,29 +621,3 @@ def emit_load_instr(self, module): def emit_unload_instr(self, module): return '' - - -# The module system used by the framework -_modules_system = None - - -def init_modules_system(modules_kind=None): - global _modules_system - - if modules_kind is None: - _modules_system = ModulesSystem(NoModImpl()) - elif modules_kind == 'tmod': - _modules_system = ModulesSystem(TModImpl()) - elif modules_kind == 'tmod4': - _modules_system = ModulesSystem(TMod4Impl()) - elif modules_kind == 'lmod': - _modules_system = ModulesSystem(LModImpl()) - else: - raise ConfigError('unknown module system: %s' % modules_kind) - - -def get_modules_system(): - if _modules_system is None: - raise ConfigError('no modules system is configured') - - return _modules_system diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index fed9a20073..ea9c89a795 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -2,13 +2,21 @@ # Basic functionality for regression tests # +__all__ = ['RegressionTest', + 'RunOnlyRegressionTest', 'CompileOnlyRegressionTest'] + + import fnmatch +import inspect +import itertools import os import shutil import reframe.core.debug as debug import reframe.core.fields as fields import reframe.core.logging as logging +import reframe.core.runtime as rt +import reframe.utility as util import reframe.utility.os_ext as os_ext from reframe.core.deferrable import deferrable, _DeferredExpression, evaluate from reframe.core.environments import Environment @@ -17,7 +25,7 @@ from reframe.core.schedulers import Job from reframe.core.schedulers.registry import getscheduler from reframe.core.shell import BashScriptBuilder -from reframe.core.systems import System, SystemPartition +from reframe.core.systems import SystemPartition from reframe.utility.sanity import assert_reference @@ -29,31 +37,30 @@ class RegressionTest: regression test goes through during its lifetime. :arg name: The name of the test. - This is the only argument that the users may specify freely. + If :class:`None`, the framework will try to assign a unique and + human-readable name to the test. + :arg prefix: The directory prefix of the test. - You should initialize this to the directory containing the file that - defines the regression test. - You can achieve this by always passing ``os.path.dirname(__file__)``. - :arg system: The system that this regression test will run on. - The framework takes care of initializing and passing correctly this - argument. - :arg resources: An object managing the framework's resources. - The framework takes care of initializing and passing correctly this - argument. - - Concrete regression test subclasses should call the base constructor as - follows: - - :: - - class MyTest(RegressionTest): - def __init__(self, my_test_args, **kwargs): - super().__init__('mytest', os.path.dirname(__file__), **kwargs) + If :class:`None`, the framework will set it to the directory containing + the test file. + + .. note:: + The ``name`` and ``prefix`` arguments are just maintained for backward + compatibility to the old (prior to 2.13) syntax of regression tests. + Users are advised to use the new simplified syntax for writing + regression tests. + Refer to the :doc:`ReFrame Tutorial ` for more information. + + This class is also directly available under the top-level :mod:`reframe` + module. + + .. versionchanged:: 2.13 + """ #: The name of the test. #: - #: :type: Alphanumeric string. - name = fields.AlphanumericField('name') + #: :type: string that can contain any character except ``/`` + name = fields.StringPatternField('name', '[^\/]+') #: List of programming environments supported by this test. #: @@ -457,29 +464,46 @@ def __init__(self, my_test_args, **kwargs): _stdout = fields.StringField('_stdout', allow_none=True) _stderr = fields.StringField('_stderr', allow_none=True) _perf_logfile = fields.StringField('_perf_logfile', allow_none=True) - _current_system = fields.TypedField('_current_system', System) _current_partition = fields.TypedField('_current_partition', SystemPartition, allow_none=True) _current_environ = fields.TypedField('_current_environ', Environment, allow_none=True) _job = fields.TypedField('_job', Job, allow_none=True) - def __init__(self, name, prefix, system, resources): - self.name = name - self.descr = name + def __new__(cls, *args, **kwargs): + obj = super().__new__(cls) + obj._prefix = os.path.abspath(os.path.dirname(inspect.getfile(cls))) + + # Create a test name from the class name and the constructor's + # arguments + name = cls.__qualname__ + #name = util.decamelize(cls.__name__) + if args or kwargs: + arg_names = map(lambda x: util.toalphanum(str(x)), + itertools.chain(args, kwargs.values())) + name += '_' + '_'.join(arg_names) + + obj.name = name + return obj + + def __init__(self, name=None, prefix=None): + if name is not None: + self.name = name + + self.descr = self.name self.valid_prog_environs = [] - self.valid_systems = [] - self.sourcepath = '' - self.prebuild_cmd = [] - self.postbuild_cmd = [] - self.executable = os.path.join('.', self.name) + self.valid_systems = [] + self.sourcepath = '' + self.prebuild_cmd = [] + self.postbuild_cmd = [] + self.executable = os.path.join('.', self.name) self.executable_opts = [] - self.pre_run = [] - self.post_run = [] - self.keep_files = [] - self.readonly_files = [] - self.tags = set() - self.maintainers = [] + self.pre_run = [] + self.post_run = [] + self.keep_files = [] + self.readonly_files = [] + self.tags = set() + self.maintainers = [] # Strict performance check, if applicable self.strict_check = True @@ -498,7 +522,9 @@ def __init__(self, name, prefix, system, resources): self.local = False # Static directories of the regression check - self._prefix = os.path.abspath(prefix) + if prefix is not None: + self._prefix = os.path.abspath(prefix) + self.sourcesdir = 'src' # Output patterns @@ -516,22 +542,20 @@ def __init__(self, name, prefix, system, resources): self.time_limit = (0, 10, 0) # Runtime information of the test - self._current_system = system self._current_partition = None - self._current_environ = None + self._current_environ = None # Associated job - self._job = None + self._job = None self.extra_resources = {} # Dynamic paths of the regression check; will be set in setup() - self._resources_mgr = resources self._stagedir = None self._stdout = None self._stderr = None - # Compilation task output - self._compile_task = None + # Compilation process output + self._compile_proc = None # Performance logging self._perf_logger = logging.null_logger @@ -565,9 +589,9 @@ def current_system(self): This is set by the framework during the initialization phase. - :type: :class:`reframe.core.systems.System`. + :type: :class:`reframe.core.runtime.HostSystem`. """ - return self._current_system + return rt.runtime().system @property def job(self): @@ -670,12 +694,12 @@ def supports_system(self, partition_name): if '*' in self.valid_systems: return True - if self._current_system.name in self.valid_systems: + if self.current_system.name in self.valid_systems: return True # Check if this is a relative name if partition_name.find(':') == -1: - partition_name = '%s:%s' % (self._current_system.name, + partition_name = '%s:%s' % (self.current_system.name, partition_name) return partition_name in self.valid_systems @@ -698,12 +722,6 @@ def is_local(self): return self.local or self._current_partition.scheduler.is_local - def _sanitize_basename(self, name): - """Create a basename safe to be used as path component - - Replace all path separator characters in `name` with underscores.""" - return name.replace(os.sep, '_') - def _setup_environ(self, environ): """Setup the current environment and load it.""" @@ -727,17 +745,15 @@ def _setup_paths(self): """Setup the check's dynamic paths.""" self.logger.debug('setting up paths') try: - self._stagedir = self._resources_mgr.stagedir( - self._sanitize_basename(self._current_partition.name), + self._stagedir = rt.runtime().resources.make_stagedir( + self._current_partition.name, self.name, - self._sanitize_basename(self._current_environ.name) - ) + self._current_environ.name) - self.outputdir = self._resources_mgr.outputdir( - self._sanitize_basename(self._current_partition.name), + self.outputdir = rt.runtime().resources.make_outputdir( + self._current_partition.name, self.name, - self._sanitize_basename(self._current_environ.name) - ) + self._current_environ.name) except OSError as e: raise PipelineError('failed to set up paths') from e @@ -767,12 +783,10 @@ def _setup_job(self, **job_opts): scheduler_type = self._current_partition.scheduler launcher_type = self._current_partition.launcher - job_name = '%s_%s_%s_%s' % ( - self.name, - self._sanitize_basename(self._current_system.name), - self._sanitize_basename(self._current_partition.name), - self._sanitize_basename(self._current_environ.name) - ) + job_name = '%s_%s_%s_%s' % (self.name, + self.current_system.name, + self._current_partition.name, + self._current_environ.name) job_script_filename = os.path.join(self._stagedir, job_name + '.sh') self._job = scheduler_type( @@ -815,7 +829,8 @@ def _setup_job(self, **job_opts): def _setup_perf_logging(self): self.logger.debug('setting up performance logging') self._perf_logfile = os.path.join( - self._resources_mgr.logdir(self._current_partition.name), + rt.runtime().resources.make_perflogdir( + self._current_partition.name), self.name + '.log' ) @@ -936,14 +951,14 @@ def compile(self, **compile_opts): includedir = os.path.dirname(staged_sourcepath) self._current_environ.include_search_path.append(includedir) - self._compile_task = self._current_environ.compile( + self._compile_proc = self._current_environ.compile( sourcepath=staged_sourcepath, executable=os.path.join(self._stagedir, self.executable), **compile_opts) self.logger.debug('compilation stdout:\n%s' % - self._compile_task.stdout) + self._compile_proc.stdout) self.logger.debug('compilation stderr:\n%s' % - self._compile_task.stderr) + self._compile_proc.stderr) self.postbuild() self.logger.debug('compilation finished') @@ -954,7 +969,7 @@ def run(self): This call is non-blocking. It simply submits the job associated with this test and returns. """ - if not self._current_system or not self._current_partition: + if not self.current_system or not self._current_partition: raise PipelineError('no system or system partition is set') with os_ext.change_dir(self._stagedir): @@ -1087,7 +1102,11 @@ def __str__(self): class RunOnlyRegressionTest(RegressionTest): - """Base class for run-only regression tests.""" + """Base class for run-only regression tests. + + This class is also directly available under the top-level :mod:`reframe` + module. + """ def compile(self, **compile_opts): """The compilation phase of the regression test pipeline. @@ -1119,6 +1138,9 @@ class CompileOnlyRegressionTest(RegressionTest): The standard output and standard error of the test will be set to those of the compilation stage. + + This class is also directly available under the top-level :mod:`reframe` + module. """ def __init__(self, *args, **kwargs): @@ -1146,10 +1168,10 @@ def compile(self, **compile_opts): try: with open(self._stdout, 'w') as f: - f.write(self._compile_task.stdout) + f.write(self._compile_proc.stdout) with open(self._stderr, 'w') as f: - f.write(self._compile_task.stderr) + f.write(self._compile_proc.stderr) except OSError as e: raise PipelineError('could not write stdout/stderr') from e diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py new file mode 100644 index 0000000000..d5a1b9d72f --- /dev/null +++ b/reframe/core/runtime.py @@ -0,0 +1,296 @@ +# +# Handling of the current host context +# + +import os +import functools +import re +import shutil +import socket +from datetime import datetime + +import reframe.core.config as config +import reframe.core.fields as fields +import reframe.utility.os_ext as os_ext +from reframe.core.exceptions import (ConfigError, + ReframeFatalError, + SpawnedProcessError, + SystemAutodetectionError, + UnknownSystemError) +from reframe.core.modules import ModulesSystem + + +class HostSystem: + """The host system of the framework. + + The host system is a representation of the system that the framework + currently runs on.If the framework is properly configured, the host + system is automatically detected. If not, it may be explicitly set by the + user. + + This class is mainly a proxy of :class:`reframe.core.systems.System` that + stores optionally a partition name and provides some additional + functionality for manipulating system partitions. + + All attributes of the :class:`reframe.core.systems.System` may be accessed + directly from this proxy. + + .. note:: + .. versionadded:: 2.13 + """ + + def __init__(self, system, partname=None): + self._system = system + self._partname = partname + + def __getattr__(self, attr): + # Delegate any failed attribute lookup to our backend + return getattr(self._system, attr) + + @property + def partitions(self): + """The partitions of this system. + + :type: :class:`list[reframe.core.systems.SystemPartition]`. + """ + + if not self._partname: + return self._system.partitions + + return [p for p in self._system.partitions if p.name == self._partname] + + def partition(self, name): + """Return the system partition ``name``. + + :type: :class:`reframe.core.systems.SystemPartition`. + """ + for p in self.partitions: + if p.name == name: + return p + + return None + + def __str__(self): + return str(self._system) + + def __repr__(self): + return 'HostSystem(%r, %r)' % (self._system, self._partname) + + +class HostResources: + """Resources associated with ReFrame execution on the current host. + + .. note:: + .. versionadded:: 2.13 + """ + + #: The prefix directory of ReFrame execution. + #: This is always an absolute path. + #: + #: :type: :class:`str` + #: + #: .. caution:: + #: Users may not set this field. + #: + prefix = fields.AbsolutePathField('prefix') + outputdir = fields.AbsolutePathField('outputdir', allow_none=True) + stagedir = fields.AbsolutePathField('stagedir', allow_none=True) + perflogdir = fields.AbsolutePathField('perflogdir', allow_none=True) + + def __init__(self, prefix=None, stagedir=None, + outputdir=None, perflogdir=None, timefmt=None): + self.prefix = prefix or '.' + self.stagedir = stagedir + self.outputdir = outputdir + self.perflogdir = perflogdir + self._timestamp = datetime.now() + self.timefmt = timefmt + + def _makedir(self, *dirs, wipeout=False): + ret = os.path.join(*dirs) + if wipeout: + shutil.rmtree(ret, True) + + os.makedirs(ret, exist_ok=True) + return ret + + @property + def timestamp(self): + return self._timestamp.strftime(self.timefmt) if self.timefmt else '' + + @property + def output_prefix(self): + """The output prefix directory of ReFrame.""" + if self.outputdir is None: + return os.path.join(self.prefix, 'output', self.timestamp) + else: + return os.path.join(self.outputdir, self.timestamp) + + @property + def stage_prefix(self): + """The stage prefix directory of ReFrame.""" + if self.stagedir is None: + return os.path.join(self.prefix, 'stage', self.timestamp) + else: + return os.path.join(self.stagedir, self.timestamp) + + @property + def perflog_prefix(self): + """The prefix directory of the performance logs of ReFrame.""" + if self.perflogdir is None: + return os.path.join(self.prefix, 'logs') + else: + return self.perflogdir + + def make_stagedir(self, *dirs, wipeout=True): + return self._makedir(self.stage_prefix, *dirs, wipeout=wipeout) + + def make_outputdir(self, *dirs, wipeout=True): + return self._makedir(self.output_prefix, *dirs, wipeout=wipeout) + + def make_perflogdir(self, *dirs, wipeout=False): + return self._makedir(self.perflog_prefix, *dirs, wipeout=wipeout) + + +class RuntimeContext: + """The runtime context of the framework. + + This class essentially groups the current host system and the associated + resources of the framework on the current system. + + There is a single instance of this class globally in the framework. + + .. note:: + .. versionadded:: 2.13 + """ + + def __init__(self, dict_config, sysdescr=None): + self._site_config = config.SiteConfiguration(dict_config) + if sysdescr is not None: + sysname, _, partname = sysdescr.partition(':') + try: + self._system = HostSystem( + self._site_config.systems[sysname], partname) + except KeyError: + raise UnknownSystemError('unknown system: %s' % + sysdescr) from None + else: + self._system = HostSystem(self._autodetect_system()) + + self._resources = HostResources( + self._system.prefix, self._system.stagedir, + self._system.outputdir, self._system.logdir) + self._modules_system = ModulesSystem.create( + self._system.modules_system) + + def _autodetect_system(self): + """Auto-detect system.""" + + # Try to detect directly the cluster name from /etc/xthostname (Cray + # specific) + try: + hostname = os_ext.run_command( + 'cat /etc/xthostname', check=True).stdout + except SpawnedProcessError: + # Try to figure it out with the standard method + hostname = socket.gethostname() + + # Go through the supported systems and try to match the hostname + for system in self._site_config.systems.values(): + for hostname_patt in system.hostnames: + if re.match(hostname_patt, hostname): + return system + + raise SystemAutodetectionError + + def mode(self, name): + try: + return self._site_config.modes[name] + except KeyError: + raise ConfigError('unknown execution mode: %s' % name) from None + + @property + def system(self): + """The current host system. + + :type: :class:`reframe.core.runtime.HostSystem` + """ + return self._system + + @property + def resources(self): + """The framework resources. + + :type: :class:`reframe.core.runtime.HostResources` + """ + return self._resources + + @property + def modules_system(self): + """The modules system used by the current host system. + + :type: :class:`reframe.core.modules.ModulesSystem`. + """ + return self._modules_system + + + +# Global resources for the current host +_runtime_context = None + + +def init_runtime(dict_config, sysname=None): + global _runtime_context + + if _runtime_context is None: + _runtime_context = RuntimeContext(dict_config, sysname) + + +def runtime(): + """Retrieve the framework's runtime context. + + :type: :class:`reframe.core.runtime.RuntimeContext` + + .. note:: + .. versionadded:: 2.13 + """ + if _runtime_context is None: + raise ReframeFatalError('no runtime context is configured') + + return _runtime_context + + +# The following utilities are useful only for the unit tests + +class temp_runtime: + """Context manager to temporarily switch to another runtime.""" + + def __init__(self, dict_config, sysname=None): + global _runtime_context + self._runtime_save = _runtime_context + if dict_config is None: + _runtime_context = None + else: + _runtime_context = RuntimeContext(dict_config, sysname) + + def __enter__(self): + return _runtime_context + + def __exit__(self, exc_type, exc_value, traceback): + global _runtime_context + _runtime_context = self._runtime_save + + +def switch_runtime(dict_config, sysname=None): + """Function decorator for temporarily changing the runtime for a function.""" + def _runtime_deco(fn): + @functools.wraps(fn) + def _fn(*args, **kwargs): + with temp_runtime(dict_config, sysname): + ret = fn(*args, **kwargs) + + return ret + + return _fn + + return _runtime_deco diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 3d747d4df8..dd013086fd 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -30,7 +30,7 @@ def __str__(self): class Job(abc.ABC): """A job descriptor. - .. note:: + .. caution:: This is an abstract class. Users may not create jobs directly. """ diff --git a/reframe/core/schedulers/local.py b/reframe/core/schedulers/local.py index 46b440933f..730bebb334 100644 --- a/reframe/core/schedulers/local.py +++ b/reframe/core/schedulers/local.py @@ -26,7 +26,7 @@ class _TimeoutExpired(ReframeError): pass -@register_scheduler('local') +@register_scheduler('local', local=True) class LocalJob(sched.Job): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index c4ce3a80ce..3e7e3ada6a 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -5,11 +5,11 @@ import reframe.core.schedulers as sched import reframe.utility.os_ext as os_ext +from reframe.core.config import settings from reframe.core.exceptions import (SpawnedProcessError, JobBlockedError, JobError) from reframe.core.logging import getlogger from reframe.core.schedulers.registry import register_scheduler -from reframe.settings import settings class SlurmJobState(sched.JobState): @@ -143,7 +143,7 @@ def prepare(self, builder): def submit(self): cmd = 'sbatch %s' % self.script_filename - completed = self._run_command(cmd, settings.job_submit_timeout) + completed = self._run_command(cmd, settings().job_submit_timeout) jobid_match = re.search('Submitted batch job (?P\d+)', completed.stdout) if not jobid_match: @@ -176,7 +176,7 @@ def _count_compatible_nodes(self, nodes): if (n.active_features >= constraints and n.partitions >= partitions and n.name not in excluded_node_names): - num_nodes += 1 + num_nodes += 1 return num_nodes @@ -270,7 +270,7 @@ def wait(self): if self._state in self._completion_states: return - intervals = itertools.cycle(settings.job_poll_intervals) + intervals = itertools.cycle(settings().job_poll_intervals) self._update_state() while self._state not in self._completion_states: time.sleep(next(intervals)) @@ -280,7 +280,7 @@ def cancel(self): super().cancel() getlogger().debug('cancelling job (id=%s)' % self._jobid) self._run_command('scancel %s' % self._jobid, - settings.job_submit_timeout) + settings().job_submit_timeout) self._is_cancelling = True def finished(self): diff --git a/reframe/core/systems.py b/reframe/core/systems.py index b797426df1..caad0f2c7a 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -7,7 +7,7 @@ class SystemPartition: """A representation of a system partition inside ReFrame.""" - _name = fields.NonWhitespaceField('_name') + _name = fields.StringPatternField('_name', '(\w|-)+') _descr = fields.StringField('_descr') _access = fields.TypedListField('_access', str) _environs = fields.TypedListField('_environs', Environment) @@ -29,25 +29,14 @@ def __init__(self, name, descr=None, scheduler=None, launcher=None, self._resources = dict(resources) self._max_jobs = max_jobs self._local_env = local_env - self._active = True # Parent system self._system = None - def enable(self): - self._active = True - - def disable(self): - self._active = False - @property def access(self): return self._access - @property - def active(self): - return self._active - @property def descr(self): """A detailed description of this partition.""" @@ -155,26 +144,18 @@ def __repr__(self): class System: """A representation of a system inside ReFrame.""" - _name = fields.NonWhitespaceField('_name') + _name = fields.StringPatternField('_name', '(\w|-)+') _descr = fields.StringField('_descr') _hostnames = fields.TypedListField('_hostnames', str) _partitions = fields.TypedListField('_partitions', SystemPartition) - _modules_system = fields.AlphanumericField('_modules_system', - allow_none=True) - - prefix = fields.StringField('prefix') - stagedir = fields.StringField('stagedir', allow_none=True) - outputdir = fields.StringField('outputdir', allow_none=True) - logdir = fields.StringField('logdir', allow_none=True) - - #: Global resources directory for this system - #: - #: You may use this directory for storing large resource files of your - #: regression tests. - #: See `here `__ on how to configure this. - #: - #: :type: :class:`str` - resourcesdir = fields.StringField('resourcesdir') + _modules_system = fields.StringPatternField('_modules_system', + '(\w|-)+', allow_none=True) + + _prefix = fields.StringField('_prefix') + _stagedir = fields.StringField('_stagedir', allow_none=True) + _outputdir = fields.StringField('_outputdir', allow_none=True) + _logdir = fields.StringField('_logdir', allow_none=True) + _resourcesdir = fields.StringField('_resourcesdir') def __init__(self, name, descr=None, hostnames=[], partitions=[], prefix='.', stagedir=None, outputdir=None, logdir=None, @@ -184,16 +165,21 @@ def __init__(self, name, descr=None, hostnames=[], partitions=[], self._hostnames = list(hostnames) self._partitions = list(partitions) self._modules_system = modules_system - self.prefix = prefix - self.stagedir = stagedir - self.outputdir = outputdir - self.logdir = logdir - self.resourcesdir = resourcesdir + self._prefix = prefix + self._stagedir = stagedir + self._outputdir = outputdir + self._logdir = logdir + self._resourcesdir = resourcesdir # Set parent system for the given partitions for p in partitions: p._system = self + @property + def name(self): + """The name of this system.""" + return self._name + @property def descr(self): """The description of this system.""" @@ -201,36 +187,51 @@ def descr(self): @property def hostnames(self): + """The hostname patterns associated with this system.""" return self._hostnames @property def modules_system(self): + """The modules system name associated with this system.""" return self._modules_system @property - def name(self): - """The name of this system.""" - return self._name + def prefix(self): + """The ReFrame prefix associated with this system.""" + return self._prefix @property - def partitions(self): - """Get all the active partitions of this system. + def stagedir(self): + """The ReFrame stage directory prefix associated with this system.""" + return self._stagedir - :returns: a list of :class:`SystemPartition`. - """ - return [p for p in self._partitions if p.active] + @property + def outputdir(self): + """The ReFrame output directory prefix associated with this system.""" + return self._outputdir - def partition(self, name): - """Get system partition with ``name``. + @property + def logdir(self): + """The ReFrame log directory prefix associated with this system.""" + return self._logdir - :returns: the requested :class:`SystemPartition`, or :class:`None` if - not found. + @property + def resourcesdir(self): + """Global resources directory for this system. + + You may use this directory for storing large resource files of your + regression tests. + See `here `__ on how to configure + this. + + :type: :class:`str` """ - for p in self._partitions: - if p.name == name and p.active: - return p + return self._resourcesdir - return None + @property + def partitions(self): + """All the system partitions associated with this system.""" + return self._partitions def add_partition(self, partition): partition._system = self diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 79e23f4191..1f5eb4f4ae 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -3,19 +3,19 @@ import sys import reframe -import reframe.frontend.config as config +import reframe.core.config as config import reframe.core.logging as logging +import reframe.core.runtime as runtime import reframe.utility.os_ext as os_ext from reframe.core.exceptions import (EnvironError, ConfigError, ReframeError, - ReframeFatalError, format_exception) -from reframe.core.modules import get_modules_system, init_modules_system + ReframeFatalError, format_exception, + SystemAutodetectionError) from reframe.frontend.argparse import ArgumentParser from reframe.frontend.executors import Runner from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.loader import RegressionCheckLoader from reframe.frontend.printer import PrettyPrinter -from reframe.frontend.resources import ResourcesManager def list_supported_systems(systems, printer): @@ -202,14 +202,12 @@ def main(): # Load configuration try: - settings = config.load_from_file(options.config_file) + settings = config.load_settings_from_file(options.config_file) except (OSError, ReframeError) as e: sys.stderr.write( '%s: could not load settings: %s\n' % (sys.argv[0], e)) sys.exit(1) - site_config = config.SiteConfiguration() - site_config.load_from_dict(settings.site_configuration) # Configure logging try: logging.configure_logging(settings.logging_config) @@ -221,63 +219,66 @@ def main(): printer = PrettyPrinter() printer.colorize = options.colorize - if options.system: - try: - sysname, sep, partname = options.system.partition(':') - system = site_config.systems[sysname] - if partname: - # Disable all partitions except partname - for p in system.partitions: - if p.name != partname: - p.disable() - - if not system.partitions: - raise KeyError(options.system) - - except KeyError: - printer.error("unknown system specified: `%s'" % options.system) - list_supported_systems(site_config.systems.values(), printer) - sys.exit(1) - else: - # Try to autodetect system - system = config.autodetect_system(site_config) - if not system: - printer.error("could not auto-detect system. Please specify " - "it manually using the `--system' option.") - list_supported_systems(site_config.systems.values(), printer) - sys.exit(1) - try: - # Init modules system - init_modules_system(system.modules_system) - except ReframeError as e: - printer.error('could not initialize the modules system: %s' % e) + runtime.init_runtime(settings.site_configuration, options.system) + except SystemAutodetectionError: + printer.error("could not auto-detect system; please use the " + "`--system' option to specify one explicitly") + sys.exit(1) + + except (ConfigError, OSError) as e: + printer.error('configuration error %s' % e) sys.exit(1) + rt = runtime.runtime() try: if options.module_map_file: - get_modules_system().load_mapping_from_file( - options.module_map_file) + rt.modules_system.load_mapping_from_file(options.module_map_file) if options.module_mappings: for m in options.module_mappings: - get_modules_system().load_mapping(m) + rt.modules_system.load_mapping(m) - except (ReframeError, OSError) as e: + except (ConfigError, OSError) as e: printer.error('could not load module mappings: %s' % e) sys.exit(1) if options.mode: try: - mode_args = site_config.modes[options.mode] + mode_args = rt.mode(options.mode) # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) options = argparser.parse_args(namespace=options) - except KeyError: - printer.error("no such execution mode: `%s'" % (options.mode)) + except ConfigError as e: + printer.error('could not obtain execution mode: %s' % e) sys.exit(1) + # Adjust system directories + if options.prefix: + # if prefix is set, reset all other directories + rt.resources.prefix = os.path.expandvars(options.prefix) + rt.resources.outputdir = None + rt.resources.stagedir = None + rt.resources.perflogdir = None + + if options.output: + rt.resources.outputdir = os.path.expandvars(options.output) + + if options.stage: + rt.resources.stagedir = os.path.expandvars(options.stage) + + if options.logdir: + rt.resources.perflogdir = os.path.expandvars(options.logdir) + + if (os_ext.samefile(rt.resources.stage_prefix, + rt.resources.output_prefix) and + not options.keep_stage_files): + printer.error('stage and output refer to the same directory; ' + 'if this is on purpose, please use also the ' + "`--keep-stage-files' option.") + sys.exit(1) + # Setup the check loader if options.checkpath: load_path = [] @@ -299,35 +300,6 @@ def main(): prefix=reframe.INSTALL_PREFIX, recurse=settings.checks_path_recurse) - # Adjust system directories - if options.prefix: - # if prefix is set, reset all other directories - system.prefix = os.path.expandvars(options.prefix) - system.outputdir = None - system.stagedir = None - system.logdir = None - - if options.output: - system.outputdir = os.path.expandvars(options.output) - - if options.stage: - system.stagedir = os.path.expandvars(options.stage) - - if options.logdir: - system.logdir = os.path.expandvars(options.logdir) - - resources = ResourcesManager(prefix=system.prefix, - output_prefix=system.outputdir, - stage_prefix=system.stagedir, - log_prefix=system.logdir, - timestamp=options.timestamp) - if (os_ext.samefile(resources.stage_prefix, resources.output_prefix) and - not options.keep_stage_files): - printer.error('stage and output refer to the same directory. ' - 'If this is on purpose, please use also the ' - "`--keep-stage-files' option.") - sys.exit(1) - printer.log_config(options) # Print command line @@ -343,13 +315,13 @@ def main(): printer.info('%03s Check search path : %s' % ('(R)' if loader.recurse else '', "'%s'" % ':'.join(loader.load_path))) - printer.info(' Stage dir prefix : %s' % resources.stage_prefix) - printer.info(' Output dir prefix : %s' % resources.output_prefix) - printer.info(' Logging dir : %s' % resources.log_prefix) + printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix) + printer.info(' Output dir prefix : %s' % rt.resources.output_prefix) + printer.info(' Logging dir : %s' % rt.resources.perflog_prefix) try: # Locate and load checks try: - checks_found = loader.load_all(system=system, resources=resources) + checks_found = loader.load_all() except OSError as e: raise ReframeError from e @@ -404,11 +376,11 @@ def main(): # Unload regression's module and load user-specified modules if settings.reframe_module: - get_modules_system().unload_module(settings.reframe_module) + rt.modules_system.unload_module(settings.reframe_module) for m in options.user_modules: try: - get_modules_system().load_module(m, force=True) + rt.modules_system.load_module(m, force=True) except EnvironError: printer.info("could not load module `%s': Skipping..." % m) @@ -450,7 +422,7 @@ def main(): max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: - runner.runall(checks_matched, system) + runner.runall(checks_matched) finally: # Print a retry report if we did any retries if runner.stats.num_failures(run=0): @@ -483,8 +455,8 @@ def main(): finally: try: if options.save_log_files: - logging.save_log_files(resources.output_prefix) + logging.save_log_files(rt.resources.output_prefix) except OSError as e: - printer.error(str(e)) + printer.error('could not save log file: %s' % e) sys.exit(1) diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 5a19f8a9e3..4dc6f7671c 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -3,6 +3,7 @@ import reframe.core.debug as debug import reframe.core.logging as logging +import reframe.core.runtime as runtime from reframe.core.environments import EnvironmentSnapshot from reframe.core.exceptions import (AbortTaskError, JobNotStartedError, ReframeFatalError, TaskExit) @@ -164,15 +165,15 @@ def policy(self): def stats(self): return self._stats - def runall(self, checks, system): + def runall(self, checks): try: self._printer.separator('short double line', 'Running %d check(s)' % len(checks)) self._printer.timestamp('Started on', 'short double line') self._printer.info() - self._runall(checks, system) + self._runall(checks) if self._max_retries: - self._retry_failed(checks, system) + self._retry_failed(checks) finally: # Print the summary line @@ -202,7 +203,7 @@ def _environ_supported(self, check, environ): else: return ret and check.supports_environ(environ.name) - def _retry_failed(self, checks, system): + def _retry_failed(self, checks): while (self._stats.num_failures() and self._current_run < self._max_retries): failed_checks = [ @@ -222,9 +223,10 @@ def _retry_failed(self, checks, system): 'Retrying %d failed check(s) (retry %d/%d)' % (len(failed_checks), self._current_run, self._max_retries) ) - self._runall(failed_checks, system) + self._runall(failed_checks) - def _runall(self, checks, system): + def _runall(self, checks): + system = runtime.runtime().system self._policy.enter() for c in checks: self._policy.enter_check(c) diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 068fc6fb7d..0b359e0012 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -9,23 +9,40 @@ from importlib.machinery import SourceFileLoader import reframe.core.debug as debug -from reframe.core.exceptions import NameConflictError +import reframe.utility as util +from reframe.core.exceptions import NameConflictError, TestLoadError from reframe.core.logging import getlogger class RegressionCheckValidator(ast.NodeVisitor): def __init__(self): - self._validated = False + self._has_import = False + self._has_regression_test = False @property def valid(self): - return self._validated + return self._has_import and self._has_regression_test - def visit_FunctionDef(self, node): - if (node.name == '_get_checks' and - node.col_offset == 0 and - node.args.kwarg): - self._validated = True + def visit_Import(self, node): + for m in node.names: + if m.name.startswith('reframe'): + self._has_import = True + + def visit_ImportFrom(self, node): + if node.module.startswith('reframe'): + self._has_import = True + + def visit_ClassDef(self, node): + for b in node.bases: + try: + # Unqualified name as in `class C(RegressionTest)` + cls_name = b.id + except AttributeError: + # Qualified name as in `class C(rfm.RegressionTest)` + cls_name = b.attr + + if 'RegressionTest' in cls_name: + self._has_regression_test = True class RegressionCheckLoader: @@ -80,16 +97,27 @@ def prefix(self): def recurse(self): return self._recurse - def load_from_module(self, module, **check_args): + def load_from_module(self, module): """Load user checks from module. This method tries to call the `_get_checks()` method of the user check and validates its return value.""" from reframe.core.pipeline import RegressionTest - # We can safely call `_get_checks()` here, since the source file is - # already validated - candidates = module._get_checks(**check_args) + old_syntax = hasattr(module, '_get_checks') + new_syntax = hasattr(module, '_rfm_gettests') + if old_syntax and new_syntax: + raise TestLoadError('%s: mixing old and new regression test ' + 'syntax is not allowed' % module.__file__) + + if not old_syntax and not new_syntax: + return [] + + if old_syntax: + candidates = module._get_checks() + else: + candidates = module._rfm_gettests() + if not isinstance(candidates, collections.abc.Sequence): return [] @@ -98,7 +126,7 @@ def load_from_module(self, module, **check_args): if not isinstance(c, RegressionTest): continue - testfile = inspect.getfile(type(c)) + testfile = module.__file__ try: conflicted = self._loaded[c.name] except KeyError: @@ -116,19 +144,17 @@ def load_from_module(self, module, **check_args): return ret def load_from_file(self, filename, **check_args): - module_name = self._module_name(filename) if not self._validate_source(filename): return [] - loader = SourceFileLoader(module_name, filename) - return self.load_from_module(loader.load_module(), **check_args) + return self.load_from_module(util.import_module_from_file(filename)) - def load_from_dir(self, dirname, recurse=False, **check_args): + def load_from_dir(self, dirname, recurse=False): checks = [] for entry in os.scandir(dirname): if recurse and entry.is_dir(): checks.extend( - self.load_from_dir(entry.path, recurse, **check_args) + self.load_from_dir(entry.path, recurse) ) if (entry.name.startswith('.') or @@ -136,11 +162,11 @@ def load_from_dir(self, dirname, recurse=False, **check_args): not entry.is_file()): continue - checks.extend(self.load_from_file(entry.path, **check_args)) + checks.extend(self.load_from_file(entry.path)) return checks - def load_all(self, **check_args): + def load_all(self): """Load all checks in self._load_path. If a prefix exists, it will be prepended to each path.""" @@ -150,9 +176,8 @@ def load_all(self, **check_args): if not os.path.exists(d): continue if os.path.isdir(d): - checks.extend(self.load_from_dir(d, self._recurse, - **check_args)) + checks.extend(self.load_from_dir(d, self._recurse)) else: - checks.extend(self.load_from_file(d, **check_args)) + checks.extend(self.load_from_file(d)) return checks diff --git a/reframe/frontend/resources.py b/reframe/frontend/resources.py deleted file mode 100644 index 8bd49fa3cc..0000000000 --- a/reframe/frontend/resources.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Regression resources management -# - -import os -import shutil -from datetime import datetime - -import reframe.core.debug as debug - - -class ResourcesManager: - def __init__(self, prefix='.', output_prefix=None, stage_prefix=None, - log_prefix=None, timestamp=None): - - # Get the timestamp - time = datetime.now().strftime(timestamp or '') - - self._prefix = os.path.abspath(prefix) - if output_prefix: - self._output_prefix = os.path.join( - os.path.abspath(output_prefix), time - ) - else: - self._output_prefix = os.path.join(self._prefix, 'output', time) - - if stage_prefix: - self._stage_prefix = os.path.join( - os.path.abspath(stage_prefix), time - ) - else: - self._stage_prefix = os.path.join(self._prefix, 'stage', time) - - # regression performance logs - if not log_prefix: - self._log_prefix = os.path.join(self._prefix, 'logs') - else: - self._log_prefix = os.path.abspath(log_prefix) - - def __repr__(self): - return debug.repr(self) - - def _makedir(self, *dirs, wipeout=False): - ret = os.path.join(*dirs) - if wipeout: - shutil.rmtree(ret, True) - - os.makedirs(ret, exist_ok=True) - return ret - - @property - def prefix(self): - return self._prefix - - @property - def output_prefix(self): - return self._output_prefix - - @property - def log_prefix(self): - return self._log_prefix - - @property - def stage_prefix(self): - return self._stage_prefix - - def stagedir(self, *dirs, wipeout=True): - return self._makedir(self._stage_prefix, *dirs, wipeout=wipeout) - - def outputdir(self, *dirs, wipeout=True): - return self._makedir(self._output_prefix, *dirs, wipeout=wipeout) - - def logdir(self, *dirs, wipeout=False): - return self._makedir(self._log_prefix, *dirs, wipeout=wipeout) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 2f54467494..ae0bb6152c 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -1,9 +1,85 @@ -import collections.abc -import os +import collections +import importlib import importlib.util +import os +import re +import sys + from collections import UserDict +def _get_module_name(filename): + barename, _ = os.path.splitext(filename) + if os.path.basename(filename) == '__init__.py': + barename = os.path.dirname(filename) + + if os.path.isabs(barename): + module_name = os.path.basename(barename) + else: + module_name = barename.replace(os.sep, '.') + + return module_name + + +def _do_import_module_from_file(filename, module_name=None): + module_name = module_name or _get_module_name(filename) + if module_name in sys.modules: + return sys.modules[module_name] + + spec = importlib.util.spec_from_file_location(module_name, filename) + if spec is None: + raise ImportError("No module named '%s'" % module_name, + name=module_name, path=filename) + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def import_module_from_file(filename): + """Import module from file.""" + + filename = os.path.normpath(os.path.expandvars(filename)) + if os.path.isdir(filename): + filename = os.path.join(filename, '__init__.py') + + module_name = _get_module_name(filename) + if os.path.isabs(filename): + return _do_import_module_from_file(filename, module_name) + + return importlib.import_module(module_name) + + +def decamelize(s): + """Decamelize the string ``s``. + + For example, ``MyBaseClass`` will be converted to ``my_base_class``. + """ + + if not isinstance(s, str): + raise TypeError('decamelize() requires a string argument') + + if not s: + return '' + + return re.sub(r'([a-z])([A-Z])', r'\1_\2', s).lower() + + +def toalphanum(s): + """Convert string ``s`` be replacing any non-alphanumeric character with + ``_``. + """ + + if not isinstance(s, str): + raise TypeError('toalphanum() requires a string argument') + + if not s: + return '' + + return re.sub(r'\W', '_', s) + + class ScopedDict(UserDict): """This is a special dict that imposes scopes on its keys. @@ -132,24 +208,3 @@ def __delitem__(self, key): def __missing__(self, key): raise KeyError(str(key)) - - -def import_module_from_file(filename): - filename = os.path.expandvars(filename) - - # Figure out a reasonable module name - # FIXME: we are not treating specially `__init__.py` - barename, _ = os.path.splitext(filename) - if os.path.isabs(barename): - module_name = os.path.basename(barename) - else: - module_name = barename.replace('/', '.') - - spec = importlib.util.spec_from_file_location(module_name, filename) - if spec is None: - raise ImportError("No module named '%s'" % module_name, - name=module_name, path=filename) - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module diff --git a/reframe/utility/os_ext.py b/reframe/utility/os_ext.py index d2acdc4e51..1330b4aaa8 100644 --- a/reframe/utility/os_ext.py +++ b/reframe/utility/os_ext.py @@ -197,6 +197,14 @@ def mkstemp_path(*args, **kwargs): return path +def force_remove_file(filename): + """Remove filename ignoring errors if the file does not exist.""" + try: + os.remove(filename) + except FileNotFoundError: + pass + + class change_dir: """Context manager which changes the current working directory to the provided one.""" diff --git a/test_reframe.py b/test_reframe.py index 9422536756..75bca77b5e 100755 --- a/test_reframe.py +++ b/test_reframe.py @@ -1,8 +1,26 @@ #!/usr/bin/env python3 -# import unittest +import argparse import nose +import sys + +import unittests.fixtures as fixtures + if __name__ == '__main__': - # unittest.main() + parser = argparse.ArgumentParser( + add_help=False, + usage='%(prog)s [REFRAME_OPTIONS...] [NOSE_OPTIONS...]') + parser.add_argument('--rfm-user-config', action='store', metavar='FILE', + help='Config file to use for native unit tests.') + parser.add_argument('--rfm-help', action='help', + help='Print this help message and exit.') + + options, rem_args = parser.parse_known_args() + if options.rfm_user_config: + fixtures.set_user_config(options.rfm_user_config) + + fixtures.init_runtime() + + sys.argv = [sys.argv[0], *rem_args] nose.main() diff --git a/tutorial/advanced/advanced_example1.py b/tutorial/advanced/advanced_example1.py index 7549fbaf0a..55863c44fa 100644 --- a/tutorial/advanced/advanced_example1.py +++ b/tutorial/advanced/advanced_example1.py @@ -1,14 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest - -class MakefileTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('preprocessor_check', os.path.dirname(__file__), - **kwargs) +@rfm.simple_test +class MakefileTest(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the use of Makefiles ' 'and compile options') self.valid_systems = ['*'] @@ -21,7 +18,3 @@ def __init__(self, **kwargs): def compile(self): self.current_environ.cppflags = '-DMESSAGE' super().compile() - - -def _get_checks(**kwargs): - return [MakefileTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example2.py b/tutorial/advanced/advanced_example2.py index 73ab818489..0d18cef18c 100644 --- a/tutorial/advanced/advanced_example2.py +++ b/tutorial/advanced/advanced_example2.py @@ -1,14 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RunOnlyRegressionTest - -class RunOnlyTest(RunOnlyRegressionTest): - def __init__(self, **kwargs): - super().__init__('run_only_check', os.path.dirname(__file__), - **kwargs) +@rfm.simple_test +class ExampleRunOnlyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the class' 'RunOnlyRegressionTest') self.valid_systems = ['*'] @@ -24,7 +21,3 @@ def __init__(self, **kwargs): lower, upper) self.maintainers = ['put-your-name-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [RunOnlyTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example3.py b/tutorial/advanced/advanced_example3.py index d362c869d9..37db613229 100644 --- a/tutorial/advanced/advanced_example3.py +++ b/tutorial/advanced/advanced_example3.py @@ -1,14 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import CompileOnlyRegressionTest - -class CompileOnlyTest(CompileOnlyRegressionTest): - def __init__(self, **kwargs): - super().__init__('compile_only_check', os.path.dirname(__file__), - **kwargs) +@rfm.simple_test +class ExampleCompileOnlyTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the class' 'CompileOnlyRegressionTest') self.valid_systems = ['*'] @@ -16,7 +13,3 @@ def __init__(self, **kwargs): self.sanity_patterns = sn.assert_not_found('warning', self.stderr) self.maintainers = ['put-your-name-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [CompileOnlyTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example4.py b/tutorial/advanced/advanced_example4.py index 246eb717b3..00fcbae3b2 100644 --- a/tutorial/advanced/advanced_example4.py +++ b/tutorial/advanced/advanced_example4.py @@ -1,14 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest - -class EnvironmentVariableTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('env_variable_check', os.path.dirname(__file__), - **kwargs) +@rfm.simple_test +class EnvironmentVariableTest(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the use' 'of environment variables provided by loaded modules') self.valid_systems = ['daint:gpu'] @@ -22,7 +19,3 @@ def __init__(self, **kwargs): def compile(self): super().compile(makefile='Makefile_example4') - - -def _get_checks(**kwargs): - return [EnvironmentVariableTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example5.py b/tutorial/advanced/advanced_example5.py index 9eec719a95..359c73bbe4 100644 --- a/tutorial/advanced/advanced_example5.py +++ b/tutorial/advanced/advanced_example5.py @@ -1,14 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RunOnlyRegressionTest - -class TimeLimitTest(RunOnlyRegressionTest): - def __init__(self, **kwargs): - super().__init__('time_limit_check', os.path.dirname(__file__), - **kwargs) +@rfm.simple_test +class TimeLimitTest(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the use' 'of a user-defined time limit') self.valid_systems = ['daint:gpu', 'daint:mc'] @@ -20,7 +17,3 @@ def __init__(self, **kwargs): r'CANCELLED.*DUE TO TIME LIMIT', self.stderr) self.maintainers = ['put-your-name-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [TimeLimitTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example6.py b/tutorial/advanced/advanced_example6.py index 1718d60dcf..0b80085bae 100644 --- a/tutorial/advanced/advanced_example6.py +++ b/tutorial/advanced/advanced_example6.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RunOnlyRegressionTest -class DeferredIterationTest(RunOnlyRegressionTest): - def __init__(self, **kwargs): - super().__init__('deferred_iteration_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class DeferredIterationTest(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the use of deferred ' 'iteration via the `map` sanity function.') self.valid_systems = ['*'] @@ -20,7 +18,3 @@ def __init__(self, **kwargs): sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers))) self.maintainers = ['put-your-name-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [DeferredIterationTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example7.py b/tutorial/advanced/advanced_example7.py index 8effde78e6..2d6d35dfe1 100644 --- a/tutorial/advanced/advanced_example7.py +++ b/tutorial/advanced/advanced_example7.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RunOnlyRegressionTest -class PrerunDemoTest(RunOnlyRegressionTest): - def __init__(self, **kwargs): - super().__init__('prerun_demo_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class PrerunDemoTest(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() self.descr = ('ReFrame tutorial demonstrating the use of ' 'pre- and post-run commands') self.valid_systems = ['*'] @@ -22,10 +20,5 @@ def __init__(self, **kwargs): sn.all(sn.map(lambda x: sn.assert_bounded(x, 50, 80), numbers)), sn.assert_found('FINISHED', self.stdout) ]) - self.maintainers = ['put-your-name-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [PrerunDemoTest(**kwargs)] diff --git a/tutorial/advanced/advanced_example8.py b/tutorial/advanced/advanced_example8.py new file mode 100644 index 0000000000..6f4d8417f0 --- /dev/null +++ b/tutorial/advanced/advanced_example8.py @@ -0,0 +1,52 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.parameterized_test([('MPI',), ('OpenMP',)]) +class MatrixVectorTest(rfm.RegressionTest): + def __init__(self, variant): + super().__init__() + self.descr = 'Matrix-vector multiplication test (%s)' % variant + self.valid_systems = ['daint:gpu', 'daint:mc'] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', + 'PrgEnv-intel', 'PrgEnv-pgi'] + self.prgenv_flags = { + 'PrgEnv-cray': '-homp', + 'PrgEnv-gnu': '-fopenmp', + 'PrgEnv-intel': '-openmp', + 'PrgEnv-pgi': '-mp' + } + + if variant == 'MPI': + self.num_tasks = 8 + self.num_tasks_per_node = 2 + self.num_cpus_per_task = 4 + self.sourcepath = 'example_matrix_vector_multiplication_mpi_openmp.c' + elif variant == 'OpenMP': + self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' + self.num_cpus_per_task = 4 + + self.variables = { + 'OMP_NUM_THREADS': str(self.num_cpus_per_task) + } + matrix_dim = 1024 + iterations = 100 + self.executable_opts = [str(matrix_dim), str(iterations)] + + expected_norm = matrix_dim + found_norm = sn.extractsingle( + r'The L2 norm of the resulting vector is:\s+(?P\S+)', + self.stdout, 'norm', float) + self.sanity_patterns = sn.all([ + sn.assert_found( + r'time for single matrix vector multiplication', self.stdout), + sn.assert_lt(sn.abs(expected_norm - found_norm), 1.0e-6) + ]) + self.maintainers = ['you-can-type-your-email-here'] + self.tags = {'tutorial'} + + def compile(self): + if self.prgenv_flags is not None: + self.current_environ.cflags = self.prgenv_flags[self.current_environ.name] + + super().compile() diff --git a/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c b/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c new file mode 120000 index 0000000000..0c2c685c39 --- /dev/null +++ b/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c @@ -0,0 +1 @@ +../../src/example_matrix_vector_multiplication_mpi_openmp.c \ No newline at end of file diff --git a/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c b/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c new file mode 120000 index 0000000000..bfee770f9f --- /dev/null +++ b/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c @@ -0,0 +1 @@ +../../src/example_matrix_vector_multiplication_openmp.c \ No newline at end of file diff --git a/tutorial/example1.py b/tutorial/example1.py index 5887beee99..15964f15dd 100644 --- a/tutorial/example1.py +++ b/tutorial/example1.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class SerialTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example1_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example1Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Simple matrix-vector multiplication example' self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -17,7 +15,3 @@ def __init__(self, **kwargs): r'time for single matrix vector multiplication', self.stdout) self.maintainers = ['you-can-type-your-email-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [SerialTest(**kwargs)] diff --git a/tutorial/example2.py b/tutorial/example2.py index 7735b06090..3bf018faf6 100644 --- a/tutorial/example2.py +++ b/tutorial/example2.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class OpenMPTestIfElse(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example2a_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example2aTest(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication example with OpenMP' self.valid_systems = ['*'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', @@ -36,10 +34,10 @@ def compile(self): super().compile() -class OpenMPTestDict(RegressionTest): +@rfm.simple_test +class Example2bTest(rfm.RegressionTest): def __init__(self, **kwargs): - super().__init__('example2b_check', - os.path.dirname(__file__), **kwargs) + super().__init__() self.descr = 'Matrix-vector multiplication example with OpenMP' self.valid_systems = ['*'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', @@ -64,7 +62,3 @@ def compile(self): prgenv_flags = self.prgenv_flags[self.current_environ.name] self.current_environ.cflags = prgenv_flags super().compile() - - -def _get_checks(**kwargs): - return [OpenMPTestIfElse(**kwargs), OpenMPTestDict(**kwargs)] diff --git a/tutorial/example3.py b/tutorial/example3.py index 922b8b19ef..2efad40cd8 100644 --- a/tutorial/example3.py +++ b/tutorial/example3.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class MPITest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example3_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example3Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication example with MPI' self.valid_systems = ['daint:gpu', 'daint:mc'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', @@ -35,7 +33,3 @@ def compile(self): prgenv_flags = self.prgenv_flags[self.current_environ.name] self.current_environ.cflags = prgenv_flags super().compile() - - -def _get_checks(**kwargs): - return [MPITest(**kwargs)] diff --git a/tutorial/example4.py b/tutorial/example4.py index abe12a4c85..94f320c324 100644 --- a/tutorial/example4.py +++ b/tutorial/example4.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class OpenACCTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example4_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example4Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication example with OpenACC' self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] @@ -28,7 +26,3 @@ def compile(self): prgenv_flags = self.prgenv_flags[self.current_environ.name] self.current_environ.cflags = prgenv_flags super().compile() - - -def _get_checks(**kwargs): - return [OpenACCTest(**kwargs)] diff --git a/tutorial/example5.py b/tutorial/example5.py index acc1de6c90..6c4827e608 100644 --- a/tutorial/example5.py +++ b/tutorial/example5.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class CudaTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example5_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example5Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication example with CUDA' self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-pgi'] @@ -19,7 +17,3 @@ def __init__(self, **kwargs): r'time for single matrix vector multiplication', self.stdout) self.maintainers = ['you-can-type-your-email-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [CudaTest(**kwargs)] diff --git a/tutorial/example6.py b/tutorial/example6.py index bdd4f31d57..74103690e2 100644 --- a/tutorial/example6.py +++ b/tutorial/example6.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class SerialNormTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example6_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example6Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication with L2 norm check' self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -28,7 +26,3 @@ def __init__(self, **kwargs): ]) self.maintainers = ['you-can-type-your-email-here'] self.tags = {'tutorial'} - - -def _get_checks(**kwargs): - return [SerialNormTest(**kwargs)] diff --git a/tutorial/example7.py b/tutorial/example7.py index 7d2f28b46b..d279046655 100644 --- a/tutorial/example7.py +++ b/tutorial/example7.py @@ -1,13 +1,11 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class CudaPerfTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('example7_check', - os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class Example7Test(rfm.RegressionTest): + def __init__(self): + super().__init__() self.descr = 'Matrix-vector multiplication (CUDA performance test)' self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-pgi'] @@ -32,7 +30,3 @@ def __init__(self, **kwargs): def compile(self): self.current_environ.cxxflags = '-O3' super().compile() - - -def _get_checks(**kwargs): - return [CudaPerfTest(**kwargs)] diff --git a/tutorial/example8.py b/tutorial/example8.py index a1034838db..939c4a3acc 100644 --- a/tutorial/example8.py +++ b/tutorial/example8.py @@ -1,13 +1,10 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class BaseMatrixVectorTest(RegressionTest): - def __init__(self, test_version, **kwargs): - super().__init__('example8_' + test_version.lower() + '_check', - os.path.dirname(__file__), **kwargs) +class BaseMatrixVectorTest(rfm.RegressionTest): + def __init__(self, test_version): + super().__init__() self.descr = '%s matrix-vector multiplication' % test_version self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -36,15 +33,17 @@ def compile(self): super().compile() +@rfm.simple_test class SerialTest(BaseMatrixVectorTest): - def __init__(self, **kwargs): - super().__init__('Serial', **kwargs) + def __init__(self): + super().__init__('Serial') self.sourcepath = 'example_matrix_vector_multiplication.c' +@rfm.simple_test class OpenMPTest(BaseMatrixVectorTest): - def __init__(self, **kwargs): - super().__init__('OpenMP', **kwargs) + def __init__(self): + super().__init__('OpenMP') self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi'] @@ -59,9 +58,10 @@ def __init__(self, **kwargs): } +@rfm.simple_test class MPITest(BaseMatrixVectorTest): - def __init__(self, **kwargs): - super().__init__('MPI', **kwargs) + def __init__(self): + super().__init__('MPI') self.valid_systems = ['daint:gpu', 'daint:mc'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi'] @@ -80,9 +80,10 @@ def __init__(self, **kwargs): } +@rfm.simple_test class OpenACCTest(BaseMatrixVectorTest): - def __init__(self, **kwargs): - super().__init__('OpenACC', **kwargs) + def __init__(self): + super().__init__('OpenACC') self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] self.sourcepath = 'example_matrix_vector_multiplication_openacc.c' @@ -94,16 +95,12 @@ def __init__(self, **kwargs): } +@rfm.simple_test class CudaTest(BaseMatrixVectorTest): - def __init__(self, **kwargs): - super().__init__('CUDA', **kwargs) + def __init__(self): + super().__init__('CUDA') self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-pgi'] self.sourcepath = 'example_matrix_vector_multiplication_cuda.cu' self.modules = ['cudatoolkit'] self.num_gpus_per_node = 1 - - -def _get_checks(**kwargs): - return [SerialTest(**kwargs), OpenMPTest(**kwargs), MPITest(**kwargs), - OpenACCTest(**kwargs), CudaTest(**kwargs)] diff --git a/unittests/fixtures.py b/unittests/fixtures.py index e7d55a0d77..6d0590652d 100644 --- a/unittests/fixtures.py +++ b/unittests/fixtures.py @@ -4,150 +4,68 @@ import os import tempfile -import reframe.frontend.config as config -from reframe.core.modules import (get_modules_system, - init_modules_system, NoModImpl) +import reframe.core.config as config +import reframe.core.modules as modules +import reframe.core.runtime as rt +from reframe.core.exceptions import UnknownSystemError + TEST_RESOURCES = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'resources') +TEST_RESOURCES_CHECKS = os.path.join(TEST_RESOURCES, 'checks') TEST_MODULES = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'modules') -TEST_SITE_CONFIG = { - 'systems': { - 'testsys': { - 'descr': 'Fake system for unit tests', - 'hostnames': ['testsys'], - 'prefix': '/foo/bar', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'resources': {}, - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'], - 'descr': 'Login nodes' - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': [], - 'resources': { - 'gpu': ['--gres=gpu:{num_gpus_per_node}'], - 'datawarp': [ - '#DW jobdw capacity={capacity}', - '#DW stage_in source={stagein_src}' - ] - }, - 'access': [], - 'environs': ['PrgEnv-gnu', 'builtin-gcc'], - 'descr': 'GPU partition', - } - } - } - }, - - 'environments': { - 'testsys:login': { - 'PrgEnv-gnu': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - }, - }, - '*': { - 'PrgEnv-gnu': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-cray': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-cray'], - }, - - 'builtin-gcc': { - 'type': 'ProgEnvironment', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - } -} - - -def init_native_modules_system(): - init_modules_system(HOST.modules_system if HOST else None) - - -# Guess current system and initialize its modules system -_config_file = os.getenv('RFM_CONFIG_FILE', 'reframe/settings.py') -settings = config.load_from_file(_config_file) -_site_config = config.SiteConfiguration() -_site_config.load_from_dict(settings.site_configuration) -HOST = config.autodetect_system(_site_config) -init_native_modules_system() - - -def get_test_config(): - """Get a regression tests setup configuration. - - Returns a tuple of system, partition and environment that you can pass to - `RegressionTest`'s setup method. - """ - site_config = config.SiteConfiguration() - site_config.load_from_dict(TEST_SITE_CONFIG) - system = site_config.systems['testsys'] - partition = system.partition('gpu') - environ = partition.environment('builtin-gcc') - return (system, partition, environ) +# Unit tests site configuration +TEST_SITE_CONFIG = None +# User supplied configuration file and site configuration +USER_CONFIG_FILE = None +USER_SITE_CONFIG = None -def generate_test_config(filename=None, - template='unittests/resources/settings_unittests.tmpl', - **subst): - if not filename: - with tempfile.NamedTemporaryFile(delete=False, suffix='.py') as fp: - filename = fp.name - if not 'modules_system' in subst: - subst['modules_system'] = None +def set_user_config(config_file): + global USER_CONFIG_FILE, USER_SITE_CONFIG - if not 'logfile' in subst: - with tempfile.NamedTemporaryFile(delete=False, suffix='.log') as fp: - subst['logfile'] = fp.name + USER_CONFIG_FILE = config_file + user_settings = config.load_settings_from_file(config_file) + USER_SITE_CONFIG = user_settings.site_configuration - with open(filename, 'w') as fw, open(template) as fr: - fw.write(fr.read().format(**subst)) - return filename, subst +def init_runtime(): + global TEST_SITE_CONFIG + settings = config.load_settings_from_file( + 'unittests/resources/settings.py') + TEST_SITE_CONFIG = settings.site_configuration + rt.init_runtime(TEST_SITE_CONFIG, 'generic') -def force_remove_file(filename): - try: - os.remove(filename) - except FileNotFoundError: - pass + +def switch_to_user_runtime(fn): + """Decorator to switch to the user supplied configuration. + + If no such configuration exists, this decorator returns the target function + untouched. + """ + if USER_SITE_CONFIG is None: + return fn + + return rt.switch_runtime(USER_SITE_CONFIG)(fn) # FIXME: This may conflict in the unlikely situation that a user defines a # system named `kesch` with a partition named `pn`. -def partition_with_scheduler(name, skip_partitions=['kesch:pn']): - """Retrieve a partition from the current system whose registered name is - ``name``. +def partition_with_scheduler(name=None, skip_partitions=['kesch:pn']): + """Retrieve a system partition from the runtime whose scheduler is registered + with ``name``. If ``name`` is :class:`None`, any partition with a non-local scheduler will be returned. Partitions specified in ``skip_partitions`` will be skipped from searching. """ - if HOST is None: - return None - - for p in HOST.partitions: + system = rt.runtime().system + for p in system.partitions: if p.fullname in skip_partitions: continue @@ -161,4 +79,5 @@ def partition_with_scheduler(name, skip_partitions=['kesch:pn']): def has_sane_modules_system(): - return not isinstance(get_modules_system().backend, NoModImpl) + return not isinstance(rt.runtime().modules_system.backend, + modules.NoModImpl) diff --git a/unittests/resources/badchecks/badargs.py b/unittests/resources/badchecks/badargs.py deleted file mode 100644 index 717f8f2520..0000000000 --- a/unittests/resources/badchecks/badargs.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - -from reframe.core.pipeline import RegressionTest - - -class EmptyTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('emptycheck', os.path.dirname(__file__), **kwargs) - - -def _get_checks(): - return [EmptyTest()] diff --git a/unittests/resources/badchecks/badentry.py b/unittests/resources/checks/bad/badentry.py similarity index 100% rename from unittests/resources/badchecks/badentry.py rename to unittests/resources/checks/bad/badentry.py diff --git a/unittests/resources/badchecks/invalid_check.py b/unittests/resources/checks/bad/invalid_check.py similarity index 100% rename from unittests/resources/badchecks/invalid_check.py rename to unittests/resources/checks/bad/invalid_check.py diff --git a/unittests/resources/badchecks/invalid_entry.py b/unittests/resources/checks/bad/invalid_entry.py similarity index 100% rename from unittests/resources/badchecks/invalid_entry.py rename to unittests/resources/checks/bad/invalid_entry.py diff --git a/unittests/resources/badchecks/invalid_iterable.py b/unittests/resources/checks/bad/invalid_iterable.py similarity index 100% rename from unittests/resources/badchecks/invalid_iterable.py rename to unittests/resources/checks/bad/invalid_iterable.py diff --git a/unittests/resources/badchecks/invalid_return.py b/unittests/resources/checks/bad/invalid_return.py similarity index 100% rename from unittests/resources/badchecks/invalid_return.py rename to unittests/resources/checks/bad/invalid_return.py diff --git a/unittests/resources/badchecks/noentry.py b/unittests/resources/checks/bad/noentry.py similarity index 100% rename from unittests/resources/badchecks/noentry.py rename to unittests/resources/checks/bad/noentry.py diff --git a/unittests/resources/badchecks/notacheck.py b/unittests/resources/checks/bad/notacheck.py similarity index 100% rename from unittests/resources/badchecks/notacheck.py rename to unittests/resources/checks/bad/notacheck.py diff --git a/unittests/resources/checks/emptycheck.py b/unittests/resources/checks/emptycheck.py new file mode 100644 index 0000000000..7b479975c0 --- /dev/null +++ b/unittests/resources/checks/emptycheck.py @@ -0,0 +1,6 @@ +import reframe as rfm + + +@rfm.simple_test +class EmptyTest(rfm.RegressionTest): + pass diff --git a/unittests/resources/frontend_checks.py b/unittests/resources/checks/frontend_checks.py similarity index 71% rename from unittests/resources/frontend_checks.py rename to unittests/resources/checks/frontend_checks.py index e453117242..0c1efa9736 100644 --- a/unittests/resources/frontend_checks.py +++ b/unittests/resources/checks/frontend_checks.py @@ -2,26 +2,25 @@ # Special checks for testing the front-end # -import os - +import reframe as rfm import reframe.utility.sanity as sn from reframe.core.exceptions import ReframeError, SanityError -from reframe.core.pipeline import RunOnlyRegressionTest -class BaseFrontendCheck(RunOnlyRegressionTest): - def __init__(self, name, **kwargs): - super().__init__(name, os.path.dirname(__file__), **kwargs) +class BaseFrontendCheck(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() self.local = True self.executable = 'echo hello && echo perf: 10' self.sanity_patterns = sn.assert_found('hello', self.stdout) - self.tags = {self.name} + self.tags = {type(self).__name__} self.maintainers = ['VK'] +@rfm.simple_test class BadSetupCheck(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -30,9 +29,10 @@ def setup(self, system, environ, **job_opts): raise ReframeError('Setup failure') +@rfm.simple_test class BadSetupCheckEarly(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.local = False @@ -41,29 +41,33 @@ def setup(self, system, environ, **job_opts): raise ReframeError('Setup failure') +@rfm.simple_test class NoSystemCheck(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_prog_environs = ['*'] +@rfm.simple_test class NoPrgEnvCheck(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] +@rfm.simple_test class SanityFailureCheck(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.sanity_patterns = sn.assert_found('foo', self.stdout) +@rfm.simple_test class PerformanceFailureCheck(BaseFrontendCheck): - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.perf_patterns = { @@ -76,11 +80,12 @@ def __init__(self, **kwargs): } +@rfm.simple_test class CustomPerformanceFailureCheck(BaseFrontendCheck): """Simulate a performance check that ignores completely logging""" - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.strict_check = False @@ -92,8 +97,8 @@ def check_performance(self): class KeyboardInterruptCheck(BaseFrontendCheck): """Simulate keyboard interrupt during test's execution.""" - def __init__(self, phase='wait', **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self, phase='wait'): + super().__init__() self.executable = 'sleep 1' self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -116,8 +121,8 @@ def wait(self): class SystemExitCheck(BaseFrontendCheck): """Simulate system exit from within a check.""" - def __init__(self, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self): + super().__init__() self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -129,8 +134,8 @@ def wait(self): class SleepCheck(BaseFrontendCheck): _next_id = 0 - def __init__(self, sleep_time, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self, sleep_time): + super().__init__() self.name = '%s_%s' % (self.name, SleepCheck._next_id) self.sourcesdir = None self.sleep_time = sleep_time @@ -150,9 +155,8 @@ def __init__(self, sleep_time, **kwargs): class RetriesCheck(BaseFrontendCheck): - - def __init__(self, run_to_pass, filename, **kwargs): - super().__init__(type(self).__name__, **kwargs) + def __init__(self, run_to_pass, filename): + super().__init__() self.sourcesdir = None self.valid_systems = ['*'] self.valid_prog_environs = ['*'] @@ -161,13 +165,3 @@ def __init__(self, run_to_pass, filename, **kwargs): self.post_run = ['((current_run++))', 'echo $current_run > %s' % filename] self.sanity_patterns = sn.assert_found('%d' % run_to_pass, self.stdout) - - -def _get_checks(**kwargs): - return [BadSetupCheck(**kwargs), - BadSetupCheckEarly(**kwargs), - NoSystemCheck(**kwargs), - NoPrgEnvCheck(**kwargs), - SanityFailureCheck(**kwargs), - PerformanceFailureCheck(**kwargs), - CustomPerformanceFailureCheck(**kwargs)] diff --git a/unittests/resources/hellocheck.py b/unittests/resources/checks/hellocheck.py similarity index 59% rename from unittests/resources/hellocheck.py rename to unittests/resources/checks/hellocheck.py index 33568c2f63..c4911593e5 100644 --- a/unittests/resources/hellocheck.py +++ b/unittests/resources/checks/hellocheck.py @@ -1,12 +1,12 @@ -import os - +import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.pipeline import RegressionTest -class HelloTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('hellocheck', os.path.dirname(__file__), **kwargs) +@rfm.simple_test +class HelloTest(rfm.RegressionTest): + def __init__(self): + super().__init__() + self.name = 'hellocheck' self.descr = 'C Hello World test' # All available systems are supported @@ -16,7 +16,3 @@ def __init__(self, **kwargs): self.tags = {'foo', 'bar'} self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self.maintainers = ['VK'] - - -def _get_checks(**kwargs): - return [HelloTest(**kwargs)] diff --git a/unittests/resources/hellocheck_copy.py b/unittests/resources/checks/hellocheck_copy.py similarity index 100% rename from unittests/resources/hellocheck_copy.py rename to unittests/resources/checks/hellocheck_copy.py diff --git a/unittests/resources/hellocheck_make.py b/unittests/resources/checks/hellocheck_make.py similarity index 95% rename from unittests/resources/hellocheck_make.py rename to unittests/resources/checks/hellocheck_make.py index d66a75d4a1..fb42fb5222 100644 --- a/unittests/resources/hellocheck_make.py +++ b/unittests/resources/checks/hellocheck_make.py @@ -1,3 +1,7 @@ +# +# We purposely use the old syntax here +# + import os import reframe.utility.sanity as sn diff --git a/unittests/resources/src/Makefile b/unittests/resources/checks/src/Makefile similarity index 100% rename from unittests/resources/src/Makefile rename to unittests/resources/checks/src/Makefile diff --git a/unittests/resources/src/Makefile.nofort b/unittests/resources/checks/src/Makefile.nofort similarity index 100% rename from unittests/resources/src/Makefile.nofort rename to unittests/resources/checks/src/Makefile.nofort diff --git a/unittests/resources/src/compiler_failure.c b/unittests/resources/checks/src/compiler_failure.c similarity index 100% rename from unittests/resources/src/compiler_failure.c rename to unittests/resources/checks/src/compiler_failure.c diff --git a/unittests/resources/src/compiler_warning.c b/unittests/resources/checks/src/compiler_warning.c similarity index 100% rename from unittests/resources/src/compiler_warning.c rename to unittests/resources/checks/src/compiler_warning.c diff --git a/unittests/resources/src/hello.c b/unittests/resources/checks/src/hello.c similarity index 100% rename from unittests/resources/src/hello.c rename to unittests/resources/checks/src/hello.c diff --git a/unittests/resources/src/hello.cpp b/unittests/resources/checks/src/hello.cpp similarity index 100% rename from unittests/resources/src/hello.cpp rename to unittests/resources/checks/src/hello.cpp diff --git a/unittests/resources/src/hello.f90 b/unittests/resources/checks/src/hello.f90 similarity index 100% rename from unittests/resources/src/hello.f90 rename to unittests/resources/checks/src/hello.f90 diff --git a/unittests/resources/src/hello.sh b/unittests/resources/checks/src/hello.sh similarity index 100% rename from unittests/resources/src/hello.sh rename to unittests/resources/checks/src/hello.sh diff --git a/unittests/resources/src/homer.txt b/unittests/resources/checks/src/homer.txt similarity index 100% rename from unittests/resources/src/homer.txt rename to unittests/resources/checks/src/homer.txt diff --git a/unittests/resources/src/sleep_deeply.sh b/unittests/resources/checks/src/sleep_deeply.sh similarity index 100% rename from unittests/resources/src/sleep_deeply.sh rename to unittests/resources/checks/src/sleep_deeply.sh diff --git a/unittests/resources/checks_unlisted/good.py b/unittests/resources/checks_unlisted/good.py new file mode 100644 index 0000000000..211aee9507 --- /dev/null +++ b/unittests/resources/checks_unlisted/good.py @@ -0,0 +1,50 @@ +# +# New-style checks for testing the registration decorators +# + +import reframe as rfm + +# We just import this individually for testing purposes +from reframe.core.pipeline import RegressionTest + + +@rfm.parameterized_test((x, y) for x in range(3) for y in range(2)) +class MyBaseTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + def __eq__(self, other): + """This is just for unit tests for convenience.""" + if not isinstance(other, MyBaseTest): + return NotImplemented + + return self.a == other.a and self.b == other.b + + def __repr__(self): + return 'MyBaseTest(%s, %s)' % (self.a, self.b) + + +@rfm.parameterized_test({'a': x, 'b': y} for x in range(3) for y in range(2)) +class AnotherBaseTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + def __eq__(self, other): + """This is just for unit tests for convenience.""" + if not isinstance(other, AnotherBaseTest): + return NotImplemented + + return self.a == other.a and self.b == other.b + + def __repr__(self): + return 'AnotherBaseTest(%s, %s)' % (self.a, self.b) + + +@rfm.simple_test +class MyTest(MyBaseTest): + def __init__(self): + super().__init__(10, 20) diff --git a/unittests/resources/checks_unlisted/kbd_interrupt.py b/unittests/resources/checks_unlisted/kbd_interrupt.py new file mode 100644 index 0000000000..1a0e82c48d --- /dev/null +++ b/unittests/resources/checks_unlisted/kbd_interrupt.py @@ -0,0 +1,22 @@ +# +# A special check to simulate a keyboard interrupt +# +# The reason this test is in a different file is just for being loaded by the +# CLI unit tests exclusively. +# + +import reframe as rfm + + +@rfm.simple_test +class KeyboardInterruptCheck(rfm.RunOnlyRegressionTest): + def __init__(self, **kwargs): + super().__init__() + self.local = True + self.executable = 'sleep 1' + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.tags = {self.name} + + def setup(self, system, environ, **job_opts): + raise KeyboardInterrupt diff --git a/unittests/resources/checks_unlisted/mixed.py b/unittests/resources/checks_unlisted/mixed.py new file mode 100644 index 0000000000..298a377016 --- /dev/null +++ b/unittests/resources/checks_unlisted/mixed.py @@ -0,0 +1,28 @@ +import os + +import reframe.core.decorators as deco +from reframe.core.pipeline import RegressionTest + + +@deco.parameterized_test((x, y) for x in range(3) for y in range(2)) +class MyBaseTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + +@deco.simple_test +class MyTest(MyBaseTest): + def __init__(self): + super().__init__(10, 20) + + +class OldStyleTest(MyBaseTest): + def __init__(self, **kwargs): + super().__init__('old_style_test', + os.path.dirname(__file__), **kwargs) + + +def _get_checks(**kwargs): + return [OldStyleTest(**kwargs)] diff --git a/unittests/resources/emptycheck.py b/unittests/resources/emptycheck.py deleted file mode 100644 index 642c140d14..0000000000 --- a/unittests/resources/emptycheck.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - -from reframe.core.pipeline import RegressionTest - - -class EmptyTest(RegressionTest): - def __init__(self, **kwargs): - super().__init__('emptycheck', os.path.dirname(__file__), **kwargs) - - -def _get_checks(**kwargs): - return [EmptyTest(**kwargs)] diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py new file mode 100644 index 0000000000..4740b52b7b --- /dev/null +++ b/unittests/resources/settings.py @@ -0,0 +1,156 @@ +# +# ReFrame settings for use in the unit tests +# + + +class ReframeSettings: + _reframe_module = 'reframe' + _job_poll_intervals = [1, 2, 3] + _job_submit_timeout = 60 + _checks_path = ['checks/'] + _checks_path_recurse = True + _site_configuration = { + 'systems': { + # Generic system configuration that allows to run ReFrame locally + # on any system. + 'generic': { + 'descr': 'Generic example system', + 'hostnames': ['localhost'], + 'partitions': { + 'login': { + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': ['builtin-gcc'], + 'descr': 'Login nodes' + }, + } + }, + 'testsys': { + # A fake system simulating a possible cluster configuration, in + # order to test different aspects of the framework. + 'descr': 'Fake system for unit tests', + 'hostnames': ['testsys'], + 'prefix': '.rfm_testing/install', + 'resourcesdir': '.rfm_testing/resources', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'resources': {}, + 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'], + 'descr': 'Login nodes' + }, + 'gpu': { + 'scheduler': 'nativeslurm', + 'modules': [], + 'resources': { + 'gpu': ['--gres=gpu:{num_gpus_per_node}'], + 'datawarp': [ + '#DW jobdw capacity={capacity}', + '#DW stage_in source={stagein_src}' + ] + }, + 'access': [], + 'environs': ['PrgEnv-gnu', 'builtin-gcc'], + 'descr': 'GPU partition', + } + } + } + }, + 'environments': { + 'testsys:login': { + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu'], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + }, + }, + '*': { + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu'], + }, + 'PrgEnv-cray': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-cray'], + }, + 'builtin': { + 'type': 'ProgEnvironment', + 'cc': 'cc', + 'cxx': '', + 'ftn': '', + }, + 'builtin-gcc': { + 'type': 'ProgEnvironment', + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + } + } + }, + 'modes': { + '*': { + 'unittest': [ + '-c', 'unittests/resources/checks/hellocheck.py', + '-p', 'builtin-gcc', + '--force-local' + ] + } + } + } + + _logging_config = { + 'level': 'DEBUG', + 'handlers': { + '.reframe_unittest.log': { + 'level': 'DEBUG', + 'format': ('[%(asctime)s] %(levelname)s: ' + '%(check_name)s: %(message)s'), + 'datefmt': '%FT%T', + 'append': False, + }, + '&1': { + 'level': 'INFO', + 'format': '%(message)s' + }, + } + } + + @property + def version(self): + return self._version + + @property + def reframe_module(self): + return self._reframe_module + + @property + def job_poll_intervals(self): + return self._job_poll_intervals + + @property + def job_submit_timeout(self): + return self._job_submit_timeout + + @property + def checks_path(self): + return self._checks_path + + @property + def checks_path_recurse(self): + return self._checks_path_recurse + + @property + def site_configuration(self): + return self._site_configuration + + @property + def logging_config(self): + return self._logging_config + + +settings = ReframeSettings() diff --git a/unittests/resources/settings_unittests.tmpl b/unittests/resources/settings_unittests.tmpl deleted file mode 100644 index e944825a72..0000000000 --- a/unittests/resources/settings_unittests.tmpl +++ /dev/null @@ -1,111 +0,0 @@ -# -# ReFrame settings template for use in the unit tests -# - - -class ReframeSettings: - _reframe_module = 'reframe' - _job_poll_intervals = [1, 2, 3] - _job_submit_timeout = 60 - _checks_path = ['checks/'] - _checks_path_recurse = True - _site_configuration = {{ - 'systems': {{ - # Generic system used also in unit tests - 'generic': {{ - 'descr': 'Generic example system', - 'modules_system': {modules_system}, - - # Adjust to your system's hostname - 'hostnames': ['localhost'], - 'partitions': {{ - 'login': {{ - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - }} - }} - }} - }}, - - 'environments': {{ - '*': {{ - 'builtin': {{ - 'type': 'ProgEnvironment', - 'cc': 'cc', - 'cxx': '', - 'ftn': '', - }}, - - 'builtin-gcc': {{ - 'type': 'ProgEnvironment', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - }} - }} - }}, - 'modes': {{ - '*': {{ - 'unittest': [ - '-c', 'unittests/resources/hellocheck.py', - '-p', 'builtin-gcc', - '--force-local' - ] - }} - }} - }} - - _logging_config = {{ - 'level': 'DEBUG', - 'handlers': {{ - '{logfile}': {{ - 'level': 'DEBUG', - 'format': ('[%(asctime)s] %(levelname)s: ' - '%(check_name)s: %(message)s'), - 'datefmt': '%FT%T', - 'append': False, - }}, - '&1': {{ - 'level': 'INFO', - 'format': '%(message)s' - }}, - }} - }} - - @property - def version(self): - return self._version - - @property - def reframe_module(self): - return self._reframe_module - - @property - def job_poll_intervals(self): - return self._job_poll_intervals - - @property - def job_submit_timeout(self): - return self._job_submit_timeout - - @property - def checks_path(self): - return self._checks_path - - @property - def checks_path_recurse(self): - return self._checks_path_recurse - - @property - def site_configuration(self): - return self._site_configuration - - @property - def logging_config(self): - return self._logging_config - - -settings = ReframeSettings() diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 351680feff..d267d65436 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -9,32 +9,34 @@ from contextlib import redirect_stdout, redirect_stderr from io import StringIO +import reframe.core.config as config +import reframe.core.runtime as rt +import reframe.utility.os_ext as os_ext import unittests.fixtures as fixtures -import reframe.frontend.config as config from reframe.core.environments import EnvironmentSnapshot -from reframe.core.modules import init_modules_system def run_command_inline(argv, funct, *args, **kwargs): + # Save current execution context argv_save = sys.argv environ_save = EnvironmentSnapshot() - captured_stdout = StringIO() - captured_stderr = StringIO() sys.argv = argv exitcode = None - print(' '.join(argv)) + + captured_stdout = StringIO() + captured_stderr = StringIO() + print(sys.argv) with redirect_stdout(captured_stdout): with redirect_stderr(captured_stderr): try: - exitcode = funct(*args, **kwargs) + with rt.temp_runtime(None): + exitcode = funct(*args, **kwargs) except SystemExit as e: exitcode = e.code finally: - # restore environment, command-line arguments, and the native - # modules system + # Restore execution context environ_save.load() sys.argv = argv_save - fixtures.init_native_modules_system() return (exitcode, captured_stdout.getvalue(), @@ -75,24 +77,20 @@ def argv(self): def setUp(self): self.prefix = tempfile.mkdtemp(dir='unittests') - self.config_file = 'custom_settings.py' self.system = 'generic:login' - self.checkpath = ['unittests/resources/hellocheck.py'] + self.checkpath = ['unittests/resources/checks/hellocheck.py'] self.environs = ['builtin-gcc'] self.local = True self.action = 'run' self.more_options = [] self.mode = None - self.config_file, subst = fixtures.generate_test_config() - self.logfile = subst['logfile'] - self.delete_config_file = True + self.config_file = 'unittests/resources/settings.py' + self.logfile = '.reframe_unittest.log' self.ignore_check_conflicts = True def tearDown(self): shutil.rmtree(self.prefix) - os.remove(self.logfile) - if self.delete_config_file: - os.remove(self.config_file) + os_ext.force_remove_file(self.logfile) def _run_reframe(self): import reframe.frontend.cli as cli @@ -131,25 +129,17 @@ def test_check_success(self): self.assertEqual(0, returncode) self.assert_log_file_is_saved() - @unittest.skipIf(not fixtures.partition_with_scheduler(None), - 'job submission not supported') + @fixtures.switch_to_user_runtime def test_check_submit_success(self): # This test will run on the auto-detected system - system = fixtures.HOST - partition = fixtures.partition_with_scheduler(None) - init_modules_system(system.modules_system) + partition = fixtures.partition_with_scheduler() + if not partition: + self.skipTest('job submission not supported') + self.config_file = fixtures.USER_CONFIG_FILE self.local = False self.system = partition.fullname - # Use the system config file here - # - # FIXME: This whole thing is quite hacky; we definitely need to - # redesign the fixtures. It is also not equivalent to the previous - # version, which monkey-patched the logging settings. - self.config_file = os.getenv('RFM_CONFIG_FILE', 'reframe/settings.py') - self.delete_config_file = False - # pick up the programming environment of the partition self.environs = [partition.environs[0].name] @@ -159,7 +149,7 @@ def test_check_submit_success(self): self.assertEqual(0, returncode) def test_check_failure(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['-t', 'BadSetupCheck'] returncode, stdout, _ = self._run_reframe() @@ -167,7 +157,7 @@ def test_check_failure(self): self.assertNotEqual(returncode, 0) def test_check_setup_failure(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['-t', 'BadSetupCheckEarly'] self.local = False @@ -177,10 +167,10 @@ def test_check_setup_failure(self): self.assertIn('FAILED', stdout) self.assertNotEqual(returncode, 0) - # FIXME: The following test is temporarily disabled. It should be - # re-enabled with the unit tests refactoring. - def _test_check_kbd_interrupt(self): - self.checkpath = ['unittests/resources/kbd_interrupt_check.py'] + def test_check_kbd_interrupt(self): + self.checkpath = [ + 'unittests/resources/checks_unlisted/kbd_interrupt.py' + ] self.more_options = ['-t', 'KeyboardInterruptCheck'] self.local = False @@ -191,7 +181,7 @@ def _test_check_kbd_interrupt(self): self.assertNotEqual(returncode, 0) def test_check_sanity_failure(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['-t', 'SanityFailureCheck'] returncode, stdout, stderr = self._run_reframe() @@ -205,7 +195,7 @@ def test_check_sanity_failure(self): ['login'], self.environs)) def test_performance_check_failure(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['-t', 'PerformanceFailureCheck'] returncode, stdout, stderr = self._run_reframe() @@ -221,13 +211,13 @@ def test_performance_check_failure(self): ['login'])) def test_skip_system_check_option(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['--skip-system-check', '-t', 'NoSystemCheck'] returncode, stdout, _ = self._run_reframe() self.assertIn('PASSED', stdout) def test_skip_prgenv_check_option(self): - self.checkpath = ['unittests/resources/frontend_checks.py'] + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['--skip-prgenv-check', '-t', 'NoPrgEnvCheck'] returncode, stdout, _ = self._run_reframe() self.assertIn('PASSED', stdout) @@ -250,6 +240,8 @@ def test_unknown_system(self): self.system = 'foo' self.checkpath = [] returncode, stdout, stderr = self._run_reframe() + print(stdout) + print(stderr) self.assertNotIn('Traceback', stdout) self.assertNotIn('Traceback', stderr) self.assertEqual(1, returncode) @@ -306,14 +298,8 @@ def test_execution_modes(self): self.assertIn('PASSED', stdout) self.assertIn('Ran 1 test case', stdout) - def test_unknown_modules_system(self): - fixtures.generate_test_config( - self.config_file, logfile=self.logfile, modules_system="'foo'") - returncode, stdout, stderr = self._run_reframe() - self.assertNotEqual(0, returncode) - def test_no_ignore_check_conflicts(self): - self.checkpath = ['unittests/resources'] + self.checkpath = ['unittests/resources/checks'] self.more_options = ['-R'] self.ignore_check_conflicts = False self.action = 'list' diff --git a/unittests/test_config.py b/unittests/test_config.py new file mode 100644 index 0000000000..396c760089 --- /dev/null +++ b/unittests/test_config.py @@ -0,0 +1,142 @@ +import copy +import unittest + +import reframe.core.config as config +import unittests.fixtures as fixtures +from reframe.core.exceptions import ConfigError + + +class TestSiteConfigurationFromDict(unittest.TestCase): + def setUp(self): + self.site_config = config.SiteConfiguration() + self.dict_config = copy.deepcopy(fixtures.TEST_SITE_CONFIG) + + def get_partition(self, system, name): + for p in system.partitions: + if p.name == name: + return p + + def test_load_success(self): + self.site_config.load_from_dict(self.dict_config) + self.assertEqual(2, len(self.site_config.systems)) + + system = self.site_config.systems['testsys'] + self.assertEqual(2, len(system.partitions)) + + part_login = self.get_partition(system, 'login') + part_gpu = self.get_partition(system, 'gpu') + self.assertIsNotNone(part_login) + self.assertIsNotNone(part_gpu) + self.assertEqual('testsys:login', part_login.fullname) + self.assertEqual('testsys:gpu', part_gpu.fullname) + self.assertEqual(3, len(part_login.environs)) + self.assertEqual(2, len(part_gpu.environs)) + + # Check that PrgEnv-gnu on login partition is resolved to the special + # version defined in the 'dom:login' section + env_login = part_login.environment('PrgEnv-gnu') + self.assertEqual('gcc', env_login.cc) + self.assertEqual('g++', env_login.cxx) + self.assertEqual('gfortran', env_login.ftn) + + # Check that the PrgEnv-gnu of the gpu partition is resolved to the + # default one + env_gpu = part_gpu.environment('PrgEnv-gnu') + self.assertEqual('cc', env_gpu.cc) + self.assertEqual('CC', env_gpu.cxx) + self.assertEqual('ftn', env_gpu.ftn) + + # Check resource instantiation + self.assertEqual(['--gres=gpu:16'], + part_gpu.get_resource('gpu', num_gpus_per_node=16)) + self.assertEqual(['#DW jobdw capacity=100GB', + '#DW stage_in source=/foo'], + part_gpu.get_resource('datawarp', + capacity='100GB', + stagein_src='/foo')) + + def test_load_failure_empty_dict(self): + dict_config = {} + self.assertRaises(ValueError, + self.site_config.load_from_dict, dict_config) + + def test_load_failure_no_environments(self): + dict_config = {'systems': {}} + self.assertRaises(ValueError, + self.site_config.load_from_dict, dict_config) + + def test_load_failure_no_systems(self): + dict_config = {'environments': {}} + self.assertRaises(ValueError, + self.site_config.load_from_dict, dict_config) + + def test_load_failure_environments_no_scoped_dict(self): + self.dict_config['environments'] = { + 'testsys': 'PrgEnv-gnu' + } + self.assertRaises(TypeError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_partitions_nodict(self): + self.dict_config['systems']['testsys']['partitions'] = ['gpu'] + self.assertRaises(ConfigError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_systems_nodict(self): + self.dict_config['systems']['testsys'] = ['gpu'] + self.assertRaises(TypeError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_partitions_nodict(self): + self.dict_config['systems']['testsys']['partitions']['login'] = 'foo' + self.assertRaises(TypeError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_partconfig_nodict(self): + self.dict_config['systems']['testsys']['partitions']['login'] = 'foo' + self.assertRaises(TypeError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_unresolved_environment(self): + self.dict_config['environments'] = { + '*': { + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu'], + } + } + } + self.assertRaises(ConfigError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_envconfig_nodict(self): + self.dict_config['environments']['*']['PrgEnv-gnu'] = 'foo' + self.assertRaises(TypeError, + self.site_config.load_from_dict, self.dict_config) + + def test_load_failure_envconfig_notype(self): + self.dict_config['environments'] = { + '*': { + 'PrgEnv-gnu': { + 'modules': ['PrgEnv-gnu'], + } + } + } + self.assertRaises(ConfigError, + self.site_config.load_from_dict, self.dict_config) + + +class TestConfigLoading(unittest.TestCase): + def test_load_normal_config(self): + config.load_settings_from_file('unittests/resources/settings.py') + + def test_load_unknown_file(self): + self.assertRaises(ConfigError, config.load_settings_from_file, 'foo') + + def test_load_no_settings(self): + self.assertRaises(ConfigError, + config.load_settings_from_file, 'unittests') + + def test_load_invalid_settings(self): + self.assertRaises(ConfigError, config.load_settings_from_file, + 'unittests/resources/invalid_settings.py') diff --git a/unittests/test_environments.py b/unittests/test_environments.py index c005567ae8..2937370095 100644 --- a/unittests/test_environments.py +++ b/unittests/test_environments.py @@ -4,8 +4,8 @@ import reframe.core.environments as renv import reframe.utility.os_ext as os_ext import unittests.fixtures as fixtures +from reframe.core.runtime import runtime from reframe.core.exceptions import CompilationError, EnvironError -from reframe.core.modules import get_modules_system class TestEnvironment(unittest.TestCase): @@ -17,21 +17,27 @@ def assertEnvironmentVariable(self, name, value): def assertModulesLoaded(self, modules): for m in modules: - self.assertTrue(get_modules_system().is_module_loaded(m)) + self.assertTrue(self.modules_system.is_module_loaded(m)) def assertModulesNotLoaded(self, modules): for m in modules: - self.assertFalse(get_modules_system().is_module_loaded(m)) + self.assertFalse(self.modules_system.is_module_loaded(m)) - def setUp(self): - get_modules_system().searchpath_add(fixtures.TEST_MODULES) + def setup_modules_system(self): + if not fixtures.has_sane_modules_system(): + self.skipTest('no modules system configured') + + self.modules_system = runtime().modules_system + self.modules_system.searchpath_add(fixtures.TEST_MODULES) # Always add a base module; this is a workaround for the modules # environment's inconsistent behaviour, that starts with an empty # LOADEDMODULES variable and ends up removing it completely if all # present modules are removed. - get_modules_system().load_module('testmod_base') + self.modules_system.load_module('testmod_base') + def setUp(self): + self.modules_system = None os.environ['_fookey1'] = 'origfoo' os.environ['_fookey1b'] = 'foovalue1' os.environ['_fookey2b'] = 'foovalue2' @@ -48,7 +54,9 @@ def setUp(self): self.environ_other.set_variable(name='_fookey11', value='value11') def tearDown(self): - get_modules_system().searchpath_remove(fixtures.TEST_MODULES) + if self.modules_system is not None: + self.modules_system.searchpath_remove(fixtures.TEST_MODULES) + self.environ_save.load() def test_setup(self): @@ -97,15 +105,15 @@ def test_load_restore(self): self.assertEnvironmentVariable(name='_fookey1', value='origfoo') if fixtures.has_sane_modules_system(): self.assertFalse( - get_modules_system().is_module_loaded('testmod_foo')) + self.modules_system.is_module_loaded('testmod_foo')) - @unittest.skipIf(not fixtures.has_sane_modules_system(), - 'no modules systems supported') + @fixtures.switch_to_user_runtime def test_load_already_present(self): - get_modules_system().load_module('testmod_boo') + self.setup_modules_system() + self.modules_system.load_module('testmod_boo') self.environ.load() self.environ.unload() - self.assertTrue(get_modules_system().is_module_loaded('testmod_boo')) + self.assertTrue(self.modules_system.is_module_loaded('testmod_boo')) def test_equal(self): env1 = renv.Environment('env1', modules=['foo', 'bar']) @@ -117,37 +125,37 @@ def test_not_equal(self): env2 = renv.Environment('env2', modules=['foo', 'bar']) self.assertNotEqual(env1, env2) - @unittest.skipIf(not fixtures.has_sane_modules_system(), - 'no modules systems supported') + @fixtures.switch_to_user_runtime def test_conflicting_environments(self): + self.setup_modules_system() envfoo = renv.Environment(name='envfoo', modules=['testmod_foo', 'testmod_boo']) envbar = renv.Environment(name='envbar', modules=['testmod_bar']) envfoo.load() envbar.load() for m in envbar.modules: - self.assertTrue(get_modules_system().is_module_loaded(m)) + self.assertTrue(self.modules_system.is_module_loaded(m)) for m in envfoo.modules: - self.assertFalse(get_modules_system().is_module_loaded(m)) + self.assertFalse(self.modules_system.is_module_loaded(m)) - @unittest.skipIf(not fixtures.has_sane_modules_system(), - 'no modules systems supported') + @fixtures.switch_to_user_runtime def test_conflict_environ_after_module_load(self): - get_modules_system().load_module('testmod_foo') + self.setup_modules_system() + self.modules_system.load_module('testmod_foo') envfoo = renv.Environment(name='envfoo', modules=['testmod_foo']) envfoo.load() envfoo.unload() - self.assertTrue(get_modules_system().is_module_loaded('testmod_foo')) + self.assertTrue(self.modules_system.is_module_loaded('testmod_foo')) - @unittest.skipIf(not fixtures.has_sane_modules_system(), - 'no modules systems supported') + @fixtures.switch_to_user_runtime def test_conflict_environ_after_module_force_load(self): - get_modules_system().load_module('testmod_foo') + self.setup_modules_system() + self.modules_system.load_module('testmod_foo') envbar = renv.Environment(name='envbar', modules=['testmod_bar']) envbar.load() envbar.unload() - self.assertTrue(get_modules_system().is_module_loaded('testmod_foo')) + self.assertTrue(self.modules_system.is_module_loaded('testmod_foo')) def test_swap(self): from reframe.core.environments import swap_environments @@ -161,11 +169,11 @@ def test_swap(self): class TestProgEnvironment(unittest.TestCase): def setUp(self): self.environ_save = renv.EnvironmentSnapshot() - self.executable = os.path.join(fixtures.TEST_RESOURCES, 'hello') + self.executable = os.path.join(fixtures.TEST_RESOURCES_CHECKS, 'hello') def tearDown(self): # Remove generated executable ingoring file-not-found errors - fixtures.force_remove_file(self.executable) + os_ext.force_remove_file(self.executable) self.environ_save.load() def assertHelloMessage(self, executable=None): @@ -174,10 +182,10 @@ def assertHelloMessage(self, executable=None): self.assertTrue(os_ext.grep_command_output(cmd=executable, pattern='Hello, World\!')) - fixtures.force_remove_file(executable) + os_ext.force_remove_file(executable) def compile_with_env(self, env, skip_fortran=False): - srcdir = os.path.join(fixtures.TEST_RESOURCES, 'src') + srcdir = os.path.join(fixtures.TEST_RESOURCES_CHECKS, 'src') env.cxxflags = '-O2' env.load() env.compile(sourcepath=os.path.join(srcdir, 'hello.c'), @@ -196,7 +204,7 @@ def compile_with_env(self, env, skip_fortran=False): env.unload() def compile_dir_with_env(self, env, skip_fortran=False): - srcdir = os.path.join(fixtures.TEST_RESOURCES, 'src') + srcdir = os.path.join(fixtures.TEST_RESOURCES_CHECKS, 'src') env.cxxflags = '-O3' env.load() diff --git a/unittests/test_fields.py b/unittests/test_fields.py index f378b1a28f..1739ec4469 100644 --- a/unittests/test_fields.py +++ b/unittests/test_fields.py @@ -1,3 +1,4 @@ +import os import unittest import reframe.core.fields as fields @@ -42,30 +43,6 @@ class FieldTester: self.assertRaises(ValueError, exec, "tester.ro = 'bar'", globals(), locals()) - def test_alphanumeric_field(self): - class FieldTester: - field1 = fields.AlphanumericField('field1', allow_none=True) - field2 = fields.AlphanumericField('field2') - - def __init__(self, value): - self.field1 = value - - tester1 = FieldTester('foo') - tester2 = FieldTester('bar') - self.assertIsInstance(FieldTester.field1, fields.AlphanumericField) - self.assertEqual('foo', tester1.field1) - self.assertEqual('bar', tester2.field1) - self.assertRaises(TypeError, FieldTester, 12) - self.assertRaises(ValueError, FieldTester, 'foo bar') - - # Setting field2 must not affect field - tester1.field2 = 'foobar' - self.assertEqual('foo', tester1.field1) - self.assertEqual('foobar', tester1.field2) - - # Setting field1 to None must be fine - tester1.field1 = None - def test_typed_field(self): class ClassA: def __init__(self, val): @@ -250,15 +227,19 @@ def __init__(self, value): self.assertRaises(TypeError, exec, 'tester.field = 13', globals(), locals()) - def test_non_whitespace_field(self): + def test_string_pattern_field(self): class FieldTester: - field = fields.NonWhitespaceField('field') + field = fields.StringPatternField('field', '\S+') - tester = FieldTester() - tester.field = 'foobar' - self.assertIsInstance(FieldTester.field, fields.NonWhitespaceField) - self.assertEqual('foobar', tester.field) - self.assertRaises(ValueError, exec, 'tester.field = "foo bar"', + def __init__(self, value): + self.field = value + + tester = FieldTester('foo123') + self.assertIsInstance(FieldTester.field, fields.StringPatternField) + self.assertEqual('foo123', tester.field) + self.assertRaises(TypeError, exec, 'tester.field = 13', + globals(), locals()) + self.assertRaises(ValueError, exec, 'tester.field = "foo 123"', globals(), locals()) def test_integer_field(self): @@ -444,6 +425,25 @@ class FieldTester: self.assertWarns(ReframeDeprecationWarning, exec, 'tester.value = 2', globals(), locals()) + def test_absolute_path_field(self): + class FieldTester: + value = fields.AbsolutePathField('value', allow_none=True) + + def __init__(self, value): + self.value = value + + tester = FieldTester('foo') + self.assertEquals(os.path.abspath('foo'), tester.value) + + # Test set with an absolute path already + tester.value = os.path.abspath('foo') + self.assertEquals(os.path.abspath('foo'), tester.value) + + # This should not raise + tester.value = None + self.assertRaises(TypeError, exec, 'tester.value = 1', + globals(), locals()) + def test_scoped_dict_field(self): class FieldTester: field = fields.ScopedDictField('field', int) diff --git a/unittests/test_loader.py b/unittests/test_loader.py index 3a0e803892..7cb6246cce 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -1,63 +1,60 @@ import os import unittest -from reframe.core.exceptions import ConfigError, NameConflictError +from reframe.core.exceptions import (ConfigError, + NameConflictError, TestLoadError) from reframe.core.systems import System from reframe.frontend.loader import RegressionCheckLoader -from reframe.frontend.resources import ResourcesManager class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True) self.loader_with_path = RegressionCheckLoader( - ['unittests/resources', 'unittests/foobar'], + ['unittests/resources/checks', 'unittests/foobar'], ignore_conflicts=True) self.loader_with_prefix = RegressionCheckLoader( - load_path=['badchecks'], - prefix=os.path.abspath('unittests/resources')) - - self.system = System('foo') - self.resources = ResourcesManager() + load_path=['bad'], + prefix=os.path.abspath('unittests/resources/checks')) def test_load_file_relative(self): checks = self.loader.load_from_file( - 'unittests/resources/emptycheck.py', - system=self.system, resources=self.resources - ) + 'unittests/resources/checks/emptycheck.py') self.assertEqual(1, len(checks)) - self.assertEqual(checks[0].name, 'emptycheck') + self.assertEqual(checks[0].name, 'EmptyTest') def test_load_file_absolute(self): checks = self.loader.load_from_file( - os.path.abspath('unittests/resources/emptycheck.py'), - system=self.system, resources=self.resources - ) + os.path.abspath('unittests/resources/checks/emptycheck.py')) self.assertEqual(1, len(checks)) - self.assertEqual(checks[0].name, 'emptycheck') + self.assertEqual(checks[0].name, 'EmptyTest') def test_load_recursive(self): - checks = self.loader.load_from_dir( - 'unittests/resources', recurse=True, - system=self.system, resources=self.resources - ) + checks = self.loader.load_from_dir('unittests/resources/checks', + recurse=True) self.assertEqual(11, len(checks)) def test_load_all(self): - checks = self.loader_with_path.load_all(system=self.system, - resources=self.resources) + checks = self.loader_with_path.load_all() self.assertEqual(10, len(checks)) def test_load_all_with_prefix(self): - checks = self.loader_with_prefix.load_all(system=self.system, - resources=self.resources) + checks = self.loader_with_prefix.load_all() self.assertEqual(1, len(checks)) + def test_load_new_syntax(self): + checks = self.loader.load_from_file( + 'unittests/resources/checks_unlisted/good.py') + self.assertEqual(13, len(checks)) + + def test_load_mixed_syntax(self): + self.assertRaises(TestLoadError, self.loader.load_from_file, + 'unittests/resources/checks_unlisted/mixed.py') + def test_conflicted_checks(self): self.loader_with_path._ignore_conflicts = False - self.assertRaises(NameConflictError, self.loader_with_path.load_all, - system=self.system, resources=self.resources) + self.assertRaises(NameConflictError, self.loader_with_path.load_all) def test_load_error(self): self.assertRaises(OSError, self.loader.load_from_file, - 'unittests/resources/foo.py') + 'unittests/resources/checks/foo.py') diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 68e68546f5..3c03064416 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -9,8 +9,6 @@ import reframe.core.logging as rlog from reframe.core.exceptions import ReframeError from reframe.core.pipeline import RegressionTest -from reframe.core.systems import System -from reframe.frontend.resources import ResourcesManager class TestLogger(unittest.TestCase): @@ -32,10 +30,7 @@ def setUp(self): # Logger adapter with an associated check self.logger_with_check = rlog.LoggerAdapter( - self.logger, RegressionTest( - 'random_check', '.', System('foosys'), ResourcesManager() - ) - ) + self.logger, RegressionTest('random_check', '.')) def tearDown(self): os.remove(self.logfile) @@ -116,9 +111,7 @@ def setUp(self): } } } - self.check = RegressionTest( - 'random_check', '.', System('gagsys'), ResourcesManager() - ) + self.check = RegressionTest('random_check', '.') def tearDown(self): if os.path.exists(self.logfile): diff --git a/unittests/test_modules.py b/unittests/test_modules.py index e0587d19eb..975f4cbab0 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -6,12 +6,12 @@ import reframe.core.modules as modules from reframe.core.environments import EnvironmentSnapshot from reframe.core.exceptions import ConfigError, EnvironError +from reframe.core.runtime import runtime from unittests.fixtures import TEST_MODULES class _TestModulesSystem(unittest.TestCase): def setUp(self): - self.modules_system = modules.get_modules_system() self.environ_save = EnvironmentSnapshot() self.modules_system.searchpath_add(TEST_MODULES) @@ -103,7 +103,7 @@ def test_emit_unload_commands(self): class TestTModModulesSystem(_TestModulesSystem): def setUp(self): try: - modules.init_modules_system('tmod') + self.modules_system = modules.ModulesSystem.create('tmod') except ConfigError: self.skipTest('tmod not supported') else: @@ -119,7 +119,7 @@ def expected_unload_instr(self, module): class TestTMod4ModulesSystem(_TestModulesSystem): def setUp(self): try: - modules.init_modules_system('tmod4') + self.modules_system = modules.ModulesSystem.create('tmod4') except ConfigError: self.skipTest('tmod4 not supported') else: @@ -135,7 +135,7 @@ def expected_unload_instr(self, module): class TestLModModulesSystem(_TestModulesSystem): def setUp(self): try: - modules.init_modules_system('lmod') + self.modules_system = modules.ModulesSystem.create('lmod') except ConfigError: self.skipTest('lmod not supported') else: @@ -151,7 +151,7 @@ def expected_unload_instr(self, module): class TestNoModModulesSystem(_TestModulesSystem): def setUp(self): try: - modules.init_modules_system() + self.modules_system = modules.ModulesSystem.create() except ConfigError: self.skipTest('nomod not supported') else: @@ -200,7 +200,8 @@ def test_name_version(self): def test_equal(self): self.assertEqual(modules.Module('foo'), modules.Module('foo')) - self.assertEqual(modules.Module('foo/1.2'), modules.Module('foo/1.2')) + self.assertEqual(modules.Module('foo/1.2'), + modules.Module('foo/1.2')) self.assertEqual(modules.Module('foo'), modules.Module('foo/1.2')) self.assertEqual(hash(modules.Module('foo')), hash(modules.Module('foo'))) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 5a8064324b..e8950255a9 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -3,36 +3,41 @@ import tempfile import unittest +import reframe.core.runtime as rt import reframe.utility.sanity as sn import unittests.fixtures as fixtures -from reframe.core.exceptions import (ReframeError, PipelineError, SanityError, +from reframe.core.exceptions import (ReframeError, ReframeSyntaxError, + PipelineError, SanityError, CompilationError) -from reframe.core.modules import get_modules_system from reframe.core.pipeline import (CompileOnlyRegressionTest, RegressionTest, RunOnlyRegressionTest) from reframe.frontend.loader import RegressionCheckLoader -from reframe.frontend.resources import ResourcesManager class TestRegressionTest(unittest.TestCase): - def setUp(self): - get_modules_system().searchpath_add(fixtures.TEST_MODULES) + def setup_local_execution(self): + self.partition = rt.runtime().system.partition('login') + self.progenv = self.partition.environment('builtin-gcc') + + def setup_remote_execution(self): + self.partition = fixtures.partition_with_scheduler() + if self.partition is None: + self.skipTest('job submission not supported') + + try: + self.progenv = self.partition.environs[0] + except IndexError: + self.skipTest('no environments configured for partition: %s' % + self.partition.fullname) - # Load a system configuration - self.system, self.partition, self.progenv = fixtures.get_test_config() + def setUp(self): + self.setup_local_execution() self.resourcesdir = tempfile.mkdtemp(dir='unittests') - self.loader = RegressionCheckLoader(['unittests/resources']) - self.resources = ResourcesManager(prefix=self.resourcesdir) + self.loader = RegressionCheckLoader(['unittests/resources/checks']) def tearDown(self): shutil.rmtree(self.resourcesdir, ignore_errors=True) - - def setup_from_site(self): - self.partition = fixtures.partition_with_scheduler(None) - - # pick the first environment of partition - if self.partition.environs: - self.progenv = self.partition.environs[0] + shutil.rmtree('.rfm_testing', ignore_errors=True) def replace_prefix(self, filename, new_prefix): basename = os.path.basename(filename) @@ -54,9 +59,7 @@ def keep_files_list(self, test, compile_only=False): def test_environ_setup(self): test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] @@ -66,7 +69,7 @@ def test_environ_setup(self): test.setup(self.partition, self.progenv) for m in test.modules: - self.assertTrue(get_modules_system().is_module_loaded(m)) + self.assertTrue(rt.runtime().modules_system.is_module_loaded(m)) for k, v in test.variables.items(): self.assertEqual(os.environ[k], v) @@ -86,27 +89,21 @@ def _run_test(self, test, compile_only=False): for f in self.keep_files_list(test, compile_only): self.assertTrue(os.path.exists(f)) - @unittest.skipIf(not fixtures.partition_with_scheduler(None), - 'job submission not supported') + @fixtures.switch_to_user_runtime def test_hellocheck(self): - self.setup_from_site() + self.setup_remote_execution() test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] self._run_test(test) - @unittest.skipIf(not fixtures.partition_with_scheduler(None), - 'job submission not supported') + @fixtures.switch_to_user_runtime def test_hellocheck_make(self): - self.setup_from_site() + self.setup_remote_execution() test = self.loader.load_from_file( - 'unittests/resources/hellocheck_make.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck_make.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] @@ -114,9 +111,7 @@ def test_hellocheck_make(self): def test_hellocheck_local(self): test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] @@ -130,28 +125,13 @@ def test_hellocheck_local(self): test.local = True self._run_test(test) - def test_hellocheck_local_slashes(self): - # Try to fool path creation by adding slashes to environment partitions - # names - from reframe.core.environments import ProgEnvironment - - self.progenv = ProgEnvironment('bad/name', self.progenv.modules, - self.progenv.variables) - - # That's a bit hacky, but we are in a unit test - self.system._name += os.sep + 'bad' - self.partition._name += os.sep + 'bad' - self.test_hellocheck_local() - def test_hellocheck_local_prepost_run(self): @sn.sanity_function def stagedir(test): return test.stagedir test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] @@ -172,9 +152,7 @@ def stagedir(test): def test_run_only_sanity(self): test = RunOnlyRegressionTest('runonlycheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.executable = './hello.sh' test.executable_opts = ['Hello, World!'] test.local = True @@ -185,32 +163,27 @@ def test_run_only_sanity(self): def test_compile_only_failure(self): test = CompileOnlyRegressionTest('compileonlycheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.sourcepath = 'compiler_failure.c' - test.valid_prog_environs = [self.progenv.name] - test.valid_systems = [self.system.name] + test.valid_prog_environs = ['*'] + test.valid_systems = ['*'] test.setup(self.partition, self.progenv) self.assertRaises(CompilationError, test.compile) def test_compile_only_warning(self): test = CompileOnlyRegressionTest('compileonlycheckwarning', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.sourcepath = 'compiler_warning.c' self.progenv.cflags = '-Wall' - test.valid_prog_environs = [self.progenv.name] - test.valid_systems = [self.system.name] + test.valid_prog_environs = ['*'] + test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'warning', test.stderr) self._run_test(test, compile_only=True) + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_supports_system(self): test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] test.valid_systems = ['*'] self.assertTrue(test.supports_system('gpu')) @@ -244,9 +217,7 @@ def test_supports_system(self): def test_supports_environ(self): test = self.loader.load_from_file( - 'unittests/resources/hellocheck.py', - system=self.system, resources=self.resources - )[0] + 'unittests/resources/checks/hellocheck.py')[0] test.valid_prog_environs = ['*'] self.assertTrue(test.supports_environ('foo1')) @@ -260,10 +231,7 @@ def test_supports_environ(self): self.assertFalse(test.supports_environ('Prgenv-foo-version1')) def test_sourcesdir_none(self): - test = RegressionTest('hellocheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + test = RegressionTest('hellocheck', 'unittests/resources/checks') test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] @@ -271,9 +239,7 @@ def test_sourcesdir_none(self): def test_sourcesdir_none_generated_sources(self): test = RegressionTest('hellocheck_generated_sources', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.sourcesdir = None test.prebuild_cmd = ["printf '#include \\n int main(){ " "printf(\"Hello, World!\\\\n\"); return 0; }' " @@ -288,9 +254,7 @@ def test_sourcesdir_none_generated_sources(self): def test_sourcesdir_none_compile_only(self): test = CompileOnlyRegressionTest('hellocheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] @@ -298,9 +262,7 @@ def test_sourcesdir_none_compile_only(self): def test_sourcesdir_none_run_only(self): test = RunOnlyRegressionTest('hellocheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.sourcesdir = None test.executable = 'echo' test.executable_opts = ["Hello, World!"] @@ -312,38 +274,35 @@ def test_sourcesdir_none_run_only(self): def test_sourcepath_abs(self): test = CompileOnlyRegressionTest('compileonlycheck', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') test.valid_prog_environs = [self.progenv.name] - test.valid_systems = [self.system.name] + test.valid_systems = ['*'] test.setup(self.partition, self.progenv) test.sourcepath = '/usr/src' self.assertRaises(PipelineError, test.compile) def test_sourcepath_upref(self): test = CompileOnlyRegressionTest('compileonlycheck', - 'unittests/resources', - resources=self.resources, - system=self.system) - test.valid_prog_environs = [self.progenv.name] - test.valid_systems = [self.system.name] + 'unittests/resources/checks') + test.valid_prog_environs = ['*'] + test.valid_systems = ['*'] test.setup(self.partition, self.progenv) test.sourcepath = '../hellosrc' self.assertRaises(PipelineError, test.compile) + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_extra_resources(self): # Load test site configuration - system, partition, progenv = fixtures.get_test_config() - test = RegressionTest('dummycheck', 'unittests/resources', - resources=self.resources, system=self.system) + test = RegressionTest('dummycheck', 'unittests/resources/checks') test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.extra_resources = { 'gpu': {'num_gpus_per_node': 2}, 'datawarp': {'capacity': '100GB', 'stagein_src': '/foo'} } - test.setup(self.partition, self.progenv) + partition = rt.runtime().system.partition('gpu') + environ = partition.environment('builtin-gcc') + test.setup(partition, environ) test.job.options += ['--foo'] expected_job_options = ['--gres=gpu:2', '#DW jobdw capacity=100GB', @@ -352,18 +311,112 @@ def test_extra_resources(self): self.assertCountEqual(expected_job_options, test.job.options) +class TestNewStyleChecks(unittest.TestCase): + def test_regression_test(self): + class MyTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + test = MyTest(1, 2) + self.assertEqual(os.path.abspath(os.path.dirname(__file__)), + test.prefix) + self.assertEqual('TestNewStyleChecks.test_regression_test.' + '.MyTest_1_2', test.name) + + def test_regression_test_strange_names(self): + class C: + def __init__(self, a): + self.a = a + + def __repr__(self): + return 'C(%s)' % self.a + + class MyTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + test = MyTest('(a*b+c)/12', C(33)) + self.assertEqual( + 'TestNewStyleChecks.test_regression_test_strange_names.' + '.MyTest__a_b_c__12_C_33_', test.name) + + def test_user_inheritance(self): + class MyBaseTest(RegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + class MyTest(MyBaseTest): + def __init__(self): + super().__init__(1, 2) + + test = MyTest() + self.assertEqual('TestNewStyleChecks.test_user_inheritance.' + '.MyTest', test.name) + + def test_runonly_test(self): + class MyTest(RunOnlyRegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + test = MyTest(1, 2) + self.assertEqual(os.path.abspath(os.path.dirname(__file__)), + test.prefix) + self.assertEqual('TestNewStyleChecks.test_runonly_test.' + '.MyTest_1_2', test.name) + + def test_compileonly_test(self): + class MyTest(CompileOnlyRegressionTest): + def __init__(self, a, b): + super().__init__() + self.a = a + self.b = b + + test = MyTest(1, 2) + self.assertEqual(os.path.abspath(os.path.dirname(__file__)), + test.prefix) + self.assertEqual('TestNewStyleChecks.test_compileonly_test.' + '.MyTest_1_2', test.name) + + def test_registration(self): + import sys + import unittests.resources.checks_unlisted.good as mod + checks = mod._rfm_gettests() + self.assertEqual(13, len(checks)) + self.assertEqual([mod.MyBaseTest(0, 0), + mod.MyBaseTest(0, 1), + mod.MyBaseTest(1, 0), + mod.MyBaseTest(1, 1), + mod.MyBaseTest(2, 0), + mod.MyBaseTest(2, 1), + mod.AnotherBaseTest(0, 0), + mod.AnotherBaseTest(0, 1), + mod.AnotherBaseTest(1, 0), + mod.AnotherBaseTest(1, 1), + mod.AnotherBaseTest(2, 0), + mod.AnotherBaseTest(2, 1), + mod.MyBaseTest(10, 20)], checks) + + class TestSanityPatterns(unittest.TestCase): + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def setUp(self): - # Load test site configuration - self.system, self.partition, self.progenv = fixtures.get_test_config() + # Set up the test runtime + self.resourcesdir = tempfile.mkdtemp(dir='unittests') + rt.runtime().resources.prefix = self.resourcesdir # Set up RegressionTest instance - self.resourcesdir = tempfile.mkdtemp(dir='unittests') - self.resources = ResourcesManager(prefix=self.resourcesdir) self.test = RegressionTest('test_performance', - 'unittests/resources', - resources=self.resources, - system=self.system) + 'unittests/resources/checks') + self.partition = rt.runtime().system.partition('gpu') + self.progenv = self.partition.environment('builtin-gcc') self.test.setup(self.partition, self.progenv) self.test.reference = { diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 321e84f4f3..35c9ff242d 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,39 +3,26 @@ import tempfile import unittest -import reframe.frontend.config as config +import reframe.core.runtime as runtime import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies from reframe.core.exceptions import JobNotStartedError -from reframe.core.modules import init_modules_system from reframe.frontend.loader import RegressionCheckLoader -from reframe.frontend.resources import ResourcesManager -from reframe.settings import settings -from unittests.resources.hellocheck import HelloTest -from unittests.resources.frontend_checks import (KeyboardInterruptCheck, - SleepCheck, BadSetupCheck, - RetriesCheck, SystemExitCheck) +from unittests.resources.checks.hellocheck import HelloTest +from unittests.resources.checks.frontend_checks import ( + KeyboardInterruptCheck, SleepCheck, + BadSetupCheck, RetriesCheck, SystemExitCheck) class TestSerialExecutionPolicy(unittest.TestCase): def setUp(self): - # Load a system configuration - settings = config.load_from_file("reframe/settings.py") - self.site_config = config.SiteConfiguration() - self.site_config.load_from_dict(settings.site_configuration) - self.system = self.site_config.systems['generic'] self.resourcesdir = tempfile.mkdtemp(dir='unittests') - self.resources = ResourcesManager(prefix=self.resourcesdir) - self.loader = RegressionCheckLoader(['unittests/resources'], + self.loader = RegressionCheckLoader(['unittests/resources/checks'], ignore_conflicts=True) - # Init modules system - init_modules_system(self.system.modules_system) - # Setup the runner self.runner = executors.Runner(policies.SerialExecutionPolicy()) - self.checks = self.loader.load_all(system=self.system, - resources=self.resources) + self.checks = self.loader.load_all() def tearDown(self): shutil.rmtree(self.resourcesdir, ignore_errors=True) @@ -56,7 +43,7 @@ def assert_all_dead(self): self.assertTrue(finished) def test_runall(self): - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(7, stats.num_cases()) @@ -67,7 +54,7 @@ def test_runall(self): def test_runall_skip_system_check(self): self.runner.policy.skip_system_check = True - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(8, stats.num_cases()) @@ -78,7 +65,7 @@ def test_runall_skip_system_check(self): def test_runall_skip_prgenv_check(self): self.runner.policy.skip_environ_check = True - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(8, stats.num_cases()) @@ -89,7 +76,7 @@ def test_runall_skip_prgenv_check(self): def test_runall_skip_sanity_check(self): self.runner.policy.skip_sanity_check = True - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(7, stats.num_cases()) @@ -100,7 +87,7 @@ def test_runall_skip_sanity_check(self): def test_runall_skip_performance_check(self): self.runner.policy.skip_performance_check = True - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(7, stats.num_cases()) @@ -111,7 +98,7 @@ def test_runall_skip_performance_check(self): def test_strict_performance_check(self): self.runner.policy.strict_check = True - self.runner.runall(self.checks, self.system) + self.runner.runall(self.checks) stats = self.runner.stats self.assertEqual(7, stats.num_cases()) @@ -122,34 +109,32 @@ def test_strict_performance_check(self): def test_force_local_execution(self): self.runner.policy.force_local = True - self.runner.runall([HelloTest(system=self.system, resources=self.resources)], - self.system) + self.runner.runall([HelloTest()]) stats = self.runner.stats for t in stats.get_tasks(): self.assertTrue(t.check.local) def test_kbd_interrupt_within_test(self): - check = KeyboardInterruptCheck(system=self.system, - resources=self.resources) + check = KeyboardInterruptCheck() self.assertRaises(KeyboardInterrupt, self.runner.runall, - [check], self.system) + [check]) stats = self.runner.stats self.assertEqual(1, stats.num_failures()) self.assert_all_dead() def test_system_exit_within_test(self): - check = SystemExitCheck(system=self.system, resources=self.resources) + check = SystemExitCheck() # This should not raise and should not exit - self.runner.runall([check], self.system) + self.runner.runall([check]) stats = self.runner.stats self.assertEqual(1, stats.num_failures()) def test_retries_bad_check(self): max_retries = 2 - checks = [BadSetupCheck(system=self.system, resources=self.resources)] + checks = [BadSetupCheck()] self.runner._max_retries = max_retries - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that the test was retried #max_retries times and failed. self.assertEqual(1, self.runner.stats.num_cases()) @@ -158,9 +143,9 @@ def test_retries_bad_check(self): def test_retries_good_check(self): max_retries = 2 - checks = [HelloTest(system=self.system, resources=self.resources)] + checks = [HelloTest()] self.runner._max_retries = max_retries - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that the test passed without retries. self.assertEqual(1, self.runner.stats.num_cases()) @@ -175,10 +160,9 @@ def test_pass_in_retries(self): with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp: fp.write('0\n') - checks = [RetriesCheck(run_to_pass, fp.name, system=self.system, - resources=self.resources)] + checks = [RetriesCheck(run_to_pass, fp.name)] self.runner._max_retries = max_retries - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that the test passed after retries in run #run_to_pass. self.assertEqual(1, self.runner.stats.num_cases()) @@ -235,7 +219,7 @@ def setUp(self): self.runner.policy.task_listeners.append(self.monitor) def set_max_jobs(self, value): - for p in self.system.partitions: + for p in runtime.runtime().system.partitions: p._max_jobs = value def read_timestamps(self, tasks): @@ -254,13 +238,9 @@ def read_timestamps(self, tasks): self.end_stamps.sort() def test_concurrency_unlimited(self): - checks = [ - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources) - ] + checks = [SleepCheck(0.5) for i in range(3)] self.set_max_jobs(len(checks)) - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) @@ -282,16 +262,10 @@ def test_concurrency_unlimited(self): def test_concurrency_limited(self): # The number of checks must be <= 2*max_jobs. - checks = [ - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources) - ] + checks = [SleepCheck(0.5) for i in range(5)] max_jobs = len(checks) - 2 self.set_max_jobs(max_jobs) - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) @@ -326,15 +300,10 @@ def test_concurrency_limited(self): self.skipTest('the system seems too loaded.') def test_concurrency_none(self): - checks = [ - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources), - SleepCheck(0.5, system=self.system, resources=self.resources) - ] - + checks = [SleepCheck(0.5) for i in range(3)] num_checks = len(checks) self.set_max_jobs(1) - self.runner.runall(checks, self.system) + self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) @@ -354,21 +323,15 @@ def test_concurrency_none(self): def _run_checks(self, checks, max_jobs): self.set_max_jobs(max_jobs) - self.assertRaises(KeyboardInterrupt, self.runner.runall, - checks, self.system) + self.assertRaises(KeyboardInterrupt, self.runner.runall, checks) self.assertEqual(4, self.runner.stats.num_cases()) self.assertEqual(4, self.runner.stats.num_failures()) self.assert_all_dead() def test_kbd_interrupt_in_wait_with_concurrency(self): - checks = [ - KeyboardInterruptCheck(system=self.system, - resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources) - ] + checks = [KeyboardInterruptCheck(), + SleepCheck(10), SleepCheck(10), SleepCheck(10)] self._run_checks(checks, 4) def test_kbd_interrupt_in_wait_with_limited_concurrency(self): @@ -377,33 +340,16 @@ def test_kbd_interrupt_in_wait_with_limited_concurrency(self): # KeyboardInterruptCheck to finish first (the corresponding wait should # trigger the failure), so as to make the framework kill the remaining # three. - checks = [ - KeyboardInterruptCheck(system=self.system, - resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources), - SleepCheck(10, system=self.system, resources=self.resources) - ] + checks = [KeyboardInterruptCheck(), + SleepCheck(10), SleepCheck(10), SleepCheck(10)] self._run_checks(checks, 2) def test_kbd_interrupt_in_setup_with_concurrency(self): - checks = [ - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - KeyboardInterruptCheck(phase='setup', - system=self.system, - resources=self.resources), - ] + checks = [SleepCheck(1), SleepCheck(1), SleepCheck(1), + KeyboardInterruptCheck(phase='setup')] self._run_checks(checks, 4) def test_kbd_interrupt_in_setup_with_limited_concurrency(self): - checks = [ - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - KeyboardInterruptCheck(phase='setup', - system=self.system, - resources=self.resources), - ] + checks = [SleepCheck(1), SleepCheck(1), SleepCheck(1), + KeyboardInterruptCheck(phase='setup')] self._run_checks(checks, 2) diff --git a/unittests/test_runtime.py b/unittests/test_runtime.py new file mode 100644 index 0000000000..8595d17b1f --- /dev/null +++ b/unittests/test_runtime.py @@ -0,0 +1,20 @@ +import unittest + +import reframe.core.runtime as rt +import unittests.fixtures as fixtures + + +class TestRuntime(unittest.TestCase): + @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') + def test_hostsystem_api(self): + system = rt.runtime().system + self.assertEqual('testsys', system.name) + self.assertEqual('Fake system for unit tests', system.descr) + self.assertEqual('.rfm_testing/resources', system.resourcesdir) + self.assertEqual(2, len(system.partitions)) + self.assertIsNotNone(system.partition('login')) + self.assertIsNotNone(system.partition('gpu')) + self.assertIsNone(system.partition('foobar')) + + # Test delegation to the underlying System + self.assertEqual('.rfm_testing/install', system.prefix) diff --git a/unittests/test_sanity_functions.py b/unittests/test_sanity_functions.py index 4d5bc8101e..5591d91e8f 100644 --- a/unittests/test_sanity_functions.py +++ b/unittests/test_sanity_functions.py @@ -6,7 +6,7 @@ import reframe.utility.sanity as sn from reframe.core.deferrable import evaluate, make_deferrable from reframe.core.exceptions import SanityError -from unittests.fixtures import TEST_RESOURCES +from unittests.fixtures import TEST_RESOURCES_CHECKS class TestDeferredBuiltins(unittest.TestCase): @@ -155,7 +155,8 @@ def test_zip(self): class TestAsserts(unittest.TestCase): def setUp(self): - self.utf16_file = os.path.join(TEST_RESOURCES, 'src', 'homer.txt') + self.utf16_file = os.path.join(TEST_RESOURCES_CHECKS, + 'src', 'homer.txt') def test_assert_true(self): self.assertTrue(sn.assert_true(True)) @@ -474,12 +475,12 @@ def myrange(n): self.assertEqual(0, sn.count(myrange(0))) def test_glob(self): - filepatt = os.path.join(TEST_RESOURCES, '*.py') + filepatt = os.path.join(TEST_RESOURCES_CHECKS, '*.py') self.assertTrue(sn.glob(filepatt)) self.assertTrue(sn.glob(make_deferrable(filepatt))) def test_iglob(self): - filepatt = os.path.join(TEST_RESOURCES, '*.py') + filepatt = os.path.join(TEST_RESOURCES_CHECKS, '*.py') self.assertTrue(sn.count(sn.iglob(filepatt))) self.assertTrue(sn.count(sn.iglob(make_deferrable(filepatt)))) @@ -494,7 +495,8 @@ def test_chain(self): class TestPatternMatchingFunctions(unittest.TestCase): def setUp(self): self.tempfile = None - self.utf16_file = os.path.join(TEST_RESOURCES, 'src', 'homer.txt') + self.utf16_file = os.path.join(TEST_RESOURCES_CHECKS, + 'src', 'homer.txt') with NamedTemporaryFile('wt', delete=False) as fp: self.tempfile = fp.name fp.write('Step: 1\n') diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 053ffb50b8..97fa1dfdee 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -7,7 +7,9 @@ import unittest from datetime import datetime +import reframe.core.runtime as rt import reframe.utility.os_ext as os_ext +import unittests.fixtures as fixtures from reframe.core.environments import Environment from reframe.core.exceptions import JobError, JobNotStartedError from reframe.core.launchers.local import LocalLauncher @@ -15,7 +17,6 @@ from reframe.core.schedulers.registry import getscheduler from reframe.core.schedulers.slurm import SlurmNode from reframe.core.shell import BashScriptBuilder -from unittests.fixtures import TEST_RESOURCES, partition_with_scheduler class _TestJob(unittest.TestCase): @@ -40,15 +41,31 @@ def tearDown(self): shutil.rmtree(self.workdir) @property - @abc.abstractmethod def job_type(self): - """Return a concrete job class.""" + return getscheduler(self.sched_name) + + @property + @abc.abstractmethod + def sched_name(self): + """Return the registered name of the scheduler.""" @property @abc.abstractmethod def launcher(self): """Return a launcher to use for this test.""" + @abc.abstractmethod + def setup_user(self, msg=None): + """Configure the test for running with the user supplied job scheduler + configuration or skip it. + """ + partition = fixtures.partition_with_scheduler(self.sched_name) + if partition is None: + msg = msg or "scheduler '%s' not configured" % self.sched_name + self.skipTest(msg) + + self.testjob.options += partition.access + @abc.abstractmethod def assertScriptSanity(self, script_file): """Assert the sanity of the produced script file.""" @@ -62,14 +79,18 @@ def test_prepare(self): self.testjob.prepare(self.builder) self.assertScriptSanity(self.testjob.script_filename) + @fixtures.switch_to_user_runtime def test_submit(self): + self.setup_user() self.testjob.prepare(self.builder) self.testjob.submit() self.assertIsNotNone(self.testjob.jobid) self.testjob.wait() self.assertEqual(0, self.testjob.exitcode) + @fixtures.switch_to_user_runtime def test_submit_timelimit(self, check_elapsed_time=True): + self.setup_user() self.testjob._command = 'sleep 10' self.testjob._time_limit = (0, 0, 2) self.testjob.prepare(self.builder) @@ -85,7 +106,9 @@ def test_submit_timelimit(self, check_elapsed_time=True): with open(self.testjob.stdout) as fp: self.assertIsNone(re.search('postrun', fp.read())) + @fixtures.switch_to_user_runtime def test_cancel(self): + self.setup_user() self.testjob._command = 'sleep 30' self.testjob.prepare(self.builder) t_job = datetime.now() @@ -106,7 +129,9 @@ def test_wait_before_submit(self): self.testjob.prepare(self.builder) self.assertRaises(JobNotStartedError, self.testjob.wait) + @fixtures.switch_to_user_runtime def test_poll(self): + self.setup_user() self.testjob._command = 'sleep 2' self.testjob.prepare(self.builder) self.testjob.submit() @@ -128,13 +153,21 @@ def assertProcessDied(self, pid): pass @property - def job_type(self): - return getscheduler('local') + def sched_name(self): + return 'local' + + @property + def sched_configured(self): + return True @property def launcher(self): return LocalLauncher() + def setup_user(self, msg=None): + # Local scheduler is by definition available + pass + def test_submit_timelimit(self): from reframe.core.schedulers.local import LOCAL_JOB_TIMEOUT @@ -200,7 +233,7 @@ def test_cancel_term_ignore(self): self.testjob._pre_run = [] self.testjob._post_run = [] - self.testjob._command = os.path.join(TEST_RESOURCES, + self.testjob._command = os.path.join(fixtures.TEST_RESOURCES_CHECKS, 'src', 'sleep_deeply.sh') self.testjob.cancel_grace_period = 2 self.testjob.prepare(self.builder) @@ -228,16 +261,19 @@ def test_cancel_term_ignore(self): class TestSlurmJob(_TestJob): @property - def job_type(self): - return getscheduler('slurm') + def sched_name(self): + return 'slurm' + + @property + def sched_configured(self): + return fixtures.partition_with_scheduler('slurm') is not None @property def launcher(self): return LocalLauncher() - def setup_from_sysconfig(self): - partition = partition_with_scheduler('slurm') - self.testjob.options += partition.access + def setup_user(self, msg=None): + super().setup_user(msg='SLURM (with sacct) not configured') def test_prepare(self): # Mock up a job submission @@ -311,31 +347,17 @@ def test_prepare_without_smt(self): with open(self.testjob.script_filename) as fp: self.assertIsNotNone(re.search(r'--hint=nomultithread', fp.read())) - @unittest.skipIf(not partition_with_scheduler('slurm'), - 'Slurm scheduler not supported') - def test_submit(self): - self.setup_from_sysconfig() - super().test_submit() - - @unittest.skipIf(not partition_with_scheduler('slurm'), - 'Slurm scheduler not supported') def test_submit_timelimit(self): # Skip this test for Slurm, since we the minimum time limit is 1min - self.skipTest("Slurm's minimum time limit is 60s") + self.skipTest("SLURM's minimum time limit is 60s") - @unittest.skipIf(not partition_with_scheduler('slurm'), - 'Slurm scheduler not supported') def test_cancel(self): from reframe.core.schedulers.slurm import SLURM_JOB_CANCELLED - self.setup_from_sysconfig() super().test_cancel() self.assertEqual(self.testjob.state, SLURM_JOB_CANCELLED) - @unittest.skipIf(not partition_with_scheduler('slurm'), - 'Slurm scheduler not supported') def test_poll(self): - self.setup_from_sysconfig() super().test_poll() @@ -471,7 +493,6 @@ def prepare_job(self): class TestSlurmFlexibleNodeAllocationExclude(TestSlurmFlexibleNodeAllocation): - def create_dummy_exclude_nodes(obj): return [obj.create_dummy_nodes()[0].name] @@ -499,12 +520,6 @@ def test_valid_constraint_partition(self): super().test_valid_constraint_partition(expected_num_tasks=None) -class TestSqueueJob(TestSlurmJob): - @property - def job_type(self): - return getscheduler('squeue') - - class TestSlurmNode(unittest.TestCase): def setUp(self): node_description = ('NodeName=nid00001 Arch=x86_64 CoresPerSocket=12 ' @@ -535,3 +550,17 @@ def test_attributes(self): def test_str(self): self.assertEqual('nid00001', str(self.node)) + + +class TestSqueueJob(TestSlurmJob): + @property + def sched_name(self): + return 'squeue' + + def setup_user(self, msg=None): + partition = (fixtures.partition_with_scheduler(self.sched_name) or + fixtures.partition_with_scheduler('slurm')) + if partition is None: + self.skipTest('SLURM not configured') + + self.testjob.options += partition.access diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 495f868ace..f1a40fa634 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1,12 +1,13 @@ import os import shutil +import sys import tempfile import unittest import reframe import reframe.core.debug as debug import reframe.core.fields as fields -import reframe.utility +import reframe.utility as util import reframe.utility.os_ext as os_ext from reframe.core.exceptions import (SpawnedProcessError, SpawnedProcessTimeout) @@ -155,6 +156,17 @@ def test_git_repo_exists(self): self.assertFalse(os_ext.git_repo_exists( 'https://github.com/eth-cscs/xxx', timeout=3)) + def test_force_remove_file(self): + with tempfile.NamedTemporaryFile(delete=False) as fp: + pass + + self.assertTrue(os.path.exists(fp.name)) + os_ext.force_remove_file(fp.name) + self.assertFalse(os.path.exists(fp.name)) + + # Try to remove a non-existent file + os_ext.force_remove_file(fp.name) + class TestCopyTree(unittest.TestCase): def setUp(self): @@ -242,28 +254,43 @@ def tearDown(self): class TestImportFromFile(unittest.TestCase): def test_load_relpath(self): - module = reframe.utility.import_module_from_file('reframe/__init__.py') + module = util.import_module_from_file('reframe/__init__.py') self.assertEqual(reframe.VERSION, module.VERSION) + self.assertEqual('reframe', module.__name__) + self.assertIs(module, sys.modules.get('reframe')) - # FIXME: we do not treat specially the `__init__.py` files - self.assertEqual('reframe.__init__', module.__name__) + def test_load_directory(self): + module = util.import_module_from_file('reframe') + self.assertEqual(reframe.VERSION, module.VERSION) + self.assertEqual('reframe', module.__name__) + self.assertIs(module, sys.modules.get('reframe')) def test_load_abspath(self): filename = os.path.abspath('reframe/__init__.py') - module = reframe.utility.import_module_from_file(filename) + module = util.import_module_from_file(filename) self.assertEqual(reframe.VERSION, module.VERSION) - - # FIXME: we do not treat specially the `__init__.py` files - self.assertEqual('__init__', module.__name__) + self.assertEqual('reframe', module.__name__) + self.assertIs(module, sys.modules.get('reframe')) def test_load_unknown_path(self): try: - reframe.utility.import_module_from_file('/foo') + util.import_module_from_file('/foo') self.fail() except ImportError as e: self.assertEqual('foo', e.name) self.assertEqual('/foo', e.path) + def test_load_twice(self): + filename = os.path.abspath('reframe/__init__.py') + module1 = util.import_module_from_file(filename) + module2 = util.import_module_from_file(filename) + self.assertIs(module1, module2) + + def test_load_namespace_package(self): + module = util.import_module_from_file('unittests/resources') + self.assertIn('unittests', sys.modules) + self.assertIn('unittests.resources', sys.modules) + class TestDebugRepr(unittest.TestCase): def test_builtin_types(self): @@ -323,6 +350,26 @@ def tearDown(self): os.rmdir(self.temp_dir) +class TestMiscUtilities(unittest.TestCase): + def test_decamelize(self): + self.assertEqual('', util.decamelize('')) + self.assertEqual('my_base_class', util.decamelize('MyBaseClass')) + self.assertEqual('my_base_class12', util.decamelize('MyBaseClass12')) + self.assertEqual('my_class_a', util.decamelize('MyClass_A')) + self.assertEqual('my_class', util.decamelize('my_class')) + self.assertRaises(TypeError, util.decamelize, None) + self.assertRaises(TypeError, util.decamelize, 12) + + def test_sanitize(self): + self.assertEqual('', util.toalphanum('')) + self.assertEqual('ab12', util.toalphanum('ab12')) + self.assertEqual('ab1_2', util.toalphanum('ab1_2')) + self.assertEqual('ab1__2', util.toalphanum('ab1**2')) + self.assertEqual('ab__12_', util.toalphanum('ab (12)')) + self.assertRaises(TypeError, util.toalphanum, None) + self.assertRaises(TypeError, util.toalphanum, 12) + + class TestScopedDict(unittest.TestCase): def test_construction(self): d = {