From 39b8da4ec99799063fbdd17d232e4df16155a4f6 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 24 May 2018 11:49:13 +0200 Subject: [PATCH 1/4] Make tests compatible with pytest * Change exception name `TestLoadError` to `LoadTestError` so that pytest doesn't treat it as a test. * Stop inheriting from `unittest.TestCase` in abstract classes used in test files. * Update documentation and requirements. * Use pytest inside the `test_reframe.py` script. --- docs/started.rst | 46 ++++++++++++++++++++---------------- reframe/core/exceptions.py | 4 ++-- reframe/frontend/loader.py | 4 ++-- requirements.txt | 2 +- test_reframe.py | 4 ++-- unittests/test_launchers.py | 16 ++++++------- unittests/test_loader.py | 4 ++-- unittests/test_modules.py | 10 ++++---- unittests/test_schedulers.py | 6 ++--- 9 files changed, 51 insertions(+), 45 deletions(-) diff --git a/docs/started.rst b/docs/started.rst index b08cfd7cf4..29a0768874 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -7,14 +7,14 @@ Requirements * Python 3.5 or higher. Python 2 is not supported. - .. note:: + .. note:: .. versionchanged:: 2.8 A functional TCL modules system is no more required. ReFrame can now operate without a modules system at all. Optional ~~~~~~~~ -* For running the unit tests of the framework, the `nose `__ Python module is needed. +* For running the unit tests of the framework, the `pytest `__ unittesting framework is needed. You are advised to run the `unit tests <#running-the-unit-tests>`__ of the framework after installing it on a new system to make sure that everything works fine. @@ -42,24 +42,30 @@ The output should look like the following: .. code:: bash - test_check_failure (unittests.test_cli.TestFrontend) ... ok - test_check_sanity_failure (unittests.test_cli.TestFrontend) ... ok - test_check_submit_success (unittests.test_cli.TestFrontend) ... SKIP: job submission not supported - test_check_success (unittests.test_cli.TestFrontend) ... ok - test_checkpath_recursion (unittests.test_cli.TestFrontend) ... ok - test_custom_performance_check_failure (unittests.test_cli.TestFrontend) ... ok - ... - test_copytree (unittests.test_utility.TestOSTools) ... ok - test_grep (unittests.test_utility.TestOSTools) ... ok - test_inpath (unittests.test_utility.TestOSTools) ... ok - test_subdirs (unittests.test_utility.TestOSTools) ... ok - test_always_true (unittests.test_utility.TestUtilityFunctions) ... ok - test_standard_threshold (unittests.test_utility.TestUtilityFunctions) ... ok - - ---------------------------------------------------------------------- - Ran 235 tests in 33.842s - - OK (SKIP=7) + collected 442 items + + unittests/test_argparser.py .. [ 0%] + unittests/test_cli.py ....s........... [ 4%] + unittests/test_config.py ............... [ 7%] + unittests/test_deferrable.py .............................................. [ 17%] + unittests/test_environments.py sss...s..... [ 20%] + unittests/test_exceptions.py ............. [ 23%] + unittests/test_fields.py .................... [ 28%] + unittests/test_launchers.py .............. [ 31%] + unittests/test_loader.py ......... [ 33%] + unittests/test_logging.py ..................... [ 38%] + unittests/test_modules.py ........ssssssssssssssss............................ [ 49%] + unittests/test_pipeline.py ....s..s......................... [ 57%] + unittests/test_policies.py ............................... [ 64%] + unittests/test_runtime.py . [ 64%] + unittests/test_sanity_functions.py ............................................... [ 75%] + .............. [ 78%] + unittests/test_schedulers.py ..........s.s......ss...................s.s......ss. [ 90%] + unittests/test_script_builders.py . [ 90%] + unittests/test_utility.py ......................................... [ 99%] + unittests/test_versioning.py .. [100%] + + ======================== 411 passed, 31 skipped in 28.10 seconds ========================= You will notice in the output that all the job submission related tests have been skipped. The test suite detects if the current system has a job submission system and is configured for ReFrame (see `Configuring ReFrame for your site `__) and it will skip all the unsupported unit tests. diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index e1eaffc95a..bf51a0483d 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -40,11 +40,11 @@ class ReframeSyntaxError(ReframeError): """Raised when the syntax of regression tests is not correct.""" -class TestLoadError(ReframeError): +class LoadTestError(ReframeError): """Raised when the regression test cannot be loaded.""" -class NameConflictError(TestLoadError): +class NameConflictError(LoadTestError): """Raised when there is a name clash in the test suite.""" diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 0b359e0012..642302016c 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -10,7 +10,7 @@ import reframe.core.debug as debug import reframe.utility as util -from reframe.core.exceptions import NameConflictError, TestLoadError +from reframe.core.exceptions import NameConflictError, LoadTestError from reframe.core.logging import getlogger @@ -107,7 +107,7 @@ def load_from_module(self, module): old_syntax = hasattr(module, '_get_checks') new_syntax = hasattr(module, '_rfm_gettests') if old_syntax and new_syntax: - raise TestLoadError('%s: mixing old and new regression test ' + raise LoadTestError('%s: mixing old and new regression test ' 'syntax is not allowed' % module.__file__) if not old_syntax and not new_syntax: diff --git a/requirements.txt b/requirements.txt index 9eccdff6b3..a1fa368492 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -nose==1.3.7 +pytest>=3.5.0 diff --git a/test_reframe.py b/test_reframe.py index 75bca77b5e..a6906b5f33 100755 --- a/test_reframe.py +++ b/test_reframe.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse -import nose +import pytest import sys import unittests.fixtures as fixtures @@ -23,4 +23,4 @@ fixtures.init_runtime() sys.argv = [sys.argv[0], *rem_args] - nose.main() + pytest.main() diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py index 63d1b11de5..1e60a90975 100644 --- a/unittests/test_launchers.py +++ b/unittests/test_launchers.py @@ -24,7 +24,7 @@ def finished(self): pass -class _TestLauncher(abc.ABC, unittest.TestCase): +class _TestLauncher(abc.ABC): """Base class for launcher tests.""" def setUp(self): @@ -80,7 +80,7 @@ def test_emit_minimal_command(self): self.assertEqual(self.expected_minimal_command, emitted_command) -class TestSrunLauncher(_TestLauncher): +class TestSrunLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): return getlauncher('srun')(options=['--foo']) @@ -94,7 +94,7 @@ def expected_minimal_command(self): return 'srun --foo ls -l' -class TestSrunallocLauncher(_TestLauncher): +class TestSrunallocLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): @@ -136,7 +136,7 @@ def expected_minimal_command(self): 'ls -l') -class TestAlpsLauncher(_TestLauncher): +class TestAlpsLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): return getlauncher('alps')(options=['--foo']) @@ -150,7 +150,7 @@ def expected_minimal_command(self): return 'aprun -B --foo ls -l' -class TestMpirunLauncher(_TestLauncher): +class TestMpirunLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): return getlauncher('mpirun')(options=['--foo']) @@ -164,7 +164,7 @@ def expected_minimal_command(self): return 'mpirun -np 1 --foo ls -l' -class TestMpiexecLauncher(_TestLauncher): +class TestMpiexecLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): return getlauncher('mpiexec')(options=['--foo']) @@ -178,7 +178,7 @@ def expected_minimal_command(self): return 'mpiexec -n 1 --foo ls -l' -class TestLauncherWrapperAlps(_TestLauncher): +class TestLauncherWrapperAlps(_TestLauncher, unittest.TestCase): @property def launcher(self): return launchers.LauncherWrapper( @@ -195,7 +195,7 @@ def expected_minimal_command(self): return 'ddt --offline aprun -B --foo ls -l' -class TestLocalLauncher(_TestLauncher): +class TestLocalLauncher(_TestLauncher, unittest.TestCase): @property def launcher(self): return getlauncher('local')(['--foo']) diff --git a/unittests/test_loader.py b/unittests/test_loader.py index 7cb6246cce..22a7b506c3 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -2,7 +2,7 @@ import unittest from reframe.core.exceptions import (ConfigError, - NameConflictError, TestLoadError) + NameConflictError, LoadTestError) from reframe.core.systems import System from reframe.frontend.loader import RegressionCheckLoader @@ -48,7 +48,7 @@ def test_load_new_syntax(self): self.assertEqual(13, len(checks)) def test_load_mixed_syntax(self): - self.assertRaises(TestLoadError, self.loader.load_from_file, + self.assertRaises(LoadTestError, self.loader.load_from_file, 'unittests/resources/checks_unlisted/mixed.py') def test_conflicted_checks(self): diff --git a/unittests/test_modules.py b/unittests/test_modules.py index 975f4cbab0..d91afe9990 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -10,7 +10,7 @@ from unittests.fixtures import TEST_MODULES -class _TestModulesSystem(unittest.TestCase): +class _TestModulesSystem: def setUp(self): self.environ_save = EnvironmentSnapshot() self.modules_system.searchpath_add(TEST_MODULES) @@ -100,7 +100,7 @@ def test_emit_unload_commands(self): self.modules_system.emit_unload_commands('m0')) -class TestTModModulesSystem(_TestModulesSystem): +class TestTModModulesSystem(_TestModulesSystem, unittest.TestCase): def setUp(self): try: self.modules_system = modules.ModulesSystem.create('tmod') @@ -116,7 +116,7 @@ def expected_unload_instr(self, module): return 'module unload %s' % module -class TestTMod4ModulesSystem(_TestModulesSystem): +class TestTMod4ModulesSystem(_TestModulesSystem, unittest.TestCase): def setUp(self): try: self.modules_system = modules.ModulesSystem.create('tmod4') @@ -132,7 +132,7 @@ def expected_unload_instr(self, module): return 'module unload %s' % module -class TestLModModulesSystem(_TestModulesSystem): +class TestLModModulesSystem(_TestModulesSystem, unittest.TestCase): def setUp(self): try: self.modules_system = modules.ModulesSystem.create('lmod') @@ -148,7 +148,7 @@ def expected_unload_instr(self, module): return 'module unload %s' % module -class TestNoModModulesSystem(_TestModulesSystem): +class TestNoModModulesSystem(_TestModulesSystem, unittest.TestCase): def setUp(self): try: self.modules_system = modules.ModulesSystem.create() diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 97fa1dfdee..afe61a1c9d 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -19,7 +19,7 @@ from reframe.core.shell import BashScriptBuilder -class _TestJob(unittest.TestCase): +class _TestJob: def setUp(self): self.workdir = tempfile.mkdtemp(dir='unittests') self.testjob = self.job_type( @@ -144,7 +144,7 @@ def test_poll_before_submit(self): self.assertRaises(JobNotStartedError, self.testjob.finished) -class TestLocalJob(_TestJob): +class TestLocalJob(_TestJob, unittest.TestCase): def assertProcessDied(self, pid): try: os.kill(pid, 0) @@ -259,7 +259,7 @@ def test_cancel_term_ignore(self): self.assertProcessDied(sleep_pid) -class TestSlurmJob(_TestJob): +class TestSlurmJob(_TestJob, unittest.TestCase): @property def sched_name(self): return 'slurm' From f361acaa9e3f6d2761d61964fbf2399c9b869e78 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 24 May 2018 11:59:48 +0200 Subject: [PATCH 2/4] Use sys.exit to return the exit status of pytest.main --- test_reframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_reframe.py b/test_reframe.py index a6906b5f33..83c9bbaa91 100755 --- a/test_reframe.py +++ b/test_reframe.py @@ -23,4 +23,4 @@ fixtures.init_runtime() sys.argv = [sys.argv[0], *rem_args] - pytest.main() + sys.exit(pytest.main()) From d7d788a99f7f4ce4832af6edb3fba3fd7a4932da Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 28 May 2018 16:19:06 +0200 Subject: [PATCH 3/4] Address PR comments --- reframe/core/exceptions.py | 4 ++-- reframe/frontend/loader.py | 7 ++++--- unittests/test_loader.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index bf51a0483d..83da1a21bc 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -40,11 +40,11 @@ class ReframeSyntaxError(ReframeError): """Raised when the syntax of regression tests is not correct.""" -class LoadTestError(ReframeError): +class RegressionTestLoadError(ReframeError): """Raised when the regression test cannot be loaded.""" -class NameConflictError(LoadTestError): +class NameConflictError(RegressionTestLoadError): """Raised when there is a name clash in the test suite.""" diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 642302016c..e8e8f01f30 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -10,7 +10,7 @@ import reframe.core.debug as debug import reframe.utility as util -from reframe.core.exceptions import NameConflictError, LoadTestError +from reframe.core.exceptions import NameConflictError, RegressionTestLoadError from reframe.core.logging import getlogger @@ -107,8 +107,9 @@ def load_from_module(self, module): old_syntax = hasattr(module, '_get_checks') new_syntax = hasattr(module, '_rfm_gettests') if old_syntax and new_syntax: - raise LoadTestError('%s: mixing old and new regression test ' - 'syntax is not allowed' % module.__file__) + raise RegressionTestLoadError('%s: mixing old and new regression ' + 'test syntax is not allowed' + % module.__file__) if not old_syntax and not new_syntax: return [] diff --git a/unittests/test_loader.py b/unittests/test_loader.py index 22a7b506c3..9f2547b65a 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -1,8 +1,8 @@ import os import unittest -from reframe.core.exceptions import (ConfigError, - NameConflictError, LoadTestError) +from reframe.core.exceptions import (ConfigError, NameConflictError, + RegressionTestLoadError) from reframe.core.systems import System from reframe.frontend.loader import RegressionCheckLoader @@ -48,7 +48,7 @@ def test_load_new_syntax(self): self.assertEqual(13, len(checks)) def test_load_mixed_syntax(self): - self.assertRaises(LoadTestError, self.loader.load_from_file, + self.assertRaises(RegressionTestLoadError, self.loader.load_from_file, 'unittests/resources/checks_unlisted/mixed.py') def test_conflicted_checks(self): From f2a896f8a82ae6f31f9d7a92d9e04f9acc23bf19 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 29 May 2018 09:13:59 +0200 Subject: [PATCH 4/4] Fix code style --- reframe/frontend/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index e8e8f01f30..c00ca426f7 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -108,8 +108,8 @@ def load_from_module(self, module): new_syntax = hasattr(module, '_rfm_gettests') if old_syntax and new_syntax: raise RegressionTestLoadError('%s: mixing old and new regression ' - 'test syntax is not allowed' - % module.__file__) + 'test syntax is not allowed' % + module.__file__) if not old_syntax and not new_syntax: return []