diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 6b2b720f67..0355c3586c 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1280,6 +1280,10 @@ General Configuration Ignore test name conflicts when loading tests. + .. deprecated:: 3.8.0 + This option will be removed in a future version. + + .. js:attribute:: .general[].trap_job_errors diff --git a/docs/manpage.rst b/docs/manpage.rst index b04327b918..40d0715e92 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -53,6 +53,9 @@ This is something that writers of regression tests should bear in mind. This option can also be set using the :envvar:`RFM_IGNORE_CHECK_CONFLICTS` environment variable or the :js:attr:`ignore_check_conflicts` general configuration parameter. + .. deprecated:: 3.8.0 + This option will be removed in a future version. + -------------- Test filtering @@ -790,6 +793,9 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: Associated configuration parameter :js:attr:`ignore_check_conflicts` general configuration parameter ================================== ================== + .. deprecated:: 3.8.0 + This environment variable will be removed in a future version. + .. envvar:: RFM_TRAP_JOB_ERRORS diff --git a/docs/tutorial_tips_tricks.rst b/docs/tutorial_tips_tricks.rst index 1dee279aa5..3192d6d9ef 100644 --- a/docs/tutorial_tips_tricks.rst +++ b/docs/tutorial_tips_tricks.rst @@ -11,7 +11,8 @@ Debugging --------- ReFrame tests are Python classes inside Python source files, so the usual debugging techniques for Python apply, but the ReFrame frontend will filter some errors and stack traces by default in order to keep the output clean. -ReFrame test files are imported, so any error that appears during import time will cause the test loading process to fail and print a stack trace pointing to the offending line. +Generally, ReFrame will not print the full stack trace for user programming errors and will not block the test loading process. +If a test has errors and cannot be loaded, an error message will be printed and the loading of the remaining tests will continue. In the following, we have inserted a small typo in the ``hello2.py`` tutorial example: .. code:: bash @@ -20,41 +21,24 @@ In the following, we have inserted a small typo in the ``hello2.py`` tutorial ex .. code-block:: none - ./bin/reframe: name error: name 'rm' is not defined - ./bin/reframe: Traceback (most recent call last): - File "/Users/karakasv/Repositories/reframe/reframe/frontend/cli.py", line 668, in main - checks_found = loader.load_all() - File "/Users/karakasv/Repositories/reframe/reframe/frontend/loader.py", line 204, in load_all - checks.extend(self.load_from_dir(d, self._recurse)) - File "/Users/karakasv/Repositories/reframe/reframe/frontend/loader.py", line 189, in load_from_dir - checks.extend(self.load_from_file(entry.path)) - File "/Users/karakasv/Repositories/reframe/reframe/frontend/loader.py", line 174, in load_from_file - return self.load_from_module(util.import_module_from_file(filename)) - File "/Users/karakasv/Repositories/reframe/reframe/utility/__init__.py", line 96, in import_module_from_file - return importlib.import_module(module_name) - File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/importlib/__init__.py", line 127, in import_module - return _bootstrap._gcd_import(name[level:], package, level) - File "", line 1006, in _gcd_import - File "", line 983, in _find_and_load - File "", line 967, in _find_and_load_unlocked - File "", line 677, in _load_unlocked - File "", line 728, in exec_module - File "", line 219, in _call_with_frames_removed - File "/Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py", line 10, in - @rm.parameterized_test(['c'], ['cpp']) - NameError: name 'rm' is not defined - + ./bin/reframe: skipping test file '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py': name error: tutorials/basics/hello/hello2.py:17: name 's' is not defined + sanity_patterns = s.assert_found(r'Hello, World\!', 'hello.out') + (rerun with '-v' for more information) + [List of matched checks] + - HelloTest (found in '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py') + Found 1 check(s) -However, if there is a Python error inside your test's constructor, ReFrame will issue a warning and keep on loading and initializing the rest of the tests. +Notice how ReFrame prints also the source code line that caused the error. +This is not always the case, however. +ReFrame cannot always track a user error back to its source and this is particularly true for the ReFrame-specific syntactic elements, such as the class `builtins `__. +In such cases, ReFrame will just print the error message but not the source code context. +In the following example, we introduce a typo in the argument of the :obj:`@run_before` decorator: .. code-block:: none - ./bin/reframe: skipping test due to errors: HelloMultiLangTest: use `-v' for more information - FILE: /Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py:13 - ./bin/reframe: skipping test due to errors: HelloMultiLangTest: use `-v' for more information - FILE: /Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py:13 + ./bin/reframe: skipping test file '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py': reframe syntax error: invalid pipeline stage specified: 'compil' (rerun with '-v' for more information) [List of matched checks] - - HelloTest (found in '/Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello1.py') + - HelloTest (found in '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py') Found 1 check(s) @@ -66,27 +50,27 @@ As suggested by the warning message, passing :option:`-v` will give you the stac .. code-block:: none - ./bin/reframe: skipping test due to errors: HelloMultiLangTest: use `-v' for more information - FILE: /Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py:13 - Traceback (most recent call last): - File "/Users/karakasv/Repositories/reframe/reframe/core/decorators.py", line 49, in _instantiate_all - ret.append(_instantiate(cls, args)) - File "/Users/karakasv/Repositories/reframe/reframe/core/decorators.py", line 32, in _instantiate - return cls(*args) - File "/Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py", line 13, in __init__ - foo - NameError: name 'foo' is not defined - - ./bin/reframe: skipping test due to errors: HelloMultiLangTest: use `-v' for more information - FILE: /Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py:13 + ./bin/reframe: skipping test file '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py': name error: tutorials/basics/hello/hello2.py:17: name 's' is not defined + sanity_patterns = s.assert_found(r'Hello, World\!', 'hello.out') + (rerun with '-v' for more information) Traceback (most recent call last): - File "/Users/karakasv/Repositories/reframe/reframe/core/decorators.py", line 49, in _instantiate_all - ret.append(_instantiate(cls, args)) - File "/Users/karakasv/Repositories/reframe/reframe/core/decorators.py", line 32, in _instantiate - return cls(*args) - File "/Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello2.py", line 13, in __init__ - foo - NameError: name 'foo' is not defined + File "/Users/user/Repositories/reframe/reframe/frontend/loader.py", line 172, in load_from_file + util.import_module_from_file(filename, force) + File "/Users/user/Repositories/reframe/reframe/utility/__init__.py", line 101, in import_module_from_file + return importlib.import_module(module_name) + File "/usr/local/Cellar/python@3.9/3.9.1_6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/importlib/__init__.py", line 127, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + File "", line 1030, in _gcd_import + File "", line 1007, in _find_and_load + File "", line 986, in _find_and_load_unlocked + File "", line 680, in _load_unlocked + File "", line 790, in exec_module + File "", line 228, in _call_with_frames_removed + File "/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py", line 11, in + class HelloMultiLangTest(rfm.RegressionTest): + File "/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py", line 17, in HelloMultiLangTest + sanity_patterns = s.assert_found(r'Hello, World\!', 'hello.out') + NameError: name 's' is not defined Loaded 1 test(s) Generated 1 test case(s) @@ -95,9 +79,8 @@ As suggested by the warning message, passing :option:`-v` will give you the stac Filtering test cases(s) by other attributes: 1 remaining Final number of test cases: 1 [List of matched checks] - - HelloTest (found in '/Users/karakasv/Repositories/reframe/tutorials/basics/hello/hello1.py') + - HelloTest (found in '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py') Found 1 check(s) - Log file(s) saved in: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-ckymcl44.log' .. tip:: diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py index c0c03e05f1..7f4756b05f 100644 --- a/reframe/core/decorators.py +++ b/reframe/core/decorators.py @@ -21,9 +21,7 @@ import reframe.utility.osext as osext import reframe.core.warnings as warn import reframe.core.hooks as hooks -from reframe.core.exceptions import (ReframeSyntaxError, - SkipTestError, - user_frame) +from reframe.core.exceptions import ReframeSyntaxError, SkipTestError, what from reframe.core.logging import getlogger from reframe.core.pipeline import RegressionTest from reframe.utility.versioning import VersionValidator @@ -51,22 +49,18 @@ def _instantiate_all(): try: if cls in mod.__rfm_skip_tests: continue - except AttributeError: mod.__rfm_skip_tests = set() try: ret.append(_instantiate(cls, args)) except SkipTestError as e: - getlogger().warning(f'skipping test {cls.__name__!r}: {e}') + getlogger().warning(f'skipping test {cls.__qualname__!r}: {e}') except Exception: - frame = user_frame(*sys.exc_info()) - filename = frame.filename if frame else 'n/a' - lineno = frame.lineno if frame else 'n/a' + exc_info = sys.exc_info() getlogger().warning( - f"skipping test {cls.__name__!r} due to errors: " - f"use `-v' for more information\n" - f" FILE: {filename}:{lineno}" + f"skipping test {cls.__qualname__!r}: {what(*exc_info)} " + f"(rerun with '-v' for more information)" ) getlogger().verbose(traceback.format_exc()) @@ -88,8 +82,11 @@ def _validate_test(cls): 'subclass of RegressionTest') if (cls.is_abstract()): - raise ValueError(f'decorated test ({cls.__qualname__!r}) has one or ' - f'more undefined parameters') + getlogger().warning( + f'skipping test {cls.__qualname__!r}: ' + f'test has one or more undefined parameters' + ) + return False conditions = [VersionValidator(v) for v in cls._rfm_required_version] if (cls._rfm_required_version and @@ -151,7 +148,7 @@ def parameterized_test(*inst): def _do_register(cls): if _validate_test(cls): if not cls.param_space.is_empty(): - raise ValueError( + raise ReframeSyntaxError( f'{cls.__qualname__!r} is already a parameterized test' ) @@ -204,7 +201,7 @@ def required_version(*versions): ) if not versions: - raise ValueError('no versions specified') + raise ReframeSyntaxError('no versions specified') conditions = [VersionValidator(v) for v in versions] @@ -246,10 +243,11 @@ def run_before(stage): from_version='3.7.0' ) if stage not in _USER_PIPELINE_STAGES: - raise ValueError(f'invalid pipeline stage specified: {stage!r}') + raise ReframeSyntaxError( + f'invalid pipeline stage specified: {stage!r}') if stage == 'init': - raise ValueError('pre-init hooks are not allowed') + raise ReframeSyntaxError('pre-init hooks are not allowed') return hooks.attach_to('pre_' + stage) @@ -268,7 +266,8 @@ def run_after(stage): from_version='3.7.0' ) if stage not in _USER_PIPELINE_STAGES: - raise ValueError(f'invalid pipeline stage specified: {stage!r}') + raise ReframeSyntaxError( + f'invalid pipeline stage specified: {stage!r}') # Map user stage names to the actual pipeline functions if needed if stage == 'init': diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index c335277074..5ff5c7976d 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -320,6 +320,22 @@ def is_exit_request(exc_type, exc_value, tb): FailureLimitError)) +def is_user_error(exc_type, exc_value, tb): + '''Check if error is a user programming error. + + A user error is any of :py:class:`AttributeError`, :py:class:`NameError`, + :py:class:`TypeError` or :py:class:`ValueError` and the exception is + thrown from user context. + ''' + + frame = user_frame(exc_type, exc_value, tb) + if frame is None: + return False + + return isinstance(exc_value, + (AttributeError, NameError, TypeError, ValueError)) + + def is_severe(exc_type, exc_value, tb): '''Check if exception is a severe one.''' @@ -330,14 +346,8 @@ def is_severe(exc_type, exc_value, tb): if isinstance(exc_value, soft_errors): return False - # Treat specially type and value errors - type_error = isinstance(exc_value, TypeError) - value_error = isinstance(exc_value, ValueError) - frame = user_frame(exc_type, exc_value, tb) - if (type_error or value_error) and frame is not None: - return False - - return True + # User errors are treated as soft + return not is_user_error(exc_type, exc_value, tb) def what(exc_type, exc_value, tb): @@ -349,14 +359,12 @@ def what(exc_type, exc_value, tb): reason = utility.decamelize(exc_type.__name__, ' ') # We need frame information for user type and value errors - frame = user_frame(exc_type, exc_value, tb) - user_type_error = isinstance(exc_value, TypeError) and frame - user_value_error = isinstance(exc_value, ValueError) and frame if isinstance(exc_value, KeyboardInterrupt): reason = 'cancelled by user' elif isinstance(exc_value, AbortTaskError): reason = f'aborted due to {type(exc_value.__cause__).__name__}' - elif user_type_error or user_value_error: + elif is_user_error(exc_type, exc_value, tb): + frame = user_frame(exc_type, exc_value, tb) relpath = os.path.relpath(frame.filename) source = ''.join(frame.code_context) reason += f': {relpath}:{frame.lineno}: {exc_value}\n{source}' diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 12bda462b1..76fc7bab76 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -50,7 +50,7 @@ def __setitem__(self, key, value): self._namespace.pop(key, None) elif key in self['_rfm_local_param_space']: - raise ValueError( + raise ReframeSyntaxError( f'cannot override parameter {key!r}' ) else: @@ -76,7 +76,7 @@ def __getitem__(self, key): except KeyError: # Handle parameter access if key in self['_rfm_local_param_space']: - raise ValueError( + raise ReframeSyntaxError( 'accessing a test parameter from the class ' 'body is disallowed' ) from None @@ -426,7 +426,7 @@ class attribute. This behavior does not apply when the assigned value return elif not var_space[name].field is value: desc = '.'.join([cls.__qualname__, name]) - raise ValueError( + raise ReframeSyntaxError( f'cannot override variable descriptor {desc!r}' ) @@ -437,7 +437,7 @@ class attribute. This behavior does not apply when the assigned value try: param_space = super().__getattribute__('_rfm_param_space') if name in param_space.params: - raise ValueError(f'cannot override parameter {name!r}') + raise ReframeSyntaxError(f'cannot override parameter {name!r}') except AttributeError: pass diff --git a/reframe/core/namespaces.py b/reframe/core/namespaces.py index eee629a7a0..6590694f1f 100644 --- a/reframe/core/namespaces.py +++ b/reframe/core/namespaces.py @@ -10,6 +10,8 @@ import abc +from reframe.core.exceptions import ReframeSyntaxError + class LocalNamespace: '''Local namespace of a regression test. @@ -59,7 +61,7 @@ def __repr__(self): def _raise_namespace_clash(self, name): '''Raise an error if there is a namespace clash.''' - raise ValueError( + raise ReframeSyntaxError( f'{name!r} is already present in the current namespace' ) @@ -165,7 +167,7 @@ def sanity(self, cls, illegal_names=None): for key in self._namespace: if key in illegal_names: - raise ValueError( + raise ReframeSyntaxError( f'{key!r} already defined in class ' f'{cls.__qualname__!r}' ) @@ -177,6 +179,6 @@ def inject(self, obj, objtype=None): ''' def __setitem__(self, key, value): - raise ValueError( + raise ReframeSyntaxError( f'cannot set item {key!r} into a {type(self).__qualname__} object' ) diff --git a/reframe/core/parameters.py b/reframe/core/parameters.py index 1cb4b05c8f..6f678269be 100644 --- a/reframe/core/parameters.py +++ b/reframe/core/parameters.py @@ -12,6 +12,7 @@ import itertools import reframe.core.namespaces as namespaces +from reframe.core.exceptions import ReframeSyntaxError class TestParam: @@ -108,7 +109,7 @@ def join(self, other, cls): self.params[key] != () and other.params[key] != ()): - raise ValueError( + raise ReframeSyntaxError( f'parameter space conflict: ' f'parameter {key!r} is defined in more than ' f'one base class of class {cls.__qualname__!r}' @@ -132,7 +133,7 @@ def extend(self, cls): # changed using the `x = parameter([...])` syntax. for key, values in cls.__dict__.items(): if key in self.params: - raise ValueError( + raise ReframeSyntaxError( f'parameter {key!r} must be modified through the built-in ' f'parameter type' ) diff --git a/reframe/core/variables.py b/reframe/core/variables.py index 867b09a99b..2727a290c1 100644 --- a/reframe/core/variables.py +++ b/reframe/core/variables.py @@ -10,8 +10,9 @@ import math import copy -import reframe.core.namespaces as namespaces import reframe.core.fields as fields +import reframe.core.namespaces as namespaces +from reframe.core.exceptions import ReframeSyntaxError class _UndefinedType: @@ -47,7 +48,7 @@ def __init__(self, *args, **kwargs): self._default_value = kwargs.pop('value', Undefined) if not issubclass(field_type, fields.Field): - raise ValueError( + raise TypeError( f'field {field_type!r} is not derived from ' f'{fields.Field.__qualname__}' ) @@ -99,7 +100,7 @@ def __getattr__(self, name): def _check_is_defined(self): if not self.is_defined(): - raise ValueError( + raise ReframeSyntaxError( f'variable {self.name} is not assigned a value' ) @@ -451,7 +452,7 @@ def join(self, other, cls): # Make doubly declared vars illegal. Note that this will be # triggered when inheriting from multiple RegressionTest classes. if key in self.vars: - raise ValueError( + raise ReframeSyntaxError( f'variable {key!r} is declared in more than one of the ' f'parent classes of class {cls.__qualname__!r}' ) @@ -478,7 +479,7 @@ def extend(self, cls): if isinstance(var, TestVar): # Disable redeclaring a variable if key in self.vars: - raise ValueError( + raise ReframeSyntaxError( f'cannot redeclare the variable {key!r}' ) @@ -495,7 +496,7 @@ def extend(self, cls): _assigned_vars.add(key) elif value is Undefined: # Cannot be set as Undefined if not a variable - raise ValueError( + raise ReframeSyntaxError( f'{key!r} has not been declared as a variable' ) @@ -518,7 +519,7 @@ def sanity(self, cls, illegal_names=None): for key in self._namespace: if key in illegal_names and key not in self._injected_vars: - raise ValueError( + raise ReframeSyntaxError( f'{key!r} already defined in class ' f'{cls.__qualname__!r}' ) diff --git a/reframe/core/warnings.py b/reframe/core/warnings.py index 9c710f1fe3..5e13aa16ed 100644 --- a/reframe/core/warnings.py +++ b/reframe/core/warnings.py @@ -63,7 +63,7 @@ def user_deprecation_warning(message, from_version='0.0.0'): ''' - # Unroll the stack and issue the warning from the first stack frame that is + # Unwind the stack and issue the warning from the first stack frame that is # outside the framework. stack_level = 1 for s in inspect.stack(): @@ -74,7 +74,6 @@ def user_deprecation_warning(message, from_version='0.0.0'): stack_level += 1 min_version = semver.VersionInfo.parse(from_version) - version = semver.VersionInfo.parse(reframe.VERSION) if version.prerelease: # Promote prereleases, so that we issue the warning also in this case diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index eaf9a883f9..387f95fff6 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -237,7 +237,8 @@ def main(): ) locate_options.add_argument( '--ignore-check-conflicts', action='store_true', - help='Skip checks with conflicting names', + help=('Skip checks with conflicting names ' + '(this option is deprecated and has no effect)'), envvar='RFM_IGNORE_CHECK_CONFLICTS', configvar='general/ignore_check_conflicts' ) @@ -600,6 +601,12 @@ def main(): printer.error(logfiles_message()) sys.exit(1) + if site_config.get('general/0/ignore_check_conflicts'): + logging.getlogger().warning( + "the 'ignore_check_conflicts' option is deprecated " + "and will be removed in the future" + ) + rt = runtime.runtime() autodetect.detect_topology() try: @@ -706,10 +713,7 @@ def main(): loader = RegressionCheckLoader( load_path=check_search_path, - recurse=check_search_recursive, - ignore_conflicts=site_config.get( - 'general/0/ignore_check_conflicts' - ) + recurse=check_search_recursive ) def print_infoline(param, value): @@ -746,11 +750,8 @@ def print_infoline(param, value): printer.info('') try: # Locate and load checks - try: - checks_found = loader.load_all() - printer.verbose(f'Loaded {len(checks_found)} test(s)') - except OSError as e: - raise errors.ReframeError from e + checks_found = loader.load_all() + printer.verbose(f'Loaded {len(checks_found)} test(s)') # Generate all possible test cases first; we will need them for # resolving dependencies after filtering diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index d3fa9175d9..6eb24dc57d 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -11,10 +11,12 @@ import collections.abc import inspect import os +import sys +import traceback import reframe.utility as util import reframe.utility.osext as osext -from reframe.core.exceptions import NameConflictError +from reframe.core.exceptions import NameConflictError, is_severe, what from reframe.core.logging import getlogger @@ -38,12 +40,11 @@ def visit_ImportFrom(self, node): class RegressionCheckLoader: - def __init__(self, load_path, recurse=False, ignore_conflicts=False): + def __init__(self, load_path, recurse=False): # Expand any environment variables and symlinks load_path = [os.path.realpath(osext.expandvars(p)) for p in load_path] self._load_path = osext.unique_abs_paths(load_path, recurse) self._recurse = recurse - self._ignore_conflicts = ignore_conflicts # Loaded tests by name; maps test names to the file that were defined self._loaded = {} @@ -85,8 +86,7 @@ def _validate_check(self, check): for attr in required_attrs: if not hasattr(check, attr): getlogger().warning( - f'{checkfile}: {attr!r} not defined for test {name!r}; ' - f'skipping...' + f'skipping test {name!r}: {attr!r} not defined' ) return False @@ -155,13 +155,10 @@ def load_from_module(self, module): self._loaded[c.name] = testfile ret.append(c) else: - msg = (f'{testfile}: test {c.name!r} ' - f'already defined in {conflicted!r}') - - if self._ignore_conflicts: - getlogger().warning(f'{msg}; skipping...') - else: - raise NameConflictError(msg) + raise NameConflictError( + f'test {c.name!r} from {testfile!r} ' + f'is already defined in {conflicted!r}' + ) getlogger().debug(f' > Loaded {len(ret)} test(s)') return ret @@ -170,9 +167,22 @@ def load_from_file(self, filename, force=False): if not self._validate_source(filename): return [] - return self.load_from_module( - util.import_module_from_file(filename, force) - ) + try: + return self.load_from_module( + util.import_module_from_file(filename, force) + ) + except Exception: + exc_info = sys.exc_info() + if not is_severe(*exc_info): + # Simply skip the file in this case + getlogger().warning( + f"skipping test file {filename!r}: {what(*exc_info)} " + f"(rerun with '-v' for more information)" + ) + getlogger().verbose(traceback.format_exc()) + return [] + else: + raise def load_from_dir(self, dirname, recurse=False, force=False): checks = [] diff --git a/unittests/resources/checks/bad/abstract_check.py b/unittests/resources/checks/bad/abstract_check.py new file mode 100644 index 0000000000..75fc27e738 --- /dev/null +++ b/unittests/resources/checks/bad/abstract_check.py @@ -0,0 +1,14 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + + +import reframe as rfm + + +@rfm.simple_test +class AbstractTest(rfm.RegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + p = parameter() diff --git a/unittests/test_cli.py b/unittests/test_cli.py index d073bf6607..6a1055e8b9 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -62,7 +62,6 @@ def _run_reframe(system='generic:default', more_options=None, mode=None, config_file='unittests/resources/settings.py', - ignore_check_conflicts=True, perflogdir=str(perflogdir)): import reframe.frontend.cli as cli @@ -95,9 +94,6 @@ def _run_reframe(system='generic:default', elif action == 'help': argv += ['-h'] - if ignore_check_conflicts: - argv += ['--ignore-check-conflicts'] - if perflogdir: argv += ['--perflogdir', perflogdir] @@ -477,27 +473,16 @@ def test_execution_modes(run_reframe): assert 'Ran 2/2 test case' in stdout -def test_no_ignore_check_conflicts(run_reframe): - returncode, *_ = run_reframe( - checkpath=['unittests/resources/checks'], - more_options=['-R'], - ignore_check_conflicts=False, - action='list' - ) - assert returncode != 0 - - def test_timestamp_option(run_reframe): from datetime import datetime timefmt = datetime.now().strftime('xxx_%F') returncode, stdout, _ = run_reframe( checkpath=['unittests/resources/checks'], - ignore_check_conflicts=False, action='list', more_options=['-R', '--timestamp=xxx_%F'] ) - assert returncode != 0 + assert returncode == 0 assert timefmt in stdout @@ -561,8 +546,7 @@ def test_filtering_multiple_criteria(run_reframe): returncode, stdout, stderr = run_reframe( checkpath=['unittests/resources/checks'], action='list', - more_options=['-t', 'foo', '-n', 'hellocheck', - '--ignore-check-conflicts'] + more_options=['-t', 'foo', '-n', 'hellocheck'] ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr diff --git a/unittests/test_loader.py b/unittests/test_loader.py index a93204f052..39e928a35f 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -7,21 +7,20 @@ import pytest import reframe as rfm -from reframe.core.exceptions import NameConflictError, ReframeSyntaxError +from reframe.core.exceptions import ReframeSyntaxError from reframe.core.warnings import ReframeDeprecationWarning from reframe.frontend.loader import RegressionCheckLoader @pytest.fixture def loader(): - return RegressionCheckLoader(['.'], ignore_conflicts=True) + return RegressionCheckLoader(['.']) @pytest.fixture def loader_with_path(): return RegressionCheckLoader( - ['unittests/resources/checks', 'unittests/foobar'], - ignore_conflicts=True + ['unittests/resources/checks', 'unittests/foobar'] ) @@ -49,12 +48,6 @@ def test_load_all(loader_with_path): assert 12 == len(checks) -def test_conflicted_checks(loader_with_path): - loader_with_path._ignore_conflicts = False - with pytest.raises(NameConflictError): - loader_with_path.load_all() - - def test_load_error(loader): with pytest.raises(OSError): loader.load_from_file('unittests/resources/checks/foo.py') @@ -62,9 +55,8 @@ def test_load_error(loader): def test_load_bad_required_version(loader): with pytest.warns(ReframeDeprecationWarning): - with pytest.raises(ValueError): - loader.load_from_file('unittests/resources/checks_unlisted/' - 'no_required_version.py') + loader.load_from_file('unittests/resources/checks_unlisted/' + 'no_required_version.py') def test_load_bad_init(loader): diff --git a/unittests/test_parameters.py b/unittests/test_parameters.py index fd48df06b4..dfc5556a06 100644 --- a/unittests/test_parameters.py +++ b/unittests/test_parameters.py @@ -9,6 +9,7 @@ import reframe as rfm +from reframe.core.exceptions import ReframeSyntaxError class NoParams(rfm.RunOnlyRegressionTest): @@ -143,13 +144,6 @@ class MyTest(ExtendParams): test = MyTest(_rfm_use_params=True) -def test_register_abstract_test(): - with pytest.raises(ValueError): - @rfm.simple_test - class MyTest(Abstract): - pass - - def test_simple_test_decorator(): @rfm.simple_test class MyTest(ExtendParams): @@ -168,7 +162,7 @@ class MyTest(ExtendParams): 'ignore::reframe.core.warnings.ReframeDeprecationWarning' ) def test_parameterized_test_is_incompatible(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): @rfm.parameterized_test(['var']) class MyTest(TwoParams): def __init__(self, var): @@ -182,7 +176,7 @@ class Spam(rfm.RegressionMixin): class Ham(rfm.RegressionMixin): P0 = parameter([2]) - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Eggs(Spam, Ham): '''Trigger error from param name clashing.''' @@ -202,20 +196,20 @@ def test_namespace_clash(): class Spam(rfm.RegressionTest): foo = variable(int) - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Ham(Spam): foo = parameter([1]) def test_double_declare(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(rfm.RegressionTest): P0 = parameter([1, 2, 3]) P0 = parameter() def test_overwrite_param(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(TwoParams): P0 = [1, 2, 3] @@ -246,7 +240,7 @@ class Bar(Base): def test_param_access(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Foo(rfm.RegressionTest): p = parameter([1, 2, 3]) x = f'accessing {p!r} in the class body is disallowed.' @@ -256,12 +250,12 @@ def test_param_space_read_only(): class Foo(rfm.RegressionMixin): pass - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): Foo.param_space['a'] = (1, 2, 3) def test_parameter_override(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): # Trigger the check from MetaNamespace class MyTest(rfm.RegressionTest): p = parameter([1, 2]) @@ -271,7 +265,7 @@ class MyTest(rfm.RegressionTest): class Foo(rfm.RegressionTest): p = parameter([1, 2]) - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Bar(Foo): p = 0 @@ -285,12 +279,12 @@ class Foo(rfm.RegressionTest): def test_override_parameter(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Foo(rfm.RegressionTest): p = parameter([1, 2]) p = 1 - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Foo(rfm.RegressionTest): p = parameter([1, 2]) p += 1 @@ -308,5 +302,5 @@ class MyTest(rfm.RegressionTest): p = parameter([1, 2, 3]) assert MyTest.p == (1, 2, 3,) - with pytest.raises(ValueError, match='cannot override parameter'): + with pytest.raises(ReframeSyntaxError, match='cannot override parameter'): MyTest.p = (4, 5,) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index ff630b5791..20030a2ab6 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -716,7 +716,7 @@ def x(self): def test_multiple_inheritance(HelloTest): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(rfm.RunOnlyRegressionTest, HelloTest): pass diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 0c4a0fb8de..9e92b24997 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -65,8 +65,7 @@ def timestamps(self): @pytest.fixture def make_loader(): def _make_loader(check_search_path): - return RegressionCheckLoader(check_search_path, - ignore_conflicts=True) + return RegressionCheckLoader(check_search_path) return _make_loader diff --git a/unittests/test_variables.py b/unittests/test_variables.py index a1c692645a..fa86fdadad 100644 --- a/unittests/test_variables.py +++ b/unittests/test_variables.py @@ -8,6 +8,7 @@ import math import reframe as rfm +from reframe.core.exceptions import ReframeSyntaxError @pytest.fixture @@ -41,19 +42,19 @@ def test_custom_variable(OneVarTest): def test_redeclare_builtin_var_clash(NoVarsTest): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(NoVarsTest): name = variable(str) def test_name_clash_builtin_property(NoVarsTest): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(NoVarsTest): current_environ = variable(str) def test_redeclare_var_clash(OneVarTest): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(OneVarTest): foo = variable(str) @@ -62,7 +63,7 @@ def test_inheritance_clash(NoVarsTest): class MyMixin(rfm.RegressionMixin): name = variable(str) - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(NoVarsTest, MyMixin): '''Trigger error from inheritance clash.''' @@ -86,13 +87,13 @@ class Spam(rfm.RegressionMixin): class Ham(rfm.RegressionMixin): v0 = variable(int, value=2) - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Eggs(Spam, Ham): '''Trigger error from var name clashing.''' def test_double_declare(): - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class MyTest(rfm.RegressionTest): v0 = variable(int, value=1) v0 = variable(float, value=0.5) @@ -111,10 +112,12 @@ class MyTest(rfm.RegressionTest): class Descriptor: '''Dummy descriptor to attempt overriding the variable descriptor.''' + def __get__(self, obj, objtype=None): return 'dummy descriptor' - with pytest.raises(ValueError, match='cannot override variable descr'): + with pytest.raises(ReframeSyntaxError, + match='cannot override variable descr'): MyTest.v0 = Descriptor() @@ -171,7 +174,7 @@ def __init__(self): def test_required_non_var(): msg = "'not_a_var' has not been declared as a variable" - with pytest.raises(ValueError, match=msg): + with pytest.raises(ReframeSyntaxError, match=msg): class Foo(rfm.RegressionTest): not_a_var = required @@ -180,7 +183,7 @@ def test_invalid_field(): class Foo: '''An invalid descriptor''' - with pytest.raises(ValueError): + with pytest.raises(TypeError): class MyTest(rfm.RegressionTest): a = variable(int, value=4, field=Foo) @@ -215,7 +218,7 @@ class Foo(rfm.RegressionMixin): x = f'accessing {my_var!r} works because it has a default value.' assert 'bananas' in getattr(Foo, 'x') - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): class Foo(rfm.RegressionMixin): my_var = variable(int) x = f'accessing {my_var!r} fails because its value is not set.' @@ -225,7 +228,7 @@ def test_var_space_is_read_only(): class Foo(rfm.RegressionMixin): pass - with pytest.raises(ValueError): + with pytest.raises(ReframeSyntaxError): Foo._rfm_var_space['v'] = 0