diff --git a/changelog/13823.feature.rst b/changelog/13823.feature.rst new file mode 100644 index 00000000000..15586d70663 --- /dev/null +++ b/changelog/13823.feature.rst @@ -0,0 +1,10 @@ +Added a :confval:`strict` configuration option to enable all strictness-related options. + +When set to ``True``, the :confval:`strict` option currently enables :confval:`strict_config`, +:confval:`strict_markers`, :confval:`strict_xfail`, and :confval:`strict_parametrization_ids`. + +The individual strictness options can be explicitly set to override the global :confval:`strict` setting. + +If new strictness options are added in the future, they will also be automatically enabled by :confval:`strict`. +Therefore, we only recommend setting ``strict=True`` if you're using a locked version of pytest, +or if you want to proactively adopt new strictness options as they are added. diff --git a/changelog/13823.improvement.rst b/changelog/13823.improvement.rst new file mode 100644 index 00000000000..91ad910a177 --- /dev/null +++ b/changelog/13823.improvement.rst @@ -0,0 +1,7 @@ +Added :confval:`strict_xfail` as an alias to the ``xfail_strict`` option, +:confval:`strict_config` as an alias to the ``--strict-config`` flag, +and :confval:`strict_markers` as an alias to the ``--strict-markers`` flag. +This makes all strictness options consistently have configuration options with the prefix ``strict_``. + +Added :meth:`Config.hasini() ` method to check whether a configuration option has been explicitly set (via configuration file or ``--override-ini``), +as opposed to just using its default value. diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index e129dd931a9..790e1d89f29 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -589,18 +589,20 @@ removed in pytest 8 (deprecated since pytest 2.4.0): - ``parser.addoption(..., type="int/string/float/complex")`` - use ``type=int`` etc. instead. -The ``--strict`` command-line option -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``--strict`` command-line option (reintroduced) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. deprecated:: 6.2 -.. versionremoved:: 8.0 +.. versionchanged:: 9.0 -The ``--strict`` command-line option has been deprecated in favor of ``--strict-markers``, which +The ``--strict`` command-line option had been deprecated in favor of ``--strict-markers``, which better conveys what the option does. -We have plans to maybe in the future to reintroduce ``--strict`` and make it an encompassing -flag for all strictness related options (``--strict-markers`` and ``--strict-config`` -at the moment, more might be introduced in the future). +In version 8.1, we accidentally un-deprecated ``--strict``. + +In version 9.0, we changed ``--strict`` to make it set the new :confval:`strict` +configuration option. It now enables all strictness related options (including +:confval:`strict_markers`). .. _cmdline-preparse-deprecated: diff --git a/doc/en/example/markers.rst b/doc/en/example/markers.rst index babcd9e2f3a..071869c07b4 100644 --- a/doc/en/example/markers.rst +++ b/doc/en/example/markers.rst @@ -286,8 +286,7 @@ For an example on how to add and work with markers from a plugin, see * Asking for existing markers via ``pytest --markers`` gives good output - * Typos in function markers are treated as an error if you use - the ``--strict-markers`` option. + * Typos in function markers are treated as an error if you use the :confval:`strict_markers` configuration option. .. _`scoped-marking`: diff --git a/doc/en/how-to/mark.rst b/doc/en/how-to/mark.rst index 33f9d18bfe3..40ee14c36fd 100644 --- a/doc/en/how-to/mark.rst +++ b/doc/en/how-to/mark.rst @@ -80,14 +80,14 @@ surprising due to mistyped names. As described in the previous section, you can the warning for custom marks by registering them in your ``pytest.ini`` file or using a custom ``pytest_configure`` hook. -When the ``--strict-markers`` command-line flag is passed, any unknown marks applied +When the :confval:`strict_markers` ini option is set, any unknown marks applied with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error. You can -enforce this validation in your project by adding ``--strict-markers`` to ``addopts``: +enforce this validation in your project by setting :confval:`strict_markers` in your configuration: .. code-block:: ini [pytest] - addopts = --strict-markers + strict_markers = True markers = slow: marks tests as slow (deselect with '-m "not slow"') serial diff --git a/doc/en/how-to/skipping.rst b/doc/en/how-to/skipping.rst index 6584b1c7b24..10c45c23ed2 100644 --- a/doc/en/how-to/skipping.rst +++ b/doc/en/how-to/skipping.rst @@ -331,12 +331,12 @@ You can change this by setting the ``strict`` keyword-only parameter to ``True`` This will make ``XPASS`` ("unexpectedly passing") results from this test to fail the test suite. You can change the default value of the ``strict`` parameter using the -``xfail_strict`` ini option: +``strict_xfail`` ini option: .. code-block:: ini [pytest] - xfail_strict=true + strict_xfail=true Ignoring xfail diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 3dfa11901ea..d5f8716d7b4 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -261,7 +261,7 @@ pytest.mark.xfail Marks a test function as *expected to fail*. -.. py:function:: pytest.mark.xfail(condition=False, *, reason=None, raises=None, run=True, strict=xfail_strict) +.. py:function:: pytest.mark.xfail(condition=False, *, reason=None, raises=None, run=True, strict=strict_xfail) :keyword Union[bool, str] condition: Condition for marking the test function as xfail (``True/False`` or a @@ -286,7 +286,7 @@ Marks a test function as *expected to fail*. that are always failing and there should be a clear indication if they unexpectedly start to pass (for example a new release of a library fixes a known bug). - Defaults to :confval:`xfail_strict`, which is ``False`` by default. + Defaults to :confval:`strict_xfail`, which is ``False`` by default. Custom marks @@ -1774,25 +1774,21 @@ passed multiple times. The expected format is ``name=value``. For example:: .. confval:: markers - When the ``--strict-markers`` or ``--strict`` command-line arguments are used, + When the :confval:`strict_markers` configuration option is set, only known markers - defined in code by core pytest or some plugin - are allowed. You can list additional markers in this setting to add them to the whitelist, - in which case you probably want to add ``--strict-markers`` to ``addopts`` + in which case you probably want to set :confval:`strict_markers` to ``True`` to avoid future regressions: .. code-block:: ini [pytest] - addopts = --strict-markers + strict_markers = True markers = slow serial - .. note:: - The use of ``--strict-markers`` is highly preferred. ``--strict`` was kept for - backward compatibility only and may be confusing for others as it only applies to - markers and not to other options. .. confval:: minversion @@ -2070,7 +2066,33 @@ passed multiple times. The expected format is ``name=value``. For example:: "auto" can be used to explicitly use the global verbosity level. -.. confval:: xfail_strict +.. confval:: strict + + If set to ``True``, enables all strictness options: + + * :confval:`strict_config` + * :confval:`strict_markers` + * :confval:`strict_xfail` + * :confval:`strict_parametrization_ids` + + Plugins may also enable their own strictness options. + + If you explicitly set an individual strictness option, it takes precedence over ``strict``. + + .. note:: + If new strictness options are added to pytest in the future, they will also be enabled by ``strict``. + We therefore only recommend using this option when using a locked version of pytest, + or if you want to proactively adopt new strictness options as they are added. + + .. code-block:: ini + + [pytest] + strict = True + + .. versionadded:: 9.0 + + +.. confval:: strict_xfail If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the test suite. @@ -2080,7 +2102,38 @@ passed multiple times. The expected format is ``name=value``. For example:: .. code-block:: ini [pytest] - xfail_strict = True + strict_xfail = True + + You can also enable this option via the :confval:`strict` option. + + .. versionchanged:: 9.0 + Renamed from ``xfail_strict`` to ``strict_xfail``. + ``xfail_strict`` is accepted as an alias for ``strict_xfail``. + + +.. confval:: strict_config + + If set to ``True``, any warnings encountered while parsing the ``pytest`` section of the configuration file will raise errors. + + .. code-block:: ini + + [pytest] + strict_config = True + + You can also enable this option via the :confval:`strict` option. + + +.. confval:: strict_markers + + If set to ``True``, markers not registered in the ``markers`` section of the configuration file will raise errors. + + .. code-block:: ini + + [pytest] + strict_markers = True + + You can also enable this option via the :confval:`strict` option. + .. confval:: strict_parametrization_ids @@ -2094,6 +2147,8 @@ passed multiple times. The expected format is ``name=value``. For example:: [pytest] strict_parametrization_ids = True + You can also enable this option via the :confval:`strict` option. + For example, .. code-block:: python diff --git a/pyproject.toml b/pyproject.toml index 964c4f449dc..09c3146d792 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -355,7 +355,7 @@ max_supported_python = "3.14" [tool.pytest.ini_options] minversion = "2.0" -addopts = "-rfEX -p pytester --strict-markers" +addopts = "-rfEX -p pytester" python_files = [ "test_*.py", "*_test.py", @@ -378,8 +378,7 @@ norecursedirs = [ "build", "dist", ] -xfail_strict = true -strict_parametrization_ids = true +strict = true filterwarnings = [ "error", "default:Using or importing the ABCs:DeprecationWarning:unittest2.*", diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 2af60fa9c3c..7a36b10b541 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1515,7 +1515,12 @@ def _validate_plugins(self) -> None: ) def _warn_or_fail_if_strict(self, message: str) -> None: - if self.known_args_namespace.strict_config: + if self.hasini("strict_config"): + strict_config = self.getini("strict_config") + else: + strict_config = self.getini("strict") + + if strict_config: raise UsageError(message) self.issue_config_time_warning(PytestConfigWarning(message), stacklevel=3) @@ -1629,6 +1634,36 @@ def getini(self, name: str) -> Any: self._inicache[canonical_name] = val = self._getini(canonical_name) return val + def hasini(self, name: str) -> bool: + """Return whether the configuration value was explicitly defined. + + Returns ``True`` if the configuration option was explicitly set + either in an :ref:`ini file ` or via command-line + override (``--override-ini``). Returns ``False`` if only the + default value would be used. + + This can be used to distinguish between a value being explicitly set + to its default versus not being set at all. + + If the specified name hasn't been registered through a prior + :func:`parser.addini ` call (usually from a + plugin), a ValueError is raised. + + .. versionadded:: 9.0 + """ + canonical_name = self._parser._ini_aliases.get(name, name) + + if canonical_name not in self._parser._inidict: + raise ValueError(f"unknown configuration value: {name!r}") + + if canonical_name in self.inicfg: + return True + for alias, target in self._parser._ini_aliases.items(): + if target == canonical_name and alias in self.inicfg: + return True + + return False + # Meant for easy monkeypatching by legacypath plugin. # Can be inlined back (with no cover removed) once legacypath is gone. def _getini_unknown_type(self, name: str, type: str, value: object): diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index afd1892b222..dfa00fa7230 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -550,3 +550,40 @@ def _split_lines(self, text, width): for line in text.splitlines(): lines.extend(textwrap.wrap(line.strip(), width)) return lines + + +class OverrideIniAction(argparse.Action): + """Custom argparse action that makes a CLI flag equivalent to overriding an + option, in addition to behaving like `store_true`. + + This can simplify things since code only needs to inspect the ini option + and not consider the CLI flag. + """ + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + nargs: int | str | None = None, + *args, + ini_option: str, + ini_value: str, + **kwargs, + ) -> None: + super().__init__(option_strings, dest, 0, *args, **kwargs) + self.ini_option = ini_option + self.ini_value = ini_value + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + *args, + **kwargs, + ) -> None: + setattr(namespace, self.dest, True) + current_overrides = getattr(namespace, "override_ini", None) + if current_overrides is None: + current_overrides = [] + current_overrides.append(f"{self.ini_option}={self.ini_value}") + setattr(namespace, "override_ini", current_overrides) diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index da5d06f83b0..6ee65b1ce7e 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -103,7 +103,7 @@ def pytest_addoption(parser: Parser) -> None: dest="override_ini", action="append", help='Override ini option with "option=value" style, ' - "e.g. `-o xfail_strict=True -o cache_dir=cache`.", + "e.g. `-o strict_xfail=True -o cache_dir=cache`.", ) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 893dee90e84..fd3e8c92fa9 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -32,6 +32,7 @@ from _pytest.config import hookimpl from _pytest.config import PytestPluginManager from _pytest.config import UsageError +from _pytest.config.argparsing import OverrideIniAction from _pytest.config.argparsing import Parser from _pytest.config.compat import PathAwareHookProxy from _pytest.outcomes import exit @@ -75,20 +76,45 @@ def pytest_addoption(parser: Parser) -> None: ) group.addoption( "--strict-config", - action="store_true", - help="Any warnings encountered while parsing the `pytest` section of the " - "configuration file raise errors", + action=OverrideIniAction, + ini_option="strict_config", + ini_value="true", + help="Enables the strict_config option", ) group.addoption( "--strict-markers", - action="store_true", - help="Markers not registered in the `markers` section of the configuration " - "file raise errors", + action=OverrideIniAction, + ini_option="strict_markers", + ini_value="true", + help="Enables the strict_markers option", ) group.addoption( "--strict", - action="store_true", - help="(Deprecated) alias to --strict-markers", + action=OverrideIniAction, + ini_option="strict", + ini_value="true", + help="Enables the strict option", + ) + parser.addini( + "strict_config", + "Any warnings encountered while parsing the `pytest` section of the " + "configuration file raise errors", + type="bool", + default=False, + ) + parser.addini( + "strict_markers", + "Markers not registered in the `markers` section of the configuration " + "file raise errors", + type="bool", + default=False, + ) + parser.addini( + "strict", + "Enables all strictness options, currently: " + "strict_config, strict_markers, strict_xfail, strict_parametrization_ids", + type="bool", + default=False, ) group = parser.getgroup("pytest-warnings") diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index d0280e26945..c65f7b58773 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -580,17 +580,21 @@ def __getattr__(self, name: str) -> MarkDecorator: # If the name is not in the set of known marks after updating, # then it really is time to issue a warning or an error. if name not in self._markers: - if self._config.option.strict_markers or self._config.option.strict: - fail( - f"{name!r} not found in `markers` configuration option", - pytrace=False, - ) - # Raise a specific error for common misspellings of "parametrize". if name in ["parameterize", "parametrise", "parameterise"]: __tracebackhide__ = True fail(f"Unknown '{name}' mark, did you mean 'parametrize'?") + if self._config.hasini("strict_markers"): + strict_markers = self._config.getini("strict_markers") + else: + strict_markers = self._config.getini("strict") + if strict_markers: + fail( + f"{name!r} not found in `markers` configuration option", + pytrace=False, + ) + warnings.warn( f"Unknown pytest.mark.{name} - is this a typo? You can register " "custom marks to avoid this warning - for details, see " diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4dbf1cc8775..a355e842fbf 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -964,7 +964,10 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: def _strict_parametrization_ids_enabled(self) -> bool: if self.config: - return bool(self.config.getini("strict_parametrization_ids")) + if self.config.hasini("strict_parametrization_ids"): + return bool(self.config.getini("strict_parametrization_ids")) + else: + return bool(self.config.getini("strict")) return False def _resolve_ids(self) -> Iterable[str | _HiddenParam]: diff --git a/src/_pytest/skipping.py b/src/_pytest/skipping.py index 6ba30c4574c..8fae911ecbb 100644 --- a/src/_pytest/skipping.py +++ b/src/_pytest/skipping.py @@ -37,11 +37,12 @@ def pytest_addoption(parser: Parser) -> None: ) parser.addini( - "xfail_strict", + "strict_xfail", "Default for the strict parameter of xfail " - "markers when not given explicitly (default: False)", + "markers when not given explicitly (default: False) (alias: xfail_strict)", default=False, type="bool", + aliases=["xfail_strict"], ) @@ -74,7 +75,7 @@ def nop(*args, **kwargs): ) config.addinivalue_line( "markers", - "xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): " + "xfail(condition, ..., *, reason=..., run=True, raises=None, strict=strict_xfail): " "mark the test function as an expected failure if any of the conditions " "evaluate to True. Optionally specify a reason for better reporting " "and run=False if you don't even want to execute the test function. " @@ -213,7 +214,12 @@ def evaluate_xfail_marks(item: Item) -> Xfail | None: """Evaluate xfail marks on item, returning Xfail if triggered.""" for mark in item.iter_markers(name="xfail"): run = mark.kwargs.get("run", True) - strict = mark.kwargs.get("strict", item.config.getini("xfail_strict")) + if "strict" in mark.kwargs: + strict = mark.kwargs["strict"] + elif item.config.hasini("strict_xfail"): + strict = item.config.getini("strict_xfail") + else: + strict = item.config.getini("strict") raises = mark.kwargs.get("raises", None) if "condition" not in mark.kwargs: conditions = mark.args diff --git a/testing/plugins_integration/pytest.ini b/testing/plugins_integration/pytest.ini index 86058fbbac8..b0eb9c3806f 100644 --- a/testing/plugins_integration/pytest.ini +++ b/testing/plugins_integration/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --strict-markers +strict_markers = True asyncio_mode = strict filterwarnings = error::pytest.PytestWarning diff --git a/testing/test_collection.py b/testing/test_collection.py index 153811dea3e..cd8e13c8790 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -2726,15 +2726,17 @@ def test_1(): pass ), ], ) +@pytest.mark.parametrize("option_name", ["strict_parametrization_ids", "strict"]) def test_strict_parametrization_ids( pytester: Pytester, x_y: Sequence[tuple[int, int]], expected_duplicates: Sequence[str], + option_name: str, ) -> None: pytester.makeini( - """ + f""" [pytest] - strict_parametrization_ids = true + {option_name} = true """ ) pytester.makepyfile( diff --git a/testing/test_config.py b/testing/test_config.py index 773bd2c927d..0296abe903a 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -355,6 +355,22 @@ def test_silence_unknown_key_warning(self, pytester: Pytester) -> None: result = pytester.runpytest() result.stdout.no_fnmatch_line("*PytestConfigWarning*") + @pytest.mark.parametrize("option_name", ["strict_config", "strict"]) + def test_strict_config_ini_option( + self, pytester: Pytester, option_name: str + ) -> None: + """Test that strict_config and strict ini options enable strict config checking.""" + pytester.makeini( + f""" + [pytest] + unknown_option = 1 + {option_name} = True + """ + ) + result = pytester.runpytest() + result.stderr.fnmatch_lines("ERROR: Unknown config option: unknown_option") + assert result.ret == pytest.ExitCode.USAGE_ERROR + @pytest.mark.filterwarnings("default::pytest.PytestConfigWarning") def test_disable_warnings_plugin_disables_config_warnings( self, pytester: Pytester @@ -1215,6 +1231,67 @@ def test_confcutdir_check_isdir(self, pytester: Pytester) -> None: def test_iter_rewritable_modules(self, names, expected) -> None: assert list(_iter_rewritable_modules(names)) == expected + def test_hasini(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("myopt", "my option", default="default_value") + parser.addini("another", "another option", default="another_default") + parser.addini("aliased_opt", "option with alias", default="alias_default", aliases=["old_alias"]) + parser.addini("no_default", "option without explicit default") + """ + ) + + # Option defined in ini file. + pytester.makeini( + """ + [pytest] + myopt = from_file + """ + ) + config = pytester.parseconfig() + assert config.hasini("myopt") is True + assert config.hasini("another") is False + + # Option set via --override-ini. + config = pytester.parseconfig("-o", "another=overridden") + assert config.hasini("myopt") is True + assert config.hasini("another") is True + + # Option with no explicit value. + assert config.hasini("no_default") is False + + # Option set via alias in ini file. + pytester.makeini( + """ + [pytest] + old_alias = set_via_alias + """ + ) + config = pytester.parseconfig() + assert config.hasini("aliased_opt") is True + assert config.hasini("old_alias") is True + + # Using canonical name when alias is set. + pytester.makeini( + """ + [pytest] + aliased_opt = set_via_canonical + """ + ) + config = pytester.parseconfig() + assert config.hasini("aliased_opt") is True + assert config.hasini("old_alias") is True + + # Override via alias. + config = pytester.parseconfig("-o", "old_alias=override_via_alias") + assert config.hasini("aliased_opt") is True + assert config.hasini("old_alias") is True + + # Unknown configuration value should raise ValueError. + with pytest.raises(ValueError, match="unknown configuration value"): + config.hasini("unknown_option") + def test_add_cleanup(self, pytester: Pytester) -> None: config = Config.fromdictargs({}, []) config._do_configure() diff --git a/testing/test_mark.py b/testing/test_mark.py index ad86a9695b0..8d76ea310eb 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -183,7 +183,9 @@ def test_hello(): reprec.assertoutcome(passed=1) -@pytest.mark.parametrize("option_name", ["--strict-markers", "--strict"]) +@pytest.mark.parametrize( + "option_name", ["--strict-markers", "--strict", "strict_markers", "strict"] +) def test_strict_prohibits_unregistered_markers( pytester: Pytester, option_name: str ) -> None: @@ -195,7 +197,16 @@ def test_hello(): pass """ ) - result = pytester.runpytest(option_name) + if option_name in ("strict_markers", "strict"): + pytester.makeini( + f""" + [pytest] + {option_name} = true + """ + ) + result = pytester.runpytest() + else: + result = pytester.runpytest(option_name) assert result.ret != 0 result.stdout.fnmatch_lines( ["'unregisteredmark' not found in `markers` configuration option"] diff --git a/testing/test_skipping.py b/testing/test_skipping.py index 558e3656524..e1e25e45468 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -684,13 +684,14 @@ def test_foo(): assert result.ret == 0 @pytest.mark.parametrize("strict_val", ["true", "false"]) + @pytest.mark.parametrize("option_name", ["strict_xfail", "strict"]) def test_strict_xfail_default_from_file( - self, pytester: Pytester, strict_val + self, pytester: Pytester, strict_val: str, option_name: str ) -> None: pytester.makeini( f""" [pytest] - xfail_strict = {strict_val} + {option_name} = {strict_val} """ ) p = pytester.makepyfile( @@ -1178,7 +1179,7 @@ def test_default_markers(pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ "*skipif(condition, ..., [*], reason=...)*skip*", - "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=xfail_strict)*expected failure*", + "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=strict_xfail)*expected failure*", ] )