Further "unknown marks warning" improvements (#5178)

Further "unknown marks warning" improvements

changelog/5023.feature.rst

@@ -0,0 +1,5 @@
+New flag ``--strict-markers`` that triggers an error when unknown markers (e.g. those not registered using the `markers option`_ in the configuration file) are used in the test suite.
+
+The existing ``--strict`` option has the same behavior currently, but can be augmented in the future for additional checks.
+
+.. _`markers option`: https://docs.pytest.org/en/latest/reference.html#confval-markers

doc/en/example/markers.rst

@@ -259,7 +259,7 @@ For an example on how to add and work with markers from a plugin, see
     * Asking for existing markers via ``pytest --markers`` gives good output
 
     * Typos in function markers are treated as an error if you use
-      the ``--strict`` option.
+      the ``--strict-markers`` option.
 
 .. _`scoped-marking`:
 

doc/en/mark.rst

@@ -41,15 +41,15 @@ marks by registering them in ``pytest.ini`` like this:
         slow
         serial
 
-When the ``--strict`` command-line flag is passed, any unknown marks applied
+When the ``--strict-markers`` command-line flag is passed, any unknown marks applied
 with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
 Marks added by pytest or by a plugin instead of the decorator will not trigger
-this error.  To enforce validation of markers, add ``--strict`` to ``addopts``:
+this error.  To enforce validation of markers, add ``--strict-markers`` to ``addopts``:
 
 .. code-block:: ini
 
     [pytest]
-    addopts = --strict
+    addopts = --strict-markers
     markers =
         slow
         serial

doc/en/reference.rst

@@ -1269,15 +1269,17 @@ passed multiple times. The expected format is ``name=value``. For example::
 
 .. confval:: markers
 
-    When the ``--strict`` command-line argument is used, only known markers -
-    defined in code by core pytest or some plugin - are allowed.
-    You can list additional markers in this setting to add them to the whitelist.
+    When the ``--strict-markers`` or ``--strict`` command-line arguments are used,
+    only known markers - defined in code by core pytest or some plugin - are allowed.
 
-    You can list one marker name per line, indented from the option name.
+    You can list additional markers in this setting to add them to the whitelist,
+    in which case you probably want to add ``--strict-markers`` to ``addopts``
+    to avoid future regressions:
 
     .. code-block:: ini
 
         [pytest]
+        addopts = --strict-markers
         markers =
             slow
             serial

src/_pytest/main.py

@@ -47,11 +47,6 @@ def pytest_addoption(parser):
         type="args",
         default=[],
     )
-    # parser.addini("dirpatterns",
-    #    "patterns specifying possible locations of test files",
-    #    type="linelist", default=["**/test_*.txt",
-    #            "**/test_*.py", "**/*_test.py"]
-    # )
     group = parser.getgroup("general", "running and selection options")
     group._addoption(
         "-x",
@@ -71,9 +66,10 @@ def pytest_addoption(parser):
         help="exit after first num failures or errors.",
     )
     group._addoption(
+        "--strict-markers",
         "--strict",
         action="store_true",
-        help="marks not registered in configuration file raise errors.",
+        help="markers not registered in the `markers` section of the configuration file raise errors.",
     )
     group._addoption(
         "-c",

src/_pytest/mark/structures.py

@@ -311,8 +311,11 @@ def __getattr__(self, name):
             # If the name is not in the set of known marks after updating,
             # then it really is time to issue a warning or an error.
             if name not in self._markers:
-                if self._config.option.strict:
-                    fail("{!r} is not a registered marker".format(name), pytrace=False)
+                if self._config.option.strict_markers:
+                    fail(
+                        "{!r} not found in `markers` configuration option".format(name),
+                        pytrace=False,
+                    )
                 else:
                     warnings.warn(
                         "Unknown pytest.mark.%s - is this a typo?  You can register "

testing/code/test_excinfo.py

@@ -11,7 +11,6 @@
 import py
 import six
 from six.moves import queue
-from test_source import astonly
 
 import _pytest
 import pytest
@@ -147,7 +146,6 @@ def test_traceback_entry_getsource(self):
         assert s.startswith("def f():")
         assert s.endswith("raise ValueError")
 
-    @astonly
     @failsonjython
     def test_traceback_entry_getsource_in_construct(self):
         source = _pytest._code.Source(

testing/code/test_source.py

@@ -16,7 +16,6 @@
 import pytest
 from _pytest._code import Source
 
-astonly = pytest.mark.nothing
 failsonjython = pytest.mark.xfail("sys.platform.startswith('java')")
 
 
@@ -227,7 +226,6 @@ def test_getstatementrange_triple_quoted(self):
         s = source.getstatement(1)
         assert s == str(source)
 
-    @astonly
     def test_getstatementrange_within_constructs(self):
         source = Source(
             """\
@@ -630,7 +628,6 @@ def test_multiline():
 
 
 class TestTry(object):
-    pytestmark = astonly
     source = """\
 try:
     raise ValueError
@@ -675,7 +672,6 @@ def test_finally(self):
 
 
 class TestIf(object):
-    pytestmark = astonly
     source = """\
 if 1:
     y = 3

testing/python/fixtures.py

@@ -1925,10 +1925,10 @@ def test_hello(arg1):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=1)
 
-    @pytest.mark.issue(226)
     @pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"])
     @pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"])
     def test_ordering_dependencies_torndown_first(self, testdir, param1, param2):
+        """#226"""
         testdir.makepyfile(
             """
             import pytest
@@ -2707,9 +2707,9 @@ def test_3():
         reprec = testdir.inline_run("-v")
         reprec.assertoutcome(passed=5)
 
-    @pytest.mark.issue(246)
     @pytest.mark.parametrize("scope", ["session", "function", "module"])
     def test_finalizer_order_on_parametrization(self, scope, testdir):
+        """#246"""
         testdir.makepyfile(
             """
             import pytest
@@ -2744,8 +2744,8 @@ def test_other():
         reprec = testdir.inline_run("-lvs")
         reprec.assertoutcome(passed=3)
 
-    @pytest.mark.issue(396)
     def test_class_scope_parametrization_ordering(self, testdir):
+        """#396"""
         testdir.makepyfile(
             """
             import pytest
@@ -2865,8 +2865,8 @@ def test_foo(fix):
         res = testdir.runpytest("-v")
         res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"])
 
-    @pytest.mark.issue(920)
     def test_deterministic_fixture_collection(self, testdir, monkeypatch):
+        """#920"""
         testdir.makepyfile(
             """
             import pytest
@@ -3649,7 +3649,6 @@ class TestScopeOrdering(object):
     """Class of tests that ensure fixtures are ordered based on their scopes (#2405)"""
 
     @pytest.mark.parametrize("variant", ["mark", "autouse"])
-    @pytest.mark.issue(github="#2405")
     def test_func_closure_module_auto(self, testdir, variant, monkeypatch):
         """Semantically identical to the example posted in #2405 when ``use_mark=True``"""
         monkeypatch.setenv("FIXTURE_ACTIVATION_VARIANT", variant)

testing/python/integration.py

@@ -393,8 +393,9 @@ def test_blah(self):
         assert not call.items
 
 
-@pytest.mark.issue(351)
 class TestParameterize(object):
+    """#351"""
+
     def test_idfn_marker(self, testdir):
         testdir.makepyfile(
             """

testing/python/metafunc.py

@@ -159,8 +159,9 @@ def func(x, y):
                 ("x", "y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"]
             )
 
-    @pytest.mark.issue(510)
     def test_parametrize_empty_list(self):
+        """#510"""
+
         def func(y):
             pass
 
@@ -262,8 +263,8 @@ def test_function():
         for val, expected in values:
             assert _idval(val, "a", 6, None, item=None, config=None) == expected
 
-    @pytest.mark.issue(250)
     def test_idmaker_autoname(self):
+        """#250"""
         from _pytest.python import idmaker
 
         result = idmaker(
@@ -356,8 +357,8 @@ def test_idmaker_enum(self):
         result = idmaker(("a", "b"), [pytest.param(e.one, e.two)])
         assert result == ["Foo.one-Foo.two"]
 
-    @pytest.mark.issue(351)
     def test_idmaker_idfn(self):
+        """#351"""
         from _pytest.python import idmaker
 
         def ids(val):
@@ -375,8 +376,8 @@ def ids(val):
         )
         assert result == ["10.0-IndexError()", "20-KeyError()", "three-b2"]
 
-    @pytest.mark.issue(351)
     def test_idmaker_idfn_unique_names(self):
+        """#351"""
         from _pytest.python import idmaker
 
         def ids(val):
@@ -459,8 +460,9 @@ def test_idmaker_with_ids_unique_names(self):
         )
         assert result == ["a0", "a1", "b0", "c", "b1"]
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect(self):
+        """#714"""
+
         def func(x, y):
             pass
 
@@ -473,8 +475,9 @@ def func(x, y):
         assert metafunc._calls[0].params == dict(x=1, y=2)
         assert metafunc._calls[1].params == dict(x=1, y=3)
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_list(self):
+        """#714"""
+
         def func(x, y):
             pass
 
@@ -483,8 +486,9 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == dict(y="b")
         assert metafunc._calls[0].params == dict(x="a")
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_all(self):
+        """#714"""
+
         def func(x, y):
             pass
 
@@ -493,8 +497,9 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == {}
         assert metafunc._calls[0].params == dict(x="a", y="b")
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_empty(self):
+        """#714"""
+
         def func(x, y):
             pass
 
@@ -503,9 +508,9 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == dict(x="a", y="b")
         assert metafunc._calls[0].params == {}
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_functional(self, testdir):
         """
+        #714
         Test parametrization with 'indirect' parameter applied on
         particular arguments. As y is is direct, its value should
         be used directly rather than being passed to the fixture
@@ -532,21 +537,23 @@ def test_simple(x,y):
         result = testdir.runpytest("-v")
         result.stdout.fnmatch_lines(["*test_simple*a-b*", "*1 passed*"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_error(self, testdir):
+        """#714"""
+
         def func(x, y):
             pass
 
         metafunc = self.Metafunc(func)
         with pytest.raises(pytest.fail.Exception):
             metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "z"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_uses_no_fixture_error_indirect_false(self, testdir):
         """The 'uses no fixture' error tells the user at collection time
         that the parametrize data they've set up doesn't correspond to the
         fixtures in their test function, rather than silently ignoring this
         and letting the test potentially pass.
+
+        #714
         """
         testdir.makepyfile(
             """
@@ -560,8 +567,8 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no argument 'y'*"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_uses_no_fixture_error_indirect_true(self, testdir):
+        """#714"""
         testdir.makepyfile(
             """
             import pytest
@@ -580,8 +587,8 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_uses_no_fixture_error_indirect_string(self, testdir):
+        """#714"""
         testdir.makepyfile(
             """
             import pytest
@@ -597,8 +604,8 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_indirect_uses_no_fixture_error_indirect_list(self, testdir):
+        """#714"""
         testdir.makepyfile(
             """
             import pytest
@@ -614,8 +621,8 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue(714)
     def test_parametrize_argument_not_in_indirect_list(self, testdir):
+        """#714"""
         testdir.makepyfile(
             """
             import pytest
@@ -1201,9 +1208,9 @@ def test_foo(x):
         reprec = testdir.runpytest()
         reprec.assert_outcomes(passed=4)
 
-    @pytest.mark.issue(463)
     @pytest.mark.parametrize("attr", ["parametrise", "parameterize", "parameterise"])
     def test_parametrize_misspelling(self, testdir, attr):
+        """#463"""
         testdir.makepyfile(
             """
             import pytest
@@ -1386,8 +1393,9 @@ def pytest_generate_tests(metafunc):
         assert output.count("preparing foo-3") == 1
 
 
-@pytest.mark.issue(308)
 class TestMarkersWithParametrization(object):
+    """#308"""
+
     def test_simple_mark(self, testdir):
         s = """
             import pytest
@@ -1575,8 +1583,8 @@ def test_increment(n, expected):
         reprec = testdir.inline_run(SHOW_PYTEST_WARNINGS_ARG)
         reprec.assertoutcome(passed=2, skipped=2)
 
-    @pytest.mark.issue(290)
     def test_parametrize_ID_generation_string_int_works(self, testdir):
+        """#290"""
         testdir.makepyfile(
             """
             import pytest