Add stacklevel tests for warnings

Resolves pytest-dev#4445 and pytest-dev#5928 (thanks to @allanlewis)

changelog/4445.bugfix.rst

@@ -0,0 +1 @@
+Fixed some warning reports produced by pytest to point to the correct location of the warning in the user's code.

changelog/5928.bugfix.rst

@@ -0,0 +1 @@
+Report ``PytestUnknownMarkWarning`` at the level of the user's code, not ``pytest``'s.

src/_pytest/hookspec.py

@@ -562,7 +562,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
 
 
 @hookspec(historic=True)
-def pytest_warning_captured(warning_message, when, item):
+def pytest_warning_captured(warning_message, when, item, location):
     """
     Process a warning captured by the internal pytest warnings plugin.
 
@@ -582,6 +582,10 @@ def pytest_warning_captured(warning_message, when, item):
         in a future release.
 
         The item being executed if ``when`` is ``"runtest"``, otherwise ``None``.
+
+    :param tuple location:
+        Holds information about the execution context of the captured warning (filename, linenumber, function).
+        ``function`` evaluates to <module> when the execution context is at the module level.
     """
 
 

src/_pytest/mark/structures.py

@@ -325,6 +325,7 @@ def __getattr__(self, name: str) -> MarkDecorator:
                     "custom marks to avoid this warning - for details, see "
                     "https://docs.pytest.org/en/latest/mark.html" % name,
                     PytestUnknownMarkWarning,
+                    2,
                 )
 
         return MarkDecorator(Mark(name, (), {}))

src/_pytest/warnings.py

@@ -149,6 +149,10 @@ def _issue_warning_captured(warning, hook, stacklevel):
         warnings.warn(warning, stacklevel=stacklevel)
     # Mypy can't infer that record=True means records is not None; help it.
     assert records is not None
+    frame = sys._getframe(stacklevel - 1)
+    location = frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name
     hook.pytest_warning_captured.call_historic(
-        kwargs=dict(warning_message=records[0], when="config", item=None)
+        kwargs=dict(
+            warning_message=records[0], when="config", item=None, location=location
+        )
     )

testing/test_warnings.py

@@ -641,3 +641,160 @@ def pytest_configure():
     assert "INTERNALERROR" not in result.stderr.str()
     warning = recwarn.pop()
     assert str(warning.message) == "from pytest_configure"
+
+
+class TestStackLevel:
+    @pytest.fixture
+    def capwarn(self, testdir):
+        class CapturedWarnings:
+            captured = []
+
+            @classmethod
+            def pytest_warning_captured(cls, warning_message, when, item, location):
+                cls.captured.append((warning_message, location))
+
+        testdir.plugins = [CapturedWarnings()]
+
+        return CapturedWarnings
+
+    def test_issue4445_rewrite(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.assertion.rewrite.py:241
+        """
+        testdir.makepyfile(some_mod="")
+        conftest = testdir.makeconftest(
+            """
+                import some_mod
+                import pytest
+
+                pytest.register_assert_rewrite("some_mod")
+            """
+        )
+        testdir.parseconfig()
+
+        # with stacklevel=5 the warning originates from register_assert_rewrite
+        # function in the created conftest.py
+        assert len(capwarn.captured) == 1
+        warning, location = capwarn.captured.pop()
+        file, lineno, func = location
+
+        assert "Module already imported" in str(warning.message)
+        assert file == str(conftest)
+        assert func == "<module>"  # the above conftest.py
+        assert lineno == 4
+
+    def test_issue4445_preparse(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.config.__init__.py:910
+        """
+        testdir.makeconftest(
+            """
+            import nothing
+            """
+        )
+        testdir.parseconfig("--help")
+
+        # with stacklevel=2 the warning should originate from config._preparse and is
+        # thrown by an errorneous conftest.py
+        assert len(capwarn.captured) == 1
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "could not load initial conftests" in str(warning.message)
+        assert "config{sep}__init__.py".format(sep=os.sep) in file
+        assert func == "_preparse"
+
+    def test_issue4445_import_plugin(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.config.__init__.py:585
+        """
+        testdir.makepyfile(
+            some_plugin="""
+            import pytest
+            pytest.skip("thing", allow_module_level=True)
+            """
+        )
+        testdir.syspathinsert()
+        testdir.parseconfig("-p", "some_plugin")
+
+        # with stacklevel=2 the warning should originate from
+        # config.PytestPluginManager.import_plugin is thrown by a skipped plugin
+
+        # During config parsing the the pluginargs are checked in a while loop
+        # that as a result of the argument count runs import_plugin twice, hence
+        # two identical warnings are captured (is this intentional?).
+        assert len(capwarn.captured) == 2
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "skipped plugin 'some_plugin': thing" in str(warning.message)
+        assert "config{sep}__init__.py".format(sep=os.sep) in file
+        assert func == "import_plugin"
+
+    def test_issue4445_resultlog(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.resultlog.py:35
+        """
+        testdir.makepyfile(
+            """
+            def test_dummy():
+                pass
+        """
+        )
+        # Use parseconfigure() because the warning in resultlog.py is triggered in
+        # the pytest_configure hook
+        testdir.parseconfigure(
+            "--result-log={dir}".format(dir=testdir.tmpdir.join("result.log"))
+        )
+
+        # with stacklevel=2 the warning originates from resultlog.pytest_configure
+        # and is thrown when --result-log is used
+        warning, location = capwarn.captured.pop()
+        file, _, func = location
+
+        assert "--result-log is deprecated" in str(warning.message)
+        assert "resultlog.py" in file
+        assert func == "pytest_configure"
+
+    def test_issue4445_cacheprovider_set(self, testdir, capwarn):
+        """#4445: Make sure the warning points to a reasonable location
+        See origin of _issue_warning_captured at: _pytest.cacheprovider.py:59
+        """
+        testdir.tmpdir.join(".pytest_cache").write("something wrong")
+        testdir.runpytest(plugins=[capwarn()])
+
+        # with stacklevel=3 the warning originates from one stacklevel above
+        # _issue_warning_captured in cacheprovider.Cache.set and is thrown
+        # when there are errors during cache folder creation
+
+        # set is called twice (in module stepwise and in cacheprovider) so emits
+        # two warnings when there are errors during cache folder creation. (is this intentional?)
+        assert len(capwarn.captured) == 2
+        warning, location = capwarn.captured.pop()
+        file, lineno, func = location
+
+        assert "could not create cache path" in str(warning.message)
+        assert "cacheprovider.py" in file
+        assert func == "set"
+
+    def test_issue4445_issue5928_mark_generator(self, testdir):
+        """#4445 and #5928: Make sure the warning from an unknown mark points to
+        the test file where this mark is used.
+        """
+        testfile = testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.unknown
+            def test_it():
+                pass
+            """
+        )
+        result = testdir.runpytest_subprocess()
+        # with stacklevel=2 the warning should originate from the above created test file
+        result.stdout.fnmatch_lines_random(
+            [
+                "*{testfile}:3*".format(testfile=str(testfile)),
+                "*Unknown pytest.mark.unknown*",
+            ]
+        )