From 747b8372eabc8909aae411937c6b80536bb214c4 Mon Sep 17 00:00:00 2001
From: Bruno Oliveira <nicoddemus@gmail.com>
Date: Sun, 13 Feb 2022 09:39:57 -0300
Subject: [PATCH] Try to import module before creating dummy modules with
 'importmode=importlib'

The dummy modules we introduce in `insert_missing_modules` (due to #7856 and #7859)
would cause problems if the dummy modules actually end up replacing modules
which could be imported normally because they are available in `PYTHONPATH`.

Now we attempt to first import the module via normal mechanisms, and only
introduce the dummy modules if the intermediary modules don't actually exist.

Close #9645
---
 changelog/9645.bugfix.rst  |  1 +
 src/_pytest/pathlib.py     | 19 ++++++++++++++-----
 testing/test_collection.py | 29 +++++++++++++++++++++++++++++
 testing/test_pathlib.py    | 19 ++++++++++++-------
 4 files changed, 56 insertions(+), 12 deletions(-)
 create mode 100644 changelog/9645.bugfix.rst

diff --git a/changelog/9645.bugfix.rst b/changelog/9645.bugfix.rst
new file mode 100644
index 00000000000..089a783dee6
--- /dev/null
+++ b/changelog/9645.bugfix.rst
@@ -0,0 +1 @@
+Fixed regression where ``--import-mode=importlib`` used together with :envvar:`PYTHONPATH` or :confval:`pythonpath` would cause import errors in test suites.
diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py
index def5fa94b2b..c5a411b5963 100644
--- a/src/_pytest/pathlib.py
+++ b/src/_pytest/pathlib.py
@@ -603,11 +603,20 @@ def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) ->
     module_parts = module_name.split(".")
     while module_name:
         if module_name not in modules:
-            module = ModuleType(
-                module_name,
-                doc="Empty module created by pytest's importmode=importlib.",
-            )
-            modules[module_name] = module
+            try:
+                # If sys.meta_path is empty, calling import_module will issue
+                # a warning and raise ModuleNotFoundError. To avoid the
+                # warning, we check sys.meta_path explicitly and raise the error
+                # ourselves to fall back to creating a dummy module.
+                if not sys.meta_path:
+                    raise ModuleNotFoundError
+                importlib.import_module(module_name)
+            except ModuleNotFoundError:
+                module = ModuleType(
+                    module_name,
+                    doc="Empty module created by pytest's importmode=importlib.",
+                )
+                modules[module_name] = module
         module_parts.pop(-1)
         module_name = ".".join(module_parts)
 
diff --git a/testing/test_collection.py b/testing/test_collection.py
index e79ae384d7a..a943a44d227 100644
--- a/testing/test_collection.py
+++ b/testing/test_collection.py
@@ -1507,6 +1507,35 @@ def test_modules_not_importable_as_side_effect(self, pytester: Pytester) -> None
             ]
         )
 
+    def test_using_python_path(self, pytester: Pytester) -> None:
+        """
+        Dummy modules created by insert_missing_modules should not get in
+        the way of modules that could be imported via python path (#9645).
+        """
+        pytester.makeini(
+            """
+            [pytest]
+            pythonpath = .
+            addopts = --import-mode importlib
+            """
+        )
+        pytester.makepyfile(
+            **{
+                "tests/__init__.py": "",
+                "tests/conftest.py": "",
+                "tests/subpath/__init__.py": "",
+                "tests/subpath/helper.py": "",
+                "tests/subpath/test_something.py": """
+                import tests.subpath.helper
+
+                def test_something():
+                    assert True
+                """,
+            }
+        )
+        result = pytester.runpytest()
+        result.stdout.fnmatch_lines("*1 passed in*")
+
 
 def test_does_not_crash_on_error_from_decorated_function(pytester: Pytester) -> None:
     """Regression test for an issue around bad exception formatting due to
diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py
index fe5e08f212f..c901dc6f435 100644
--- a/testing/test_pathlib.py
+++ b/testing/test_pathlib.py
@@ -562,15 +562,20 @@ def test_module_name_from_path(self, tmp_path: Path) -> None:
         result = module_name_from_path(Path("/home/foo/test_foo.py"), Path("/bar"))
         assert result == "home.foo.test_foo"
 
-    def test_insert_missing_modules(self) -> None:
-        modules = {"src.tests.foo": ModuleType("src.tests.foo")}
-        insert_missing_modules(modules, "src.tests.foo")
-        assert sorted(modules) == ["src", "src.tests", "src.tests.foo"]
+    def test_insert_missing_modules(
+        self, monkeypatch: MonkeyPatch, tmp_path: Path
+    ) -> None:
+        monkeypatch.chdir(tmp_path)
+        # Use 'xxx' and 'xxy' as parent names as they are unlikely to exist and
+        # don't end up being imported.
+        modules = {"xxx.tests.foo": ModuleType("xxx.tests.foo")}
+        insert_missing_modules(modules, "xxx.tests.foo")
+        assert sorted(modules) == ["xxx", "xxx.tests", "xxx.tests.foo"]
 
         mod = ModuleType("mod", doc="My Module")
-        modules = {"src": mod}
-        insert_missing_modules(modules, "src")
-        assert modules == {"src": mod}
+        modules = {"xxy": mod}
+        insert_missing_modules(modules, "xxy")
+        assert modules == {"xxy": mod}
 
         modules = {}
         insert_missing_modules(modules, "")