Add support for namespace packages

Now subpackages that are part of a namespace package are imported under their full namespace-package name, instead of just under their subpackage name.

changelog/11475.feature.rst

@@ -0,0 +1,3 @@
+pytest now correctly identifies modules that are part of `namespace packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages>`__, for example when importing user-level modules for doctesting.
+
+Previously pytest was not aware of namespace packages, so running a doctest from a subpackage that is part of a namespace package would import just the subpackage (for example ``app.models``) instead of its full path (for example ``com.company.app.models``).

src/_pytest/pathlib.py

@@ -528,7 +528,7 @@ def import_path(
             pkg_root, module_name = resolve_pkg_root_and_module_name(
                 path, consider_ns_packages=True
             )
-        except ValueError:
+        except CouldNotResolvePathError:
             pass
         else:
             mod = _import_module_using_spec(
@@ -551,8 +551,10 @@ def import_path(
         return mod
 
     try:
-        pkg_root, module_name = resolve_pkg_root_and_module_name(path)
-    except ValueError:
+        pkg_root, module_name = resolve_pkg_root_and_module_name(
+            path, consider_ns_packages=True
+        )
+    except CouldNotResolvePathError:
         pkg_root, module_name = path.parent, path.stem
 
     # Change sys.path permanently: restoring it at the end of this function would cause surprising
@@ -749,30 +751,42 @@ def resolve_pkg_root_and_module_name(
 
     Passing the full path to `models.py` will yield Path("src") and "app.core.models".
 
-    If consider_ns_packages is True, then we additionally check if the top-level directory
-    without __init__.py is reachable from sys.path; if it is, it is then considered a namespace package:
+    If consider_ns_packages is True, then we additionally check upwards in the hierarchy
+    until we find a directory that is reachable from sys.path, which marks it as a namespace package:
 
     https://packaging.python.org/en/latest/guides/packaging-namespace-packages
 
-    This is not the default because we need to analyze carefully if it is safe to assume this
-    for all imports.
-
-    Raises ValueError if the given path does not belong to a package (missing any __init__.py files).
+    Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files).
     """
     pkg_path = resolve_package_path(path)
     if pkg_path is not None:
-        pkg_root = pkg_path.parent
-        # pkg_root.parent does not contain a __init__.py file, as per resolve_package_path,
-        # but if it is reachable from sys.argv, it should be considered a namespace package.
         # https://packaging.python.org/en/latest/guides/packaging-namespace-packages/
-        if consider_ns_packages and str(pkg_root.parent) in sys.path:
-            pkg_root = pkg_root.parent
+        pkg_root = pkg_path.parent
+        if consider_ns_packages:
+            # Go upwards in the hierarchy, if we find a parent path included
+            # in sys.path, it means the package found by resolve_package_path()
+            # actually belongs to a namespace package.
+            for parent in pkg_path.parents:
+                # If any of the parent paths has a __init__.py, it means it is not
+                # a namespace package (see the docs linked above).
+                if (parent / "__init__.py").is_file():
+                    break
+                if str(parent) in sys.path:
+                    # Point the pkg_root to the root of the namespace package.
+                    pkg_root = parent
+                    break
+
         names = list(path.with_suffix("").relative_to(pkg_root).parts)
         if names[-1] == "__init__":
             names.pop()
         module_name = ".".join(names)
         return pkg_root, module_name
-    raise ValueError(f"Could not resolve for {path}")
+
+    raise CouldNotResolvePathError(f"Could not resolve for {path}")
+
+
+class CouldNotResolvePathError(Exception):
+    """Custom exception raised by resolve_pkg_root_and_module_name."""
 
 
 def scandir(

testing/test_pathlib.py

@@ -16,6 +16,7 @@
 from _pytest.monkeypatch import MonkeyPatch
 from _pytest.pathlib import bestrelpath
 from _pytest.pathlib import commonpath
+from _pytest.pathlib import CouldNotResolvePathError
 from _pytest.pathlib import ensure_deletable
 from _pytest.pathlib import fnmatch_ex
 from _pytest.pathlib import get_extended_length_path_str
@@ -262,20 +263,6 @@ def test_check_filepath_consistency(
         assert orig == p
         assert issubclass(ImportPathMismatchError, ImportError)
 
-    def test_issue131_on__init__(self, tmp_path: Path) -> None:
-        # __init__.py files may be namespace packages, and thus the
-        # __file__ of an imported module may not be ourselves
-        # see issue
-        tmp_path.joinpath("proja").mkdir()
-        p1 = tmp_path.joinpath("proja", "__init__.py")
-        p1.touch()
-        tmp_path.joinpath("sub", "proja").mkdir(parents=True)
-        p2 = tmp_path.joinpath("sub", "proja", "__init__.py")
-        p2.touch()
-        m1 = import_path(p1, root=tmp_path)
-        m2 = import_path(p2, root=tmp_path)
-        assert m1 == m2
-
     def test_ensuresyspath_append(self, tmp_path: Path) -> None:
         root1 = tmp_path / "root1"
         root1.mkdir()
@@ -613,7 +600,7 @@ def test_resolve_pkg_root_and_module_name(
         (tmp_path / "src/app/core").mkdir(parents=True)
         models_py = tmp_path / "src/app/core/models.py"
         models_py.touch()
-        with pytest.raises(ValueError):
+        with pytest.raises(CouldNotResolvePathError):
             _ = resolve_pkg_root_and_module_name(models_py)
 
         # Create the __init__.py files, it should now resolve to a proper module name.
@@ -918,3 +905,102 @@ def test_safe_exists(tmp_path: Path) -> None:
         side_effect=ValueError("name too long"),
     ):
         assert safe_exists(p) is False
+
+
+class TestNamespacePackages:
+    """Test import_path support when importing from properly namespace packages."""
+
+    def setup_directories(
+        self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester
+    ) -> Tuple[Path, Path]:
+        # Set up a namespace package "com.company", containing
+        # two subpackages, "app" and "calc".
+        (tmp_path / "src/dist1/com/company/app/core").mkdir(parents=True)
+        (tmp_path / "src/dist1/com/company/app/__init__.py").touch()
+        (tmp_path / "src/dist1/com/company/app/core/__init__.py").touch()
+        models_py = tmp_path / "src/dist1/com/company/app/core/models.py"
+        models_py.touch()
+
+        (tmp_path / "src/dist2/com/company/calc/algo").mkdir(parents=True)
+        (tmp_path / "src/dist2/com/company/calc/__init__.py").touch()
+        (tmp_path / "src/dist2/com/company/calc/algo/__init__.py").touch()
+        algorithms_py = tmp_path / "src/dist2/com/company/calc/algo/algorithms.py"
+        algorithms_py.touch()
+
+        # Validate the namespace package by importing it in a Python subprocess.
+        r = pytester.runpython_c(
+            dedent(
+                f"""
+                import sys
+                sys.path.append(r{str(tmp_path / "src/dist1")!r})
+                sys.path.append(r{str(tmp_path / "src/dist2")!r})
+                import com.company.app.core.models
+                import com.company.calc.algo.algorithms
+                """
+            )
+        )
+        assert r.ret == 0
+
+        monkeypatch.syspath_prepend(tmp_path / "src/dist1")
+        monkeypatch.syspath_prepend(tmp_path / "src/dist2")
+        return models_py, algorithms_py
+
+    @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"])
+    def test_resolve_pkg_root_and_module_name_ns_multiple_levels(
+        self,
+        tmp_path: Path,
+        monkeypatch: MonkeyPatch,
+        pytester: Pytester,
+        import_mode: str,
+    ) -> None:
+        models_py, algorithms_py = self.setup_directories(
+            tmp_path, monkeypatch, pytester
+        )
+
+        pkg_root, module_name = resolve_pkg_root_and_module_name(
+            models_py, consider_ns_packages=True
+        )
+        assert (pkg_root, module_name) == (
+            tmp_path / "src/dist1",
+            "com.company.app.core.models",
+        )
+
+        mod = import_path(models_py, mode=import_mode, root=tmp_path)
+        assert mod.__name__ == "com.company.app.core.models"
+        assert mod.__file__ == str(models_py)
+
+        pkg_root, module_name = resolve_pkg_root_and_module_name(
+            algorithms_py, consider_ns_packages=True
+        )
+        assert (pkg_root, module_name) == (
+            tmp_path / "src/dist2",
+            "com.company.calc.algo.algorithms",
+        )
+
+        mod = import_path(algorithms_py, mode=import_mode, root=tmp_path)
+        assert mod.__name__ == "com.company.calc.algo.algorithms"
+        assert mod.__file__ == str(algorithms_py)
+
+    @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"])
+    def test_incorrect_namespace_package(
+        self,
+        tmp_path: Path,
+        monkeypatch: MonkeyPatch,
+        pytester: Pytester,
+        import_mode: str,
+    ) -> None:
+        models_py, algorithms_py = self.setup_directories(
+            tmp_path, monkeypatch, pytester
+        )
+        # Namespace packages must not have an __init__.py at any of its
+        # directories; if it does, we then fall back to importing just the
+        # part of the package containing the __init__.py files.
+        (tmp_path / "src/dist1/com/__init__.py").touch()
+
+        pkg_root, module_name = resolve_pkg_root_and_module_name(
+            models_py, consider_ns_packages=True
+        )
+        assert (pkg_root, module_name) == (
+            tmp_path / "src/dist1/com/company",
+            "app.core.models",
+        )