From 2a7aca9c53b5ef4b46e606a1ba1edf86eb42c77a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 24 Feb 2024 10:20:36 -0300 Subject: [PATCH] Add support for namespace packages Now subpackages that are part of a namespace package are imported under their full namespace-package name, instead of just under their subpackage name. --- changelog/11475.feature.rst | 3 + src/_pytest/pathlib.py | 44 +++++++++----- testing/test_pathlib.py | 116 +++++++++++++++++++++++++++++++----- 3 files changed, 133 insertions(+), 30 deletions(-) create mode 100644 changelog/11475.feature.rst diff --git a/changelog/11475.feature.rst b/changelog/11475.feature.rst new file mode 100644 index 00000000000..6b73c81586a --- /dev/null +++ b/changelog/11475.feature.rst @@ -0,0 +1,3 @@ +pytest now correctly identifies modules that are part of `namespace packages `__, for example when importing user-level modules for doctesting. + +Previously pytest was not aware of namespace packages, so running a doctest from a subpackage that is part of a namespace package would import just the subpackage (for example ``app.models``) instead of its full path (for example ``com.company.app.models``). diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index f1fc1366341..63ead200726 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -528,7 +528,7 @@ def import_path( pkg_root, module_name = resolve_pkg_root_and_module_name( path, consider_ns_packages=True ) - except ValueError: + except CouldNotResolvePathError: pass else: mod = _import_module_using_spec( @@ -551,8 +551,10 @@ def import_path( return mod try: - pkg_root, module_name = resolve_pkg_root_and_module_name(path) - except ValueError: + pkg_root, module_name = resolve_pkg_root_and_module_name( + path, consider_ns_packages=True + ) + except CouldNotResolvePathError: pkg_root, module_name = path.parent, path.stem # Change sys.path permanently: restoring it at the end of this function would cause surprising @@ -749,30 +751,42 @@ def resolve_pkg_root_and_module_name( Passing the full path to `models.py` will yield Path("src") and "app.core.models". - If consider_ns_packages is True, then we additionally check if the top-level directory - without __init__.py is reachable from sys.path; if it is, it is then considered a namespace package: + If consider_ns_packages is True, then we additionally check upwards in the hierarchy + until we find a directory that is reachable from sys.path, which marks it as a namespace package: https://packaging.python.org/en/latest/guides/packaging-namespace-packages - This is not the default because we need to analyze carefully if it is safe to assume this - for all imports. - - Raises ValueError if the given path does not belong to a package (missing any __init__.py files). + Raises CouldNotResolvePathError if the given path does not belong to a package (missing any __init__.py files). """ pkg_path = resolve_package_path(path) if pkg_path is not None: - pkg_root = pkg_path.parent - # pkg_root.parent does not contain a __init__.py file, as per resolve_package_path, - # but if it is reachable from sys.argv, it should be considered a namespace package. # https://packaging.python.org/en/latest/guides/packaging-namespace-packages/ - if consider_ns_packages and str(pkg_root.parent) in sys.path: - pkg_root = pkg_root.parent + pkg_root = pkg_path.parent + if consider_ns_packages: + # Go upwards in the hierarchy, if we find a parent path included + # in sys.path, it means the package found by resolve_package_path() + # actually belongs to a namespace package. + for parent in pkg_path.parents: + # If any of the parent paths has a __init__.py, it means it is not + # a namespace package (see the docs linked above). + if (parent / "__init__.py").is_file(): + break + if str(parent) in sys.path: + # Point the pkg_root to the root of the namespace package. + pkg_root = parent + break + names = list(path.with_suffix("").relative_to(pkg_root).parts) if names[-1] == "__init__": names.pop() module_name = ".".join(names) return pkg_root, module_name - raise ValueError(f"Could not resolve for {path}") + + raise CouldNotResolvePathError(f"Could not resolve for {path}") + + +class CouldNotResolvePathError(Exception): + """Custom exception raised by resolve_pkg_root_and_module_name.""" def scandir( diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 29f2a743d2e..33fda9563c2 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -16,6 +16,7 @@ from _pytest.monkeypatch import MonkeyPatch from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath +from _pytest.pathlib import CouldNotResolvePathError from _pytest.pathlib import ensure_deletable from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import get_extended_length_path_str @@ -262,20 +263,6 @@ def test_check_filepath_consistency( assert orig == p assert issubclass(ImportPathMismatchError, ImportError) - def test_issue131_on__init__(self, tmp_path: Path) -> None: - # __init__.py files may be namespace packages, and thus the - # __file__ of an imported module may not be ourselves - # see issue - tmp_path.joinpath("proja").mkdir() - p1 = tmp_path.joinpath("proja", "__init__.py") - p1.touch() - tmp_path.joinpath("sub", "proja").mkdir(parents=True) - p2 = tmp_path.joinpath("sub", "proja", "__init__.py") - p2.touch() - m1 = import_path(p1, root=tmp_path) - m2 = import_path(p2, root=tmp_path) - assert m1 == m2 - def test_ensuresyspath_append(self, tmp_path: Path) -> None: root1 = tmp_path / "root1" root1.mkdir() @@ -613,7 +600,7 @@ def test_resolve_pkg_root_and_module_name( (tmp_path / "src/app/core").mkdir(parents=True) models_py = tmp_path / "src/app/core/models.py" models_py.touch() - with pytest.raises(ValueError): + with pytest.raises(CouldNotResolvePathError): _ = resolve_pkg_root_and_module_name(models_py) # Create the __init__.py files, it should now resolve to a proper module name. @@ -918,3 +905,102 @@ def test_safe_exists(tmp_path: Path) -> None: side_effect=ValueError("name too long"), ): assert safe_exists(p) is False + + +class TestNamespacePackages: + """Test import_path support when importing from properly namespace packages.""" + + def setup_directories( + self, tmp_path: Path, monkeypatch: MonkeyPatch, pytester: Pytester + ) -> Tuple[Path, Path]: + # Set up a namespace package "com.company", containing + # two subpackages, "app" and "calc". + (tmp_path / "src/dist1/com/company/app/core").mkdir(parents=True) + (tmp_path / "src/dist1/com/company/app/__init__.py").touch() + (tmp_path / "src/dist1/com/company/app/core/__init__.py").touch() + models_py = tmp_path / "src/dist1/com/company/app/core/models.py" + models_py.touch() + + (tmp_path / "src/dist2/com/company/calc/algo").mkdir(parents=True) + (tmp_path / "src/dist2/com/company/calc/__init__.py").touch() + (tmp_path / "src/dist2/com/company/calc/algo/__init__.py").touch() + algorithms_py = tmp_path / "src/dist2/com/company/calc/algo/algorithms.py" + algorithms_py.touch() + + # Validate the namespace package by importing it in a Python subprocess. + r = pytester.runpython_c( + dedent( + f""" + import sys + sys.path.append(r{str(tmp_path / "src/dist1")!r}) + sys.path.append(r{str(tmp_path / "src/dist2")!r}) + import com.company.app.core.models + import com.company.calc.algo.algorithms + """ + ) + ) + assert r.ret == 0 + + monkeypatch.syspath_prepend(tmp_path / "src/dist1") + monkeypatch.syspath_prepend(tmp_path / "src/dist2") + return models_py, algorithms_py + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_resolve_pkg_root_and_module_name_ns_multiple_levels( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1", + "com.company.app.core.models", + ) + + mod = import_path(models_py, mode=import_mode, root=tmp_path) + assert mod.__name__ == "com.company.app.core.models" + assert mod.__file__ == str(models_py) + + pkg_root, module_name = resolve_pkg_root_and_module_name( + algorithms_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2", + "com.company.calc.algo.algorithms", + ) + + mod = import_path(algorithms_py, mode=import_mode, root=tmp_path) + assert mod.__name__ == "com.company.calc.algo.algorithms" + assert mod.__file__ == str(algorithms_py) + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_incorrect_namespace_package( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + import_mode: str, + ) -> None: + models_py, algorithms_py = self.setup_directories( + tmp_path, monkeypatch, pytester + ) + # Namespace packages must not have an __init__.py at any of its + # directories; if it does, we then fall back to importing just the + # part of the package containing the __init__.py files. + (tmp_path / "src/dist1/com/__init__.py").touch() + + pkg_root, module_name = resolve_pkg_root_and_module_name( + models_py, consider_ns_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist1/com/company", + "app.core.models", + )