From c0324aea24545baea127af922c9d3a45f9bd2304 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 12:08:17 +0100 Subject: [PATCH 1/9] Add _typeshed.importlib to house implicit protocols --- stdlib/_typeshed/importlib.pyi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 stdlib/_typeshed/importlib.pyi diff --git a/stdlib/_typeshed/importlib.pyi b/stdlib/_typeshed/importlib.pyi new file mode 100644 index 000000000000..a6f3d4f7c106 --- /dev/null +++ b/stdlib/_typeshed/importlib.pyi @@ -0,0 +1,18 @@ +# Implicit protocols used in importlib +# We intentionally omit deprecated and optional methods + +from collections.abc import Sequence +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Protocol + +__all__ = ["LoaderProtocol", "MetaPathFinderProtocol", "PathEntryFinderProtocol"] + +class LoaderProtocol(Protocol): + def load_module(self, fullname: str, /) -> ModuleType: ... + +class MetaPathFinderProtocol(Protocol): + def find_spec(self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... + +class PathEntryFinderProtocol(Protocol): + def find_spec(self, fullname: str, target: ModuleType | None = ...) -> ModuleSpec | None: ... From ae8ee9dccf54626c0d8a115057f4d1b7b9d1a01e Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 12:20:52 +0100 Subject: [PATCH 2/9] Replace definitions that rely on importlib.abc.* with protocols --- stdlib/_typeshed/importlib.pyi | 4 ++-- stdlib/importlib/abc.pyi | 4 ++-- stdlib/importlib/util.pyi | 5 +++-- stdlib/pkgutil.pyi | 12 ++++++------ stdlib/sys/__init__.pyi | 13 ++++--------- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/stdlib/_typeshed/importlib.pyi b/stdlib/_typeshed/importlib.pyi index a6f3d4f7c106..d75effb67d39 100644 --- a/stdlib/_typeshed/importlib.pyi +++ b/stdlib/_typeshed/importlib.pyi @@ -1,5 +1,5 @@ -# Implicit protocols used in importlib -# We intentionally omit deprecated and optional methods +# Implicit protocols used in importlib. +# We intentionally omit deprecated and optional methods. from collections.abc import Sequence from importlib.machinery import ModuleSpec diff --git a/stdlib/importlib/abc.pyi b/stdlib/importlib/abc.pyi index 75e78ed59172..b00986abfb2d 100644 --- a/stdlib/importlib/abc.pyi +++ b/stdlib/importlib/abc.pyi @@ -64,7 +64,7 @@ class SourceLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta): # The base classes differ starting in 3.10: if sys.version_info >= (3, 10): - # Please keep in sync with sys._MetaPathFinder + # Please keep in sync with _typeshed.MetaPathFinderProtocol class MetaPathFinder(metaclass=ABCMeta): if sys.version_info < (3, 12): def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... @@ -85,7 +85,7 @@ if sys.version_info >= (3, 10): def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ... else: - # Please keep in sync with sys._MetaPathFinder + # Please keep in sync with _typeshed.MetaPathFinderProtocol class MetaPathFinder(Finder): def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... def invalidate_caches(self) -> None: ... diff --git a/stdlib/importlib/util.pyi b/stdlib/importlib/util.pyi index 6608f70d4469..2492c76d5c6c 100644 --- a/stdlib/importlib/util.pyi +++ b/stdlib/importlib/util.pyi @@ -3,6 +3,7 @@ import importlib.machinery import sys import types from _typeshed import ReadableBuffer, StrOrBytesPath +from _typeshed.importlib import LoaderProtocol from collections.abc import Callable from typing import Any from typing_extensions import ParamSpec @@ -23,13 +24,13 @@ def source_from_cache(path: str) -> str: ... def decode_source(source_bytes: ReadableBuffer) -> str: ... def find_spec(name: str, package: str | None = None) -> importlib.machinery.ModuleSpec | None: ... def spec_from_loader( - name: str, loader: importlib.abc.Loader | None, *, origin: str | None = None, is_package: bool | None = None + name: str, loader: LoaderProtocol | None, *, origin: str | None = None, is_package: bool | None = None ) -> importlib.machinery.ModuleSpec | None: ... def spec_from_file_location( name: str, location: StrOrBytesPath | None = None, *, - loader: importlib.abc.Loader | None = None, + loader: LoaderProtocol | None = None, submodule_search_locations: list[str] | None = ..., ) -> importlib.machinery.ModuleSpec | None: ... def module_from_spec(spec: importlib.machinery.ModuleSpec) -> types.ModuleType: ... diff --git a/stdlib/pkgutil.pyi b/stdlib/pkgutil.pyi index 4a0c8d101b7a..1fcc8db3bfab 100644 --- a/stdlib/pkgutil.pyi +++ b/stdlib/pkgutil.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import SupportsRead +from _typeshed.importlib import LoaderProtocol, MetaPathFinderProtocol, PathEntryFinderProtocol from collections.abc import Callable, Iterable, Iterator -from importlib.abc import Loader, MetaPathFinder, PathEntryFinder from typing import IO, Any, NamedTuple, TypeVar from typing_extensions import deprecated @@ -23,7 +23,7 @@ if sys.version_info < (3, 12): _PathT = TypeVar("_PathT", bound=Iterable[str]) class ModuleInfo(NamedTuple): - module_finder: MetaPathFinder | PathEntryFinder + module_finder: MetaPathFinderProtocol | PathEntryFinderProtocol name: str ispkg: bool @@ -37,11 +37,11 @@ if sys.version_info < (3, 12): def __init__(self, fullname: str, file: IO[str], filename: str, etc: tuple[str, str, int]) -> None: ... @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") -def find_loader(fullname: str) -> Loader | None: ... -def get_importer(path_item: str) -> PathEntryFinder | None: ... +def find_loader(fullname: str) -> LoaderProtocol | None: ... +def get_importer(path_item: str) -> MetaPathFinderProtocol | PathEntryFinderProtocol | None: ... @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") -def get_loader(module_or_name: str) -> Loader | None: ... -def iter_importers(fullname: str = "") -> Iterator[MetaPathFinder | PathEntryFinder]: ... +def get_loader(module_or_name: str) -> LoaderProtocol | None: ... +def iter_importers(fullname: str = "") -> Iterator[MetaPathFinderProtocol | PathEntryFinderProtocol]: ... def iter_modules(path: Iterable[str] | None = None, prefix: str = "") -> Iterator[ModuleInfo]: ... def read_code(stream: SupportsRead[bytes]) -> Any: ... # undocumented def walk_packages( diff --git a/stdlib/sys/__init__.pyi b/stdlib/sys/__init__.pyi index 353e20c4b2e1..5867c9a9d510 100644 --- a/stdlib/sys/__init__.pyi +++ b/stdlib/sys/__init__.pyi @@ -1,9 +1,8 @@ import sys from _typeshed import OptExcInfo, ProfileFunction, TraceFunction, structseq +from _typeshed.importlib import MetaPathFinderProtocol, PathEntryFinderProtocol from builtins import object as _object from collections.abc import AsyncGenerator, Callable, Sequence -from importlib.abc import PathEntryFinder -from importlib.machinery import ModuleSpec from io import TextIOWrapper from types import FrameType, ModuleType, TracebackType from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final @@ -15,10 +14,6 @@ _T = TypeVar("_T") _ExitCode: TypeAlias = str | int | None _OptExcInfo: TypeAlias = OptExcInfo # noqa: Y047 # TODO: obsolete, remove fall 2022 or later -# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` -class _MetaPathFinder(Protocol): - def find_spec(self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... - # ----- sys variables ----- if sys.platform != "win32": abiflags: str @@ -44,13 +39,13 @@ if sys.version_info >= (3, 12): last_exc: BaseException # or undefined. maxsize: int maxunicode: int -meta_path: list[_MetaPathFinder] +meta_path: list[MetaPathFinderProtocol] modules: dict[str, ModuleType] if sys.version_info >= (3, 10): orig_argv: list[str] path: list[str] -path_hooks: list[Callable[[str], PathEntryFinder]] -path_importer_cache: dict[str, PathEntryFinder | None] +path_hooks: list[Callable[[str], PathEntryFinderProtocol]] +path_importer_cache: dict[str, PathEntryFinderProtocol | None] platform: str if sys.version_info >= (3, 9): platlibdir: str From a25120b4b1d31e689cc05048573494476fa8e446 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 13:00:24 +0100 Subject: [PATCH 3/9] Add regression tests for importlib --- stdlib/@tests/test_cases/check_importlib.py | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/stdlib/@tests/test_cases/check_importlib.py b/stdlib/@tests/test_cases/check_importlib.py index 8ff52a050fe0..4ee9a6fd90ce 100644 --- a/stdlib/@tests/test_cases/check_importlib.py +++ b/stdlib/@tests/test_cases/check_importlib.py @@ -1,7 +1,14 @@ +from __future__ import annotations + import importlib.abc +import importlib.util import pathlib import sys import zipfile +from collections.abc import Sequence +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing_extensions import Self # Assert that some Path classes are Traversable. if sys.version_info >= (3, 9): @@ -11,3 +18,30 @@ def traverse(t: importlib.abc.Traversable) -> None: traverse(pathlib.Path()) traverse(zipfile.Path("")) + + +class MetaFinder: + @classmethod + def find_spec(cls, fullname: str, path: Sequence[str] | None, target: ModuleType | None = None) -> ModuleSpec | None: + return None # simplified mock for demonstration purposes only + + +class PathFinder: + @classmethod + def path_hook(cls, path_entry: str) -> type[Self]: + return cls # simplified mock for demonstration purposes only + + @classmethod + def find_spec(cls, fullname: str, target: ModuleType | None = None) -> ModuleSpec | None: + return None # simplified mock for demonstration purposes only + + +class Loader: + @classmethod + def load_module(cls, fullname: str) -> ModuleType: + return ModuleType(fullname) + + +sys.meta_path.append(MetaFinder) +sys.path_hooks.append(PathFinder.path_hook) +importlib.util.spec_from_loader("tmpfile", Loader) From dde456e7ad50b45dae152b81dc982012fc53db08 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 13:25:50 +0100 Subject: [PATCH 4/9] Avoid using an existing name in test --- stdlib/@tests/test_cases/check_importlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/@tests/test_cases/check_importlib.py b/stdlib/@tests/test_cases/check_importlib.py index 4ee9a6fd90ce..17eefdafc971 100644 --- a/stdlib/@tests/test_cases/check_importlib.py +++ b/stdlib/@tests/test_cases/check_importlib.py @@ -44,4 +44,4 @@ def load_module(cls, fullname: str) -> ModuleType: sys.meta_path.append(MetaFinder) sys.path_hooks.append(PathFinder.path_hook) -importlib.util.spec_from_loader("tmpfile", Loader) +importlib.util.spec_from_loader("xxxx42xxxx", Loader) From a189e293c3d1c2dff6229df514edaa91e3efabf0 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 14:05:07 +0100 Subject: [PATCH 5/9] Fix regression in pkgutil.get_importer --- stdlib/pkgutil.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/pkgutil.pyi b/stdlib/pkgutil.pyi index 1fcc8db3bfab..7e7fa4fda9a1 100644 --- a/stdlib/pkgutil.pyi +++ b/stdlib/pkgutil.pyi @@ -38,7 +38,7 @@ if sys.version_info < (3, 12): @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") def find_loader(fullname: str) -> LoaderProtocol | None: ... -def get_importer(path_item: str) -> MetaPathFinderProtocol | PathEntryFinderProtocol | None: ... +def get_importer(path_item: str) -> PathEntryFinderProtocol | None: ... @deprecated("Use importlib.util.find_spec() instead. Will be removed in Python 3.14.") def get_loader(module_or_name: str) -> LoaderProtocol | None: ... def iter_importers(fullname: str = "") -> Iterator[MetaPathFinderProtocol | PathEntryFinderProtocol]: ... From 6cddd52181c370e327d128f1e020e71fdaf1f892 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 14:12:58 +0100 Subject: [PATCH 6/9] Make protocol arguments positional only as suggested in the review --- stdlib/_typeshed/importlib.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/_typeshed/importlib.pyi b/stdlib/_typeshed/importlib.pyi index d75effb67d39..a4e56cdaff62 100644 --- a/stdlib/_typeshed/importlib.pyi +++ b/stdlib/_typeshed/importlib.pyi @@ -15,4 +15,4 @@ class MetaPathFinderProtocol(Protocol): def find_spec(self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... class PathEntryFinderProtocol(Protocol): - def find_spec(self, fullname: str, target: ModuleType | None = ...) -> ModuleSpec | None: ... + def find_spec(self, fullname: str, target: ModuleType | None = ..., /) -> ModuleSpec | None: ... From 45b7a5c9ae08beeefd06fb80a1238a3ca1174446 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 15:50:57 +0100 Subject: [PATCH 7/9] Use _typeshed.importlib.Loader protocol in pkg_resources stub --- stubs/setuptools/pkg_resources/__init__.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stubs/setuptools/pkg_resources/__init__.pyi b/stubs/setuptools/pkg_resources/__init__.pyi index 89d9b7c51ae0..d48dd86f6a0e 100644 --- a/stubs/setuptools/pkg_resources/__init__.pyi +++ b/stubs/setuptools/pkg_resources/__init__.pyi @@ -1,6 +1,7 @@ import types import zipimport from _typeshed import Incomplete, StrPath, Unused +from _typeshed.importlib import LoaderProtocol from collections.abc import Callable, Generator, Iterable, Iterator, Sequence from io import BytesIO from itertools import chain @@ -359,7 +360,7 @@ def evaluate_marker(text: str, extra: Incomplete | None = None) -> bool: ... class NullProvider: egg_name: str | None egg_info: str | None - loader: types._LoaderProtocol | None + loader: LoaderProtocol | None module_path: str | None def __init__(self, module: _ModuleLike) -> None: ... From 7075e5e9607a8bd98912a7f9ebb3bf65cbfa1314 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 10 May 2024 16:19:01 +0100 Subject: [PATCH 8/9] Avoid duplication for LoaderProtocol --- stdlib/types.pyi | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/stdlib/types.pyi b/stdlib/types.pyi index f2d79b7f3ade..52af9ef8fa01 100644 --- a/stdlib/types.pyi +++ b/stdlib/types.pyi @@ -1,5 +1,6 @@ import sys from _typeshed import SupportsKeysAndGetItem +from _typeshed.importlib import LoaderProtocol from collections.abc import ( AsyncGenerator, Awaitable, @@ -16,7 +17,7 @@ from collections.abc import ( from importlib.machinery import ModuleSpec # pytype crashes if types.MappingProxyType inherits from collections.abc.Mapping instead of typing.Mapping -from typing import Any, ClassVar, Literal, Mapping, Protocol, TypeVar, final, overload # noqa: Y022 +from typing import Any, ClassVar, Literal, Mapping, TypeVar, final, overload # noqa: Y022 from typing_extensions import ParamSpec, Self, TypeVarTuple, deprecated __all__ = [ @@ -318,15 +319,12 @@ class SimpleNamespace: def __setattr__(self, name: str, value: Any, /) -> None: ... def __delattr__(self, name: str, /) -> None: ... -class _LoaderProtocol(Protocol): - def load_module(self, fullname: str, /) -> ModuleType: ... - class ModuleType: __name__: str __file__: str | None @property def __dict__(self) -> dict[str, Any]: ... # type: ignore[override] - __loader__: _LoaderProtocol | None + __loader__: LoaderProtocol | None __package__: str | None __path__: MutableSequence[str] __spec__: ModuleSpec | None From eecac02ef7fdbe82b9dc57fa3ca4bb9d341116d7 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Sun, 12 May 2024 11:47:11 +0200 Subject: [PATCH 9/9] Correct comment --- stdlib/importlib/abc.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/importlib/abc.pyi b/stdlib/importlib/abc.pyi index b00986abfb2d..3937481159dc 100644 --- a/stdlib/importlib/abc.pyi +++ b/stdlib/importlib/abc.pyi @@ -64,7 +64,7 @@ class SourceLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta): # The base classes differ starting in 3.10: if sys.version_info >= (3, 10): - # Please keep in sync with _typeshed.MetaPathFinderProtocol + # Please keep in sync with _typeshed.importlib.MetaPathFinderProtocol class MetaPathFinder(metaclass=ABCMeta): if sys.version_info < (3, 12): def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... @@ -85,7 +85,7 @@ if sys.version_info >= (3, 10): def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ... else: - # Please keep in sync with _typeshed.MetaPathFinderProtocol + # Please keep in sync with _typeshed.importlib.MetaPathFinderProtocol class MetaPathFinder(Finder): def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ... def invalidate_caches(self) -> None: ...