From 5067f08affc912860b20f03fa5476d536e986745 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 22 May 2022 22:07:49 -0400 Subject: [PATCH] Remove dependency on `pkg_resources` from `setuptools` (#1536) Avoid using `importlib.util.find_spec()` to avoid actually importing parent packages. --- ChangeLog | 4 +++ astroid/interpreter/_import/spec.py | 11 ++++---- astroid/interpreter/_import/util.py | 40 +++++++++++++++++++++-------- astroid/manager.py | 3 ++- setup.cfg | 1 - tests/unittest_manager.py | 22 +++++++++------- 6 files changed, 55 insertions(+), 26 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1f4f1480e..0011d3c54 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,10 @@ Release date: TBA Closes #1512 +* Remove dependency on ``pkg_resources`` from ``setuptools``. + + Closes #1103 + * Allowed ``AstroidManager.clear_cache`` to reload necessary brain plugins. * Fixed incorrect inferences after rebuilding the builtins module, e.g. by calling diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index ced634c0c..a8dc7cf51 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -10,6 +10,7 @@ import importlib.machinery import importlib.util import os +import pathlib import sys import zipimport from collections.abc import Sequence @@ -147,7 +148,7 @@ def contribute_to_path(self, spec, processed): # Builtin. return None - if _is_setuptools_namespace(spec.location): + if _is_setuptools_namespace(Path(spec.location)): # extend_path is called, search sys.path for module/packages # of this name see pkgutil.extend_path documentation path = [ @@ -179,7 +180,7 @@ def contribute_to_path(self, spec, processed): class ExplicitNamespacePackageFinder(ImportlibFinder): - """A finder for the explicit namespace packages, generated through pkg_resources.""" + """A finder for the explicit namespace packages.""" def find_module(self, modname, module_parts, processed, submodule_path): if processed: @@ -253,12 +254,12 @@ def contribute_to_path(self, spec, processed): ) -def _is_setuptools_namespace(location): +def _is_setuptools_namespace(location: pathlib.Path) -> bool: try: - with open(os.path.join(location, "__init__.py"), "rb") as stream: + with open(location / "__init__.py", "rb") as stream: data = stream.read(4096) except OSError: - return None + return False else: extend_path = b"pkgutil" in data and b"extend_path" in data declare_namespace = ( diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index ce3da7eac..53c6922c3 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,15 +2,35 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt -try: - import pkg_resources -except ImportError: - pkg_resources = None # type: ignore[assignment] +import sys +from functools import lru_cache +from importlib.util import _find_spec_from_path -def is_namespace(modname): - return ( - pkg_resources is not None - and hasattr(pkg_resources, "_namespace_packages") - and modname in pkg_resources._namespace_packages - ) +@lru_cache(maxsize=4096) +def is_namespace(modname: str) -> bool: + if modname in sys.builtin_module_names: + return False + + found_spec = None + + # find_spec() attempts to import parent packages when given dotted paths. + # That's unacceptable here, so we fallback to _find_spec_from_path(), which does + # not, but requires instead that each single parent ('astroid', 'nodes', etc.) + # be specced from left to right. + processed_components = [] + last_parent = None + for component in modname.split("."): + processed_components.append(component) + working_modname = ".".join(processed_components) + try: + found_spec = _find_spec_from_path(working_modname, last_parent) + except ValueError: + # executed .pth files may not have __spec__ + return True + last_parent = working_modname + + if found_spec is None: + return False + + return found_spec.origin is None diff --git a/astroid/manager.py b/astroid/manager.py index eb65944ca..c796cfbc9 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -18,7 +18,7 @@ from astroid.const import BRAIN_MODULES_DIRECTORY from astroid.exceptions import AstroidBuildingError, AstroidImportError -from astroid.interpreter._import import spec +from astroid.interpreter._import import spec, util from astroid.modutils import ( NoSourceFile, _cache_normalize_path_, @@ -384,6 +384,7 @@ def clear_cache(self) -> None: for lru_cache in ( LookupMixIn.lookup, _cache_normalize_path_, + util.is_namespace, ObjectModel.attributes, ): lru_cache.cache_clear() diff --git a/setup.cfg b/setup.cfg index fa7c436e0..1e2341a88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,6 @@ packages = find: install_requires = lazy_object_proxy>=1.4.0 wrapt>=1.11,<2 - setuptools>=20.0 typed-ast>=1.4.0,<2.0;implementation_name=="cpython" and python_version<"3.8" typing-extensions>=3.10;python_version<"3.10" python_requires = >=3.7.2 diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 8c8dd4bf3..0bce34871 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -10,12 +10,11 @@ from collections.abc import Iterator from contextlib import contextmanager -import pkg_resources - import astroid from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.interpreter._import import util from astroid.modutils import is_standard_module from astroid.nodes import Const from astroid.nodes.scoped_nodes import ClassDef @@ -111,6 +110,16 @@ def test_ast_from_namespace_pkgutil(self) -> None: def test_ast_from_namespace_pkg_resources(self) -> None: self._test_ast_from_old_namespace_package_protocol("pkg_resources") + def test_identify_old_namespace_package_protocol(self) -> None: + # Like the above cases, this package follows the old namespace package protocol + # astroid currently assumes such packages are in sys.modules, so import it + # pylint: disable-next=import-outside-toplevel + import tests.testdata.python3.data.path_pkg_resources_1.package.foo as _ # noqa + + self.assertTrue( + util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1") + ) + def test_implicit_namespace_package(self) -> None: data_dir = os.path.dirname(resources.find("data/namespace_pep_420")) contribute = os.path.join(data_dir, "contribute_to_namespace") @@ -131,7 +140,6 @@ def test_implicit_namespace_package(self) -> None: def test_namespace_package_pth_support(self) -> None: pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = [] try: module = self.manager.ast_from_module_name("foogle.fax") @@ -141,18 +149,14 @@ def test_namespace_package_pth_support(self) -> None: with self.assertRaises(AstroidImportError): self.manager.ast_from_module_name("foogle.moogle") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def test_nested_namespace_import(self) -> None: pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = ["foogle.crank"] - pkg_resources._namespace_packages["foogle.crank"] = [] try: self.manager.ast_from_module_name("foogle.crank") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def test_namespace_and_file_mismatch(self) -> None: @@ -161,12 +165,10 @@ def test_namespace_and_file_mismatch(self) -> None: self.assertEqual(ast.name, "unittest") pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = [] try: with self.assertRaises(AstroidImportError): self.manager.ast_from_module_name("unittest.foogle.fax") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def _test_ast_from_zip(self, archive: str) -> None: @@ -323,6 +325,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None: lrus = ( astroid.nodes.node_classes.LookupMixIn.lookup, astroid.modutils._cache_normalize_path_, + util.is_namespace, astroid.interpreter.objectmodel.ObjectModel.attributes, ) @@ -332,6 +335,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None: # Generate some hits and misses ClassDef().lookup("garbage") is_standard_module("unittest", std_path=["garbage_path"]) + util.is_namespace("unittest") astroid.interpreter.objectmodel.ObjectModel().attributes() # Did the hits or misses actually happen?