From 22bca184df3d2fbbfa2f32eab89fbb9ef076d97d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 1 May 2022 10:25:56 -0400 Subject: [PATCH 01/20] Remove dependence on `pkg_resources` from `setuptools` --- ChangeLog | 4 ++++ astroid/interpreter/_import/spec.py | 2 +- astroid/interpreter/_import/util.py | 21 ++++++++++++--------- setup.cfg | 1 - tests/unittest_manager.py | 9 --------- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5e9e31e115..651762ca2a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,10 @@ Release date: TBA Closes #1512 +* Remove dependency on ``pkg_resources`` from ``setuptools``. + + Closes #1103 + * Rename ``ModuleSpec`` -> ``module_type`` constructor parameter to match attribute name and improve typing. Use ``type`` instead. diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 9bafe3b00f..7a1de6cdb1 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -171,7 +171,7 @@ def contribute_to_path(self, spec, processed): class ExplicitNamespacePackageFinder(ImportlibFinder): - """A finder for the explicit namespace packages, generated through pkg_resources.""" + """A finder for the explicit namespace packages.""" def find_module(self, modname, module_parts, processed, submodule_path): if processed: diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index ce3da7eac2..c82186ba62 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,15 +2,18 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt -try: - import pkg_resources -except ImportError: - pkg_resources = None # type: ignore[assignment] +from importlib.util import find_spec def is_namespace(modname): - return ( - pkg_resources is not None - and hasattr(pkg_resources, "_namespace_packages") - and modname in pkg_resources._namespace_packages - ) + try: + found_spec = find_spec(modname) + except ValueError: + # execution of .pth files may not have __spec__ + return True + if found_spec is None: + return + # origin can be either a string on older Python versions + # or None in case it is a namespace package: + # https://github.com/python/cpython/pull/5481 + return found_spec.origin in {None, "namespace"} diff --git a/setup.cfg b/setup.cfg index e30d844d86..2d79726e8c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,6 @@ packages = find: install_requires = lazy_object_proxy>=1.4.0 wrapt>=1.11,<2 - setuptools>=20.0 typed-ast>=1.4.0,<2.0;implementation_name=="cpython" and python_version<"3.8" typing-extensions>=3.10;python_version<"3.10" python_requires = >=3.6.2 diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 4785975692..88bba7c084 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -10,8 +10,6 @@ from contextlib import contextmanager from typing import Iterator -import pkg_resources - import astroid from astroid import manager, test_utils from astroid.const import IS_JYTHON @@ -128,7 +126,6 @@ def test_implicit_namespace_package(self) -> None: def test_namespace_package_pth_support(self) -> None: pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = [] try: module = self.manager.ast_from_module_name("foogle.fax") @@ -138,18 +135,14 @@ def test_namespace_package_pth_support(self) -> None: with self.assertRaises(AstroidImportError): self.manager.ast_from_module_name("foogle.moogle") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def test_nested_namespace_import(self) -> None: pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = ["foogle.crank"] - pkg_resources._namespace_packages["foogle.crank"] = [] try: self.manager.ast_from_module_name("foogle.crank") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def test_namespace_and_file_mismatch(self) -> None: @@ -158,12 +151,10 @@ def test_namespace_and_file_mismatch(self) -> None: self.assertEqual(ast.name, "unittest") pth = "foogle_fax-0.12.5-py2.7-nspkg.pth" site.addpackage(resources.RESOURCE_PATH, pth, []) - pkg_resources._namespace_packages["foogle"] = [] try: with self.assertRaises(AstroidImportError): self.manager.ast_from_module_name("unittest.foogle.fax") finally: - del pkg_resources._namespace_packages["foogle"] sys.modules.pop("foogle") def _test_ast_from_zip(self, archive: str) -> None: From f53159ea09fc7c7106f4ca321d2ddeaf66938693 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 1 May 2022 12:36:58 -0400 Subject: [PATCH 02/20] Ensure parent packages are not imported --- astroid/interpreter/_import/util.py | 33 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index c82186ba62..6343090e9d 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,15 +2,34 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt -from importlib.util import find_spec +from importlib.util import _find_spec_from_path -def is_namespace(modname): - try: - found_spec = find_spec(modname) - except ValueError: - # execution of .pth files may not have __spec__ - return True +def is_namespace(modname: str) -> bool: + found_spec = None + + # find_spec() attempts to import parent packages when given dotted paths. + # That's unacceptable here, so we fallback to _find_spec_from_path(), which does + # not, but requires instead that each single parent ('astroid', 'nodes', etc.) + # be specced from left to right. + working_modname = "" + last_parent = None + for component in modname.split("."): + if working_modname: + working_modname += "." + component + else: + # First component + working_modname = component + try: + found_spec = _find_spec_from_path(working_modname, last_parent) + except ( + AttributeError, # TODO: remove AttributeError when 3.7+ is min + ValueError, + ): + # executed .pth files may not have __spec__ + return True + last_parent = working_modname + if found_spec is None: return # origin can be either a string on older Python versions From 0c536f1a513f4e6666aef17fc1f69192c37e2bf7 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 1 May 2022 12:46:05 -0400 Subject: [PATCH 03/20] consistent return --- astroid/interpreter/_import/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 6343090e9d..d483359a37 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -31,7 +31,7 @@ def is_namespace(modname: str) -> bool: last_parent = working_modname if found_spec is None: - return + return False # origin can be either a string on older Python versions # or None in case it is a namespace package: # https://github.com/python/cpython/pull/5481 From 52eb0955e4941b2d0f77540712810402a0827a52 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Tue, 3 May 2022 18:46:51 -0400 Subject: [PATCH 04/20] Pretend we've already dropped python3.6 and what is the result? --- astroid/interpreter/_import/util.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index d483359a37..78c6a48573 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -22,17 +22,12 @@ def is_namespace(modname: str) -> bool: working_modname = component try: found_spec = _find_spec_from_path(working_modname, last_parent) - except ( - AttributeError, # TODO: remove AttributeError when 3.7+ is min - ValueError, - ): + except ValueError: # executed .pth files may not have __spec__ return True last_parent = working_modname if found_spec is None: return False - # origin can be either a string on older Python versions - # or None in case it is a namespace package: - # https://github.com/python/cpython/pull/5481 - return found_spec.origin in {None, "namespace"} + + return found_spec.origin == "namespace" From c8e2764359c1a718e1ce0d0dd9763a46c6a356be Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 15:57:30 -0400 Subject: [PATCH 05/20] Handle old setuptools namespace package protocol --- astroid/interpreter/_import/spec.py | 16 +--------------- astroid/interpreter/_import/util.py | 25 +++++++++++++++++++++++++ tests/unittest_manager.py | 6 ++++++ 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index da4ea55096..33735e4111 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -144,7 +144,7 @@ def contribute_to_path(self, spec, processed): # Builtin. return None - if _is_setuptools_namespace(spec.location): + if util._is_setuptools_namespace(spec.location): # extend_path is called, search sys.path for module/packages # of this name see pkgutil.extend_path documentation path = [ @@ -253,20 +253,6 @@ def contribute_to_path(self, spec, processed): ) -def _is_setuptools_namespace(location): - try: - with open(os.path.join(location, "__init__.py"), "rb") as stream: - data = stream.read(4096) - except OSError: - return None - else: - extend_path = b"pkgutil" in data and b"extend_path" in data - declare_namespace = ( - b"pkg_resources" in data and b"declare_namespace(__name__)" in data - ) - return extend_path or declare_namespace - - @lru_cache() def _cached_set_diff(left, right): result = set(left) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 78c6a48573..fcd1cd7d1e 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,9 +2,25 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt +import os +import pathlib from importlib.util import _find_spec_from_path +def _is_setuptools_namespace(location): + try: + with open(os.path.join(location, "__init__.py"), "rb") as stream: + data = stream.read(4096) + except OSError: + return None + else: + extend_path = b"pkgutil" in data and b"extend_path" in data + declare_namespace = ( + b"pkg_resources" in data and b"declare_namespace(__name__)" in data + ) + return extend_path or declare_namespace + + def is_namespace(modname: str) -> bool: found_spec = None @@ -30,4 +46,13 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False + if found_spec.submodule_search_locations and any( + _is_setuptools_namespace(directory) + for directory in pathlib.Path( + found_spec.submodule_search_locations._path[0] + ).iterdir() + if directory.is_dir() + ): + return True + return found_spec.origin == "namespace" diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 3e4b6091c0..8cb81fd4fb 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -14,6 +14,7 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.interpreter._import import util from astroid.nodes import Const from . import resources @@ -107,6 +108,11 @@ def test_ast_from_namespace_pkgutil(self) -> None: def test_ast_from_namespace_pkg_resources(self) -> None: self._test_ast_from_old_namespace_package_protocol("pkg_resources") + def test_identify_old_namespace_package_protocol(self) -> None: + self.assertTrue( + util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1") + ) + def test_implicit_namespace_package(self) -> None: data_dir = os.path.dirname(resources.find("data/namespace_pep_420")) contribute = os.path.join(data_dir, "contribute_to_namespace") From 46a5d2970b36ef0e64500e14447c758e1ce87b87 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 16:03:01 -0400 Subject: [PATCH 06/20] add import back --- tests/unittest_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 8cb81fd4fb..19e47b6907 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -109,6 +109,10 @@ def test_ast_from_namespace_pkg_resources(self) -> None: self._test_ast_from_old_namespace_package_protocol("pkg_resources") def test_identify_old_namespace_package_protocol(self) -> None: + # Like the above cases, this package follows the old namespace package protocol + # astroid currently assumes such packages are in sys.modules, so import it + import tests.testdata.python3.data.path_pkg_resources_1.package.foo # noqa + self.assertTrue( util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1") ) From 4394f6d2b61322e0d1ec8bb6b81b340ae2d579bc Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 16:16:05 -0400 Subject: [PATCH 07/20] fix crash --- astroid/interpreter/_import/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index fcd1cd7d1e..a40b2c36c6 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -49,7 +49,7 @@ def is_namespace(modname: str) -> bool: if found_spec.submodule_search_locations and any( _is_setuptools_namespace(directory) for directory in pathlib.Path( - found_spec.submodule_search_locations._path[0] + found_spec.submodule_search_locations[0] ).iterdir() if directory.is_dir() ): From ee77bfda4693a75bbafa5ede36a4da6696efdbe2 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 17:04:23 -0400 Subject: [PATCH 08/20] Consistent return --- astroid/interpreter/_import/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index a40b2c36c6..f6561748e9 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -7,12 +7,12 @@ from importlib.util import _find_spec_from_path -def _is_setuptools_namespace(location): +def _is_setuptools_namespace(location) -> bool: try: with open(os.path.join(location, "__init__.py"), "rb") as stream: data = stream.read(4096) except OSError: - return None + return False else: extend_path = b"pkgutil" in data and b"extend_path" in data declare_namespace = ( From d116d52d9b8a144afd0830b511a24268795e0d8d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 17:17:37 -0400 Subject: [PATCH 09/20] Avoid the need for a python3.7 workaround by iterating all search locations --- astroid/interpreter/_import/util.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index f6561748e9..f9d7adf30b 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -46,13 +46,13 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False - if found_spec.submodule_search_locations and any( - _is_setuptools_namespace(directory) - for directory in pathlib.Path( - found_spec.submodule_search_locations[0] - ).iterdir() - if directory.is_dir() - ): - return True + if found_spec.submodule_search_locations is not None: + for search_location in found_spec.submodule_search_locations: + if any( + _is_setuptools_namespace(directory) + for directory in pathlib.Path(search_location).iterdir() + if directory.is_dir() + ): + return True return found_spec.origin == "namespace" From 2054a5683c685af7cb53a3142f8cd1c29b21e202 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 14 May 2022 17:23:44 -0400 Subject: [PATCH 10/20] pragma --- tests/unittest_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 0da3e6e3be..553f2a6992 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -113,7 +113,8 @@ def test_ast_from_namespace_pkg_resources(self) -> None: def test_identify_old_namespace_package_protocol(self) -> None: # Like the above cases, this package follows the old namespace package protocol # astroid currently assumes such packages are in sys.modules, so import it - import tests.testdata.python3.data.path_pkg_resources_1.package.foo # noqa + # pylint: disable-next=import-outside-toplevel + import tests.testdata.python3.data.path_pkg_resources_1.package.foo as _ # noqa self.assertTrue( util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1") From 2697e10c07dc8c98fb3871ed4b7fce23e11d0460 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 09:17:26 -0400 Subject: [PATCH 11/20] add caching --- astroid/interpreter/_import/util.py | 2 ++ astroid/manager.py | 3 ++- tests/unittest_manager.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index f9d7adf30b..3700012267 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -4,6 +4,7 @@ import os import pathlib +from functools import lru_cache from importlib.util import _find_spec_from_path @@ -21,6 +22,7 @@ def _is_setuptools_namespace(location) -> bool: return extend_path or declare_namespace +@lru_cache(maxsize=4096) def is_namespace(modname: str) -> bool: found_spec = None diff --git a/astroid/manager.py b/astroid/manager.py index 23330d5b95..6e7dcf4cdf 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -17,7 +17,7 @@ from astroid.const import BRAIN_MODULES_DIRECTORY from astroid.exceptions import AstroidBuildingError, AstroidImportError -from astroid.interpreter._import import spec +from astroid.interpreter._import import spec, util from astroid.modutils import ( NoSourceFile, _cache_normalize_path_, @@ -382,6 +382,7 @@ def clear_cache(self) -> None: for lru_cache in ( LookupMixIn.lookup, _cache_normalize_path_, + util.is_namespace, ObjectModel.attributes, ): lru_cache.cache_clear() diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 553f2a6992..6c105a12bb 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -325,6 +325,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None: lrus = ( astroid.nodes.node_classes.LookupMixIn.lookup, astroid.modutils._cache_normalize_path_, + util.is_namespace, astroid.interpreter.objectmodel.ObjectModel.attributes, ) @@ -334,6 +335,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None: # Generate some hits and misses ClassDef().lookup("garbage") is_standard_module("unittest", std_path=["garbage_path"]) + util.is_namespace("unittest") astroid.interpreter.objectmodel.ObjectModel().attributes() # Did the hits or misses actually happen? From c46e4438d13d4d018651543ba7d4bbdbc2327f09 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 16:55:29 -0400 Subject: [PATCH 12/20] better order of conditions --- astroid/interpreter/_import/util.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 3700012267..d4f8ae208c 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,13 +2,15 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt +from __future__ import annotations + import os import pathlib from functools import lru_cache from importlib.util import _find_spec_from_path -def _is_setuptools_namespace(location) -> bool: +def _is_setuptools_namespace(location: str | pathlib.Path) -> bool: try: with open(os.path.join(location, "__init__.py"), "rb") as stream: data = stream.read(4096) @@ -48,6 +50,9 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False + if found_spec.origin == "namespace": + return True + if found_spec.submodule_search_locations is not None: for search_location in found_spec.submodule_search_locations: if any( @@ -57,4 +62,4 @@ def is_namespace(modname: str) -> bool: ): return True - return found_spec.origin == "namespace" + return False From 6d088abe805d6f60103124b2c207f6687d4f7a53 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 17:02:03 -0400 Subject: [PATCH 13/20] cast to Path --- astroid/interpreter/_import/spec.py | 2 +- astroid/interpreter/_import/util.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index a3e7cc8272..0b34b4c6d0 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -147,7 +147,7 @@ def contribute_to_path(self, spec, processed): # Builtin. return None - if util._is_setuptools_namespace(spec.location): + if util._is_setuptools_namespace(Path(spec.location)): # extend_path is called, search sys.path for module/packages # of this name see pkgutil.extend_path documentation path = [ diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index d4f8ae208c..9987b7074d 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -4,15 +4,14 @@ from __future__ import annotations -import os import pathlib from functools import lru_cache from importlib.util import _find_spec_from_path -def _is_setuptools_namespace(location: str | pathlib.Path) -> bool: +def _is_setuptools_namespace(location: pathlib.Path) -> bool: try: - with open(os.path.join(location, "__init__.py"), "rb") as stream: + with open(location / "__init__.py", "rb") as stream: data = stream.read(4096) except OSError: return False From 7aa7da459e1181167fef1b6f5f0516ce37b83235 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 17:09:17 -0400 Subject: [PATCH 14/20] more concise implementation --- astroid/interpreter/_import/util.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 9987b7074d..b4bebf9033 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -31,14 +31,11 @@ def is_namespace(modname: str) -> bool: # That's unacceptable here, so we fallback to _find_spec_from_path(), which does # not, but requires instead that each single parent ('astroid', 'nodes', etc.) # be specced from left to right. - working_modname = "" + processed_components = [] last_parent = None for component in modname.split("."): - if working_modname: - working_modname += "." + component - else: - # First component - working_modname = component + processed_components.append(component) + working_modname = ".".join(processed_components) try: found_spec = _find_spec_from_path(working_modname, last_parent) except ValueError: From fa6094062099de924f95242f1f1971add0e39f10 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 17:24:44 -0400 Subject: [PATCH 15/20] None, not "namespace" --- astroid/interpreter/_import/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index b4bebf9033..44d3b8e344 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -46,7 +46,7 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False - if found_spec.origin == "namespace": + if found_spec.origin is None: return True if found_spec.submodule_search_locations is not None: From d0b90f48d8bf2e5223649209b3837686f8b1b19a Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 17:43:32 -0400 Subject: [PATCH 16/20] Distinguish builtins from namespace packages --- astroid/interpreter/_import/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 44d3b8e344..38e76e0ee4 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -6,6 +6,7 @@ import pathlib from functools import lru_cache +from importlib.machinery import BuiltinImporter from importlib.util import _find_spec_from_path @@ -46,6 +47,10 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False + if found_spec.loader is BuiltinImporter: + # TODO(Py39): remove, since found_spec.origin will be "built-in" + return False + if found_spec.origin is None: return True From f30e100645e4d42e901c31d0a188befb66220cc7 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 17:54:04 -0400 Subject: [PATCH 17/20] Remove unnecessary search --- astroid/interpreter/_import/util.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 38e76e0ee4..e8f85855b0 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -51,16 +51,4 @@ def is_namespace(modname: str) -> bool: # TODO(Py39): remove, since found_spec.origin will be "built-in" return False - if found_spec.origin is None: - return True - - if found_spec.submodule_search_locations is not None: - for search_location in found_spec.submodule_search_locations: - if any( - _is_setuptools_namespace(directory) - for directory in pathlib.Path(search_location).iterdir() - if directory.is_dir() - ): - return True - - return False + return found_spec.origin is None From 3dbe761e92bd58fa1466770a815b27a8f011b670 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 19:40:43 -0400 Subject: [PATCH 18/20] Easier check for builtins --- astroid/interpreter/_import/util.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index e8f85855b0..503b0b0bfa 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -5,8 +5,8 @@ from __future__ import annotations import pathlib +import sys from functools import lru_cache -from importlib.machinery import BuiltinImporter from importlib.util import _find_spec_from_path @@ -26,6 +26,9 @@ def _is_setuptools_namespace(location: pathlib.Path) -> bool: @lru_cache(maxsize=4096) def is_namespace(modname: str) -> bool: + if modname in sys.builtin_module_names: + return False + found_spec = None # find_spec() attempts to import parent packages when given dotted paths. @@ -47,8 +50,4 @@ def is_namespace(modname: str) -> bool: if found_spec is None: return False - if found_spec.loader is BuiltinImporter: - # TODO(Py39): remove, since found_spec.origin will be "built-in" - return False - return found_spec.origin is None From d92c48eccf48ea7e50cb8696e2746a2070bdbc1d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sun, 15 May 2022 19:43:12 -0400 Subject: [PATCH 19/20] Reduce size of diff --- astroid/interpreter/_import/spec.py | 17 ++++++++++++++++- astroid/interpreter/_import/util.py | 15 --------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 0b34b4c6d0..74a0e8081f 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -10,6 +10,7 @@ import importlib.machinery import importlib.util import os +import pathlib import sys import zipimport from collections.abc import Sequence @@ -147,7 +148,7 @@ def contribute_to_path(self, spec, processed): # Builtin. return None - if util._is_setuptools_namespace(Path(spec.location)): + if _is_setuptools_namespace(Path(spec.location)): # extend_path is called, search sys.path for module/packages # of this name see pkgutil.extend_path documentation path = [ @@ -256,6 +257,20 @@ def contribute_to_path(self, spec, processed): ) +def _is_setuptools_namespace(location: pathlib.Path) -> bool: + try: + with open(location / "__init__.py", "rb") as stream: + data = stream.read(4096) + except OSError: + return False + else: + extend_path = b"pkgutil" in data and b"extend_path" in data + declare_namespace = ( + b"pkg_resources" in data and b"declare_namespace(__name__)" in data + ) + return extend_path or declare_namespace + + @lru_cache() def _cached_set_diff(left, right): result = set(left) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index 503b0b0bfa..bb9a273aa3 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -4,26 +4,11 @@ from __future__ import annotations -import pathlib import sys from functools import lru_cache from importlib.util import _find_spec_from_path -def _is_setuptools_namespace(location: pathlib.Path) -> bool: - try: - with open(location / "__init__.py", "rb") as stream: - data = stream.read(4096) - except OSError: - return False - else: - extend_path = b"pkgutil" in data and b"extend_path" in data - declare_namespace = ( - b"pkg_resources" in data and b"declare_namespace(__name__)" in data - ) - return extend_path or declare_namespace - - @lru_cache(maxsize=4096) def is_namespace(modname: str) -> bool: if modname in sys.builtin_module_names: From 01b00a26380adadb9c37a0ca606d807883bd3455 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Mon, 16 May 2022 07:23:24 -0400 Subject: [PATCH 20/20] Remove unused __future__ --- astroid/interpreter/_import/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astroid/interpreter/_import/util.py b/astroid/interpreter/_import/util.py index bb9a273aa3..53c6922c33 100644 --- a/astroid/interpreter/_import/util.py +++ b/astroid/interpreter/_import/util.py @@ -2,8 +2,6 @@ # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt -from __future__ import annotations - import sys from functools import lru_cache from importlib.util import _find_spec_from_path