From e7869f05751561958b946b562093397027f6d5fa Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Sat, 18 Sep 2021 16:58:56 -0700 Subject: [PATCH] Search sys.path for PEP-561 compliant packages Closes #5701 --- mypy/main.py | 6 ++--- mypy/modulefinder.py | 48 ++++++++----------------------------- mypy/pyinfo.py | 31 +++++++++++++++++------- mypy/test/testcmdline.py | 3 +++ test-data/unit/cmdline.test | 24 +++++++++++++++++++ 5 files changed, 63 insertions(+), 49 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 3d9836587250..62e19c35c6e5 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -16,7 +16,7 @@ from mypy import util from mypy.modulefinder import ( BuildSource, FindModuleCache, SearchPaths, - get_site_packages_dirs, mypy_path, + get_search_dirs, mypy_path, ) from mypy.find_sources import create_source_list, InvalidSourceList from mypy.fscache import FileSystemCache @@ -1033,10 +1033,10 @@ def set_strict_flags() -> None: # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE - egg_dirs, site_packages = get_site_packages_dirs(options.python_executable) + egg_dirs, site_packages, sys_path = get_search_dirs(options.python_executable) search_paths = SearchPaths((os.getcwd(),), tuple(mypy_path() + options.mypy_path), - tuple(egg_dirs + site_packages), + tuple(egg_dirs + site_packages + sys_path), ()) targets = [] # TODO: use the same cache that the BuildManager will diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 94d2dd34c16e..f0207184737b 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -30,7 +30,7 @@ 'SearchPaths', [('python_path', Tuple[str, ...]), # where user code is found ('mypy_path', Tuple[str, ...]), # from $MYPYPATH or config variable - ('package_path', Tuple[str, ...]), # from get_site_packages_dirs() + ('package_path', Tuple[str, ...]), # from get_search_dirs() ('typeshed_path', Tuple[str, ...]), # paths in typeshed ]) @@ -608,28 +608,7 @@ def default_lib_path(data_dir: str, @functools.lru_cache(maxsize=None) -def get_prefixes(python_executable: Optional[str]) -> Tuple[str, str]: - """Get the sys.base_prefix and sys.prefix for the given python. - - This runs a subprocess call to get the prefix paths of the given Python executable. - To avoid repeatedly calling a subprocess (which can be slow!) we - lru_cache the results. - """ - if python_executable is None: - return '', '' - elif python_executable == sys.executable: - # Use running Python's package dirs - return pyinfo.getprefixes() - else: - # Use subprocess to get the package directory of given Python - # executable - return ast.literal_eval( - subprocess.check_output([python_executable, pyinfo.__file__, 'getprefixes'], - stderr=subprocess.PIPE).decode()) - - -@functools.lru_cache(maxsize=None) -def get_site_packages_dirs(python_executable: Optional[str]) -> Tuple[List[str], List[str]]: +def get_search_dirs(python_executable: Optional[str]) -> Tuple[List[str], List[str], List[str]]: """Find package directories for given python. This runs a subprocess call, which generates a list of the egg directories, and the site @@ -638,17 +617,17 @@ def get_site_packages_dirs(python_executable: Optional[str]) -> Tuple[List[str], """ if python_executable is None: - return [], [] + return [], [], [] elif python_executable == sys.executable: # Use running Python's package dirs - site_packages = pyinfo.getsitepackages() + site_packages, sys_path = pyinfo.getsearchdirs() else: # Use subprocess to get the package directory of given Python # executable - site_packages = ast.literal_eval( - subprocess.check_output([python_executable, pyinfo.__file__, 'getsitepackages'], + site_packages, sys_path = ast.literal_eval( + subprocess.check_output([python_executable, pyinfo.__file__, 'getsearchdirs'], stderr=subprocess.PIPE).decode()) - return expand_site_packages(site_packages) + return expand_site_packages(site_packages) + (sys_path,) def expand_site_packages(site_packages: List[str]) -> Tuple[List[str], List[str]]: @@ -781,10 +760,8 @@ def compute_search_paths(sources: List[BuildSource], if options.python_version[0] == 2: mypypath = add_py2_mypypath_entries(mypypath) - egg_dirs, site_packages = get_site_packages_dirs(options.python_executable) - base_prefix, prefix = get_prefixes(options.python_executable) - is_venv = base_prefix != prefix - for site_dir in site_packages: + egg_dirs, site_packages, sys_path = get_search_dirs(options.python_executable) + for site_dir in site_packages + sys_path: assert site_dir not in lib_path if (site_dir in mypypath or any(p.startswith(site_dir + os.path.sep) for p in mypypath) or @@ -793,15 +770,10 @@ def compute_search_paths(sources: List[BuildSource], print("See https://mypy.readthedocs.io/en/stable/running_mypy.html" "#how-mypy-handles-imports for more info", file=sys.stderr) sys.exit(1) - elif site_dir in python_path and (is_venv and not site_dir.startswith(prefix)): - print("{} is in the PYTHONPATH. Please change directory" - " so it is not.".format(site_dir), - file=sys.stderr) - sys.exit(1) return SearchPaths(python_path=tuple(reversed(python_path)), mypy_path=tuple(mypypath), - package_path=tuple(egg_dirs + site_packages), + package_path=tuple(egg_dirs + site_packages + sys_path), typeshed_path=tuple(lib_path)) diff --git a/mypy/pyinfo.py b/mypy/pyinfo.py index ab2d3286bd5c..6f60bbca4051 100644 --- a/mypy/pyinfo.py +++ b/mypy/pyinfo.py @@ -6,8 +6,10 @@ library found in Python 2. This file is run each mypy run, so it should be kept as fast as possible. """ +import os import site import sys +import sysconfig if __name__ == '__main__': sys.path = sys.path[1:] # we don't want to pick up mypy.types @@ -17,12 +19,27 @@ from typing import List, Tuple -def getprefixes(): - # type: () -> Tuple[str, str] - return getattr(sys, "base_prefix", sys.prefix), sys.prefix +def getsearchdirs(): + # type: () -> Tuple[List[str], List[str]] + site_packages = _getsitepackages() + # Do not include things from the standard library + # because those should come from typeshed. + stdlib_zip = os.path.join( + sys.base_exec_prefix, + getattr(sys, "platlibdir", "lib"), + "python{}{}.zip".format(sys.version_info.major, sys.version_info.minor) + ) + stdlib = sysconfig.get_path("stdlib") + stdlib_ext = os.path.join(stdlib, "lib-dynload") + cwd = os.path.abspath(os.getcwd()) + excludes = set(site_packages + [cwd, stdlib_zip, stdlib, stdlib_ext]) -def getsitepackages(): + abs_sys_path = (os.path.abspath(p) for p in sys.path) + return (site_packages, [p for p in abs_sys_path if p not in excludes]) + + +def _getsitepackages(): # type: () -> List[str] res = [] if hasattr(site, 'getsitepackages'): @@ -37,10 +54,8 @@ def getsitepackages(): if __name__ == '__main__': - if sys.argv[-1] == 'getsitepackages': - print(repr(getsitepackages())) - elif sys.argv[-1] == 'getprefixes': - print(repr(getprefixes())) + if sys.argv[-1] == 'getsearchdirs': + print(repr(getsearchdirs())) else: print("ERROR: incorrect argument to pyinfo.py.", file=sys.stderr) sys.exit(1) diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 9fafb1f36cae..917ae82112bc 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -56,7 +56,10 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: fixed = [python3_path, '-m', 'mypy'] env = os.environ.copy() env.pop('COLUMNS', None) + extra_path = os.path.join(os.path.abspath(test_temp_dir), 'pypath') env['PYTHONPATH'] = PREFIX + if os.path.isdir(extra_path): + env['PYTHONPATH'] += ':' + extra_path process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index f29c183f20ba..2056c7fabfe8 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -365,6 +365,30 @@ main.py:6: error: Unsupported operand types for + ("int" and "str") main.py:7: error: Module has no attribute "y" main.py:8: error: Unsupported operand types for + (Module and "int") +[case testConfigFollowImportsSysPath] +# cmd: mypy main.py +[file main.py] +from a import x +x + 0 +x + '' # E +import a +a.x + 0 +a.x + '' # E +a.y # E +a + 0 # E +[file mypy.ini] +\[mypy] +follow_imports = normal +[file pypath/a/__init__.py] +x = 0 +x += '' # Error reported here +[file pypath/a/py.typed] +[out] +main.py:3: error: Unsupported operand types for + ("int" and "str") +main.py:6: error: Unsupported operand types for + ("int" and "str") +main.py:7: error: Module has no attribute "y" +main.py:8: error: Unsupported operand types for + (Module and "int") + [case testConfigFollowImportsSilent] # cmd: mypy main.py [file main.py]