From b432da7a5e7135cb39756976eab20577954a1fb3 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 13 Aug 2020 11:14:52 -0700 Subject: [PATCH] Add `--python-infer-string-imports` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/dependency_inference/rules.py | 50 +++++++++++++------ .../python/dependency_inference/rules_test.py | 30 ++++++++--- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/python/pants/backend/python/dependency_inference/rules.py b/src/python/pants/backend/python/dependency_inference/rules.py index 0dbce4c8d22..b41ee9f346e 100644 --- a/src/python/pants/backend/python/dependency_inference/rules.py +++ b/src/python/pants/backend/python/dependency_inference/rules.py @@ -3,7 +3,7 @@ import itertools from pathlib import PurePath -from typing import cast +from typing import List, cast from pants.backend.python.dependency_inference import module_mapper from pants.backend.python.dependency_inference.import_parser import find_python_imports @@ -44,6 +44,17 @@ def register_options(cls, register): "Infer a target's imported dependencies by parsing import statements from sources." ), ) + register( + "--string-imports", + default=False, + type=bool, + help=( + "Infer a target's dependencies based on strings that look like dynamic imports, " + "such as `importlib.import_module('example.subdir.Foo')`. This can be useful if " + "you use lots of dynamic imports, such as with Django apps. To ignore any false " + "positives, put `!{bad_address}` in the `dependencies` field of your target." + ), + ) register( "--inits", default=True, @@ -69,6 +80,10 @@ def register_options(cls, register): def imports(self) -> bool: return cast(bool, self.options.imports) + @property + def string_imports(self) -> bool: + return cast(bool, self.options.string_imports) + @property def inits(self) -> bool: return cast(bool, self.options.inits) @@ -95,23 +110,28 @@ async def infer_python_dependencies( for fp in stripped_sources.snapshot.files ) digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest) - imports_per_file = tuple( - find_python_imports(file_content.content.decode(), module_name=module.module) - for file_content, module in zip(digest_contents, modules) - ) - owner_per_import = await MultiGet( - Get(PythonModuleOwner, PythonModule(imported_module)) - for file_imports in imports_per_file - for imported_module in file_imports.explicit_imports - if imported_module not in combined_stdlib - ) + + owner_requests: List[Get[PythonModuleOwner, PythonModule]] = [] + for file_content, module in zip(digest_contents, modules): + file_imports_obj = find_python_imports( + file_content.content.decode(), module_name=module.module + ) + detected_imports = ( + file_imports_obj.all_imports + if python_inference.string_imports + else file_imports_obj.explicit_imports + ) + owner_requests.extend( + Get(PythonModuleOwner, PythonModule(imported_module)) + for imported_module in detected_imports + if imported_module not in combined_stdlib + ) + + owner_per_import = await MultiGet(owner_requests) result = ( owner.address for owner in owner_per_import - if ( - owner.address - and owner.address.maybe_convert_to_base_target() != request.sources_field.address - ) + if owner.address and owner.address != request.sources_field.address ) return InferredDependencies(result, sibling_dependencies_inferrable=True) diff --git a/src/python/pants/backend/python/dependency_inference/rules_test.py b/src/python/pants/backend/python/dependency_inference/rules_test.py index c6bddbd9149..7bf68bd77ea 100644 --- a/src/python/pants/backend/python/dependency_inference/rules_test.py +++ b/src/python/pants/backend/python/dependency_inference/rules_test.py @@ -50,9 +50,6 @@ def target_types(cls): return [PythonLibrary, PythonRequirementLibrary, PythonTests] def test_infer_python_imports(self) -> None: - options_bootstrapper = create_options_bootstrapper( - args=["--backend-packages=pants.backend.python", "--source-root-patterns=src/python"] - ) self.add_to_build_file( "3rdparty/python", dedent( @@ -65,8 +62,8 @@ def test_infer_python_imports(self) -> None: ), ) - self.create_file("src/python/no_owner/f.py") - self.add_to_build_file("src/python/no_owner", "python_library()") + self.create_file("src/python/str_import/subdir/f.py") + self.add_to_build_file("src/python/str_import/subdir", "python_library()") self.create_file("src/python/util/dep.py") self.add_to_build_file("src/python/util", "python_library()") @@ -89,12 +86,21 @@ def test_infer_python_imports(self) -> None: import typing # Import from another file in the same target. from app import main + + # Dynamic string import. + importlib.import_module('str_import.subdir.f') """ ), ) self.add_to_build_file("src/python", "python_library()") - def run_dep_inference(address: Address) -> InferredDependencies: + def run_dep_inference( + address: Address, *, enable_string_imports: bool = False + ) -> InferredDependencies: + args = ["--backend-packages=pants.backend.python", "--source-root-patterns=src/python"] + if enable_string_imports: + args.append("--python-infer-string-imports") + options_bootstrapper = create_options_bootstrapper(args=args) target = self.request_single_product( WrappedTarget, Params(address, options_bootstrapper) ).target @@ -103,12 +109,11 @@ def run_dep_inference(address: Address) -> InferredDependencies: Params(InferPythonDependencies(target[PythonSources]), options_bootstrapper), ) - # NB: We do not infer `src/python/app.py`, even though it's used by `src/python/f2.py`, - # because it is part of the requested address. normal_address = Address("src/python") assert run_dep_inference(normal_address) == InferredDependencies( [ Address("3rdparty/python", target_name="Django"), + Address("src/python", relative_file_path="app.py"), Address("src/python/util", relative_file_path="dep.py", target_name="util"), ], sibling_dependencies_inferrable=True, @@ -121,6 +126,15 @@ def run_dep_inference(address: Address) -> InferredDependencies: [Address("src/python", relative_file_path="app.py", target_name="python")], sibling_dependencies_inferrable=True, ) + assert run_dep_inference( + generated_subtarget_address, enable_string_imports=True + ) == InferredDependencies( + [ + Address("src/python", relative_file_path="app.py", target_name="python"), + Address("src/python/str_import/subdir", relative_file_path="f.py"), + ], + sibling_dependencies_inferrable=True, + ) def test_infer_python_inits(self) -> None: options_bootstrapper = create_options_bootstrapper(