Merge b432da7 into 8e73969

src/python/pants/backend/python/dependency_inference/rules.py

@@ -3,7 +3,7 @@
 
 import itertools
 from pathlib import PurePath
-from typing import cast
+from typing import List, cast
 
 from pants.backend.python.dependency_inference import module_mapper
 from pants.backend.python.dependency_inference.import_parser import find_python_imports
@@ -44,6 +44,17 @@ def register_options(cls, register):
                 "Infer a target's imported dependencies by parsing import statements from sources."
             ),
         )
+        register(
+            "--string-imports",
+            default=False,
+            type=bool,
+            help=(
+                "Infer a target's dependencies based on strings that look like dynamic imports, "
+                "such as `importlib.import_module('example.subdir.Foo')`. This can be useful if "
+                "you use lots of dynamic imports, such as with Django apps. To ignore any false "
+                "positives, put `!{bad_address}` in the `dependencies` field of your target."
+            ),
+        )
         register(
             "--inits",
             default=True,
@@ -69,6 +80,10 @@ def register_options(cls, register):
     def imports(self) -> bool:
         return cast(bool, self.options.imports)
 
+    @property
+    def string_imports(self) -> bool:
+        return cast(bool, self.options.string_imports)
+
     @property
     def inits(self) -> bool:
         return cast(bool, self.options.inits)
@@ -95,23 +110,28 @@ async def infer_python_dependencies(
         for fp in stripped_sources.snapshot.files
     )
     digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest)
-    imports_per_file = tuple(
-        find_python_imports(file_content.content.decode(), module_name=module.module)
-        for file_content, module in zip(digest_contents, modules)
-    )
-    owner_per_import = await MultiGet(
-        Get(PythonModuleOwner, PythonModule(imported_module))
-        for file_imports in imports_per_file
-        for imported_module in file_imports.explicit_imports
-        if imported_module not in combined_stdlib
-    )
+
+    owner_requests: List[Get[PythonModuleOwner, PythonModule]] = []
+    for file_content, module in zip(digest_contents, modules):
+        file_imports_obj = find_python_imports(
+            file_content.content.decode(), module_name=module.module
+        )
+        detected_imports = (
+            file_imports_obj.all_imports
+            if python_inference.string_imports
+            else file_imports_obj.explicit_imports
+        )
+        owner_requests.extend(
+            Get(PythonModuleOwner, PythonModule(imported_module))
+            for imported_module in detected_imports
+            if imported_module not in combined_stdlib
+        )
+
+    owner_per_import = await MultiGet(owner_requests)
     result = (
         owner.address
         for owner in owner_per_import
-        if (
-            owner.address
-            and owner.address.maybe_convert_to_base_target() != request.sources_field.address
-        )
+        if owner.address and owner.address != request.sources_field.address
     )
     return InferredDependencies(result, sibling_dependencies_inferrable=True)
 

src/python/pants/backend/python/dependency_inference/rules_test.py

@@ -50,9 +50,6 @@ def target_types(cls):
         return [PythonLibrary, PythonRequirementLibrary, PythonTests]
 
     def test_infer_python_imports(self) -> None:
-        options_bootstrapper = create_options_bootstrapper(
-            args=["--backend-packages=pants.backend.python", "--source-root-patterns=src/python"]
-        )
         self.add_to_build_file(
             "3rdparty/python",
             dedent(
@@ -65,8 +62,8 @@ def test_infer_python_imports(self) -> None:
             ),
         )
 
-        self.create_file("src/python/no_owner/f.py")
-        self.add_to_build_file("src/python/no_owner", "python_library()")
+        self.create_file("src/python/str_import/subdir/f.py")
+        self.add_to_build_file("src/python/str_import/subdir", "python_library()")
 
         self.create_file("src/python/util/dep.py")
         self.add_to_build_file("src/python/util", "python_library()")
@@ -89,12 +86,21 @@ def test_infer_python_imports(self) -> None:
                 import typing
                 # Import from another file in the same target.
                 from app import main
+
+                # Dynamic string import.
+                importlib.import_module('str_import.subdir.f')
                 """
             ),
         )
         self.add_to_build_file("src/python", "python_library()")
 
-        def run_dep_inference(address: Address) -> InferredDependencies:
+        def run_dep_inference(
+            address: Address, *, enable_string_imports: bool = False
+        ) -> InferredDependencies:
+            args = ["--backend-packages=pants.backend.python", "--source-root-patterns=src/python"]
+            if enable_string_imports:
+                args.append("--python-infer-string-imports")
+            options_bootstrapper = create_options_bootstrapper(args=args)
             target = self.request_single_product(
                 WrappedTarget, Params(address, options_bootstrapper)
             ).target
@@ -103,12 +109,11 @@ def run_dep_inference(address: Address) -> InferredDependencies:
                 Params(InferPythonDependencies(target[PythonSources]), options_bootstrapper),
             )
 
-        # NB: We do not infer `src/python/app.py`, even though it's used by `src/python/f2.py`,
-        # because it is part of the requested address.
         normal_address = Address("src/python")
         assert run_dep_inference(normal_address) == InferredDependencies(
             [
                 Address("3rdparty/python", target_name="Django"),
+                Address("src/python", relative_file_path="app.py"),
                 Address("src/python/util", relative_file_path="dep.py", target_name="util"),
             ],
             sibling_dependencies_inferrable=True,
@@ -121,6 +126,15 @@ def run_dep_inference(address: Address) -> InferredDependencies:
             [Address("src/python", relative_file_path="app.py", target_name="python")],
             sibling_dependencies_inferrable=True,
         )
+        assert run_dep_inference(
+            generated_subtarget_address, enable_string_imports=True
+        ) == InferredDependencies(
+            [
+                Address("src/python", relative_file_path="app.py", target_name="python"),
+                Address("src/python/str_import/subdir", relative_file_path="f.py"),
+            ],
+            sibling_dependencies_inferrable=True,
+        )
 
     def test_infer_python_inits(self) -> None:
         options_bootstrapper = create_options_bootstrapper(