Deprecate python_requirements and poetry_requirements using old macro in favor of target generation

pants.toml

@@ -69,6 +69,7 @@ remote_store_address = "grpcs://cache.toolchain.com:443"
 remote_instance_name = "main"
 remote_auth_plugin = "toolchain.pants.auth.plugin:toolchain_auth_plugin"
 
+use_deprecated_python_macros = false
 
 [anonymous-telemetry]
 enabled = true

src/python/pants/backend/codegen/protobuf/scala/BUILD

@@ -4,4 +4,4 @@
 python_sources(dependencies=[":resources"])
 resources(name="resources", sources=["*.scala", "scalapbc.default.lockfile.txt"])
 
-python_tests(name="tests")
+python_tests(name="tests", timeout=120)

src/python/pants/backend/python/dependency_inference/rules_test.py

@@ -17,7 +17,8 @@
     infer_python_conftest_dependencies,
     infer_python_init_dependencies,
 )
-from pants.backend.python.macros.python_requirements_caof import PythonRequirementsCAOF
+from pants.backend.python.macros import python_requirements
+from pants.backend.python.macros.python_requirements import PythonRequirementsTargetGenerator
 from pants.backend.python.target_types import (
     PythonRequirementsFileTarget,
     PythonRequirementTarget,
@@ -260,14 +261,15 @@ def test_infer_python_strict(caplog) -> None:
         rules=[
             *import_rules(),
             *target_types_rules.rules(),
+            *python_requirements.rules(),
             QueryRule(InferredDependencies, [InferPythonImportDependencies]),
         ],
         target_types=[
             PythonSourcesGeneratorTarget,
             PythonRequirementTarget,
             PythonRequirementsFileTarget,
+            PythonRequirementsTargetGenerator,
         ],
-        context_aware_object_factories={"python_requirements": PythonRequirementsCAOF},
     )
 
     rule_runner.write_files(
@@ -337,7 +339,7 @@ def run_dep_inference(
             "src/python/requirements.txt": "venezuelan_beaver_cheese==1.0.0",
             "src/python/BUILD": dedent(
                 """\
-                    python_requirements()
+                    python_requirements(name='reqs')
                     python_sources()
                 """
             ),

src/python/pants/backend/python/macros/common_fields.py

@@ -0,0 +1,82 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from typing import Any, ClassVar, Dict, Iterable, Tuple
+
+from packaging.utils import canonicalize_name as canonicalize_project_name
+
+from pants.backend.python.target_types import (
+    PythonRequirementModulesField,
+    PythonRequirementTarget,
+    PythonRequirementTypeStubModulesField,
+    normalize_module_mapping,
+)
+from pants.engine.addresses import Address
+from pants.engine.target import DictStringToStringSequenceField, OverridesField
+from pants.util.frozendict import FrozenDict
+
+
+class ModuleMappingField(DictStringToStringSequenceField):
+    alias = "module_mapping"
+    help = (
+        "A mapping of requirement names to a list of the modules they provide.\n\n"
+        'For example, `{"ansicolors": ["colors"]}`.\n\n'
+        "Any unspecified requirements will use a default. See the "
+        f"`{PythonRequirementModulesField.alias}` field from the `{PythonRequirementTarget.alias}` "
+        f"target for more information."
+    )
+    value: FrozenDict[str, tuple[str, ...]]
+    default: ClassVar[FrozenDict[str, tuple[str, ...]]] = FrozenDict()
+
+    @classmethod
+    def compute_value(  # type: ignore[override]
+        cls, raw_value: Dict[str, Iterable[str]], address: Address
+    ) -> FrozenDict[str, Tuple[str, ...]]:
+        value_or_default = super().compute_value(raw_value, address)
+        return normalize_module_mapping(value_or_default)
+
+
+class TypeStubsModuleMappingField(DictStringToStringSequenceField):
+    alias = "type_stubs_module_mapping"
+    help = (
+        "A mapping of type-stub requirement names to a list of the modules they provide.\n\n"
+        'For example, `{"types-requests": ["requests"]}`.\n\n'
+        "If the requirement is not specified _and_ its name looks like a type stub, Pants will "
+        f"use a default. See the `{PythonRequirementTypeStubModulesField.alias}` field from the "
+        f"`{PythonRequirementTarget.alias}` target for more information."
+    )
+    value: FrozenDict[str, tuple[str, ...]]
+    default: ClassVar[FrozenDict[str, tuple[str, ...]]] = FrozenDict()
+
+    @classmethod
+    def compute_value(  # type: ignore[override]
+        cls, raw_value: Dict[str, Iterable[str]], address: Address
+    ) -> FrozenDict[str, Tuple[str, ...]]:
+        value_or_default = super().compute_value(raw_value, address)
+        return normalize_module_mapping(value_or_default)
+
+
+class RequirementsOverrideField(OverridesField):
+    help = (
+        "Override the field values for generated `python_requirement` targets.\n\n"
+        "Expects a dictionary of requirements to a dictionary for the "
+        "overrides. You may either use a string for a single requirement, "
+        "or a string tuple for multiple requirements. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "```\n"
+        "overrides={\n"
+        '  "django": {"dependencies": ["#setuptools"]]},\n'
+        '  "ansicolors": {"description": "pretty colors"]},\n'
+        '  ("ansicolors, "django"): {"tags": ["overridden"]},\n'
+        "}\n"
+        "```\n\n"
+        "Every overridden requirement is validated to be generated by this target.\n\n"
+        "You can specify the same requirement in multiple keys, so long as you don't "
+        "override the same field more than one time for the requirement."
+    )
+
+    def flatten_and_normalize(self) -> dict[str, dict[str, Any]]:
+        return {canonicalize_project_name(req): v for req, v in super().flatten().items()}

src/python/pants/backend/python/macros/deprecation_fixers.py

@@ -14,6 +14,7 @@
     DeprecationFixerRequest,
     RewrittenBuildFile,
     RewrittenBuildFileRequest,
+    UpdateBuildFilesSubsystem,
 )
 from pants.engine.addresses import Address, Addresses, AddressInput, BuildFileAddress
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
@@ -26,6 +27,7 @@
     UnexpandedTargets,
 )
 from pants.engine.unions import UnionRule
+from pants.option.global_options import GlobalOptions
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 from pants.util.strutil import bullet_list
@@ -47,8 +49,12 @@ class MacroRenames:
     generated: FrozenDict[Address, tuple[Address, str]]
 
 
+class MacroRenamesRequest:
+    pass
+
+
 @rule(desc="Determine how to rename Python macros to target generators", level=LogLevel.DEBUG)
-async def determine_macro_changes(all_targets: AllTargets) -> MacroRenames:
+async def determine_macro_changes(all_targets: AllTargets, _: MacroRenamesRequest) -> MacroRenames:
     # Strategy: Find `python_requirement` targets who depend on a `_python_requirements_file`
     # target to figure out which macros we have. Note that context-aware object factories (CAOFs)
     # are not actual targets and are "erased", so this is the way to find the macros.
@@ -144,10 +150,23 @@ class UpdatePythonMacrosRequest(DeprecationFixerRequest):
 
 
 @rule(desc="Change Python macros to target generators", level=LogLevel.DEBUG)
-def maybe_update_macros_references(
+async def maybe_update_macros_references(
     request: UpdatePythonMacrosRequest,
-    renames: MacroRenames,
+    global_options: GlobalOptions,
+    update_build_files_subsystem: UpdateBuildFilesSubsystem,
 ) -> RewrittenBuildFile:
+    if not update_build_files_subsystem.fix_python_macros:
+        return RewrittenBuildFile(request.path, request.lines, ())
+
+    if not global_options.options.use_deprecated_python_macros:
+        raise ValueError(
+            "`--update-build-files-fix-python-macros` specified when "
+            "`[GLOBAL].use_deprecated_python_macros` is already set to false, which means that "
+            "there is nothing left to fix."
+        )
+
+    renames = await Get(MacroRenames, MacroRenamesRequest())
+
     changed_generator_aliases = set()
 
     def maybe_update(input_lines: tuple[str, ...]) -> list[str]:

src/python/pants/backend/python/macros/deprecation_fixers_test.py

@@ -12,6 +12,7 @@
 from pants.backend.python.macros.deprecation_fixers import (
     GeneratorRename,
     MacroRenames,
+    MacroRenamesRequest,
     UpdatePythonMacrosRequest,
 )
 from pants.backend.python.macros.pipenv_requirements_caof import PipenvRequirementsCAOF
@@ -27,10 +28,10 @@
 
 @pytest.fixture
 def rule_runner() -> RuleRunner:
-    return RuleRunner(
+    rule_runner = RuleRunner(
         rules=(
             *deprecation_fixers.rules(),
-            QueryRule(MacroRenames, []),
+            QueryRule(MacroRenames, [MacroRenamesRequest]),
             QueryRule(RewrittenBuildFile, [UpdatePythonMacrosRequest]),
         ),
         target_types=[GenericTarget, PythonRequirementsFileTarget, PythonRequirementTarget],
@@ -39,7 +40,10 @@ def rule_runner() -> RuleRunner:
             "poetry_requirements": PoetryRequirementsCAOF,
             "pipenv_requirements": PipenvRequirementsCAOF,
         },
+        use_deprecated_python_macros=True,
     )
+    rule_runner.set_options(["--update-build-files-fix-python-macros"])
+    return rule_runner
 
 
 def test_determine_macro_changes(rule_runner: RuleRunner, caplog) -> None:
@@ -71,7 +75,7 @@ def test_determine_macro_changes(rule_runner: RuleRunner, caplog) -> None:
             "pipenv/BUILD": "pipenv_requirements()",
         }
     )
-    renames = rule_runner.request(MacroRenames, [])
+    renames = rule_runner.request(MacroRenames, [MacroRenamesRequest()])
     assert renames.generators == (
         GeneratorRename("BUILD", "python_requirements", "reqs"),
         GeneratorRename("pipenv/BUILD", "pipenv_requirements", None),

src/python/pants/backend/python/macros/pipenv_requirements.py

@@ -0,0 +1,153 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import json
+
+from packaging.utils import canonicalize_name as canonicalize_project_name
+
+from pants.backend.python.macros.common_fields import (
+    ModuleMappingField,
+    RequirementsOverrideField,
+    TypeStubsModuleMappingField,
+)
+from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.target_types import (
+    PythonRequirementModulesField,
+    PythonRequirementsField,
+    PythonRequirementsFileSourcesField,
+    PythonRequirementsFileTarget,
+    PythonRequirementTarget,
+    PythonRequirementTypeStubModulesField,
+)
+from pants.engine.addresses import Address
+from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
+from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    Dependencies,
+    GeneratedTargets,
+    GenerateTargetsRequest,
+    InvalidFieldException,
+    SingleSourceField,
+    StringField,
+    Target,
+)
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+
+
+class PipenvSourceField(SingleSourceField):
+    default = "Pipfile.lock"
+    required = False
+
+
+class PipenvPipfileTargetField(StringField):
+    alias = "pipfile_target"
+    help = "Deprecated: no longer necessary."
+    removal_version = "2.11.0.dev0"
+    removal_hint = "This field is no longer necessary."
+
+
+class PipenvRequirementsTargetGenerator(Target):
+    alias = "pipenv_requirements"
+    help = "Generate a `python_requirement` for each entry in `Pipenv.lock`."
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        ModuleMappingField,
+        TypeStubsModuleMappingField,
+        PipenvSourceField,
+        PipenvPipfileTargetField,
+        RequirementsOverrideField,
+    )
+
+
+class GenerateFromPipenvRequirementsRequest(GenerateTargetsRequest):
+    generate_from = PipenvRequirementsTargetGenerator
+
+
+# TODO(#10655): add support for PEP 440 direct references (aka VCS style).
+# TODO(#10655): differentiate between Pipfile vs. Pipfile.lock.
+@rule(desc="Generate `python_requirement` targets from Pipfile.lock", level=LogLevel.DEBUG)
+async def generate_from_pipenv_requirement(
+    request: GenerateFromPipenvRequirementsRequest,
+) -> GeneratedTargets:
+    generator = request.generator
+    lock_rel_path = generator[PipenvSourceField].value
+    lock_full_path = generator[PipenvSourceField].file_path
+
+    file_tgt = PythonRequirementsFileTarget(
+        {PythonRequirementsFileSourcesField.alias: lock_rel_path},
+        Address(
+            generator.address.spec_path,
+            target_name=generator.address.target_name,
+            relative_file_path=lock_rel_path,
+        ),
+    )
+
+    digest_contents = await Get(
+        DigestContents,
+        PathGlobs(
+            [lock_full_path],
+            glob_match_error_behavior=GlobMatchErrorBehavior.error,
+            description_of_origin=f"{generator}'s field `{PipenvSourceField.alias}`",
+        ),
+    )
+    lock_info = json.loads(digest_contents[0].content)
+
+    module_mapping = generator[ModuleMappingField].value
+    stubs_mapping = generator[TypeStubsModuleMappingField].value
+    overrides = generator[RequirementsOverrideField].flatten_and_normalize()
+
+    def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:
+        if info.get("extras"):
+            raw_req += f"[{','.join(info['extras'])}]"
+        raw_req += info.get("version", "")
+        if info.get("markers"):
+            raw_req += f";{info['markers']}"
+
+        parsed_req = PipRequirement.parse(raw_req)
+        normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
+        tgt_overrides = overrides.pop(normalized_proj_name, {})
+        if Dependencies.alias in tgt_overrides:
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
+                file_tgt.address.spec
+            ]
+
+        # TODO: Consider letting you set metadata in the target generator and having it pass down
+        #  to all generated targets. Especially useful for compatible_resolves.
+        return PythonRequirementTarget(
+            {
+                PythonRequirementsField.alias: [parsed_req],
+                PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
+                PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
+                    normalized_proj_name
+                ),
+                # This may get overridden by `tgt_overrides`, which will have already added in
+                # the file tgt.
+                Dependencies.alias: [file_tgt.address.spec],
+                **tgt_overrides,
+            },
+            generator.address.create_generated(parsed_req.project_name),
+        )
+
+    result = tuple(
+        generate_tgt(req, info)
+        for req, info in {**lock_info.get("default", {}), **lock_info.get("develop", {})}.items()
+    ) + (file_tgt,)
+
+    if overrides:
+        raise InvalidFieldException(
+            f"Unused key in the `overrides` field for {request.generator.address}: "
+            f"{sorted(overrides)}"
+        )
+
+    return GeneratedTargets(generator, result)
+
+
+def rules():
+    return (
+        *collect_rules(),
+        UnionRule(GenerateTargetsRequest, GenerateFromPipenvRequirementsRequest),
+    )

src/python/pants/backend/python/macros/pipenv_requirements_caof_test.py

@@ -20,6 +20,7 @@ def rule_runner() -> RuleRunner:
     return RuleRunner(
         target_types=[PythonRequirementTarget, PythonRequirementsFileTarget],
         context_aware_object_factories={"pipenv_requirements": PipenvRequirementsCAOF},
+        use_deprecated_python_macros=True,
     )