Add pants_requirements target generator and deprecate `pants_requir…

…ement` macro (#13512)

Progress towards #12915.

This new target generator is much more useful. It leans into the reality that Pants 2 only releases `pantsbuild.pants` and `pantsbuild.pants.testutil`, and that almost always you want to use both when developing plugins. It also removes irrelevant fields like `dist` and `modules`.

The version is also more flexible now, as described in the new `help` message.

--

Note that this is our first time we're using target generator syntax `dir:tgt#gen` in non-Go code. Even though #12917 is not decided, there seems to be consensus on using target generator syntax for file-less targets like `python_requirement`.

[ci skip-rust]

src/python/pants/backend/plugin_development/BUILD

@@ -0,0 +1,6 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+
+python_tests(name="tests")

src/python/pants/backend/plugin_development/pants_requirements.py

@@ -0,0 +1,100 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.target_types import (
+    PythonRequirementModulesField,
+    PythonRequirementsField,
+    PythonRequirementTarget,
+)
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    BoolField,
+    GeneratedTargets,
+    GenerateTargetsRequest,
+    Target,
+)
+from pants.engine.unions import UnionRule
+from pants.version import MAJOR_MINOR, PANTS_SEMVER
+
+
+class PantsRequirementsTestutilField(BoolField):
+    alias = "testutil"
+    default = True
+    help = "If true, include `pantsbuild.pants.testutil` to write tests for your plugin."
+
+
+class PantsRequirementsTargetGenerator(Target):
+    alias = "pants_requirements"
+    help = (
+        "Generate `python_requirement` targets for Pants itself to use with Pants plugins.\n\n"
+        "This is useful when writing plugins so that you can build and test your "
+        "plugin using Pants. The generated targets will have the correct version based on the "
+        "`version` in your `pants.toml`, and they will work with dependency inference.\n\n"
+        "Because the Plugin API is not yet stable, the version is set automatically for you "
+        "to improve stability. If you're currently using a dev release, the version will be set to "
+        "that exact dev release. If you're using a release candidate (rc) or stable release, the "
+        "version will allow any non-dev-release release within the release series, e.g. "
+        f"`>={MAJOR_MINOR}.0rc0,<{PANTS_SEMVER.major}.{PANTS_SEMVER.minor + 1}`.\n\n"
+        "(If this versioning scheme does not work for you, you can directly create "
+        "`python_requirement` targets for `pantsbuild.pants` and `pantsbuild.pants.testutil`. We "
+        "also invite you to share your ideas at "
+        "https://github.com/pantsbuild/pants/issues/new/choose)"
+    )
+    core_fields = (*COMMON_TARGET_FIELDS, PantsRequirementsTestutilField)
+
+
+class GenerateFromPantsRequirementsRequest(GenerateTargetsRequest):
+    generate_from = PantsRequirementsTargetGenerator
+
+
+def determine_version() -> str:
+    # Because the Plugin API is not stable, it can have breaking changes in-between dev releases.
+    # Technically, it can also have breaking changes between rcs in the same release series, but
+    # this is much less likely.
+    #
+    # So, we require exact matches when developing against a dev release, but only require
+    # matching the release series if on an rc or stable release.
+    #
+    # If this scheme does not work for users, they can:
+    #
+    #    1. Use a `python_requirement` directly
+    #    2. Add a new `version` field to this target generator.
+    #    3. Fork this target generator.
+    return (
+        f"=={PANTS_SEMVER}"
+        if PANTS_SEMVER.is_devrelease
+        else (
+            f">={PANTS_SEMVER.major}.{PANTS_SEMVER.minor}.0rc0,"
+            f"<{PANTS_SEMVER.major}.{PANTS_SEMVER.minor + 1}"
+        )
+    )
+
+
+@rule
+def generate_from_pants_requirements(
+    request: GenerateFromPantsRequirementsRequest,
+) -> GeneratedTargets:
+    generator = request.generator
+    version = determine_version()
+
+    def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
+        return PythonRequirementTarget(
+            {
+                PythonRequirementsField.alias: (f"{dist}{version}",),
+                PythonRequirementModulesField.alias: (module,),
+            },
+            generator.address.create_generated(dist),
+        )
+
+    result = [create_tgt("pantsbuild.pants", "pants")]
+    if generator[PantsRequirementsTestutilField].value:
+        result.append(create_tgt("pantsbuild.pants.testutil", "pants.testutil"))
+    return GeneratedTargets(generator, result)
+
+
+def rules():
+    return (
+        *collect_rules(),
+        UnionRule(GenerateTargetsRequest, GenerateFromPantsRequirementsRequest),
+    )

src/python/pants/backend/plugin_development/pants_requirements_test.py

@@ -0,0 +1,74 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import pytest
+from packaging.version import Version
+
+from pants.backend.plugin_development import pants_requirements
+from pants.backend.plugin_development.pants_requirements import (
+    GenerateFromPantsRequirementsRequest,
+    PantsRequirementsTargetGenerator,
+    determine_version,
+)
+from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.target_types import PythonRequirementModulesField, PythonRequirementsField
+from pants.engine.addresses import Address
+from pants.engine.target import GeneratedTargets
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.mark.parametrize(
+    "pants_version,expected",
+    (
+        ("2.4.0.dev1", "==2.4.0.dev1"),
+        ("2.4.0rc1", ">=2.4.0rc0,<2.5"),
+        ("2.4.0", ">=2.4.0rc0,<2.5"),
+    ),
+)
+def test_determine_version(monkeypatch, pants_version: str, expected: str) -> None:
+    monkeypatch.setattr(pants_requirements, "PANTS_SEMVER", Version(pants_version))
+    assert determine_version() == expected
+
+
+def test_target_generator() -> None:
+    rule_runner = RuleRunner(
+        rules=(
+            *pants_requirements.rules(),
+            QueryRule(GeneratedTargets, [GenerateFromPantsRequirementsRequest]),
+        ),
+        target_types=[PantsRequirementsTargetGenerator],
+    )
+
+    rule_runner.write_files(
+        {
+            "BUILD": (
+                "pants_requirements(name='default')\n"
+                "pants_requirements(name='no_testutil', testutil=False)\n"
+            )
+        }
+    )
+
+    generator = rule_runner.get_target(Address("", target_name="default"))
+    result = rule_runner.request(
+        GeneratedTargets, [GenerateFromPantsRequirementsRequest(generator)]
+    )
+    assert len(result) == 2
+    pants_req = next(t for t in result.values() if t.address.generated_name == "pantsbuild.pants")
+    testutil_req = next(
+        t for t in result.values() if t.address.generated_name == "pantsbuild.pants.testutil"
+    )
+    assert pants_req[PythonRequirementModulesField].value == ("pants",)
+    assert testutil_req[PythonRequirementModulesField].value == ("pants.testutil",)
+    assert pants_req[PythonRequirementsField].value == (
+        PipRequirement.parse(f"pantsbuild.pants{determine_version()}"),
+    )
+    assert testutil_req[PythonRequirementsField].value == (
+        PipRequirement.parse(f"pantsbuild.pants.testutil{determine_version()}"),
+    )
+
+    generator = rule_runner.get_target(Address("", target_name="no_testutil"))
+    result = rule_runner.request(
+        GeneratedTargets, [GenerateFromPantsRequirementsRequest(generator)]
+    )
+    assert len(result) == 1
+    assert next(iter(result.keys())).generated_name == "pantsbuild.pants"

src/python/pants/backend/plugin_development/register.py

@@ -0,0 +1,13 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.plugin_development import pants_requirements
+from pants.backend.plugin_development.pants_requirements import PantsRequirementsTargetGenerator
+
+
+def rules():
+    return pants_requirements.rules()
+
+
+def target_types():
+    return [PantsRequirementsTargetGenerator]

src/python/pants/backend/python/macros/pants_requirement_caof.py

@@ -5,6 +5,7 @@
 from typing import Iterable, Optional
 
 from pants.base.build_environment import pants_version
+from pants.base.deprecated import warn_or_error
 from pants.base.exceptions import TargetDefinitionException
 from pants.build_graph.address import Address
 from pants.util.meta import classproperty
@@ -45,6 +46,29 @@ def __call__(
         :param modules: The modules exposed by the dist, e.g. `['pants.testutil']`. This defaults
             to the name of the dist without the leading `pantsbuild`.
         """
+        warn_or_error(
+            "2.10.0.dev0",
+            "the `pants_requirement` macro",
+            (
+                "Use the target `pants_requirements` instead. First, add "
+                "`pants.backend.plugin_development` to `[GLOBAL].backend_packages` in "
+                "`pants.toml`. Then, delete all `pants_requirement` calls and replace them with "
+                "a single `pants_requirements(name='pants')`.\n\n"
+                "By default, `pants_requirements` will generate a `python_requirement` target for "
+                "both `pantsbuild.pants` and `pantsbuild.pants.testutil`.\n\n"
+                "The address for the generated targets will be different, e.g. "
+                "`pants-plugins:pants#pantsbuild.pants` rather than "
+                "`pants-plugins:pantsbuild.pants`. If you're using dependency inference, you "
+                "should not need to update anything.\n\n"
+                "The version of Pants is more useful now. If you're using a dev release, the "
+                "version will be the exact release you're on, like before, to reduce the risk of "
+                "a Plugin API change breaking your plugin. But if you're using a release candidate "
+                "or stable release, the version will now be any non-dev release in the release "
+                "series, e.g. any release candidate or stable release in Pants 2.9. This allows "
+                "consumers of your plugin to use different patch versions than what you release "
+                "the plugin with."
+            ),
+        )
         name = name or dist or os.path.basename(self._parse_context.rel_path)
         dist = dist or "pantsbuild.pants"
         if not (dist == "pantsbuild.pants" or dist.startswith("pantsbuild.pants.")):

src/python/pants/init/BUILD

@@ -20,6 +20,7 @@ target(
         "src/python/pants/backend/experimental/scala",
         "src/python/pants/backend/experimental/terraform",
         "src/python/pants/backend/google_cloud_function/python",
+        "src/python/pants/backend/plugin_development",
         "src/python/pants/backend/project_info",
         "src/python/pants/backend/python",
         "src/python/pants/backend/python/lint/bandit",