From 5e7e0301eeffc4232c399065ca1cc730710adffd Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:23:45 -0700 Subject: [PATCH] Allow changing the versioning scheme for `python_distribution` first-party dependencies (Cherry-pick of #10977) [ci skip-rust] [ci skip-build-wheels] # Conflicts: # src/python/pants/backend/python/goals/setup_py_test.py --- .../pants/backend/python/goals/setup_py.py | 54 +++++++++++++++---- .../backend/python/goals/setup_py_test.py | 37 +++++++++++-- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/python/pants/backend/python/goals/setup_py.py b/src/python/pants/backend/python/goals/setup_py.py index 4480719cd157..56ea0fea44a2 100644 --- a/src/python/pants/backend/python/goals/setup_py.py +++ b/src/python/pants/backend/python/goals/setup_py.py @@ -1,6 +1,7 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import enum import io import itertools import logging @@ -72,6 +73,7 @@ ) from pants.engine.unions import UnionMembership, UnionRule, union from pants.option.custom_types import shell_str +from pants.option.subsystem import Subsystem from pants.python.python_setup import PythonSetup from pants.util.logging import LogLevel from pants.util.memo import memoized_property @@ -308,6 +310,45 @@ class RunSetupPyResult: output: Digest # The state of the chroot after running setup.py. +@enum.unique +class FirstPartyDependencyVersionScheme(enum.Enum): + EXACT = "exact" # i.e., == + COMPATIBLE = "compatible" # i.e., ~= + ANY = "any" # i.e., no specifier + + +class PythonDistributionSubsystem(Subsystem): + """Options for packaging wheels/sdists from a `python_distribution` target.""" + + options_scope = "python-distribution" + + @classmethod + def register_options(cls, register): + super().register_options(register) + register( + "--first-party-dependency-version-scheme", + type=FirstPartyDependencyVersionScheme, + default=FirstPartyDependencyVersionScheme.EXACT, + help=( + "What version to set in `install_requires` when a `python_distribution` depends on " + "other `python_distribution`s. If `exact`, will use `==`. If `compatible`, will " + "use `~=`. If `any`, will leave off the version. See " + "https://www.python.org/dev/peps/pep-0440/#version-specifiers." + ), + ) + + def first_party_dependency_version(self, version: str) -> str: + """Return the version string (e.g. '~=4.0') for a first-party dependency. + + If the user specified to use "any" version, then this will return an empty string. + """ + scheme = self.options.first_party_dependency_version_scheme + if scheme == FirstPartyDependencyVersionScheme.ANY: + return "" + specifier = "==" if scheme == FirstPartyDependencyVersionScheme.EXACT else "~=" + return f"{specifier}{version}" + + class SetupPySubsystem(GoalSubsystem): """Deprecated in favor of the `package` goal.""" @@ -714,7 +755,9 @@ async def get_sources(request: SetupPySourcesRequest) -> SetupPySources: @rule(desc="Compute distribution's 3rd party requirements") async def get_requirements( - dep_owner: DependencyOwner, union_membership: UnionMembership + dep_owner: DependencyOwner, + union_membership: UnionMembership, + python_distribution_subsystem: PythonDistributionSubsystem, ) -> ExportedTargetRequirements: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([dep_owner.exported_target.target.address]) @@ -736,13 +779,6 @@ async def get_requirements( # if U is in the owned deps then we'll pick up R through U. And if U is not in the owned deps # then it's owned by an exported target ET, and so R will be in the requirements for ET, and we # will require ET. - # - # TODO: Note that this logic doesn't account for indirection via dep aggregator targets, of type - # `target`. But we don't have those in v2 (yet) anyway. Plus, as we move towards buildgen and/or - # stricter build graph hygiene, it makes sense to require that targets directly declare their - # true dependencies. Plus, in the specific realm of setup-py, since we must exclude indirect - # deps across exported target boundaries, it's not a big stretch to just insist that - # requirements must be direct deps. direct_deps_tgts = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in owned_by_us ) @@ -758,7 +794,7 @@ async def get_requirements( Get(SetupKwargs, OwnedDependency(tgt)) for tgt in owned_by_others ) req_strs.extend( - f"{kwargs.name}=={kwargs.version}" + f"{kwargs.name}{python_distribution_subsystem.first_party_dependency_version(kwargs.version)}" for kwargs in set(kwargs_for_exported_targets_we_depend_on) ) diff --git a/src/python/pants/backend/python/goals/setup_py_test.py b/src/python/pants/backend/python/goals/setup_py_test.py index eee0f8538dbe..046f1e0443e2 100644 --- a/src/python/pants/backend/python/goals/setup_py_test.py +++ b/src/python/pants/backend/python/goals/setup_py_test.py @@ -11,11 +11,13 @@ DependencyOwner, ExportedTarget, ExportedTargetRequirements, + FirstPartyDependencyVersionScheme, InvalidEntryPoint, InvalidSetupPyArgs, NoOwnerError, OwnedDependencies, OwnedDependency, + PythonDistributionSubsystem, SetupKwargs, SetupKwargsRequest, SetupPyChroot, @@ -44,7 +46,7 @@ from pants.engine.addresses import Address from pants.engine.fs import Snapshot from pants.engine.internals.scheduler import ExecutionError -from pants.engine.rules import rule +from pants.engine.rules import SubsystemRule, rule from pants.engine.target import Targets from pants.engine.unions import UnionRule from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -93,6 +95,7 @@ def chroot_rule_runner() -> RuleRunner: get_exporting_owner, *python_sources.rules(), setup_kwargs_plugin, + SubsystemRule(PythonDistributionSubsystem), UnionRule(SetupKwargsRequest, PluginSetupKwargsRequest), QueryRule(SetupPyChroot, (SetupPyChrootRequest,)), ] @@ -366,6 +369,7 @@ def test_get_requirements() -> None: get_requirements, get_owned_dependencies, get_exporting_owner, + SubsystemRule(PythonDistributionSubsystem), QueryRule(ExportedTargetRequirements, (DependencyOwner,)), ] ) @@ -432,16 +436,39 @@ def test_get_requirements() -> None: ), ) - def assert_requirements(expected_req_strs, addr): - tgt = rule_runner.get_target(Address.parse(addr)) + def assert_requirements( + expected_req_strs, + addr: Address, + *, + version_scheme: FirstPartyDependencyVersionScheme = FirstPartyDependencyVersionScheme.EXACT, + ): + rule_runner.set_options( + [f"--python-distribution-first-party-dependency-version-scheme={version_scheme.value}"] + ) + tgt = rule_runner.get_target(addr) reqs = rule_runner.request( ExportedTargetRequirements, [DependencyOwner(ExportedTarget(tgt))], ) assert sorted(expected_req_strs) == list(reqs) - assert_requirements(["ext1==1.22.333", "ext2==4.5.6"], "src/python/foo/bar:bar-dist") - assert_requirements(["ext3==0.0.1", "bar==9.8.7"], "src/python/foo/corge:corge-dist") + assert_requirements( + ["ext1==1.22.333", "ext2==4.5.6"], Address("src/python/foo/bar", target_name="bar-dist") + ) + assert_requirements( + ["ext3==0.0.1", "bar==9.8.7"], Address("src/python/foo/corge", target_name="corge-dist") + ) + + assert_requirements( + ["ext3==0.0.1", "bar~=9.8.7"], + Address("src/python/foo/corge", target_name="corge-dist"), + version_scheme=FirstPartyDependencyVersionScheme.COMPATIBLE, + ) + assert_requirements( + ["ext3==0.0.1", "bar"], + Address("src/python/foo/corge", target_name="corge-dist"), + version_scheme=FirstPartyDependencyVersionScheme.ANY, + ) def test_owned_dependencies() -> None: