Support multiple disjoint Python resolves via [python].experimental_resolves

src/python/pants/backend/awslambda/python/target_types.py

@@ -11,6 +11,8 @@
     PythonModuleOwnersRequest,
 )
 from pants.backend.python.dependency_inference.rules import PythonInferSubsystem, import_rules
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.target_types import PythonResolveField
 from pants.core.goals.package import OutputPathField
 from pants.engine.addresses import Address
 from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths
@@ -130,7 +132,9 @@ class InjectPythonLambdaHandlerDependency(InjectDependenciesRequest):
 
 @rule(desc="Inferring dependency from the python_awslambda `handler` field")
 async def inject_lambda_handler_dependency(
-    request: InjectPythonLambdaHandlerDependency, python_infer_subsystem: PythonInferSubsystem
+    request: InjectPythonLambdaHandlerDependency,
+    python_infer_subsystem: PythonInferSubsystem,
+    python_setup: PythonSetup,
 ) -> InjectedDependencies:
     if not python_infer_subsystem.entry_points:
         return InjectedDependencies()
@@ -143,7 +147,12 @@ async def inject_lambda_handler_dependency(
         ),
     )
     module, _, _func = handler.val.partition(":")
-    owners = await Get(PythonModuleOwners, PythonModuleOwnersRequest(module, resolve=None))
+    owners = await Get(
+        PythonModuleOwners,
+        PythonModuleOwnersRequest(
+            module, resolve=original_tgt.target[PythonResolveField].normalized_value(python_setup)
+        ),
+    )
     address = original_tgt.target.address
     explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
         owners.ambiguous,
@@ -202,6 +211,7 @@ class PythonAWSLambda(Target):
         PythonAwsLambdaDependencies,
         PythonAwsLambdaHandlerField,
         PythonAwsLambdaRuntime,
+        PythonResolveField,
     )
     help = (
         "A self-contained Python function suitable for uploading to AWS Lambda.\n\n"

src/python/pants/backend/google_cloud_function/python/target_types.py

@@ -12,6 +12,8 @@
     PythonModuleOwnersRequest,
 )
 from pants.backend.python.dependency_inference.rules import PythonInferSubsystem, import_rules
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.target_types import PythonResolveField
 from pants.core.goals.package import OutputPathField
 from pants.engine.addresses import Address
 from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths
@@ -133,6 +135,7 @@ class InjectPythonCloudFunctionHandlerDependency(InjectDependenciesRequest):
 async def inject_cloud_function_handler_dependency(
     request: InjectPythonCloudFunctionHandlerDependency,
     python_infer_subsystem: PythonInferSubsystem,
+    python_setup: PythonSetup,
 ) -> InjectedDependencies:
     if not python_infer_subsystem.entry_points:
         return InjectedDependencies()
@@ -147,7 +150,12 @@ async def inject_cloud_function_handler_dependency(
         ),
     )
     module, _, _func = handler.val.partition(":")
-    owners = await Get(PythonModuleOwners, PythonModuleOwnersRequest(module, resolve=None))
+    owners = await Get(
+        PythonModuleOwners,
+        PythonModuleOwnersRequest(
+            module, resolve=original_tgt.target[PythonResolveField].normalized_value(python_setup)
+        ),
+    )
     address = original_tgt.target.address
     explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
         owners.ambiguous,
@@ -229,6 +237,7 @@ class PythonGoogleCloudFunction(Target):
         PythonGoogleCloudFunctionHandlerField,
         PythonGoogleCloudFunctionRuntime,
         PythonGoogleCloudFunctionType,
+        PythonResolveField,
     )
     help = (
         "A self-contained Python function suitable for uploading to Google Cloud Function.\n\n"

src/python/pants/backend/python/dependency_inference/rules.py

@@ -193,12 +193,7 @@ async def infer_python_dependencies_via_imports(
         ),
     )
 
-    resolve = (
-        tgt[PythonResolveField].normalized_value(python_setup)
-        if tgt.has_field(PythonResolveField)
-        else None
-    )
-
+    resolve = tgt[PythonResolveField].normalized_value(python_setup)
     owners_per_import = await MultiGet(
         Get(PythonModuleOwners, PythonModuleOwnersRequest(imported_module, resolve=resolve))
         for imported_module in parsed_imports

src/python/pants/backend/python/lint/flake8/rules.py

@@ -11,7 +11,7 @@
     Flake8FirstPartyPlugins,
 )
 from pants.backend.python.subsystems.setup import PythonSetup
-from pants.backend.python.util_rules import pex_from_targets
+from pants.backend.python.util_rules import pex
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
 from pants.core.goals.lint import REPORT_DIR, LintRequest, LintResult, LintResults
@@ -137,4 +137,4 @@ async def flake8_lint(
 
 
 def rules():
-    return [*collect_rules(), UnionRule(LintRequest, Flake8Request), *pex_from_targets.rules()]
+    return [*collect_rules(), UnionRule(LintRequest, Flake8Request), *pex.rules()]

src/python/pants/backend/python/subsystems/setup.py

@@ -138,12 +138,36 @@ def register_options(cls, register):
             default={"python-default": "3rdparty/python/default_lock.txt"},
             help=(
                 "A mapping of logical names to lockfile paths used in your project.\n\n"
-                "For now, things only work properly if you define a single resolve and set "
-                "`[python].experimental_default_resolve` to that value. We are close to "
-                "properly supporting multiple (disjoint) resolves.\n\n"
-                "To generate a lockfile, run `./pants generate-lockfiles --resolve=<name>` or "
-                "`./pants generate-lockfiles` to generate for all resolves (including tool "
-                "lockfiles).\n\n"
+                "Many organizations only need a single resolve for their whole project, which is "
+                "a good default and the simplest thing to do. However, you may need multiple "
+                "resolves, such as if you use two conflicting versions of a requirement in "
+                "your repository.\n\n"
+                "For now, Pants only has first-class support for disjoint resolves, meaning that "
+                "you cannot ergonomically set a `python_source` target, for example, to work "
+                "with multiple resolve. Practically, this means that you cannot yet reuse common "
+                "code, such as util files, across projects using different resolves. Support for "
+                "overlapping resolves is coming soon.\n\n"
+                "If you only need a single resolve, optionally override this option's default if "
+                "you would like to write your project's lockfile to a different path. Make "
+                "sure that `[python].experimental_default_resolve` is set to the same resolve "
+                "name. Then, run `./pants generate-lockfiles --resolve=<name>` to generate the "
+                "lockfile.\n\n"
+                "If you need multiple resolves:\n\n"
+                "  1. Define via this option multiple resolve "
+                "names and their lockfile path. The names should be meaningful to your "
+                "repository, such as `data-science` or `pants-plugins`.\n"
+                "  2. Set the default with "
+                "`[python].experimental_default_resolve`.\n"
+                "  3. Update your `python_requirement` targets with the "
+                "`experimental_compatible_resolves` field to declare which resolve(s) they should "
+                "be available in. (Often you'll set this via the `python_requirements` or "
+                "`poetry_requirements` target generators)\n"
+                "  4. Run `./pants generate-lockfiles --resolve=<name1> --resolve<name2>` to "
+                "generate the lockfiles (or `./pants generate-lockfiles` to generate tool "
+                "lockfiles too). If the results aren't what you'd expect, adjust the prior step.\n"
+                "  5. Update all relevant targets like `python_source` / `python_sources`, "
+                "`python_test` / `python_tests`, and `pex_binary` to set the "
+                "`experimental_resolve` field.\n\n"
                 "Only applies if `[python].enable_resolves` is true.\n\n"
                 "This option is experimental and may change without the normal deprecation policy."
             ),

src/python/pants/backend/python/target_types.py

@@ -109,9 +109,11 @@ class PythonResolveField(StringField, AsyncFieldMixin):
     help = (
         "The resolve from `[python].experimental_resolves` to use.\n\n"
         "If not defined, will default to `[python].default_resolve`.\n\n"
-        "Only applies if `[python].enable_resolves` is true.\n\n"
+        "All dependencies must share the same resolve. This means that you can only depend on "
+        "first-party targets like `python_source` that set their `experimental_resolve` field "
+        "to the same value, and on `python_requirement` targets that include the resolve in "
+        "their `experimental_compatible_resolves` field.\n\n"
         "This field is experimental and may change without the normal deprecation policy."
-        # TODO: Document expectations for dependencies once we validate that.
     )
 
     def normalized_value(self, python_setup: PythonSetup) -> str:
@@ -136,31 +138,6 @@ def resolve_and_lockfile(self, python_setup: PythonSetup) -> tuple[str, str] | N
         return (resolve, python_setup.resolves[resolve])
 
 
-class PythonCompatibleResolvesField(StringSequenceField, AsyncFieldMixin):
-    alias = "experimental_compatible_resolves"
-    required = False
-    help = (
-        "The set of resolves from `[python].experimental_resolves` that this target is "
-        "compatible with.\n\n"
-        "If not defined, will default to `[python].default_resolve`.\n\n"
-        "Only applies if `[python].enable_resolves` is true.\n\n"
-        "This field is experimental and may change without the normal deprecation policy."
-        # TODO: Document expectations for dependencies once we validate that.
-    )
-
-    def normalized_value(self, python_setup: PythonSetup) -> tuple[str, ...]:
-        """Get the value after applying the default and validating every key is recognized."""
-        value_or_default = self.value or (python_setup.default_resolve,)
-        invalid_resolves = set(value_or_default) - set(python_setup.resolves)
-        if invalid_resolves:
-            raise UnrecognizedResolveNamesError(
-                sorted(invalid_resolves),
-                python_setup.resolves.keys(),
-                description_of_origin=f"the field `{self.alias}` in the target {self.address}",
-            )
-        return value_or_default
-
-
 # -----------------------------------------------------------------------------------------------
 # `pex_binary` and `pex_binaries` target
 # -----------------------------------------------------------------------------------------------
@@ -775,6 +752,7 @@ class PythonSourceTarget(Target):
         *COMMON_TARGET_FIELDS,
         InterpreterConstraintsField,
         Dependencies,
+        PythonResolveField,
         PythonSourceField,
     )
     help = "A single Python source file."
@@ -812,6 +790,7 @@ class PythonTestUtilsGeneratorTarget(Target):
         *COMMON_TARGET_FIELDS,
         InterpreterConstraintsField,
         Dependencies,
+        PythonResolveField,
         PythonTestUtilsGeneratingSourcesField,
         PythonSourcesOverridesField,
     )
@@ -831,6 +810,7 @@ class PythonSourcesGeneratorTarget(Target):
         *COMMON_TARGET_FIELDS,
         InterpreterConstraintsField,
         Dependencies,
+        PythonResolveField,
         PythonSourcesGeneratingSourcesField,
         PythonSourcesOverridesField,
     )
@@ -975,20 +955,33 @@ def normalize_module_mapping(
     return FrozenDict({canonicalize_project_name(k): tuple(v) for k, v in (mapping or {}).items()})
 
 
-class PythonRequirementCompatibleResolvesField(PythonCompatibleResolvesField):
+class PythonRequirementCompatibleResolvesField(StringSequenceField, AsyncFieldMixin):
+    alias = "experimental_compatible_resolves"
+    required = False
     help = (
         "The resolves from `[python].experimental_resolves` that this requirement should be "
         "included in.\n\n"
         "If not defined, will default to `[python].default_resolve`.\n\n"
         "When generating a lockfile for a particular resolve via the `generate-lockfiles` goal, "
         "it will include all requirements that are declared compatible with that resolve. "
-        "First-party targets like `python_source` and `pex_binary` then declare which resolve(s) "
-        "they use via the `experimental_resolve` and `experimental_compatible_resolves` field; so, "
-        "for your first-party code to use a particular `python_requirement` target, that "
-        "requirement must be included in the resolve(s) "
+        "First-party targets like `python_source` and `pex_binary` then declare which resolve "
+        "they use via the `experimental_resolve` field; so, for your first-party code to use a "
+        "particular `python_requirement` target, that requirement must be included in the resolve "
         "used by that code."
     )
 
+    def normalized_value(self, python_setup: PythonSetup) -> tuple[str, ...]:
+        """Get the value after applying the default and validating every key is recognized."""
+        value_or_default = self.value or (python_setup.default_resolve,)
+        invalid_resolves = set(value_or_default) - set(python_setup.resolves)
+        if invalid_resolves:
+            raise UnrecognizedResolveNamesError(
+                sorted(invalid_resolves),
+                python_setup.resolves.keys(),
+                description_of_origin=f"the field `{self.alias}` in the target {self.address}",
+            )
+        return value_or_default
+
 
 class PythonRequirementTarget(Target):
     alias = "python_requirement"