Add the ability for Pants to provide Python via a union (with a pyenv…

… impl) (#18352)

Fixes #10447

Adding the ability for an optional union (`PythonProvider`) implementer
to return a `PythonExecutable` (newly outfitted with
`append_only_caches`) so that Python for user code can be supplied
completely by rule-code. Additionally added a `pyenv` union
implementation.

The dirty details are:
- Python is essentially NOT relocatable (hats off to IndyGreg). Python
itself, as well as ecosystem tools, are baked with assumptions about
absolute paths.
- Python installation is not atomic. That means we do our work behind a
lock. We run the bash script on every session, but it should be fast as
it only does the work if the target python doesnt exist.

---------

Co-authored-by: Andreas Stenius <git@astekk.se>

src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py

@@ -192,6 +192,7 @@ async def parse_python_dependencies(
                 file,
             ],
             input_digest=input_digest,
+            append_only_caches=python_interpreter.append_only_caches,
             description=f"Determine Python dependencies for {request.source.address}",
             env=parser_script.env,
             level=LogLevel.DEBUG,

src/python/pants/backend/python/goals/run_helper.py

@@ -27,6 +27,7 @@
 from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.rules import Get, MultiGet
 from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest
+from pants.util.frozendict import FrozenDict
 
 
 def _in_chroot(relpath: str) -> str:
@@ -112,12 +113,18 @@ async def _create_python_source_run_request(
         **complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
         "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
     }
+    append_only_caches = (
+        FrozenDict({}) if venv_pex.append_only_caches is None else venv_pex.append_only_caches
+    )
 
     return RunRequest(
         digest=merged_digest,
         args=[_in_chroot(venv_pex.pex.argv0)],
         extra_env=extra_env,
-        append_only_caches=complete_pex_environment.append_only_caches,
+        append_only_caches={
+            **complete_pex_environment.append_only_caches,
+            **append_only_caches,
+        },
     )
 
 

src/python/pants/backend/python/providers/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/python/providers/experimental/pyenv/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/python/providers/experimental/pyenv/register.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+from pants.backend.python.providers.pyenv.rules import rules as pyenv_rules
+from pants.backend.python.providers.pyenv.target_types import PyenvInstall
+
+
+def target_types():
+    return [PyenvInstall]
+
+
+def rules():
+    return pyenv_rules()

src/python/pants/backend/python/providers/pyenv/BUILD

@@ -0,0 +1,12 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+python_tests(
+    name="tests",
+    overrides={
+        "rules_integration_test.py": {
+            "timeout": 600,
+        }
+    },
+)