pex: add include_requirements flag to avoid resolving and placing 3rdparty requirements into PEX files

src/python/pants/backend/python/goals/run_helper.py

@@ -17,7 +17,11 @@
 )
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexRequest
-from pants.backend.python.util_rules.pex_environment import PexEnvironment, PythonExecutable
+from pants.backend.python.util_rules.pex_environment import (
+    PexEnvironment,
+    PexSubsystem,
+    PythonExecutable,
+)
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -41,6 +45,7 @@ async def _create_python_source_run_request(
     *,
     entry_point_field: PexEntryPointField,
     pex_env: PexEnvironment,
+    pex_subsystem: PexSubsystem,
     run_in_sandbox: bool,
     pex_path: Iterable[Pex] = (),
     console_script: Optional[ConsoleScript] = None,
@@ -66,6 +71,7 @@ async def _create_python_source_run_request(
                 addresses,
                 output_filename=f"{pex_filename}.pex",
                 internal_only=True,
+                include_requirements=pex_subsystem.include_requirements,
                 include_source_files=False,
                 include_local_dists=True,
                 # `PEX_EXTRA_SYS_PATH` should contain this entry_point's module.

src/python/pants/backend/python/goals/run_python_source.py

@@ -21,7 +21,7 @@
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import Pex, PexRequest
-from pants.backend.python.util_rules.pex_environment import PexEnvironment
+from pants.backend.python.util_rules.pex_environment import PexEnvironment, PexSubsystem
 from pants.backend.python.util_rules.pex_from_targets import rules as pex_from_targets_rules
 from pants.core.goals.run import (
     RunDebugAdapterRequest,
@@ -66,13 +66,17 @@ def _executable_main(self) -> Optional[Executable]:
 
 @rule(level=LogLevel.DEBUG)
 async def create_python_source_run_request(
-    field_set: PythonSourceFieldSet, pex_env: PexEnvironment, python_setup: PythonSetup
+    field_set: PythonSourceFieldSet,
+    pex_env: PexEnvironment,
+    python_setup: PythonSetup,
+    pex_subsystem: PexSubsystem,
 ) -> RunRequest:
     return await _create_python_source_run_request(
         field_set.address,
         entry_point_field=PexEntryPointField(field_set.source.value, field_set.address),
         executable=field_set._executable_main(),
         pex_env=pex_env,
+        pex_subsystem=pex_subsystem,
         run_in_sandbox=field_set.should_use_sandbox(python_setup),
     )
 

src/python/pants/backend/python/util_rules/pex_environment.py

@@ -81,6 +81,34 @@ def iter_path_entries():
         ),
         advanced=True,
     )
+    include_requirements = BoolOption(
+        default=True,
+        help=softwrap(
+            """
+            Set globally whether 3rd party requirements should be included into PEX files.
+
+            This may be helpful if you want to skip attempting to install any requirement. 
+            For instance, you may disable including requirements when your builds rely 
+            on having system packages installed in your build environment accessible on 
+            a certain path or if your dependencies are not provided via Python packages.
+
+            For example, with this setting disabled, you could do:
+
+            ```
+            $ PEX_EXTRA_SYS_PATH="/usr/lib/python3.11/site-packages:/usr/lib64/python3.11/site-packages" pants test ::
+            ```
+
+            and any requirements discovered with the dependency inference will not be resolved 
+            and placed into the PEX executables that would be packaged before running the relevant 
+            build steps. Required dependencies will be searched at those extra system paths you have
+            provided during the Pants goal execution. You would need to make sure the extra environment 
+            variables are going to be accessible to your build targets by setting the `extra_env_vars` 
+            argument in their declarations (or using the `__defaults__` symbol to apply the
+            argument values to targets in the filesystem tree under that BUILD file's directory).           
+            """
+        ),
+        advanced=True,
+    )
     emit_warnings = BoolOption(
         default=False,
         help=softwrap(

src/python/pants/backend/python/util_rules/pex_from_targets.py

@@ -30,6 +30,7 @@
     PexRequest,
 )
 from pants.backend.python.util_rules.pex import rules as pex_rules
+from pants.backend.python.util_rules.pex_environment import PexSubsystem
 from pants.backend.python.util_rules.pex_requirements import (
     EntireLockfile,
     LoadedLockfile,
@@ -782,8 +783,10 @@ def __init__(
 @rule
 async def generalize_requirements_pex_request(
     request: RequirementsPexRequest,
+    pex_subsystem: PexSubsystem,
 ) -> PexFromTargetsRequest:
     return PexFromTargetsRequest(
+        include_requirements=pex_subsystem.include_requirements,
         addresses=sorted(request.addresses),
         output_filename="requirements.pex",
         internal_only=True,