Ensure that changing platforms invalidates pex binary creation (#6202)

Currently, the platforms used to build pexes are not fingerprinted. This means that running ./pants binary some/python:bin twice with different platform settings will not recreate the pex on the second run. Instead it will retain the platform settings of the first run. This forces clean-alls when changing platform flags.

This patch causes the platforms flag to be fingerprinted and adds its owning subsystem to python binary create's subsystem dependencies.

contrib/python/src/python/pants/contrib/python/checks/tasks/checkstyle/checkstyle.py

@@ -14,7 +14,7 @@
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
-from pants.backend.python.tasks import pex_build_util
+from pants.backend.python.tasks.pex_build_util import PexBuilderWrapper
 from pants.base.build_environment import get_buildroot, pants_version
 from pants.base.deprecated import deprecated_conditional
 from pants.base.exceptions import TaskError
@@ -122,37 +122,36 @@ def checker_pex(self, interpreter):
     if not os.path.exists(pex_path):
       with self.context.new_workunit(name='build-checker'):
         with safe_concurrent_creation(pex_path) as chroot:
-          builder = PEXBuilder(path=chroot, interpreter=interpreter)
+          pex_builder = PexBuilderWrapper(
+            PEXBuilder(path=chroot, interpreter=interpreter),
+            PythonRepos.global_instance(),
+            PythonSetup.global_instance(), self.context.log)
+
           # Constraining is required to guard against the case where the user
           # has a pexrc file set.
-          builder.add_interpreter_constraint(str(interpreter.identity.requirement))
+          pex_builder.add_interpreter_constraint(str(interpreter.identity.requirement))
 
           if pants_dev_mode:
-            pex_build_util.dump_sources(builder, tgt=self.checker_target, log=self.context.log)
+            pex_builder.add_sources_from(self.checker_target)
             req_libs = [tgt for tgt in self.checker_target.closure()
                         if isinstance(tgt, PythonRequirementLibrary)]
-            pex_build_util.dump_requirement_libs(builder,
-                                                 interpreter=interpreter,
-                                                 req_libs=req_libs,
-                                                 log=self.context.log)
+
+            pex_builder.add_requirement_libs_from(req_libs=req_libs)
           else:
             try:
               # The checker is already on sys.path, eg: embedded in pants.pex.
               working_set = WorkingSet(entries=sys.path)
               for dist in working_set.resolve([Requirement.parse(self._CHECKER_REQ)]):
-                for req in dist.requires():
-                  builder.add_requirement(req)
-                builder.add_distribution(dist)
-              builder.add_requirement(self._CHECKER_REQ)
+                pex_builder.add_direct_requirements(dist.requires())
+                pex_builder.add_distribution(dist)
+              pex_builder.add_direct_requirements([self._CHECKER_REQ])
             except DistributionNotFound:
               # We need to resolve the checker from a local or remote distribution repo.
-              pex_build_util.dump_requirements(builder,
-                                               interpreter=interpreter,
-                                               reqs=[PythonRequirement(self._CHECKER_REQ)],
-                                               log=self.context.log)
+              pex_builder.add_resolved_requirements(
+                [PythonRequirement(self._CHECKER_REQ)])
 
-          builder.set_entry_point(self._CHECKER_ENTRYPOINT)
-          builder.freeze()
+          pex_builder.set_entry_point(self._CHECKER_ENTRYPOINT)
+          pex_builder.freeze()
 
     return PEX(pex_path, interpreter=interpreter)
 

contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py

@@ -15,8 +15,8 @@
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_target import PythonTarget
-from pants.backend.python.tasks.pex_build_util import (dump_requirement_libs, dump_sources,
-                                                       has_python_requirements, has_python_sources)
+from pants.backend.python.tasks.pex_build_util import (PexBuilderWrapper, has_python_requirements,
+                                                       has_python_sources)
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
 from pants.base.exceptions import TaskError
 from pants.base.generator import Generator, TemplateData
@@ -217,9 +217,13 @@ def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
     if not os.path.isdir(reqs_pex_path):
       req_libs = [t for t in vt.target.closure() if has_python_requirements(t)]
       with safe_concurrent_creation(reqs_pex_path) as safe_path:
-        builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True)
-        dump_requirement_libs(builder, interpreter, req_libs, self.context.log)
-        builder.freeze()
+        pex_builder = PexBuilderWrapper(
+          PEXBuilder(safe_path, interpreter=interpreter, copy=True),
+          PythonRepos.global_instance(),
+          PythonSetup.global_instance(),
+          self.context.log)
+        pex_builder.add_requirement_libs_from(req_libs)
+        pex_builder.freeze()
     return PEX(reqs_pex_path, interpreter=interpreter)
 
   def _source_pex_for_versioned_target_closure(self, interpreter, vt):
@@ -230,8 +234,12 @@ def _source_pex_for_versioned_target_closure(self, interpreter, vt):
     return PEX(source_pex_path, interpreter=interpreter)
 
   def _build_source_pex(self, interpreter, path, targets):
-    builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
+    pex_builder = PexBuilderWrapper(
+      PEXBuilder(path=path, interpreter=interpreter, copy=True),
+      PythonRepos.global_instance(),
+      PythonSetup.global_instance(),
+      self.context.log)
     for target in targets:
       if has_python_sources(target):
-        dump_sources(builder, target, self.context.log)
-    builder.freeze()
+        pex_builder.add_sources_from(target)
+    pex_builder.freeze()

src/python/pants/backend/python/subsystems/python_setup.py

@@ -31,6 +31,7 @@ def register_options(cls, register):
     register('--wheel-version', advanced=True, default='0.31.1',
              help='The wheel version for this python environment.')
     register('--platforms', advanced=True, type=list, metavar='<platform>', default=['current'],
+             fingerprint=True,
              help='A list of platforms to be supported by this python environment. Each platform'
                   'is a string, as returned by pkg_resources.get_supported_platform().')
     register('--interpreter-cache-dir', advanced=True, default=None, metavar='<dir>',

src/python/pants/backend/python/tasks/gather_sources.py

@@ -11,7 +11,7 @@
 from pex.pex_builder import PEXBuilder
 from twitter.common.collections import OrderedSet
 
-from pants.backend.python.tasks.pex_build_util import (dump_sources, has_python_sources,
+from pants.backend.python.tasks.pex_build_util import (PexBuilderWrapper, has_python_sources,
                                                        has_resources, is_python_target)
 from pants.base.exceptions import TaskError
 from pants.invalidation.cache_manager import VersionedTargetSet
@@ -82,7 +82,12 @@ def _get_pex_for_versioned_targets(self, interpreter, versioned_targets):
     return PEX(source_pex_path, interpreter=interpreter)
 
   def _build_pex(self, interpreter, path, targets):
-    builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
+    pex_builder = PexBuilderWrapper(
+      PEXBuilder(path=path, interpreter=interpreter, copy=True),
+      python_repos_subsystem=None,
+      python_setup_subsystem=None,
+      log=self.context.log)
+
     for target in targets:
-      dump_sources(builder, target, self.context.log)
-    builder.freeze()
+      pex_builder.add_sources_from(target)
+    pex_builder.freeze()

src/python/pants/backend/python/tasks/isort_prep.py

@@ -14,7 +14,7 @@
 from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
-from pants.backend.python.tasks.pex_build_util import dump_requirement_libs
+from pants.backend.python.tasks.pex_build_util import PexBuilderWrapper
 from pants.base.build_environment import get_buildroot
 from pants.base.workunit import WorkUnitLabel
 from pants.build_graph.address import Address
@@ -54,13 +54,14 @@ def create_requirements(cls, context, workdir):
       @classmethod
       def build_isort_pex(cls, context, interpreter, pex_path, requirements_lib):
         with safe_concurrent_creation(pex_path) as chroot:
-          builder = PEXBuilder(path=chroot, interpreter=interpreter)
-          dump_requirement_libs(builder=builder,
-                                interpreter=interpreter,
-                                req_libs=[requirements_lib],
-                                log=context.log)
-          builder.set_script('isort')
-          builder.freeze()
+          pex_builder = PexBuilderWrapper(
+            PEXBuilder(path=chroot, interpreter=interpreter),
+            PythonRepos.global_instance(),
+            PythonSetup.global_instance(),
+            context.log)
+          pex_builder.add_requirement_libs_from(req_libs=[requirements_lib])
+          pex_builder.set_script('isort')
+          pex_builder.freeze()
 
     def __init__(self, pex_path, interpreter=None):
       self._pex = PEX(pex_path, interpreter=interpreter)