use a pex to satisfy setup_requires for python_dist() targets

examples/src/python/example/python_distribution/hello/pants_setup_requires/BUILD

@@ -1,10 +1,18 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+main_file = 'main.py'
+
 python_dist(
-  sources=globs('*.py'),
+  sources=rglobs('*.py', exclude=[main_file]),
   setup_requires=[
     # FIXME: this currently fails because setup_requires doesn't fetch transitive deps!
     'testprojects/pants-plugins/3rdparty/python/pants',
   ],
 )
+
+python_binary(
+  name='bin',
+  sources=[main_file],
+  dependencies=[':pants_setup_requires'],
+)

examples/src/python/example/python_distribution/hello/pants_setup_requires/main.py

@@ -0,0 +1,13 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import pkg_resources
+
+
+hello_module_version = pkg_resources.get_distribution('hello_again').version
+
+if __name__ == '__main__':
+  print(hello_module_version)

examples/src/python/example/python_distribution/hello/pants_setup_requires/setup.py

@@ -4,13 +4,12 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import os
 from setuptools import setup, find_packages
 
 from pants.version import VERSION as pants_version
 
 setup(
-  name='hello',
+  name='hello_again',
   # FIXME: test the wheel version in a unit test!
   version=pants_version,
   packages=find_packages(),

src/python/pants/backend/native/subsystems/conan.py

@@ -5,20 +5,9 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
-import os
-
-from pex.interpreter import PythonInterpreter
-from pex.pex import PEX
-from pex.pex_builder import PEXBuilder
-from pex.pex_info import PexInfo
 
 from pants.backend.python.python_requirement import PythonRequirement
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
-from pants.backend.python.tasks.pex_build_util import dump_requirements
-from pants.base.build_environment import get_pants_cachedir
 from pants.binaries.executable_pex_tool import ExecutablePexTool
-from pants.util.dirutil import safe_concurrent_creation
 from pants.util.memo import memoized_property
 
 
@@ -49,20 +38,16 @@ class Conan(ExecutablePexTool):
     'deprecation>=2.0, <2.1'
   )
 
-  @classmethod
-  def implementation_version(cls):
-    return super(Conan, cls).implementation_version() + [('Conan', 0)]
-
-  @classmethod
-  def subsystem_dependencies(cls):
-    return super(Conan, cls).subsystem_dependencies() + (PythonRepos, PythonSetup)
-
   @classmethod
   def register_options(cls, register):
     super(Conan, cls).register_options(register)
     register('--conan-requirements', type=list, default=cls.default_conan_requirements,
              advanced=True, help='The requirements used to build the conan client pex.')
 
+  @classmethod
+  def implementation_version(cls):
+    return super(Conan, cls).implementation_version() + [('Conan', 0)]
+
   @memoized_property
-  def pex_tool_requirements(self):
+  def base_requirements(self):
     return [PythonRequirement(req) for req in self.get_options().conan_requirements]

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -4,21 +4,21 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import os
 from builtins import str
 from collections import defaultdict
 
-from wheel.install import WheelFile
+from pex.pex import PEX
 
 from pants.backend.native.config.environment import CppToolchain, CToolchain, Platform
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.subsystems.xcode_cli_tools import MIN_OSX_VERSION_ARG
 from pants.backend.native.targets.native_library import NativeLibrary
+from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_distribution import PythonDistribution
-from pants.backend.python.tasks.pex_build_util import resolve_multi
 from pants.base.exceptions import IncompatiblePlatformsError
+from pants.binaries.executable_pex_tool import ExecutablePexTool
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
 from pants.util.objects import SubclassesOf, datatype
@@ -138,6 +138,25 @@ def check_build_for_current_platform_only(self, targets):
       .format(str(platforms_with_sources)))
 
 
+class BuildSetupRequiresPex(ExecutablePexTool):
+  options_scope = 'build-setup-requires-pex'
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup.scoped(cls),)
+
+  @property
+  def base_requirements(self):
+    return [
+      PythonRequirement('setuptools'),
+      PythonRequirement('wheel'),
+    ]
+
+  @memoized_property
+  def _python_setup(self):
+    return PythonSetup.scoped_instance(self)
+
+
 class SetupPyNativeTools(datatype([
     ('c_toolchain', CToolchain),
     ('cpp_toolchain', CppToolchain),
@@ -149,44 +168,16 @@ class SetupPyNativeTools(datatype([
   """
 
 
-class SetupRequiresSiteDir(datatype(['site_dir'])): pass
-
-
-# TODO: This could be formulated as an @rule if targets and `PythonInterpreter` are made available
-# to the v2 engine.
-def ensure_setup_requires_site_dir(reqs_to_resolve, interpreter, site_dir,
-                                   platforms=None):
-  if not reqs_to_resolve:
-    return None
-
-  setup_requires_dists = resolve_multi(interpreter, reqs_to_resolve, platforms, None)
-
-  # FIXME: there's no description of what this does or why it's necessary.
-  overrides = {
-    'purelib': site_dir,
-    'headers': os.path.join(site_dir, 'headers'),
-    'scripts': os.path.join(site_dir, 'bin'),
-    'platlib': site_dir,
-    'data': site_dir
-  }
-
-  # The `python_dist` target builds for the current platform only.
-  # FIXME: why does it build for the current platform only?
-  for obj in setup_requires_dists['current']:
-    wf = WheelFile(obj.location)
-    wf.install(overrides=overrides, force=True)
-
-  return SetupRequiresSiteDir(site_dir)
-
-
 # TODO: It might be pretty useful to have an Optional TypeConstraint.
 class SetupPyExecutionEnvironment(datatype([
     # If None, don't set PYTHONPATH in the setup.py environment.
-    'setup_requires_site_dir',
+    ('setup_requires_pex', PEX),
     # If None, don't execute in the toolchain environment.
     'setup_py_native_tools',
 ])):
 
+
+
   _SHARED_CMDLINE_ARGS = {
     'darwin': lambda: [
       MIN_OSX_VERSION_ARG,
@@ -200,9 +191,6 @@ class SetupPyExecutionEnvironment(datatype([
   def as_environment(self):
     ret = {}
 
-    if self.setup_requires_site_dir:
-      ret['PYTHONPATH'] = self.setup_requires_site_dir.site_dir
-
     # FIXME(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose
     # executables more hygienically. We should probably be composing each datatype's members, and
     # only creating an environment at the very end.

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -16,10 +16,10 @@
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.native.tasks.link_shared_libraries import SharedLibrary
 from pants.backend.python.python_requirement import PythonRequirement
-from pants.backend.python.subsystems.python_native_code import (PythonNativeCode,
+from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex,
+                                                                PythonNativeCode,
                                                                 SetupPyExecutionEnvironment,
-                                                                SetupPyNativeTools,
-                                                                ensure_setup_requires_site_dir)
+                                                                SetupPyNativeTools)
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.tasks.pex_build_util import is_local_python_dist
 from pants.backend.python.tasks.setup_py import SetupPyRunner
@@ -44,6 +44,8 @@ class BuildLocalPythonDistributions(Task):
   # This will contain the sources used to build the python_dist().
   _DIST_SOURCE_SUBDIR = 'python_dist_subdir'
 
+  setup_requires_pex_filename = 'setup-requires.pex'
+
   # This defines the output directory when building the dist, so we know where the output wheel is
   # located. It is a subdirectory of `_DIST_SOURCE_SUBDIR`.
   _DIST_OUTPUT_DIR = 'dist'
@@ -67,6 +69,7 @@ def implementation_version(cls):
   @classmethod
   def subsystem_dependencies(cls):
     return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (
+      BuildSetupRequiresPex.scoped(cls),
       PythonNativeCode.scoped(cls),
     )
 
@@ -80,6 +83,10 @@ def _platform(cls):
   def _python_native_code_settings(self):
     return PythonNativeCode.scoped_instance(self)
 
+  @memoized_property
+  def _build_setup_requires_pex_settings(self):
+    return BuildSetupRequiresPex.scoped_instance(self)
+
   # FIXME(#5869): delete this and get Subsystems from options, when that is possible.
   def _request_single(self, product, subject):
     # NB: This is not supposed to be exposed to Tasks yet -- see #4769 to track the status of
@@ -222,25 +229,27 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t
       # We are including a platform-specific shared lib in this dist, so mark it as such.
       is_platform_specific = True
 
+    versioned_target_fingerprint = versioned_target.cache_key.hash
+
     setup_requires_dir = os.path.join(results_dir, self._SETUP_REQUIRES_SITE_SUBDIR)
     setup_reqs_to_resolve = self._get_setup_requires_to_resolve(dist_target)
     if setup_reqs_to_resolve:
       self.context.log.debug('python_dist target(s) with setup_requires detected. '
                              'Installing setup requirements: {}\n\n'
                              .format([req.key for req in setup_reqs_to_resolve]))
 
-    setup_requires_site_dir = ensure_setup_requires_site_dir(
-      setup_reqs_to_resolve, interpreter, setup_requires_dir, platforms=['current'])
-    if setup_requires_site_dir:
-      self.context.log.debug('Setting PYTHONPATH with setup_requires site directory: {}'
-                             .format(setup_requires_site_dir))
+    setup_reqs_pex_path = os.path.join(
+      setup_requires_dir,
+      'setup-requires-{}.pex'.format(versioned_target_fingerprint))
+    setup_requires_pex = self._build_setup_requires_pex_settings.bootstrap(
+      interpreter, setup_reqs_pex_path, extra_reqs=setup_reqs_to_resolve)
+    self.context.log.debug('Using pex file as setup.py interpreter: {}'
+                           .format(setup_requires_pex))
 
     setup_py_execution_environment = SetupPyExecutionEnvironment(
-      setup_requires_site_dir=setup_requires_site_dir,
+      setup_requires_pex=setup_requires_pex,
       setup_py_native_tools=native_tools)
 
-    versioned_target_fingerprint = versioned_target.cache_key.hash
-
     self._create_dist(
       dist_target,
       dist_output_dir,
@@ -280,29 +289,36 @@ def _create_dist(self, dist_tgt, dist_target_dir, interpreter,
     setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv(
       snapshot_fingerprint, is_platform_specific)
 
+    setup_requires_interpreter = PythonInterpreter(
+      binary=setup_py_execution_environment.setup_requires_pex.path(),
+      identity=interpreter.identity,
+      extras=interpreter.extras)
+
     setup_runner = SetupPyRunner(
       source_dir=dist_target_dir,
       setup_command=setup_py_snapshot_version_argv,
-      interpreter=interpreter)
+      file_input='setup.py',
+      interpreter=setup_requires_interpreter)
 
     setup_py_env = setup_py_execution_environment.as_environment()
     with environment_as(**setup_py_env):
       # Build a whl using SetupPyRunner and return its absolute path.
-      was_installed_successfully = setup_runner.run()
-      # FIXME: Make a run_raising_error() method in SetupPyRunner that doesn't print directly to
-      # stderr like pex does (better: put this in pex itself).
-      if not was_installed_successfully:
+      try:
+        setup_runner.run()
+      except SetupPyRunner.SetupPyRunnerError as e:
         raise self.BuildLocalPythonDistributionsError(
           "Installation of python distribution from target {target} into directory {into_dir} "
           "failed.\n"
           "The chosen interpreter was: {interpreter}.\n"
           "The execution environment was: {env}.\n"
-          "The setup command was: {command}."
+          "The setup command was: {command}.\n{err}"
           .format(target=dist_tgt,
                   into_dir=dist_target_dir,
-                  interpreter=interpreter,
+                  interpreter=setup_requires_interpreter,
                   env=setup_py_env,
-                  command=setup_py_snapshot_version_argv))
+                  command=setup_py_snapshot_version_argv,
+                  err=e),
+          e)
 
   def _inject_synthetic_dist_requirements(self, dist, req_lib_addr):
     """Inject a synthetic requirements library that references a local wheel.

src/python/pants/backend/python/tasks/setup_py.py

@@ -14,8 +14,10 @@
 from collections import OrderedDict, defaultdict
 
 from pex.compatibility import string, to_bytes
+from pex.executor import Executor
 from pex.installer import InstallerBase, Packager
 from pex.interpreter import PythonInterpreter
+from pex.tracer import TRACER
 from twitter.common.collections import OrderedSet
 from twitter.common.dirutil.chroot import Chroot
 
@@ -51,8 +53,11 @@
 class SetupPyRunner(InstallerBase):
   _EXTRAS = ('setuptools', 'wheel')
 
-  def __init__(self, source_dir, setup_command, **kw):
+  class SetupPyRunnerError(Exception): pass
+
+  def __init__(self, source_dir, setup_command, file_input=None, **kw):
     self.__setup_command = setup_command
+    self._file_input = file_input
     super(SetupPyRunner, self).__init__(source_dir, **kw)
 
   def mixins(self):
@@ -74,6 +79,42 @@ def mixins(self):
   def _setup_command(self):
     return self.__setup_command
 
+  def run(self):
+    """???/ripped from the pex installer class
+
+    Used so we can control the command line, and raise an exception on failure.
+    """
+    if self._installed is not None:
+      return self._installed
+
+    if self._file_input is None:
+      stdin_payload = self.bootstrap_script.encode('ascii')
+      init_args = [self._interpreter.binary, '-']
+    else:
+      stdin_payload = None
+      init_args = [self._interpreter.binary, self._file_input, '--']
+
+    full_command = init_args + self._setup_command()
+
+    with TRACER.timed('Installing %s' % self._install_tmp, V=2):
+      try:
+        Executor.execute(full_command,
+                         env=self._interpreter.sanitized_environment(),
+                         cwd=self._source_dir,
+                         stdin_payload=stdin_payload)
+        self._installed = True
+      except Executor.NonZeroExit as e:
+        self._installed = False
+        name = os.path.basename(self._source_dir)
+        raise self.SetupPyRunnerError(
+          '**** Failed to install {name} (caused by: {error}\n):'
+          'stdout:\n{stdout}\nstderr:\n{stderr}\n'
+          .format(name=name,
+                  error=e,
+                  stdout=e.stdout,
+                  stderr=e.stderr))
+        return self._installed
+
 
 class TargetAncestorIterator(object):
   """Supports iteration of target ancestor lineages."""