Support de-vendoring for installs. (#666)

Now that pex vendors setuptools and wheel, we build / install sdists with
potentially fragile import semantics for these two distributions. In
particular, we trip up against this when trying to build cryptography.

Robustify installs by de-vendoring setuptools and wheel when used in an
install context. In order to support de-vendoring make the method of
import conditional on an environment variable instead of attempting to
re-write vendored imports at runtime.

Fixes #661

.travis.yml

@@ -19,12 +19,30 @@ x-pyenv-shard: &x-pyenv-shard
     "${PYENV}" global ${PYENV_VERSION}
     pip install -U tox "setuptools>=36"
 
+x-py27: &x-py27 PYENV_VERSION=2.7.15
+
+x-py37: &x-py37 PYENV_VERSION=3.7.0
+
+x-pypy: &x-pypy PYENV_VERSION=pypy2.7-6.0.0
+
 x-linux-shard: &x-linux-shard
   <<: *x-pyenv-shard
   os: linux
   sudo: false
   dist: trusty
 
+x-linux-27-shard: &x-linux-27-shard
+  <<: *x-linux-shard
+  env:
+    - *env
+    - *x-py27
+
+x-linux-pypy-shard: &x-linux-pypy-shard
+  <<: *x-linux-shard
+  env:
+    - *env
+    - *x-pypy
+
 # Python 3.7 requires at least OpenSSL 1.0.2:
 #   https://docs.python.org/3/whatsnew/3.7.html#platform-support-removals.
 # Travis' `trusty` image does not get us there and, at the time of writing, the beta `xenial`
@@ -41,12 +59,12 @@ x-linux-37-shard: &x-linux-37-shard
         - openssl
   env:
     - *env
+    - *x-py37
     - OPENSSL_VERSION=1.0.2p
     - OPENSSL_DIR="${HOME}/.pyenv_pex_openssl"
     - LD_LIBRARY_PATH="${OPENSSL_DIR}/lib"
     - SSL_CERT_DIR=/usr/lib/ssl/certs
     - PYTHON_CONFIGURE_OPTS="--with-openssl=${OPENSSL_DIR}"
-    - PYENV_VERSION=3.7.0
     - TOX_TESTENV_PASSENV=SSL_CERT_DIR
   cache:
     <<: *cache
@@ -72,56 +90,55 @@ x-osx-shard: &x-osx-shard
   os: osx
   osx_image: xcode9.4
 
-x-py27: &x-py27
-  - *env
-  - PYENV_VERSION=2.7.15
+x-osx-ssl: &x-osx-ssl >
+  CPPFLAGS=-I/usr/local/opt/openssl/include
+  LDFLAGS=-L/usr/local/opt/openssl/lib
 
-x-py37: &x-py37
-  - *env
-  - PYENV_VERSION=3.7.0
+x-osx-27-shard: &x-osx-27-shard
+  <<: *x-osx-shard
+  env:
+    - *env
+    - *x-py27
+    - *x-osx-ssl
 
-x-pypy: &x-pypy
-  - *env
-  - PYENV_VERSION=pypy2.7-6.0.0
+x-osx-37-shard: &x-osx-37-shard
+  <<: *x-osx-shard
+  env:
+    - *env
+    - *x-py37
+    - *x-osx-ssl
 
 # NB: Travis partitions caches using a combination of os, language amd env vars. As such, we do not
 # use TOXENV and instead pass the toxenv via -e on the command line. This helps ensure we share
 # caches as much as possible (eg: all linux python 2.7.15 shards share a cache).
 matrix:
   include:
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=style
-      env: *x-py27
       script: tox -ve style
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=isort-check
-      env: *x-py27
       script: tox -ve isort-check
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=vendor-check
-      env: *x-py27
       script: tox -ve vendor-check
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=py27
-      env: *x-py27
       script: tox -ve py27
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=py27-subprocess
-      env: *x-py27
       script: tox -ve py27-subprocess
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=py27-requests
-      env: *x-py27
       script: tox -ve py27-requests
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=py27-requests-cachecontrol
-      env: *x-py27
       script: tox -ve py27-requests-cachecontrol
 
     - <<: *x-linux-shard
@@ -157,41 +174,34 @@ matrix:
       name: TOXENV=py37-requests-cachecontrol
       script: tox -ve py37-requests-cachecontrol
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-pypy-shard
       name: TOXENV=pypy
-      env: *x-pypy
       script: tox -ve pypy
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-27-shard
       name: TOXENV=py27-integration
-      env: *x-py27
       script: tox -ve py27-integration
 
     - <<: *x-linux-37-shard
       name: TOXENV=py37-integration
       script: tox -ve py37-integration
 
-    - <<: *x-linux-shard
+    - <<: *x-linux-pypy-shard
       name: TOXENV=pypy-integration
-      env: *x-pypy
       script: tox -ve pypy-integration
 
-    - <<: *x-osx-shard
+    - <<: *x-osx-27-shard
       name: TOXENV=py27-requests
-      env: *x-py27
       script: tox -ve py27-requests
 
-    - <<: *x-osx-shard
+    - <<: *x-osx-37-shard
       name: TOXENV=py37-requests
-      env: *x-py37
       script: tox -ve py37-requests
 
-    - <<: *x-osx-shard
+    - <<: *x-osx-27-shard
       name: TOXENV=py27-integration
-      env: *x-py27
       script: tox -ve py27-integration
 
-    - <<: *x-osx-shard
+    - <<: *x-osx-37-shard
       name: TOXENV=py37-integration
-      env: *x-py37
       script: tox -ve py37-integration

pex/commands/bdist_pex.py

@@ -5,6 +5,7 @@
 
 import os
 import shlex
+import subprocess
 import sys
 from distutils import log
 from distutils.core import Command
@@ -122,8 +123,9 @@ def run(self):
         log.info('Writing environment pex into %s' % target)
 
       log.debug('Building pex via: {}'.format(' '.join(cmd)))
-      process = Executor.open_process(cmd, env=env)
+      process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
       _, stderr = process.communicate()
       result = process.returncode
       if result != 0:
-        die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr), result)
+        die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')),
+            result)

pex/installer.py

@@ -11,6 +11,7 @@
 from pex.compatibility import WINDOWS
 from pex.executor import Executor
 from pex.interpreter import PythonInterpreter
+from pex.orderedset import OrderedSet
 from pex.tracer import TRACER
 
 __all__ = (
@@ -22,12 +23,12 @@ def after_installation(function):
   def function_wrapper(self, *args, **kw):
     self._installed = self.run()
     if not self._installed:
-      raise InstallerBase.InstallFailure('Failed to install %s' % self._source_dir)
+      raise SetuptoolsInstallerBase.InstallFailure('Failed to install %s' % self._source_dir)
     return function(self, *args, **kw)
   return function_wrapper
 
 
-class InstallerBase(object):
+class SetuptoolsInstallerBase(object):
   class Error(Exception): pass
   class InstallFailure(Error): pass
   class IncapableInterpreter(Error): pass
@@ -36,57 +37,59 @@ def __init__(self, source_dir, interpreter=None, install_dir=None):
     """Create an installer from an unpacked source distribution in source_dir."""
     self._source_dir = source_dir
     self._install_tmp = install_dir or safe_mkdtemp()
+    self._interpreter = interpreter or PythonInterpreter.get()
     self._installed = None
 
-    from pex import vendor
-    self._interpreter = vendor.setup_interpreter(distributions=self.mixins,
-                                                 interpreter=interpreter or PythonInterpreter.get())
-    if not self._interpreter.satisfies(self.mixins):
-      raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % (
-          self._interpreter.binary, self.__class__.__name__))
-
   @property
   def mixins(self):
-    """Return a list of requirements to load into the setup script prior to invocation."""
-    raise NotImplementedError()
+    """Return a list of extra distribution names required by the `setup_command`."""
+    return []
 
   @property
   def install_tmp(self):
     return self._install_tmp
 
-  def _setup_command(self):
-    """the setup command-line to run, to be implemented by subclasses."""
+  def setup_command(self):
+    """The setup command-line to run, to be implemented by subclasses."""
     raise NotImplementedError
 
   @property
-  def bootstrap_script(self):
+  def setup_py_wrapper(self):
+    # NB: It would be more direct to just over-write setup.py by pre-pending the setuptools import.
+    # We cannot do this however because we would then run afoul of setup.py files in the wild with
+    # from __future__ imports. This mode of injecting the import works around that issue.
     return """
-import sys
-sys.path.insert(0, {root!r})
-
-# Expose vendored mixin path_items (setuptools, wheel, etc.) directly to the package's setup.py.
-from pex import third_party
-third_party.install(root={root!r}, expose={mixins!r})
+# We need to allow setuptools to monkeypatch distutils in case the underlying setup.py uses
+# distutils; otherwise, we won't have access to distutils commands installed via the
+# `distutils.commands` `entrypoints` setup metadata (which is only supported by setuptools).
+# The prime example here is `bdist_wheel` offered by the wheel dist.
+import setuptools
 
 # Now execute the package's setup.py such that it sees itself as a setup.py executed via
 # `python setup.py ...`
+import sys
 __file__ = 'setup.py'
 sys.argv[0] = __file__
 with open(__file__, 'rb') as fp:
   exec(fp.read())
-""".format(root=third_party.isolated(), mixins=self.mixins)
+"""
 
   def run(self):
     if self._installed is not None:
       return self._installed
 
     with TRACER.timed('Installing %s' % self._install_tmp, V=2):
-      command = [self._interpreter.binary, '-sE', '-'] + self._setup_command()
+      env = self._interpreter.sanitized_environment()
+      mixins = OrderedSet(['setuptools'] + self.mixins)
+      env['PYTHONPATH'] = os.pathsep.join(third_party.expose(mixins))
+      env['__PEX_UNVENDORED__'] = '1'
+
+      command = [self._interpreter.binary, '-s', '-'] + self.setup_command()
       try:
         Executor.execute(command,
-                         env=self._interpreter.sanitized_environment(),
+                         env=env,
                          cwd=self._source_dir,
-                         stdin_payload=self.bootstrap_script.encode('ascii'))
+                         stdin_payload=self.setup_py_wrapper.encode('ascii'))
         self._installed = True
       except Executor.NonZeroExit as e:
         self._installed = False
@@ -101,11 +104,8 @@ def cleanup(self):
     safe_rmtree(self._install_tmp)
 
 
-class DistributionPackager(InstallerBase):
-  @property
-  def mixins(self):
-    return ['setuptools']
-
+class DistributionPackager(SetuptoolsInstallerBase):
+  @after_installation
   def find_distribution(self):
     dists = os.listdir(self.install_tmp)
     if len(dists) == 0:
@@ -119,24 +119,22 @@ def find_distribution(self):
 class Packager(DistributionPackager):
   """Create a source distribution from an unpacked setup.py-based project."""
 
-  def _setup_command(self):
+  def setup_command(self):
     if WINDOWS:
       return ['sdist', '--formats=zip', '--dist-dir=%s' % self._install_tmp]
     else:
       return ['sdist', '--formats=gztar', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def sdist(self):
     return self.find_distribution()
 
 
 class EggInstaller(DistributionPackager):
   """Create an egg distribution from an unpacked setup.py-based project."""
 
-  def _setup_command(self):
+  def setup_command(self):
     return ['bdist_egg', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def bdist(self):
     return self.find_distribution()
 
@@ -146,11 +144,10 @@ class WheelInstaller(DistributionPackager):
 
   @property
   def mixins(self):
-    return ['setuptools', 'wheel']
+    return ['wheel']
 
-  def _setup_command(self):
+  def setup_command(self):
     return ['bdist_wheel', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def bdist(self):
     return self.find_distribution()