Support de-vendoring for installs.

Now that pex vendors setuptools and wheel, we build / install sdists with
potentially fragile import semantics for these two distributions. In
particular, we trip up against this when trying to build cryptography.

Robustify installs by de-vendoring setuptools and wheel when used in an
install context. In order to support de-vendoring make the method of
import conditional on an environment variable instead of attempting to
re-write vendored imports at runtime.

Fixes pex-tool#661

pex/commands/bdist_pex.py

@@ -5,6 +5,7 @@
 
 import os
 import shlex
+import subprocess
 import sys
 from distutils import log
 from distutils.core import Command
@@ -122,7 +123,7 @@ def run(self):
         log.info('Writing environment pex into %s' % target)
 
       log.debug('Building pex via: {}'.format(' '.join(cmd)))
-      process = Executor.open_process(cmd, env=env)
+      process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
       _, stderr = process.communicate()
       result = process.returncode
       if result != 0:

pex/installer.py

@@ -36,19 +36,13 @@ def __init__(self, source_dir, interpreter=None, install_dir=None):
     """Create an installer from an unpacked source distribution in source_dir."""
     self._source_dir = source_dir
     self._install_tmp = install_dir or safe_mkdtemp()
+    self._interpreter = interpreter or PythonInterpreter.get()
     self._installed = None
 
-    from pex import vendor
-    self._interpreter = vendor.setup_interpreter(distributions=self.mixins,
-                                                 interpreter=interpreter or PythonInterpreter.get())
-    if not self._interpreter.satisfies(self.mixins):
-      raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % (
-          self._interpreter.binary, self.__class__.__name__))
-
   @property
   def mixins(self):
     """Return a list of requirements to load into the setup script prior to invocation."""
-    raise NotImplementedError()
+    return ['setuptools']
 
   @property
   def install_tmp(self):
@@ -59,34 +53,41 @@ def _setup_command(self):
     raise NotImplementedError
 
   @property
-  def bootstrap_script(self):
+  def setup_py_wrapper(self):
+    # NB: It would be more direct to just over-write setup.py by pre-pending the setuptools import.
+    # We cannot do this however because we would then run afoul of setup.py file in the wild with
+    # from __future__ imports. This mode of injecting the import works around that issue.
     return """
-import sys
-sys.path.insert(0, {root!r})
-
-# Expose vendored mixin path_items (setuptools, wheel, etc.) directly to the package's setup.py.
-from pex import third_party
-third_party.install(root={root!r}, expose={mixins!r})
+# We need to allow setuptools to monkeypatch distutils in case the underlying setup.py uses
+# distutils; otherwise, we won't have access to distutils commands installed vi the
+# `distutils.commands` `entrypoints` setup metadata (which is only supported by setuptools).
+# The prime example here is `bdist_wheel` offered by the wheel dist.
+import setuptools
 
 # Now execute the package's setup.py such that it sees itself as a setup.py executed via
 # `python setup.py ...`
+import sys
 __file__ = 'setup.py'
 sys.argv[0] = __file__
 with open(__file__, 'rb') as fp:
   exec(fp.read())
-""".format(root=third_party.isolated(), mixins=self.mixins)
+"""
 
   def run(self):
     if self._installed is not None:
       return self._installed
 
     with TRACER.timed('Installing %s' % self._install_tmp, V=2):
-      command = [self._interpreter.binary, '-sE', '-'] + self._setup_command()
+      env = self._interpreter.sanitized_environment()
+      env['PYTHONPATH'] = os.pathsep.join(third_party.expose(self.mixins))
+      env['__PEX_UNVENDORED__'] = '1'
+
+      command = [self._interpreter.binary, '-s', '-'] + self._setup_command()
       try:
         Executor.execute(command,
-                         env=self._interpreter.sanitized_environment(),
+                         env=env,
                          cwd=self._source_dir,
-                         stdin_payload=self.bootstrap_script.encode('ascii'))
+                         stdin_payload=self.setup_py_wrapper.encode('ascii'))
         self._installed = True
       except Executor.NonZeroExit as e:
         self._installed = False
@@ -102,10 +103,7 @@ def cleanup(self):
 
 
 class DistributionPackager(InstallerBase):
-  @property
-  def mixins(self):
-    return ['setuptools']
-
+  @after_installation
   def find_distribution(self):
     dists = os.listdir(self.install_tmp)
     if len(dists) == 0:
@@ -125,7 +123,6 @@ def _setup_command(self):
     else:
       return ['sdist', '--formats=gztar', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def sdist(self):
     return self.find_distribution()
 
@@ -136,7 +133,6 @@ class EggInstaller(DistributionPackager):
   def _setup_command(self):
     return ['bdist_egg', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def bdist(self):
     return self.find_distribution()
 
@@ -151,6 +147,5 @@ def mixins(self):
   def _setup_command(self):
     return ['bdist_wheel', '--dist-dir=%s' % self._install_tmp]
 
-  @after_installation
   def bdist(self):
     return self.find_distribution()

pex/third_party/__init__.py

@@ -208,20 +208,35 @@ def install_vendored(cls, prefix, root=None, expose=None):
 
     if expose:
       # But only expose the bits needed.
-      path_by_key = OrderedDict((spec.key, spec.relpath) for spec in vendor.iter_vendor_specs()
-                                if spec.key in expose)
-      path_by_key['pex'] = root  # The pex distribution itself is trivially available to expose.
-
-      unexposed = set(expose) - set(path_by_key.keys())
-      if unexposed:
-        raise ValueError('The following vendored dists are not available to expose: {}'
-                         .format(', '.join(sorted(unexposed))))
-
-      exposed_paths = path_by_key.values()
-      for exposed_path in exposed_paths:
-        sys.path.insert(0, os.path.join(root, exposed_path))
+      exposed_paths = []
+      for path in cls.expose(expose, root):
+        sys.path.insert(0, path)
+        exposed_paths.append(os.path.relpath(path, root))
+
       vendor_importer._expose(exposed_paths)
 
+  @classmethod
+  def expose(cls, dists, root=None):
+    from pex import vendor
+
+    root = cls._abs_root(root)
+
+    def iter_available():
+      yield 'pex', root  # The pex distribution itself is trivially available to expose.
+      for spec in vendor.iter_vendor_specs():
+        yield spec.key, spec.relpath
+
+    path_by_key = OrderedDict((key, relpath) for key, relpath in iter_available() if key in dists)
+
+    unexposed = set(dists) - set(path_by_key.keys())
+    if unexposed:
+      raise ValueError('The following vendored dists are not available to expose: {}'
+                       .format(', '.join(sorted(unexposed))))
+
+    exposed_paths = path_by_key.values()
+    for exposed_path in exposed_paths:
+      yield os.path.join(root, exposed_path)
+
   @classmethod
   def install(cls, uninstallable, prefix, path_items, root=None, warning=None):
     """Install an importer for modules found under ``path_items`` at the given import ``prefix``.
@@ -396,5 +411,13 @@ def install(root=None, expose=None):
   VendorImporter.install_vendored(prefix=import_prefix(), root=root, expose=expose)
 
 
+def expose(dists):
+  from pex.common import safe_delete
+
+  for path in VendorImporter.expose(dists, root=isolated()):
+    safe_delete(os.path.join(path, '__init__.py'))
+    yield path
+
+
 # Implicitly install an importer for vendored code on the first import of pex.third_party.
 install()

pex/vendor/__init__.py

@@ -143,7 +143,11 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names):
             pkg_file = os.path.join(pkg_path, '__init__.py')
             src = os.path.join(VendorSpec.ROOT, pkg_file)
             dest = os.path.join(dest_basedir, pkg_file)
-            chroot.copy(src, dest, label)
+            if os.path.exists(src):
+              chroot.copy(src, dest, label)
+            else:
+              # We delete `pex/vendor/_vendored/<dist>/__init__.py` when isolating third_party.
+              chroot.touch(dest, label)
           for name in vendored_names:
             vendor_module_names[name] = True
             TRACER.log('Vendoring {} from {} @ {}'.format(name, spec, spec.target_dir), V=3)

pex/vendor/__main__.py

@@ -61,6 +61,16 @@ def _find_literal_node(statement, call_argument):
     elif isinstance(call_argument.value, LiteralyEvaluable):
       return call_argument.value
 
+  @staticmethod
+  def _modify_import(original, modified):
+    indent = ' ' * (modified.absolute_bounding_box.top_left.column - 1)
+    return os.linesep.join(indent + line for line in (
+      'if "__PEX_UNVENDORED__" in __import__("os").environ:',
+      '  {}  # vendor:skip'.format(original),
+      'else:',
+      '  {}'.format(modified),
+    ))
+
   @classmethod
   def for_path_items(cls, prefix, path_items):
     pkg_names = frozenset(pkg_name for _, pkg_name, _ in pkgutil.iter_modules(path=path_items))
@@ -98,10 +108,13 @@ def _modify__import__calls(self, red_baron):  # noqa: We want __import__ as part
           root_package = value.split('.')[0]
           if root_package in self._packages:
             raw_value.replace('{!r}'.format(self._prefix + '.' + value))
+
+            parent.replace(self._modify_import(original, parent))
             yield original, parent
 
   def _modify_import_statements(self, red_baron):
     for import_node in red_baron.find_all('ImportNode'):
+      modified = False
       if self._skip(import_node):
         continue
 
@@ -129,6 +142,7 @@ def _modify_import_statements(self, red_baron):
         # imported). This ensures the code can traverse from the re-named root - `a` in this
         # example, through middle nodes (`a.b`) all the way to the leaf target (`a.b.c`).
 
+        modified = True
         def prefixed_fullname():
           return '{prefix}.{module}'.format(prefix=self._prefix,
                                             module='.'.join(map(str, import_module)))
@@ -144,6 +158,8 @@ def prefixed_fullname():
                                                          root=root_package.value)
           import_module.target = root_package.value
 
+      if modified:
+        import_node.replace(self._modify_import(original, import_node))
         yield original, import_node
 
   def _modify_from_import_statements(self, red_baron):
@@ -161,6 +177,8 @@ def _modify_from_import_statements(self, red_baron):
       if root_package.value in self._packages:
         root_package.replace('{prefix}.{root}'.format(prefix=self._prefix,
                                                       root=root_package.value))
+
+        from_import_node.replace(self._modify_import(original, from_import_node))
         yield original, from_import_node
 
 

tests/test_bdist_pex.py

@@ -1,41 +1,15 @@
 # Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-
 import os
 import stat
 import subprocess
+import sys
+from contextlib import contextmanager
 from textwrap import dedent
 
-import pex.third_party.pkg_resources as pkg_resources
 from pex.common import open_zip
-from pex.installer import DistributionPackager, WheelInstaller, after_installation
-from pex.interpreter import PythonInterpreter
-from pex.testing import temporary_content
-
-
-class BdistPexInstaller(DistributionPackager):
-
-  def __init__(self, source_dir, interpreter, bdist_args=None):
-    self._bdist_args = list(bdist_args) if bdist_args else []
-    super(BdistPexInstaller, self).__init__(source_dir=source_dir, interpreter=interpreter)
-
-  @property
-  def mixins(self):
-    return ['setuptools', 'pex']
-
-  def _setup_command(self):
-    return ['bdist_pex', '--bdist-dir={}'.format(self._install_tmp)] + self._bdist_args
-
-  @after_installation
-  def bdist(self):
-    return self.find_distribution()
-
-
-def bdist_pex_installer(source_dir, bdist_args=None):
-  pex_dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex'))
-  interpreter = PythonInterpreter.get()
-  interpreter = interpreter.with_extra(pex_dist.key, pex_dist.version, pex_dist.location)
-  return BdistPexInstaller(source_dir=source_dir, interpreter=interpreter, bdist_args=bdist_args)
+from pex.installer import WheelInstaller
+from pex.testing import temporary_content, temporary_dir
 
 
 def bdist_pex_setup_py(**kwargs):
@@ -47,6 +21,18 @@ def bdist_pex_setup_py(**kwargs):
   """.format(kwargs=kwargs))
 
 
+@contextmanager
+def bdist_pex(project_dir, bdist_args=None):
+  with temporary_dir() as dist_dir:
+    cmd = [sys.executable, 'setup.py', 'bdist_pex', '--bdist-dir={}'.format(dist_dir)]
+    if bdist_args:
+      cmd.extend(bdist_args)
+    subprocess.check_call(cmd, cwd=project_dir)
+    dists = os.listdir(dist_dir)
+    assert len(dists) == 1
+    yield os.path.join(dist_dir, dists[0])
+
+
 def assert_entry_points(entry_points):
   setup_py = bdist_pex_setup_py(name='my_app',
                                 version='0.0.0',
@@ -59,12 +45,12 @@ def do_something():
     """)
 
   with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir:
-    my_app_pex = bdist_pex_installer(project_dir).bdist()
-    process = subprocess.Popen([my_app_pex], stdout=subprocess.PIPE)
-    stdout, _ = process.communicate()
-    assert '{pex_root}' not in os.listdir(project_dir)
-    assert 0 == process.returncode
-    assert stdout == b'hello world!\n'
+    with bdist_pex(project_dir) as my_app_pex:
+      process = subprocess.Popen([my_app_pex], stdout=subprocess.PIPE)
+      stdout, _ = process.communicate()
+      assert '{pex_root}' not in os.listdir(project_dir)
+      assert 0 == process.returncode
+      assert stdout == b'hello world!\n'
 
 
 def assert_pex_args_shebang(shebang):
@@ -75,9 +61,9 @@ def assert_pex_args_shebang(shebang):
 
   with temporary_content({'setup.py': setup_py}) as project_dir:
     pex_args = '--pex-args=--python-shebang="{}"'.format(shebang)
-    my_app_pex = bdist_pex_installer(project_dir, bdist_args=[pex_args]).bdist()
-    with open(my_app_pex, 'rb') as fp:
-      assert fp.readline().decode().rstrip() == shebang
+    with bdist_pex(project_dir, bdist_args=[pex_args]) as my_app_pex:
+      with open(my_app_pex, 'rb') as fp:
+        assert fp.readline().decode().rstrip() == shebang
 
 
 def test_entry_points_dict():
@@ -126,13 +112,12 @@ def test_unwriteable_contents():
                                               install_requires=['my_app'])
     with temporary_content({'setup.py': uses_my_app_setup_py}) as uses_my_app_project_dir:
       pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl))
-      uses_my_app_pex = bdist_pex_installer(uses_my_app_project_dir, bdist_args=[pex_args]).bdist()
-
-      with open_zip(uses_my_app_pex) as zf:
-        unwriteable_sos = [path for path in zf.namelist()
-                           if path.endswith('my_app/unwriteable.so')]
-        assert 1 == len(unwriteable_sos)
-        unwriteable_so = unwriteable_sos.pop()
-        zf.extract(unwriteable_so, path=uses_my_app_project_dir)
-        extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so)
-        assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode)
+      with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex:
+        with open_zip(uses_my_app_pex) as zf:
+          unwriteable_sos = [path for path in zf.namelist()
+                             if path.endswith('my_app/unwriteable.so')]
+          assert 1 == len(unwriteable_sos)
+          unwriteable_so = unwriteable_sos.pop()
+          zf.extract(unwriteable_so, path=uses_my_app_project_dir)
+          extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so)
+          assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode)