From d1df852016f0011941a0fc0d13d9a82cc35ae856 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 6 Feb 2019 20:03:51 -0700 Subject: [PATCH] Support de-vendoring for installs. Now that pex vendors setuptools and wheel, we build / install sdists with potentially fragile import semantics for these two distributions. In particular, we trip up against this when trying to build cryptography. Robustify installs by de-vendoring setuptools and wheel when used in an install context. In order to support de-vendoring make the method of import conditional on an environment variable instead of attempting to re-write vendored imports at runtime. Fixes #661 --- pex/commands/bdist_pex.py | 3 +- pex/installer.py | 47 ++++++++++----------- pex/third_party/__init__.py | 47 +++++++++++++++------ pex/vendor/__init__.py | 6 ++- pex/vendor/__main__.py | 18 ++++++++ tests/test_bdist_pex.py | 83 +++++++++++++++---------------------- 6 files changed, 115 insertions(+), 89 deletions(-) diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index c5eee2d61..14565749d 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -5,6 +5,7 @@ import os import shlex +import subprocess import sys from distutils import log from distutils.core import Command @@ -122,7 +123,7 @@ def run(self): log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) - process = Executor.open_process(cmd, env=env) + process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: diff --git a/pex/installer.py b/pex/installer.py index 3ae8e98e5..907d8b83d 100644 --- a/pex/installer.py +++ b/pex/installer.py @@ -36,19 +36,13 @@ def __init__(self, source_dir, interpreter=None, install_dir=None): """Create an installer from an unpacked source distribution in source_dir.""" self._source_dir = source_dir self._install_tmp = install_dir or safe_mkdtemp() + self._interpreter = interpreter or PythonInterpreter.get() self._installed = None - from pex import vendor - self._interpreter = vendor.setup_interpreter(distributions=self.mixins, - interpreter=interpreter or PythonInterpreter.get()) - if not self._interpreter.satisfies(self.mixins): - raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % ( - self._interpreter.binary, self.__class__.__name__)) - @property def mixins(self): """Return a list of requirements to load into the setup script prior to invocation.""" - raise NotImplementedError() + return ['setuptools'] @property def install_tmp(self): @@ -59,34 +53,41 @@ def _setup_command(self): raise NotImplementedError @property - def bootstrap_script(self): + def setup_py_wrapper(self): + # NB: It would be more direct to just over-write setup.py by pre-pending the setuptools import. + # We cannot do this however because we would then run afoul of setup.py file in the wild with + # from __future__ imports. This mode of injecting the import works around that issue. return """ -import sys -sys.path.insert(0, {root!r}) - -# Expose vendored mixin path_items (setuptools, wheel, etc.) directly to the package's setup.py. -from pex import third_party -third_party.install(root={root!r}, expose={mixins!r}) +# We need to allow setuptools to monkeypatch distutils in case the underlying setup.py uses +# distutils; otherwise, we won't have access to distutils commands installed vi the +# `distutils.commands` `entrypoints` setup metadata (which is only supported by setuptools). +# The prime example here is `bdist_wheel` offered by the wheel dist. +import setuptools # Now execute the package's setup.py such that it sees itself as a setup.py executed via # `python setup.py ...` +import sys __file__ = 'setup.py' sys.argv[0] = __file__ with open(__file__, 'rb') as fp: exec(fp.read()) -""".format(root=third_party.isolated(), mixins=self.mixins) +""" def run(self): if self._installed is not None: return self._installed with TRACER.timed('Installing %s' % self._install_tmp, V=2): - command = [self._interpreter.binary, '-sE', '-'] + self._setup_command() + env = self._interpreter.sanitized_environment() + env['PYTHONPATH'] = os.pathsep.join(third_party.expose(self.mixins)) + env['__PEX_UNVENDORED__'] = '1' + + command = [self._interpreter.binary, '-s', '-'] + self._setup_command() try: Executor.execute(command, - env=self._interpreter.sanitized_environment(), + env=env, cwd=self._source_dir, - stdin_payload=self.bootstrap_script.encode('ascii')) + stdin_payload=self.setup_py_wrapper.encode('ascii')) self._installed = True except Executor.NonZeroExit as e: self._installed = False @@ -102,10 +103,7 @@ def cleanup(self): class DistributionPackager(InstallerBase): - @property - def mixins(self): - return ['setuptools'] - + @after_installation def find_distribution(self): dists = os.listdir(self.install_tmp) if len(dists) == 0: @@ -125,7 +123,6 @@ def _setup_command(self): else: return ['sdist', '--formats=gztar', '--dist-dir=%s' % self._install_tmp] - @after_installation def sdist(self): return self.find_distribution() @@ -136,7 +133,6 @@ class EggInstaller(DistributionPackager): def _setup_command(self): return ['bdist_egg', '--dist-dir=%s' % self._install_tmp] - @after_installation def bdist(self): return self.find_distribution() @@ -151,6 +147,5 @@ def mixins(self): def _setup_command(self): return ['bdist_wheel', '--dist-dir=%s' % self._install_tmp] - @after_installation def bdist(self): return self.find_distribution() diff --git a/pex/third_party/__init__.py b/pex/third_party/__init__.py index e845b13a8..c7f39efd8 100644 --- a/pex/third_party/__init__.py +++ b/pex/third_party/__init__.py @@ -208,20 +208,35 @@ def install_vendored(cls, prefix, root=None, expose=None): if expose: # But only expose the bits needed. - path_by_key = OrderedDict((spec.key, spec.relpath) for spec in vendor.iter_vendor_specs() - if spec.key in expose) - path_by_key['pex'] = root # The pex distribution itself is trivially available to expose. - - unexposed = set(expose) - set(path_by_key.keys()) - if unexposed: - raise ValueError('The following vendored dists are not available to expose: {}' - .format(', '.join(sorted(unexposed)))) - - exposed_paths = path_by_key.values() - for exposed_path in exposed_paths: - sys.path.insert(0, os.path.join(root, exposed_path)) + exposed_paths = [] + for path in cls.expose(expose, root): + sys.path.insert(0, path) + exposed_paths.append(os.path.relpath(path, root)) + vendor_importer._expose(exposed_paths) + @classmethod + def expose(cls, dists, root=None): + from pex import vendor + + root = cls._abs_root(root) + + def iter_available(): + yield 'pex', root # The pex distribution itself is trivially available to expose. + for spec in vendor.iter_vendor_specs(): + yield spec.key, spec.relpath + + path_by_key = OrderedDict((key, relpath) for key, relpath in iter_available() if key in dists) + + unexposed = set(dists) - set(path_by_key.keys()) + if unexposed: + raise ValueError('The following vendored dists are not available to expose: {}' + .format(', '.join(sorted(unexposed)))) + + exposed_paths = path_by_key.values() + for exposed_path in exposed_paths: + yield os.path.join(root, exposed_path) + @classmethod def install(cls, uninstallable, prefix, path_items, root=None, warning=None): """Install an importer for modules found under ``path_items`` at the given import ``prefix``. @@ -396,5 +411,13 @@ def install(root=None, expose=None): VendorImporter.install_vendored(prefix=import_prefix(), root=root, expose=expose) +def expose(dists): + from pex.common import safe_delete + + for path in VendorImporter.expose(dists, root=isolated()): + safe_delete(os.path.join(path, '__init__.py')) + yield path + + # Implicitly install an importer for vendored code on the first import of pex.third_party. install() diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py index c80f6a58b..d331e1d16 100644 --- a/pex/vendor/__init__.py +++ b/pex/vendor/__init__.py @@ -143,7 +143,11 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names): pkg_file = os.path.join(pkg_path, '__init__.py') src = os.path.join(VendorSpec.ROOT, pkg_file) dest = os.path.join(dest_basedir, pkg_file) - chroot.copy(src, dest, label) + if os.path.exists(src): + chroot.copy(src, dest, label) + else: + # We delete `pex/vendor/_vendored//__init__.py` when isolating third_party. + chroot.touch(dest, label) for name in vendored_names: vendor_module_names[name] = True TRACER.log('Vendoring {} from {} @ {}'.format(name, spec, spec.target_dir), V=3) diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py index 1bda990a4..b06f8b680 100644 --- a/pex/vendor/__main__.py +++ b/pex/vendor/__main__.py @@ -61,6 +61,16 @@ def _find_literal_node(statement, call_argument): elif isinstance(call_argument.value, LiteralyEvaluable): return call_argument.value + @staticmethod + def _modify_import(original, modified): + indent = ' ' * (modified.absolute_bounding_box.top_left.column - 1) + return os.linesep.join(indent + line for line in ( + 'if "__PEX_UNVENDORED__" in __import__("os").environ:', + ' {} # vendor:skip'.format(original), + 'else:', + ' {}'.format(modified), + )) + @classmethod def for_path_items(cls, prefix, path_items): pkg_names = frozenset(pkg_name for _, pkg_name, _ in pkgutil.iter_modules(path=path_items)) @@ -98,10 +108,13 @@ def _modify__import__calls(self, red_baron): # noqa: We want __import__ as part root_package = value.split('.')[0] if root_package in self._packages: raw_value.replace('{!r}'.format(self._prefix + '.' + value)) + + parent.replace(self._modify_import(original, parent)) yield original, parent def _modify_import_statements(self, red_baron): for import_node in red_baron.find_all('ImportNode'): + modified = False if self._skip(import_node): continue @@ -129,6 +142,7 @@ def _modify_import_statements(self, red_baron): # imported). This ensures the code can traverse from the re-named root - `a` in this # example, through middle nodes (`a.b`) all the way to the leaf target (`a.b.c`). + modified = True def prefixed_fullname(): return '{prefix}.{module}'.format(prefix=self._prefix, module='.'.join(map(str, import_module))) @@ -144,6 +158,8 @@ def prefixed_fullname(): root=root_package.value) import_module.target = root_package.value + if modified: + import_node.replace(self._modify_import(original, import_node)) yield original, import_node def _modify_from_import_statements(self, red_baron): @@ -161,6 +177,8 @@ def _modify_from_import_statements(self, red_baron): if root_package.value in self._packages: root_package.replace('{prefix}.{root}'.format(prefix=self._prefix, root=root_package.value)) + + from_import_node.replace(self._modify_import(original, from_import_node)) yield original, from_import_node diff --git a/tests/test_bdist_pex.py b/tests/test_bdist_pex.py index e7449715a..35ede3bec 100644 --- a/tests/test_bdist_pex.py +++ b/tests/test_bdist_pex.py @@ -1,41 +1,15 @@ # Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - import os import stat import subprocess +import sys +from contextlib import contextmanager from textwrap import dedent -import pex.third_party.pkg_resources as pkg_resources from pex.common import open_zip -from pex.installer import DistributionPackager, WheelInstaller, after_installation -from pex.interpreter import PythonInterpreter -from pex.testing import temporary_content - - -class BdistPexInstaller(DistributionPackager): - - def __init__(self, source_dir, interpreter, bdist_args=None): - self._bdist_args = list(bdist_args) if bdist_args else [] - super(BdistPexInstaller, self).__init__(source_dir=source_dir, interpreter=interpreter) - - @property - def mixins(self): - return ['setuptools', 'pex'] - - def _setup_command(self): - return ['bdist_pex', '--bdist-dir={}'.format(self._install_tmp)] + self._bdist_args - - @after_installation - def bdist(self): - return self.find_distribution() - - -def bdist_pex_installer(source_dir, bdist_args=None): - pex_dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex')) - interpreter = PythonInterpreter.get() - interpreter = interpreter.with_extra(pex_dist.key, pex_dist.version, pex_dist.location) - return BdistPexInstaller(source_dir=source_dir, interpreter=interpreter, bdist_args=bdist_args) +from pex.installer import WheelInstaller +from pex.testing import temporary_content, temporary_dir def bdist_pex_setup_py(**kwargs): @@ -47,6 +21,18 @@ def bdist_pex_setup_py(**kwargs): """.format(kwargs=kwargs)) +@contextmanager +def bdist_pex(project_dir, bdist_args=None): + with temporary_dir() as dist_dir: + cmd = [sys.executable, 'setup.py', 'bdist_pex', '--bdist-dir={}'.format(dist_dir)] + if bdist_args: + cmd.extend(bdist_args) + subprocess.check_call(cmd, cwd=project_dir) + dists = os.listdir(dist_dir) + assert len(dists) == 1 + yield os.path.join(dist_dir, dists[0]) + + def assert_entry_points(entry_points): setup_py = bdist_pex_setup_py(name='my_app', version='0.0.0', @@ -59,12 +45,12 @@ def do_something(): """) with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir: - my_app_pex = bdist_pex_installer(project_dir).bdist() - process = subprocess.Popen([my_app_pex], stdout=subprocess.PIPE) - stdout, _ = process.communicate() - assert '{pex_root}' not in os.listdir(project_dir) - assert 0 == process.returncode - assert stdout == b'hello world!\n' + with bdist_pex(project_dir) as my_app_pex: + process = subprocess.Popen([my_app_pex], stdout=subprocess.PIPE) + stdout, _ = process.communicate() + assert '{pex_root}' not in os.listdir(project_dir) + assert 0 == process.returncode + assert stdout == b'hello world!\n' def assert_pex_args_shebang(shebang): @@ -75,9 +61,9 @@ def assert_pex_args_shebang(shebang): with temporary_content({'setup.py': setup_py}) as project_dir: pex_args = '--pex-args=--python-shebang="{}"'.format(shebang) - my_app_pex = bdist_pex_installer(project_dir, bdist_args=[pex_args]).bdist() - with open(my_app_pex, 'rb') as fp: - assert fp.readline().decode().rstrip() == shebang + with bdist_pex(project_dir, bdist_args=[pex_args]) as my_app_pex: + with open(my_app_pex, 'rb') as fp: + assert fp.readline().decode().rstrip() == shebang def test_entry_points_dict(): @@ -126,13 +112,12 @@ def test_unwriteable_contents(): install_requires=['my_app']) with temporary_content({'setup.py': uses_my_app_setup_py}) as uses_my_app_project_dir: pex_args = '--pex-args=--disable-cache --no-pypi -f {}'.format(os.path.dirname(my_app_whl)) - uses_my_app_pex = bdist_pex_installer(uses_my_app_project_dir, bdist_args=[pex_args]).bdist() - - with open_zip(uses_my_app_pex) as zf: - unwriteable_sos = [path for path in zf.namelist() - if path.endswith('my_app/unwriteable.so')] - assert 1 == len(unwriteable_sos) - unwriteable_so = unwriteable_sos.pop() - zf.extract(unwriteable_so, path=uses_my_app_project_dir) - extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so) - assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode) + with bdist_pex(uses_my_app_project_dir, bdist_args=[pex_args]) as uses_my_app_pex: + with open_zip(uses_my_app_pex) as zf: + unwriteable_sos = [path for path in zf.namelist() + if path.endswith('my_app/unwriteable.so')] + assert 1 == len(unwriteable_sos) + unwriteable_so = unwriteable_sos.pop() + zf.extract(unwriteable_so, path=uses_my_app_project_dir) + extract_dest = os.path.join(uses_my_app_project_dir, unwriteable_so) + assert UNWRITEABLE_PERMS == stat.S_IMODE(os.stat(extract_dest).st_mode)