From 62ad71030177631df9750fef9e9edc0ff442171d Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 1 Apr 2015 14:39:45 +1300 Subject: [PATCH] Issue #2140: Build wheels automatically Building wheels before installing elminates a cause of broken environments - where install fails after we've already installed one or more packages. If a package fails to wheel, we run setup.py install as normally. --- CHANGES.txt | 2 + docs/reference/pip_install.rst | 13 ++++ pip/commands/install.py | 32 +++++++- pip/req/req_set.py | 13 +++- pip/wheel.py | 77 +++++++++++++++---- tests/conftest.py | 7 +- tests/data/packages/README.txt | 5 +- .../requires_wheelbroken_upper/__init__.py | 0 .../requires_wheelbroken_upper/setup.py | 5 ++ tests/functional/test_install.py | 23 ++++++ 10 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py create mode 100644 tests/data/packages/requires_wheelbroken_upper/setup.py diff --git a/CHANGES.txt b/CHANGES.txt index 51ee9eb3413..87e66e26df2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,8 @@ * Ignores bz2 archives if Python wasn't compiled with bz2 support. Fixes :issue:`497` +* Build Wheels prior to installing from sdist, caching them in the pip cache + directory to speed up subsequent installs. (:pull:`2618`) **6.1.1 (2015-04-07)** diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 726d04ebb82..19744280b7f 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -16,6 +16,15 @@ Description .. pip-command-description:: install +Overview +++++++++ + +Pip install has several stages: + +1. Resolve dependencies. What will be installed is determined here. +2. Build wheels. All the dependencies that can be are built into wheels. +3. Install the packages (and uninstall anything being upgraded/replaced). + Installation Order ++++++++++++++++++ @@ -452,6 +461,10 @@ implement the following command:: This should implement the complete process of installing the package in "editable" mode. +All packages will be attempted to built into wheels:: + + setup.py bdist_wheel -d XXX + One further ``setup.py`` command is invoked by ``pip install``:: setup.py clean diff --git a/pip/commands/install.py b/pip/commands/install.py index 53707e4a5c1..f524a40a4d8 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -6,10 +6,15 @@ import tempfile import shutil import warnings +try: + import wheel +except ImportError: + wheel = None from pip.req import RequirementSet -from pip.locations import virtualenv_no_global, distutils_scheme from pip.basecommand import RequirementCommand +from pip.locations import ( + virtualenv_no_global, distutils_scheme, WHEEL_CACHE_DIR) from pip.index import PackageFinder from pip.exceptions import ( InstallationError, CommandError, PreviousBuildDirError, @@ -18,6 +23,7 @@ from pip.utils import ensure_dir from pip.utils.build import BuildDirectory from pip.utils.deprecation import RemovedInPip8Warning +from pip.wheel import WheelBuilder logger = logging.getLogger(__name__) @@ -234,16 +240,20 @@ def run(self, options, args): with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) - build_delete = (not (options.no_clean or options.build_dir)) with BuildDirectory(options.build_dir, delete=build_delete) as build_dir: + if not options.cache_dir or options.download_dir: + wheel_download_dir = None + else: + wheel_download_dir = WHEEL_CACHE_DIR() requirement_set = RequirementSet( build_dir=build_dir, src_dir=options.src_dir, download_dir=options.download_dir, upgrade=options.upgrade, as_egg=options.as_egg, + installing_wheels=wheel_download_dir is not None, ignore_installed=options.ignore_installed, ignore_dependencies=options.ignore_dependencies, force_reinstall=options.force_reinstall, @@ -252,6 +262,7 @@ def run(self, options, args): session=session, pycompile=options.compile, isolated=options.isolated_mode, + wheel_download_dir=wheel_download_dir, ) self.populate_requirement_set( @@ -262,7 +273,22 @@ def run(self, options, args): return try: - requirement_set.prepare_files(finder) + if options.download_dir or not wheel: + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + requirement_set.prepare_files(finder) + else: + # build wheels before install. + wb = WheelBuilder( + requirement_set, + finder, + build_options=[], + global_options=[], + ) + # Ignore the result: a failed wheel will be + # installed from the sdist/vcs whatever. + wb.build(autobuilding=True) if not options.download_dir: requirement_set.install( diff --git a/pip/req/req_set.py b/pip/req/req_set.py index f2ba4f2355d..0c8a1906ae7 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -139,7 +139,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, ignore_installed=False, as_egg=False, target_dir=None, ignore_dependencies=False, force_reinstall=False, use_user_site=False, session=None, pycompile=True, - isolated=False, wheel_download_dir=None): + isolated=False, wheel_download_dir=None, + installing_wheels=False): """Create a RequirementSet. :param wheel_download_dir: Where still-packed .whl files should be @@ -149,6 +150,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, :param download_dir: Where still packed archives should be written to. If None they are not saved, and are deleted immediately after unpacking. + :param installing_wheels: If True, wheels will be getting installed and + should not be marked for pip deletion. """ if session is None: raise TypeError( @@ -159,10 +162,12 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, self.build_dir = build_dir self.src_dir = src_dir # XXX: download_dir and wheel_download_dir overlap semantically and may - # be combinable. + # be combined if we're willing to have non-wheel archives present in + # the wheelhouse output by 'pip wheel'. self.download_dir = download_dir self.upgrade = upgrade self.ignore_installed = ignore_installed + self.installing_wheels = installing_wheels self.force_reinstall = force_reinstall self.requirements = Requirements() # Mapping of alias: real_name @@ -446,11 +451,13 @@ def _prepare_file(self, finder, req_to_install): self.wheel_download_dir: # when doing 'pip wheel` download_dir = self.wheel_download_dir + only_download = not self.installing_wheels else: download_dir = self.download_dir + only_download = True unpack_url( req_to_install.link, req_to_install.source_dir, - download_dir, True, session=self.session, + download_dir, only_download, session=self.session, ) except requests.HTTPError as exc: logger.critical( diff --git a/pip/wheel.py b/pip/wheel.py index d689903fc57..77dcd7fe1ff 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -13,6 +13,7 @@ import shutil import stat import sys +import tempfile import warnings from base64 import urlsafe_b64encode @@ -20,11 +21,14 @@ from pip._vendor.six import StringIO +import pip +from pip.download import path_to_url, unpack_url from pip.exceptions import InvalidWheelFilename, UnsupportedWheel -from pip.locations import distutils_scheme +from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME from pip import pep425tags from pip.utils import ( - call_subprocess, ensure_dir, make_path_relative, captured_stdout) + call_subprocess, ensure_dir, make_path_relative, captured_stdout, + rmtree) from pip.utils.logging import indent_log from pip._vendor.distlib.scripts import ScriptMaker from pip._vendor import pkg_resources @@ -550,8 +554,25 @@ def __init__(self, requirement_set, finder, build_options=None, self.global_options = global_options or [] def _build_one(self, req): - """Build one wheel.""" + """Build one wheel. + :return: The filename of the built wheel, or None if the build failed. + """ + tempd = tempfile.mkdtemp('pip-wheel-') + try: + if self.__build_one(req, tempd): + try: + wheel_name = os.listdir(tempd)[0] + wheel_path = os.path.join(self.wheel_dir, wheel_name) + os.rename(os.path.join(tempd, wheel_name), wheel_path) + return wheel_path + except: + return None + return None + finally: + rmtree(tempd) + + def __build_one(self, req, tempd): base_args = [ sys.executable, '-c', "import setuptools;__file__=%r;" @@ -561,7 +582,7 @@ def _build_one(self, req): logger.info('Running setup.py bdist_wheel for %s', req.name) logger.info('Destination directory: %s', self.wheel_dir) - wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \ + wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ + self.build_options try: call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False) @@ -570,10 +591,14 @@ def _build_one(self, req): logger.error('Failed building wheel for %s', req.name) return False - def build(self): - """Build wheels.""" + def build(self, autobuilding=True): + """Build wheels. - # unpack and constructs req set + :param unpack: If True, replace the sdist we built from the with the + newly built wheel, in preparation for installation. + :return: True if all the wheels built correctly. + """ + # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() @@ -581,13 +606,15 @@ def build(self): buildset = [] for req in reqset: if req.is_wheel: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) + if autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name) elif req.editable: logger.info( - 'Skipping %s, due to being editable', req.name, - ) + 'Skipping bdist_wheel for %s, due to being editable', + req.name) + elif autobuilding and not req.source_dir: + pass else: buildset.append(req) @@ -602,8 +629,32 @@ def build(self): with indent_log(): build_success, build_failure = [], [] for req in buildset: - if self._build_one(req): + wheel_file = self._build_one(req) + if wheel_file: build_success.append(req) + if autobuilding: + # XXX: This is mildly duplicative with prepare_files, + # but not close enough to pull out to a single common + # method. + # The code below assumes temporary source dirs - + # prevent it doing bad things. + if req.source_dir and not os.path.exists(os.path.join( + req.source_dir, PIP_DELETE_MARKER_FILENAME)): + raise Exception("bad source dir - missing marker") + # Delete the source we built the wheel from + req.remove_temporary_source() + # set the build directory again - name is known from + # the work prepare_files did. + req.source_dir = req.build_location( + self.requirement_set.build_dir) + # Update the link for this. + req.link = pip.index.Link( + path_to_url(wheel_file), trusted=True) + assert req.link.is_wheel + # extract the wheel into the dir + unpack_url( + req.link, req.source_dir, None, False, + session=self.requirement_set.session) else: build_failure.append(req) diff --git a/tests/conftest.py b/tests/conftest.py index 84019dd06bc..1b1bb542307 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import py import pytest +from pip import locations + from tests.lib import SRC_DIR, TestData from tests.lib.path import Path from tests.lib.scripttest import PipTestEnvironment @@ -119,7 +121,7 @@ def isolate(tmpdir): @pytest.fixture -def virtualenv(tmpdir, monkeypatch): +def virtualenv(tmpdir, monkeypatch, isolate): """ Return a virtual environment which is unique to each test function invocation created inside of a sub directory of the test function's @@ -148,6 +150,9 @@ def virtualenv(tmpdir, monkeypatch): pip_source_dir=pip_src, ) + # Clean out our cache. + shutil.rmtree(locations.WHEEL_CACHE_DIR()) + # Undo our monkeypatching of shutil monkeypatch.undo() diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 9e89e911439..7fd5cd3a7fc 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -104,4 +104,7 @@ requires_simple_extra-0.1-py2.py3-none-any.whl ---------------------------------------------- requires_simple_extra[extra] requires simple==1.0 - +requires_wheelbroken_upper +-------------------------- +Requires wheelbroken and upper - used for testing implicit wheel building +during install. diff --git a/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py b/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/requires_wheelbroken_upper/setup.py b/tests/data/packages/requires_wheelbroken_upper/setup.py new file mode 100644 index 00000000000..255cf2219eb --- /dev/null +++ b/tests/data/packages/requires_wheelbroken_upper/setup.py @@ -0,0 +1,5 @@ +import setuptools +setuptools.setup( + name="requires_wheelbroken_upper", + version="0", + install_requires=['wheelbroken', 'upper']) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index a294d598fcd..bb0f70fa048 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,6 +7,7 @@ import pytest +from pip.locations import WHEEL_CACHE_DIR from pip.utils import rmtree from tests.lib import (pyversion, pyversion_tuple, _create_test_package, _create_svn_repo, path_to_url) @@ -669,3 +670,25 @@ def test_install_wheel_broken(script, data): res = script.pip( 'install', '--no-index', '-f', data.find_links, 'wheelbroken') assert "Successfully installed wheelbroken-0.1" in str(res), str(res) + + +def test_install_builds_wheels(script, data): + # NB This incidentally tests a local tree + tarball inputs + # primary coverage for vcs and editables is from those dedicated tests. + script.pip('install', 'wheel') + to_install = data.packages.join('requires_wheelbroken_upper') + res = script.pip( + 'install', '--no-index', '-f', data.find_links, + to_install) + expected = ("Successfully installed requires-wheelbroken-upper-0" + " upper-2.0 wheelbroken-0.1") + # Must have installed it all + assert expected in str(res), str(res) + wheels = os.listdir(WHEEL_CACHE_DIR()) + # and built wheels into the cache + assert wheels != [], str(res) + # and installed from the wheels + assert "Running setup.py install for upper" not in str(res), str(res) + assert "Running setup.py install for requires" not in str(res), str(res) + # wheelbroken has to run install + assert "Running setup.py install for wheelb" in str(res), str(res)