Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Issue #2140: Build wheels automatically
Building wheels before installing elminates a cause of broken environments -
where install fails after we've already installed one or more packages.

If a package fails to wheel, we run setup.py install as normally.
  • Loading branch information
rbtcollins committed Apr 13, 2015
1 parent 4926409 commit 08acb66
Show file tree
Hide file tree
Showing 10 changed files with 199 additions and 27 deletions.
3 changes: 3 additions & 0 deletions CHANGES.txt
Expand Up @@ -22,6 +22,9 @@
* Support ``--install-option`` and ``--global-option`` per requirement in
requirement files (:pull:`2537`)

* Build Wheels prior to installing from sdist, caching them in the pip cache
directory to speed up subsequent installs. (:pull:`2618`)

**6.1.1 (2015-04-07)**

* No longer ignore dependencies which have been added to the standard library,
Expand Down
16 changes: 16 additions & 0 deletions docs/reference/pip_install.rst
Expand Up @@ -16,6 +16,15 @@ Description
.. pip-command-description:: install


Overview
++++++++

Pip install has several stages:

1. Resolve dependencies. What will be installed is determined here.
2. Build wheels. All the dependencies that can be are built into wheels.
3. Install the packages (and uninstall anything being upgraded/replaced).

Installation Order
++++++++++++++++++

Expand Down Expand Up @@ -404,6 +413,9 @@ that pip will not attempt to build a better wheel for Python's that would have
supported it, once any generic wheel is built. To correct this, make sure that
the wheel's are built with Python specific tags - e.g. pp on Pypy.

When no wheels are found for an sdist, pip will attempt to build a wheel
automatically and insert it into the wheel cache.


Hash Verification
+++++++++++++++++
Expand Down Expand Up @@ -499,6 +511,10 @@ implement the following command::
This should implement the complete process of installing the package in
"editable" mode.

All packages will be attempted to built into wheels::

setup.py bdist_wheel -d XXX

One further ``setup.py`` command is invoked by ``pip install``::

setup.py clean
Expand Down
25 changes: 22 additions & 3 deletions pip/commands/install.py
Expand Up @@ -6,10 +6,14 @@
import tempfile
import shutil
import warnings
try:
import wheel
except ImportError:
wheel = None

from pip.req import RequirementSet
from pip.locations import virtualenv_no_global, distutils_scheme
from pip.basecommand import RequirementCommand
from pip.locations import virtualenv_no_global, distutils_scheme
from pip.index import PackageFinder
from pip.exceptions import (
InstallationError, CommandError, PreviousBuildDirError,
Expand All @@ -18,6 +22,7 @@
from pip.utils import ensure_dir
from pip.utils.build import BuildDirectory
from pip.utils.deprecation import RemovedInPip8Warning
from pip.wheel import WheelBuilder


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -233,7 +238,6 @@ def run(self, options, args):
with self._build_session(options) as session:

finder = self._build_package_finder(options, index_urls, session)

build_delete = (not (options.no_clean or options.build_dir))
with BuildDirectory(options.build_dir,
delete=build_delete) as build_dir:
Expand Down Expand Up @@ -262,7 +266,22 @@ def run(self, options, args):
return

try:
requirement_set.prepare_files(finder)
if options.download_dir or not wheel:
# on -d don't do complex things like building
# wheels, and don't try to build wheels when wheel is
# not installed.
requirement_set.prepare_files(finder)
else:
# build wheels before install.
wb = WheelBuilder(
requirement_set,
finder,
build_options=[],
global_options=[],
)
# Ignore the result: a failed wheel will be
# installed from the sdist/vcs whatever.
wb.build(autobuilding=True)

if not options.download_dir:
requirement_set.install(
Expand Down
3 changes: 2 additions & 1 deletion pip/req/req_set.py
Expand Up @@ -162,7 +162,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
self.build_dir = build_dir
self.src_dir = src_dir
# XXX: download_dir and wheel_download_dir overlap semantically and may
# be combinable.
# be combined if we're willing to have non-wheel archives present in
# the wheelhouse output by 'pip wheel'.
self.download_dir = download_dir
self.upgrade = upgrade
self.ignore_installed = ignore_installed
Expand Down
111 changes: 92 additions & 19 deletions pip/wheel.py
Expand Up @@ -14,6 +14,7 @@
import shutil
import stat
import sys
import tempfile
import warnings

from base64 import urlsafe_b64encode
Expand All @@ -22,12 +23,13 @@
from pip._vendor.six import StringIO

import pip
from pip.download import path_to_url
from pip.download import path_to_url, unpack_url
from pip.exceptions import InvalidWheelFilename, UnsupportedWheel
from pip.locations import distutils_scheme
from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME
from pip import pep425tags
from pip.utils import (
call_subprocess, ensure_dir, make_path_relative, captured_stdout)
call_subprocess, ensure_dir, make_path_relative, captured_stdout,
rmtree)
from pip.utils.logging import indent_log
from pip._vendor.distlib.scripts import ScriptMaker
from pip._vendor import pkg_resources
Expand All @@ -49,6 +51,12 @@ def _cache_for_filename(cache_dir, sdistfilename):
to cache them in, and then consult that directory when looking up
cache hits.
We only insert things into the cache if they have plausible version
numbers, so that we don't contaminate the cache with things that were not
unique. E.g. ./package might have dozens of installs done for it and build
a version of 0.0...and if we built and cached a wheel, we'd end up using
the same wheel even if the source has been edited.
:param cache_dir: The cache_dir being used by pip.
:param sdistfilename: The filename of the sdist for which this will cache
wheels.
Expand Down Expand Up @@ -590,16 +598,35 @@ class WheelBuilder(object):
"""Build wheels from a RequirementSet."""

def __init__(self, requirement_set, finder, build_options=None,
global_options=None):
global_options=None, cache_root=None):
self.requirement_set = requirement_set
self.finder = finder
self.wheel_dir = requirement_set.wheel_download_dir
self._cache_root = requirement_set._cache_root
self._wheel_dir = requirement_set.wheel_download_dir
self.build_options = build_options or []
self.global_options = global_options or []

def _build_one(self, req):
"""Build one wheel."""
def _build_one(self, req, output_dir):
"""Build one wheel.
:return: The filename of the built wheel, or None if the build failed.
"""
tempd = tempfile.mkdtemp('pip-wheel-')
try:
if self.__build_one(req, tempd):
try:
wheel_name = os.listdir(tempd)[0]
wheel_path = os.path.join(output_dir, wheel_name)
os.rename(os.path.join(tempd, wheel_name), wheel_path)
logger.info('Stored in directory: %s', output_dir)
return wheel_path
except:
return None
return None
finally:
rmtree(tempd)

def __build_one(self, req, tempd):
base_args = [
sys.executable, '-c',
"import setuptools;__file__=%r;"
Expand All @@ -608,8 +635,8 @@ def _build_one(self, req):
] + list(self.global_options)

logger.info('Running setup.py bdist_wheel for %s', req.name)
logger.info('Destination directory: %s', self.wheel_dir)
wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \
logger.debug('Destination directory: %s', tempd)
wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
+ self.build_options
try:
call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False)
Expand All @@ -618,25 +645,40 @@ def _build_one(self, req):
logger.error('Failed building wheel for %s', req.name)
return False

def build(self):
"""Build wheels."""
def build(self, autobuilding=False):
"""Build wheels.
# unpack and constructs req set
:param unpack: If True, replace the sdist we built from the with the

This comment has been minimized.

Copy link
@akaihola

akaihola Nov 25, 2015

Contributor

Parameter name unpack in the docstring doesn't match autobuilding in the function signature.

newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
assert self._wheel_dir or (autobuilding and self._cache_root)
# unpack sdists and constructs req set
self.requirement_set.prepare_files(self.finder)

reqset = self.requirement_set.requirements.values()

buildset = []
for req in reqset:
if req.is_wheel:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name)
elif req.editable:
logger.info(
'Skipping %s, due to being editable', req.name,
)
if not autobuilding:
logger.info(
'Skipping bdist_wheel for %s, due to being editable',
req.name)
elif autobuilding and not req.source_dir:
pass
else:
if autobuilding:
link = req.link
base, ext = link.splitext()
if pip.index.egg_info_matches(base, None, link) is None:
# Doesn't look like a package - don't autobuild a wheel
# because we'll have no way to lookup the result sanely
continue
buildset.append(req)

if not buildset:
Expand All @@ -650,8 +692,39 @@ def build(self):
with indent_log():
build_success, build_failure = [], []
for req in buildset:
if self._build_one(req):
if autobuilding:
output_dir = _cache_for_filename(
self._cache_root, req.link.filename)
ensure_dir(output_dir)
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(req, output_dir)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.requirement_set.build_dir)
# Update the link for this.
req.link = pip.index.Link(
path_to_url(wheel_file), trusted=True)
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=self.requirement_set.session)
else:
build_failure.append(req)

Expand Down
7 changes: 6 additions & 1 deletion tests/conftest.py
Expand Up @@ -4,6 +4,8 @@
import py
import pytest

from pip.utils import appdirs

from tests.lib import SRC_DIR, TestData
from tests.lib.path import Path
from tests.lib.scripttest import PipTestEnvironment
Expand Down Expand Up @@ -119,7 +121,7 @@ def isolate(tmpdir):


@pytest.fixture
def virtualenv(tmpdir, monkeypatch):
def virtualenv(tmpdir, monkeypatch, isolate):
"""
Return a virtual environment which is unique to each test function
invocation created inside of a sub directory of the test function's
Expand Down Expand Up @@ -148,6 +150,9 @@ def virtualenv(tmpdir, monkeypatch):
pip_source_dir=pip_src,
)

# Clean out our cache: creating the venv injects wheels into it.
shutil.rmtree(appdirs.user_cache_dir("pip"))

# Undo our monkeypatching of shutil
monkeypatch.undo()

Expand Down
5 changes: 5 additions & 0 deletions tests/data/packages/README.txt
Expand Up @@ -103,3 +103,8 @@ Is an empty package which install_requires the simple and simple2 packages.
requires_simple_extra-0.1-py2.py3-none-any.whl
----------------------------------------------
requires_simple_extra[extra] requires simple==1.0

requires_wheelbroken_upper
--------------------------
Requires wheelbroken and upper - used for testing implicit wheel building
during install.
Empty file.
5 changes: 5 additions & 0 deletions tests/data/packages/requires_wheelbroken_upper/setup.py
@@ -0,0 +1,5 @@
import setuptools
setuptools.setup(
name="requires_wheelbroken_upper",
version="0",
install_requires=['wheelbroken', 'upper'])

0 comments on commit 08acb66

Please sign in to comment.