From f26da27db8cbe565860d432eebfea430f40a7dfb Mon Sep 17 00:00:00 2001 From: "tdos.apone" Date: Sun, 2 Jul 2017 18:02:27 -0500 Subject: [PATCH] Merge pull request #18 from tgarc/master revert PR9; add --no-deps to pip wheel Revert PR9; it was never actually needed in the first place it just fixed an issue by side effect. 1) add --no-deps to pip wheel step in order to avoid building wheels for dependencies. This also addresses the issues raised in PR #11. 2) Modify the pip install step across platforms to allow installation of dependencies from the index. (This is required since we will no longer be building wheels for dependencies). --- cibuildwheel/__main__.py | 3 +++ cibuildwheel/linux.py | 36 +++++++++++++++++++++--------------- cibuildwheel/macos.py | 39 +++++++++++++++++++++++---------------- cibuildwheel/windows.py | 26 +++++++++++++++++--------- 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 21fdc73df..cf15f57ca 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -107,6 +107,9 @@ def main(): print_preamble(platform, build_options) + if not os.path.exists(output_dir): + os.mkdir(output_dir) + if platform == 'linux': cibuildwheel.linux.build(**build_options) elif platform == 'windows': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 8b7147ffc..af6415aa3 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -53,30 +53,33 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be cd /project for PYBIN in {pybin_paths}; do + # Setup + rm -rf /tmp/built_wheel + rm -rf /tmp/delocated_wheel + mkdir /tmp/built_wheel + mkdir /tmp/delocated_wheel + if [ ! -z {before_build} ]; then PATH=$PYBIN:$PATH sh -c {before_build} fi - # install the package first to take care of dependencies - "$PYBIN/pip" install . - - "$PYBIN/pip" wheel --no-deps . -w /tmp/linux_wheels - done + # Build that wheel + "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps + built_wheel=(/tmp/built_wheel/*.whl) - for whl in /tmp/linux_wheels/*.whl; do - if [[ "$whl" == *none-any.whl ]]; then - # pure python wheel - just copy to the output - cp "$whl" /output + # Delocate the wheel + # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns + # the first element + if [[ "$built_wheel" == *none-any.whl ]]; then + # pure python wheel - just copy + mv "$built_wheel" /tmp/delocated_wheel else - auditwheel repair "$whl" -w /output + auditwheel repair "$built_wheel" -w /tmp/delocated_wheel fi - done + delocated_wheel=(/tmp/delocated_wheel/*.whl) - # Install packages and test - for PYBIN in {pybin_paths}; do # Install the wheel we just built - "$PYBIN/pip" install {package_name} \ - --upgrade --force-reinstall --no-deps --no-index -f /output + "$PYBIN/pip" install "$delocated_wheel" # Install any requirements to run the tests if [ ! -z "{test_requires}" ]; then @@ -89,6 +92,9 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be PATH=$PYBIN:$PATH sh -c {test_command} popd fi + + # we're all done here; move it to output + mv "$delocated_wheel" /output done '''.format( package_name=package_name, diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 4296191c5..d98b9731b 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,5 +1,5 @@ from __future__ import print_function -import os, subprocess, shlex, sys +import os, subprocess, shlex, sys, shutil from collections import namedtuple from glob import glob try: @@ -53,31 +53,35 @@ def shell(args, env=None, cwd=None): shell([pip, 'install', 'wheel'], env=env) shell([pip, 'install', 'delocate'], env=env) + # setup dirs + if os.path.exists('/tmp/built_wheel'): + shutil.rmtree('/tmp/built_wheel') + os.makedirs('/tmp/built_wheel') + if os.path.exists('/tmp/delocated_wheel'): + shutil.rmtree('/tmp/delocated_wheel') + os.makedirs('/tmp/delocated_wheel') + # run the before_build command if before_build: before_build_prepared = prepare_command(before_build, python=python, pip=pip) shell(shlex.split(before_build_prepared), env=env) - # install the package first to take care of dependencies - shell([pip, 'install', project_dir], env=env) - - # build the wheel to temp dir - temp_wheel_dir = '/tmp/tmpwheel%s' % config.version - shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir, '--no-deps'], env=env) - temp_wheel = glob(temp_wheel_dir+'/*.whl')[0] + # build the wheel + shell([pip, 'wheel', project_dir, '-w', '/tmp/built_wheel', '--no-deps'], env=env) + built_wheel = glob('/tmp/built_wheel/*.whl')[0] - if temp_wheel.endswith('none-any.whl'): - # pure python wheel - just copy to output_dir - shell(['cp', temp_wheel, output_dir], env=env) + if built_wheel.endswith('none-any.whl'): + # pure python wheel - just move + shutil.move(built_wheel, '/tmp/delocated_wheel') else: # list the dependencies - shell(['delocate-listdeps', temp_wheel], env=env) + shell(['delocate-listdeps', built_wheel], env=env) # rebuild the wheel with shared libraries included and place in output dir - shell(['delocate-wheel', '-w', output_dir, temp_wheel], env=env) + shell(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env) + delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] - # now install the package from the generated wheel - shell([pip, 'install', package_name, '--upgrade', '--force-reinstall', - '--no-deps', '--no-index', '--find-links', output_dir], env=env) + # install the wheel + shell([pip, 'install', delocated_wheel], env=env) # test the wheel if test_requires: @@ -89,3 +93,6 @@ def shell(args, env=None, cwd=None): abs_project_dir = os.path.abspath(project_dir) test_command_absolute = test_command.format(project=abs_project_dir) shell(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env) + + # we're all done here; move it to output + shutil.move(delocated_wheel, output_dir) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index ed6454d53..32a17853f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,10 +1,11 @@ from __future__ import print_function -import os, tempfile, subprocess, sys +import os, tempfile, subprocess, sys, shutil try: from urllib2 import urlopen except ImportError: from urllib.request import urlopen from collections import namedtuple +from glob import glob from .util import prepare_command @@ -21,7 +22,7 @@ def shell(args, env=None, cwd=None): # print the command executing for the logs print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args - return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + return subprocess.check_output(' '.join(args), env=env, cwd=cwd) PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) python_configurations = [ @@ -37,11 +38,19 @@ def shell(args, env=None, cwd=None): PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'), ] + temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') + built_wheel_dir = os.path.join(temp_dir, 'built_wheel') + for config in python_configurations: if skip(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue + # setup dirs + if os.path.exists(built_wheel_dir): + shutil.rmtree(built_wheel_dir) + os.makedirs(built_wheel_dir) + env = os.environ.copy() # set up environment variables for run_with_env env['PYTHON_VERSION'] = config.version @@ -66,16 +75,12 @@ def shell(args, env=None, cwd=None): before_build_prepared = prepare_command(before_build, python='python', pip='pip') shell([before_build_prepared], env=env) - # install the package first to take care of dependencies - shell(['pip', 'install', project_dir], env=env) - # build the wheel - shell(['pip', 'wheel', project_dir, '-w', output_dir, '--no-deps'], env=env) + shell(['pip', 'wheel', project_dir, '-w', built_wheel_dir, '--no-deps'], env=env) + built_wheel = glob(built_wheel_dir+'/*.whl')[0] # install the wheel - shell(['pip', 'install', package_name, '--upgrade', - '--force-reinstall', '--no-deps', '--no-index', '-f', - output_dir], env=env) + shell(['pip', 'install', built_wheel], env=env) # test the wheel if test_requires: @@ -87,3 +92,6 @@ def shell(args, env=None, cwd=None): abs_project_dir = os.path.abspath(project_dir) test_command_absolute = test_command.format(project=abs_project_dir) shell([test_command_absolute], cwd='c:\\', env=env) + + # we're all done here; move it to output + shutil.move(built_wheel, output_dir)