diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83ee1f9d2..2ae8ce957 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,18 @@ jobs: run: | python bin/sample_build.py + - name: Get some sample wheels + run: | + python -m test.test_projects test.test_0_basic.basic_project sample_proj + cibuildwheel --output-dir wheelhouse sample_proj + env: + CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + + - uses: actions/upload-artifact@v2 + with: + name: sample_wheels + path: wheelhouse + - name: Test cibuildwheel run: | python ./bin/run_tests.py diff --git a/bin/run_tests.py b/bin/run_tests.py index b46bfbe6f..9e8e9075b 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=2400', 'test']) diff --git a/bin/sample_build.py b/bin/sample_build.py index 3389f535a..8fad4ed8c 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -22,4 +22,4 @@ options.project_python_path, project_dir ], check=True) - sys.exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) + sys.exit(subprocess.run([sys.executable, '-m', 'cibuildwheel'], cwd=project_dir).returncode) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8f65f28b4..3b7e2af75 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -137,7 +137,7 @@ def main() -> None: if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': - repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}' elif platform == 'windows': repair_command_default = '' else: @@ -339,7 +339,7 @@ def get_build_identifiers( elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector, architectures) elif platform == 'macos': - python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) + python_configurations = cibuildwheel.macos.get_python_configurations(build_selector, architectures) else: assert_never(platform) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index f7bf4354b..63e3a8bc4 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -15,11 +15,17 @@ class Architecture(Enum): # mac/linux archs x86_64 = 'x86_64' + + # linux archs i686 = 'i686' aarch64 = 'aarch64' ppc64le = 'ppc64le' s390x = 's390x' + # mac archs + universal2 = 'universal2' + arm64 = 'arm64' + # windows archs x86 = 'x86' AMD64 = 'AMD64' @@ -46,11 +52,18 @@ def parse_config(config: str, platform: PlatformName) -> 'Set[Architecture]': def auto_archs(platform: PlatformName) -> 'Set[Architecture]': native_architecture = Architecture(platform_module.machine()) result = {native_architecture} + if platform == 'linux' and native_architecture == Architecture.x86_64: # x86_64 machines can run i686 docker containers result.add(Architecture.i686) + if platform == 'windows' and native_architecture == Architecture.AMD64: result.add(Architecture.x86) + + if platform == 'macos' and native_architecture == Architecture.arm64: + # arm64 can build and test both archs of a universal2 wheel. + result.add(Architecture.universal2) + return result @staticmethod @@ -58,7 +71,7 @@ def all_archs(platform: PlatformName) -> 'Set[Architecture]': if platform == 'linux': return {Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x} elif platform == 'macos': - return {Architecture.x86_64} + return {Architecture.x86_64, Architecture.arm64, Architecture.universal2} elif platform == 'windows': return {Architecture.x86, Architecture.AMD64} else: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 13354f439..5d19dbd20 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -94,6 +94,7 @@ def build(options: BuildOptions) -> None: env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' env = options.environment.as_dictionary(env, executor=docker.environment_executor) before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) @@ -228,7 +229,7 @@ def build(options: BuildOptions) -> None: docker.copy_out(container_output_dir, options.output_dir) log.step_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) sys.exit(1) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 8fe67abc1..60e1b3c21 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -23,6 +23,8 @@ 'win32': 'Windows 32bit', 'win_amd64': 'Windows 64bit', 'macosx_x86_64': 'macOS x86_64', + 'macosx_universal2': 'macOS Universal 2 - x86_64 and arm64', + 'macosx_arm64': 'macOS arm64 - Apple Silicon', } @@ -109,15 +111,23 @@ def step_end(self, success: bool = True) -> None: self.step_start_time = None - def error(self, error: Union[BaseException, str]) -> None: + def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) - print() + self.error(error) + def warning(self, message: str) -> None: + if self.fold_mode == 'github': + print(f'::warning::{message}\n', file=sys.stderr) + else: + c = self.colors + print(f'{c.yellow}Warning{c.end}: {message}\n', file=sys.stderr) + + def error(self, error: Union[BaseException, str]) -> None: if self.fold_mode == 'github': - print(f'::error::{error}') + print(f'::error::{error}\n', file=sys.stderr) else: c = self.colors - print(f'{c.bright_red}Error{c.end} {error}') + print(f'{c.bright_red}Error{c.end}: {error}\n', file=sys.stderr) def _start_fold_group(self, name: str) -> None: self._end_fold_group() diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cf6919951..e6395449f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,15 +1,18 @@ import os +import platform +import re import shlex import shutil import subprocess import sys import tempfile from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence +from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, cast +from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log -from .typing import PathOrStr +from .typing import Literal, PathOrStr from .util import ( BuildOptions, BuildSelector, @@ -21,6 +24,7 @@ prepare_command, read_python_configs, resources_dir, + unwrap, ) @@ -34,21 +38,72 @@ def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: O return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) +def get_macos_version() -> Tuple[int, int]: + ''' + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (10, 14) <= (10, 16) == True + (11, 2) <= (11, 0) != True + ''' + version_str, _, _ = platform.mac_ver() + version = tuple(map(int, version_str.split(".")[:2])) + return cast(Tuple[int, int], version) + + +def get_macos_sdks() -> List[str]: + output = subprocess.check_output( + ['xcodebuild', '-showsdks'], + universal_newlines=True, + ) + + return [m.group(1) for m in re.finditer(r'-sdk (macosx\S+)', output)] + + class PythonConfiguration(NamedTuple): version: str identifier: str url: str -def get_python_configurations( - build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector, + architectures: Set[Architecture]) -> List[PythonConfiguration]: full_python_configs = read_python_configs('macos') python_configurations = [PythonConfiguration(**item) for item in full_python_configs] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + # filter out configs that don't match any of the selected architectures + python_configurations = [c for c in python_configurations + if any(c.identifier.endswith(a.value) for a in architectures)] + + # skip builds as required by BUILD/SKIP + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + + # When running on macOS 11 and x86_64, the reported OS is '10.16', but + # there is no such OS - it really means macOS 11. + if get_macos_version() >= (10, 16): + if any(c.identifier.startswith('pp') for c in python_configurations): + # pypy doesn't work on macOS 11 yet + # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 + log.warning(unwrap(''' + PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by + adding "pp*-macosx*" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + + if any(c.identifier.startswith('cp35') for c in python_configurations): + # CPython 3.5 doesn't work on macOS 11 + log.warning(unwrap(''' + CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython + 3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] + + return python_configurations SYMLINKS_DIR = Path('/tmp/cibw_bin') @@ -150,6 +205,9 @@ def setup_python(python_configuration: PythonConfiguration, env.pop('__PYVENV_LAUNCHER__', None) env = environment.as_dictionary(prev_environment=env) + # we version pip ourselves, so we don't care about pip version checking + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # check what version we're on call(['which', 'python'], env=env) call(['python', '--version'], env=env) @@ -180,6 +238,41 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + if python_configuration.version == '3.9': + if python_configuration.identifier.endswith('x86_64'): + # even on the macos11.0 Python installer, on the x86_64 side it's + # compatible back to 10.9. + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64') + env.setdefault('ARCHFLAGS', '-arch x86_64') + elif python_configuration.identifier.endswith('arm64'): + # macOS 11 is the first OS with arm64 support, so the wheels + # have that as a minimum. + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64') + env.setdefault('ARCHFLAGS', '-arch arm64') + elif python_configuration.identifier.endswith('universal2'): + if get_macos_version() < (10, 16): + # we can do universal2 builds on macos 10.15, but we need to + # set ARCHFLAGS otherwise CPython sets it to `-arch x86_64` + env.setdefault('ARCHFLAGS', '-arch arm64 -arch x86_64') + + if python_configuration.identifier.endswith('arm64') or python_configuration.identifier.endswith('universal2'): + if get_macos_version() < (10, 16) and 'SDKROOT' not in env: + # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but + # needs the correct SDK selected. + sdks = get_macos_sdks() + + # Different versions of Xcode contain different SDK versions... + # we're happy with anything newer than macOS 11.0 + arm64_compatible_sdks = [s for s in sdks if not s.startswith('macosx10.')] + + if not arm64_compatible_sdks: + log.warning(unwrap(''' + SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later + to build universal2 or arm64 wheels. + ''')) + else: + env.setdefault('SDKROOT', arm64_compatible_sdks[0]) + log.step('Installing build tools...') call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) @@ -198,7 +291,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) for config in python_configurations: log.build_start(config.identifier) @@ -242,60 +335,130 @@ def build(options: BuildOptions) -> None: if options.repair_command: log.step('Repairing wheel...') - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + + if config.identifier.endswith('universal2'): + delocate_archs = 'x86_64,arm64' + elif config.identifier.endswith('arm64'): + delocate_archs = 'arm64' + else: + delocate_archs = 'x86_64' + + repair_command_prepared = prepare_command( + options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + delocate_archs=delocate_archs, + ) call(repair_command_prepared, env=env, shell=True) else: shutil.move(str(built_wheel), repaired_wheel_dir) repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - if options.test_command and options.test_selector(config.identifier): - log.step('Testing wheel...') - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) - - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) - - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) - - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + log.step_end() - # clean up - shutil.rmtree(venv_dir) + if options.test_command and options.test_selector(config.identifier): + machine_arch = platform.machine() + testing_archs: List[Literal['x86_64', 'arm64']] = [] + + if config.identifier.endswith('_arm64'): + testing_archs = ['arm64'] + elif config.identifier.endswith('_universal2'): + testing_archs = ['x86_64', 'arm64'] + else: + testing_archs = ['x86_64'] + + for testing_arch in testing_archs: + if config.identifier.endswith('_universal2'): + arch_specific_identifier = f'{config.identifier}:{testing_arch}' + if not options.test_selector(arch_specific_identifier): + continue + + if machine_arch == 'x86_64' and testing_arch == 'arm64': + if config.identifier.endswith('_arm64'): + log.warning(unwrap(''' + While arm64 wheels can be built on x86_64, they cannot be tested. The + ability to test the arm64 wheels will be added in a future release of + cibuildwheel, once Apple Silicon CI runners are widely available. To + silence this warning, set `CIBW_TEST_SKIP: *-macosx_arm64`. + ''')) + elif config.identifier.endswith('_universal2'): + log.warning(unwrap(''' + While universal2 wheels can be built on x86_64, the arm64 part of them + cannot currently be tested. The ability to test the arm64 part of a + universal2 wheel will be added in a future release of cibuildwheel, once + Apple Silicon CI runners are widely available. To silence this warning, + set `CIBW_TEST_SKIP: *-macosx_universal2:arm64`. + ''')) + else: + raise RuntimeError('unreachable') + + # skip this test + continue + + log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) + + arch_prefix = [] + if testing_arch != machine_arch: + if machine_arch == 'arm64' and testing_arch == 'x86_64': + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ['arch', '-x86_64'] + else: + raise RuntimeError("don't know how to emulate {testing_arch} on {machine_arch}") + + # define a custom 'call' function that adds the arch prefix each time + def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: + if isinstance(args, str): + args = ' '.join(arch_prefix) + ' ' + args + else: + args = [*arch_prefix, *args] + return call(args, **kwargs) + + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call_with_arch(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) + + # check that we are using the Python from the virtual environment + call_with_arch(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) + + # install the wheel + call_with_arch(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call_with_arch(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call_with_arch(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + + # clean up + shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') sys.exit(1) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index e43bc1de1..41d06d45c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -41,7 +41,9 @@ python_configurations = [ { identifier = "cp36-macosx_x86_64", version = "3.6", url = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" }, { identifier = "cp37-macosx_x86_64", version = "3.7", url = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" }, { identifier = "cp38-macosx_x86_64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg" }, - { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg" }, + { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" }, + { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" }, + { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" }, { identifier = "pp27-macosx_x86_64", version = "2.7", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" }, { identifier = "pp36-macosx_x86_64", version = "3.6", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" }, { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" }, diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 1baf71a9e..9e0bc5d82 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,8 +1,8 @@ import fnmatch import itertools import os +import re import ssl -import sys import textwrap import urllib.request from enum import Enum @@ -18,13 +18,7 @@ from .environment import ParsedEnvironment from .typing import PathOrStr, PlatformName -if sys.version_info < (3, 9): - from importlib_resources import files -else: - from importlib.resources import files - - -resources_dir = files('cibuildwheel') / 'resources' +resources_dir = Path(__file__).parent / 'resources' get_pip_script = resources_dir / 'get-pip.py' install_certifi_script = resources_dir / "install_certifi.py" @@ -228,3 +222,15 @@ def detect_ci_provider() -> Optional[CIProvider]: return CIProvider.other else: return None + + +def unwrap(text: str) -> str: + ''' + Unwraps multi-line text to a single line + ''' + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + # remove consecutive whitespace + return re.sub(r'\s+', ' ', text) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4d3487ca4..45f0141ab 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -143,6 +143,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain str(installation_path / 'Scripts'), env['PATH'] ]) + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # update env with results from CIBW_ENVIRONMENT env = environment.as_dictionary(prev_environment=env) @@ -327,5 +329,5 @@ def build(options: BuildOptions) -> None: shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') sys.exit(1) diff --git a/docs/extra.css b/docs/extra.css index 0a4dad6b5..e7d035aea 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -83,5 +83,9 @@ h1, h2, h3, h4, h5, h6 { padding-top: 0.9em; } +.toctree-l3 { + border-left: 10px solid transparent; +} + /* import font awesome 4 for icons */ @import url(https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css); diff --git a/docs/faq.md b/docs/faq.md index 799059d95..95a99be9c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -44,6 +44,31 @@ Linux), and the other architectures are emulated automatically. {% include "../examples/github-with-qemu.yml" %} ``` +### Building Apple Silicon wheels on Intel {: #apple-silicon} + +`cibuildwheel` supports cross-compiling `universal2` and `arm64` wheels on `x86_64` runners. + +These wheels are not built by default, but can be enabled by setting the [`CIBW_ARCHS_MACOS` option](options.md#archs) to `x86_64 arm64 universal2`. Cross-compilation is provided by the Xcode toolchain. + +!!! important + When cross-compiling on Intel, it is not possible to test `arm64` and the `arm64` part of a `universal2` wheel. + + `cibuildwheel` will raise a warning to notify you of this - these warnings be be silenced by skipping testing on these platforms: `CIBW_TEST_SKIP: *_arm64 *_universal2:arm64`. + +Hopefully, this is a temporary situation. Once we have widely available Apple Silicon CI runners, we can build and test `arm64` and `universal2` wheels more natively. That's why `universal2` wheels are not yet built by default, and require opt-in by setting `CIBW_ARCHS_MACOS`. + +!!! note + Your runner image needs Xcode Command Line Tools 12.2 or later to build `universal2` and `arm64`. + + So far, only CPython 3.9 supports `universal2` and `arm64` wheels. + +Here's an example Github Actions workflow with a job that builds for Apple Silicon: + +> .github/workflows/build_macos.yml +```yml +{% include "../examples/github-apple-silicon.yml" %} +``` + ### Building packages with optional C extensions `cibuildwheel` defines the environment variable `CIBUILDWHEEL` to the value `1` allowing projects for which the C extension is optional to make it mandatory when building wheels. diff --git a/docs/options.md b/docs/options.md index 12d190319..dc46a1108 100644 --- a/docs/options.md +++ b/docs/options.md @@ -64,7 +64,7 @@ Default: `auto` For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet. -This option can also be set using the command-line option `--platform`. +This option can also be set using the [command-line option](#command-line) `--platform`. ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip} @@ -155,24 +155,38 @@ CIBW_SKIP: pp* } -### `CIBW_ARCHS_LINUX` {: #archs} -> Build non-native architectures +### `CIBW_ARCHS` {: #archs} +> Change the architectures built on your machine by default. -A space-separated list of architectures to build. Use this in conjunction with -emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][binfmt], to build architectures other than those your -machine natively supports. +A space-separated list of architectures to build. -Options: `auto` `native` `all` `x86_64` `i686` `aarch64` `ppc64le` `s390x` +On macOS, this option can be used to cross-compile between `x86_64`, +`universal2` and `arm64` for Apple Silicon support. -Default: `auto`, meaning the native archs supported on the build machine. For -example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`. +On Linux, this option can be used to build non-native architectures under +emulation. See [this guide](faq.md#emulation) for more information. -`native` will only build on the exact architecture you currently are on; it will -not add `i686` for `x86_64`. +Options: -`all` will expand to all known architectures; remember to use build selectors -to limit builds for each job; this list could grow in the future. +- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` +- macOS: `x86_64` `arm64` `universal2` +- Windows: `AMD64` `x86` +- `auto`: The default archs for your machine - see the table below. +- `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). +- `all` : expands to all the architectures supported on this OS. You may want + to use [CIBW_BUILD](#build-skip) with this option to target specific + architectures via build selectors. + +Default: `auto` + +| Runner | `native` | `auto` +|---|---|--- +| Linux / Intel | `x86_64` | `x86_64` `i686` +| Windows / Intel | `AMD64` | `AMD64` `x86` +| macOS / Intel | `x86_64` | `x86_64` +| macOS / AppleĀ Silicon | `arm64` | `arm64` `universal2` + +If not listed above, `auto` is the same as `native`. [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt @@ -180,10 +194,20 @@ to limit builds for each job; this list could grow in the future. #### Examples ```yaml -# On an intel runner with qemu installed, build Intel and ARM wheels +# Build `universal2` and `arm64` wheels on an Intel runner. +# Note that the `arm64` wheel and the `arm64` part of the `universal2` +# wheel cannot be tested in this configuration. +CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" + +# On an Linux Intel runner with qemu installed, build Intel and ARM wheels CIBW_ARCHS_LINUX: "auto aarch64" ``` +Platform-specific variants also available:
+`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` + +This option can also be set using the [command-line option](#command-line) `--archs`. + ## Build customization ### `CIBW_ENVIRONMENT` {: #environment} @@ -306,7 +330,7 @@ CIBW_BEFORE_BUILD: "{package}/script/prepare_for_build.sh" Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` -- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'` +- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'` - on Windows: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. @@ -315,7 +339,8 @@ The command is run on each built wheel (except for pure Python ones) before test The following placeholders must be used inside the command and will be replaced by `cibuildwheel`: - `{wheel}` for the absolute path to the built wheel -- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel. +- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel +- `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. @@ -521,11 +546,16 @@ CIBW_TEST_EXTRAS: test,qt This will skip testing on any identifiers that match the given skip patterns (see [`CIBW_SKIP`](#build-skip)). This can be used to mask out tests for wheels that have missing dependencies upstream that are slow or hard to build, or to mask up slow tests on emulated architectures. +With macOS `universal2` wheels, you can also skip the the individual archs inside the wheel using an `:arch` suffix. For example, `cp39-macosx_universal2:x86_64` or `cp39-macosx_universal2:arm64`. + #### Examples ```yaml # Will avoid testing on emulated architectures CIBW_TEST_SKIP: "*-manylinux_{aarch64,ppc64le,s390x}" + +# Skip trying to test arm64 builds on Intel Macs +CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" ``` @@ -547,7 +577,7 @@ CIBW_BUILD_VERBOSITY: 1 ``` -## Command line options +## Command line options {: #command-line} ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] @@ -591,10 +621,6 @@ optional arguments: ```