From 4380c47beeae04332b5b9050a814a898c1d634e9 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Wed, 1 May 2024 22:33:13 +0200 Subject: [PATCH] chore(ci): build and test macOS universal2 wheels on macOS arm64 (#6117) ### Description Build and test macOS universal2 wheels on macOS arm64 Removes x86_64 wheels. ### Motivation and Context `macos-latest` runners moved to macOS 14 running on arm64 hardware. This is breaking the macOS release pipeline. This PR rewrites the macOS release pipeline in 2 parts: - Build universal2 wheels on `macos-latest` - Test both arm64 & x86_64 parts of the universal2 wheel on `macos-latest` The x86_64 wheels have been removed as the universal2 ones can be used on x86_64. close #6105 --------- Signed-off-by: mayeut Signed-off-by: Justin Chu Co-authored-by: Justin Chu (cherry picked from commit 152988042cd8d1151883fa82fecf126b27f61e4d) --- .github/workflows/release_mac.yml | 85 +++++++++++++++++++------------ docs/CIPipelines.md | 2 +- setup.py | 4 +- 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml index a4878f9e70b..64701ee8afb 100644 --- a/.github/workflows/release_mac.yml +++ b/.github/workflows/release_mac.yml @@ -10,9 +10,9 @@ on: branches: [main, rel-*] workflow_dispatch: -# Use MACOSX_DEPLOYMENT_TARGET=10.15 to produce compatible wheel +# Use MACOSX_DEPLOYMENT_TARGET=11.0 to produce compatible wheel env: - MACOSX_DEPLOYMENT_TARGET: 10.15 + MACOSX_DEPLOYMENT_TARGET: 11.0 permissions: contents: read @@ -24,8 +24,6 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - host-architecture: ['x64'] - target-architecture: ['x86_64', 'universal2'] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -39,7 +37,6 @@ jobs: uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.host-architecture }} - name: Install Python dependencies run: | @@ -51,12 +48,7 @@ jobs: CC: "clang" CXX: "clang++" ONNX_ML: 1 - CMAKE_OSX_ARCHITECTURES: ${{ matrix.target-architecture == 'x86_64' && 'x86_64' || 'arm64;x86_64' }} - # Currently GitHub Action agent is using macos-11, we rename the wheels - # to use the MACOSX_DEPLOYMENT_TARGET - # Rename e.g. onnx-1.15.0-cp38-cp38-macosx_11_0_x86_64.whl - # to onnx-1.15.0-cp38-cp38-macosx_10_15_universal2.whl - ONNX_WHEEL_PLATFORM_NAME: macosx_10_15_${{ matrix.target-architecture }} + CMAKE_OSX_ARCHITECTURES: "arm64;x86_64" CMAKE_ARGS: "-DONNX_USE_LITE_PROTO=ON" run: | # Install Protobuf from source @@ -68,47 +60,73 @@ jobs: fi python -m build --wheel - for file in dist/*.whl; do - python -m pip install --upgrade $file; - done - - name: Test the installed wheel + - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: macos-wheel-${{ matrix.python-version }} + path: dist + + test: + needs: build + runs-on: macos-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + target-architecture: ['x86_64', 'arm64'] + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies run: | - pytest + arch -${{ matrix.target-architecture }} python -m pip install -q --upgrade pip + arch -${{ matrix.target-architecture }} python -m pip install -q -r requirements-release.txt - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - name: wheels + name: macos-wheel-${{ matrix.python-version }} path: dist + - name: Test the wheel + run: | + arch -${{ matrix.target-architecture }} python -m pip install --upgrade dist/*.whl + arch -${{ matrix.target-architecture }} pytest + - name: Upload wheel to PyPI weekly - if: github.event_name == 'schedule' # Only triggered by weekly event + if: github.event_name == 'schedule' && matrix.target-architecture == 'arm64' # Only triggered by weekly event run: | twine upload --verbose dist/*.whl --repository-url https://upload.pypi.org/legacy/ -u ${{ secrets.ONNXWEEKLY_USERNAME }} -p ${{ secrets.ONNXWEEKLY_TOKEN }} - name: Verify ONNX with the latest numpy if: ${{ always() }} run: | - python -m pip uninstall -y numpy onnx && python -m pip install numpy - for file in dist/*.whl; do python -m pip install --upgrade $file; done - pytest + arch -${{ matrix.target-architecture }} python -m pip uninstall -y numpy onnx + arch -${{ matrix.target-architecture }} python -m pip install numpy + arch -${{ matrix.target-architecture }} python -m pip install --upgrade dist/*.whl + arch -${{ matrix.target-architecture }} pytest - name: Verify ONNX with the latest protobuf if: ${{ always() }} run: | - python -m pip uninstall -y protobuf onnx && python -m pip install protobuf - for file in dist/*.whl; do python -m pip install --upgrade $file; done - pytest + arch -${{ matrix.target-architecture }} python -m pip uninstall -y protobuf onnx + arch -${{ matrix.target-architecture }} python -m pip install protobuf + arch -${{ matrix.target-architecture }} python -m pip install --upgrade dist/*.whl + arch -${{ matrix.target-architecture }} pytest - name: Verify ONNX with the minimumly supported packages - if: ${{ always() }} + if: always() && (matrix.target-architecture == 'x86_64' || (matrix.python-version != '3.8' && matrix.python-version != '3.9')) run: | - python -m pip uninstall -y numpy protobuf onnx && python -m pip install -r requirements-min.txt - for file in dist/*.whl; do python -m pip install --upgrade $file; done - pytest + arch -${{ matrix.target-architecture }} python -m pip uninstall -y numpy protobuf onnx + arch -${{ matrix.target-architecture }} python -m pip install -r requirements-min.txt + arch -${{ matrix.target-architecture }} python -m pip install --upgrade dist/*.whl + arch -${{ matrix.target-architecture }} pytest # Only triggered by weekly event on certain CI - name: Build and upload source distribution to PyPI weekly - if: github.event_name == 'schedule' && matrix.python-version == '3.10' && matrix.target-architecture == 'x86_64' + if: github.event_name == 'schedule' && matrix.python-version == '3.10' && matrix.target-architecture == 'arm64' run: | # Build and upload source distribution to PyPI git clean -xdf @@ -125,9 +143,10 @@ jobs: - name: Verify ONNX with ONNX Runtime PyPI package if: matrix.python-version != '3.12' run: | - python -m pip uninstall -y protobuf numpy && python -m pip install -q -r requirements-release.txt - python -m pip install -q onnxruntime + arch -${{ matrix.target-architecture }} python -m pip uninstall -y protobuf numpy + arch -${{ matrix.target-architecture }} python -m pip install -q -r requirements-release.txt + arch -${{ matrix.target-architecture }} python -m pip install -q onnxruntime export ORT_MAX_IR_SUPPORTED_VERSION=9 export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3 export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=20 - pytest + arch -${{ matrix.target-architecture }} pytest diff --git a/docs/CIPipelines.md b/docs/CIPipelines.md index 8f7a44b0cc6..89d3abfc79e 100644 --- a/docs/CIPipelines.md +++ b/docs/CIPipelines.md @@ -19,7 +19,7 @@ SPDX-License-Identifier: Apache-2.0 [WindowsRelease](/.github/workflows/release_win.yml) |
  • Main branch
  • Release branch
  • Weekly(1)
|
  • Latest Windows
  • x86 and x64
  • ONNX_USE_LITE_PROTO=ON
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
  • ONNX_USE_MSVC_STATIC_RUNTIME=OFF
|
  • Release Windows wheel
  • Release onnx-weekly package
  • Verify with different dependency versions - latest and min supported numpy version, latest and min supported protobuf version(2)
  • Verify ONNX with the latest [ONNX Runtime PyPI package](https://pypi.org/project/onnxruntime/)(3).
| [LinuxRelease_aarch64](/.github/workflows/release_linux_aarch64.yml) |
  • Main branch
  • Release branch
  • Weekly
  |
  • Latest manylinux2014_aarch64
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
  • ONNX_USE_LITE_PROTO=ON
|
  • Release Linux aarch64 wheel
  • Release onnx-weekly package
  • Verify with different dependency versions - latest numpy version, latest and min supported protobuf version
  • Verify ONNX with the latest ONNX Runtime PyPI package
| [LinuxRelease_x86_64](/.github/workflows/release_linux_x86_64.yml) |
  • Main branch
  • Release branch
  • Weekly
|
  • Latest LinuxRelease_x86_64
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
  • ONNX_USE_LITE_PROTO=ON
|
  • Release Linux x86_64 wheel
  • Release onnx-weekly package
  • Test TEST_HUB=1(4)
  • Verify with different dependency versions - latest numpy version, latest and min supported protobuf version
  • Verify ONNX with the latest ONNX Runtime PyPI package.
| - [MacRelease](/.github/workflows/release_win.yml) |
  • Main branch
  • Release branch
  • Weekly
|
  • macos-11
  • MACOSX_DEPLOYMENT_TARGET=10.12(5)
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
  • ONNX_USE_LITE_PROTO=ON
|
  • Release Mac wheel
  • Release onnx-weekly package
  • Verify with different dependency versions - latest numpy version, latest and min supported protobuf version
  • Verify ONNX with the latest ONNX Runtime PyPI package.
  • Test source distribution generation
  • Test build with source distribution
  • Release onnx-weekly source distribution
| + [MacRelease](/.github/workflows/release_mac.yml) |
  • Main branch
  • Release branch
  • Weekly
|
  • macos-latest
  • MACOSX_DEPLOYMENT_TARGET=11.0
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
  • ONNX_USE_LITE_PROTO=ON
|
  • Release Mac wheel
  • Release onnx-weekly package
  • Verify with different dependency versions - latest numpy version, latest and min supported protobuf version
  • Verify ONNX with the latest ONNX Runtime PyPI package.
  • Test source distribution generation
  • Test build with source distribution
  • Release onnx-weekly source distribution
| [Weekly CI with latest onnx.checker](/.github/workflows/weekly_mac_ci.yml) | weekly(6) |
  • macos-latest
  • MACOSX_DEPLOYMENT_TARGET=10.12
  • ONNX_USE_PROTOBUF_SHARED_LIBS=OFF
  • ONNX_ML=1
|
  • Test latest ONNX checker
  • Test latest ONNX shape inference
  • With all models from [onnx/models](https://github.com/onnx/models)(7)
| [Reuse](/.github/workflows/reuse.yml) | Every PR | |
  • Checks for Copyright and License header
  • More information could be found at: https://reuse.software/
  • If no license is to be added, or the checker does not recognize it, it must be configured under .reuse/dep5.
| [Dependabot](/.github/dependabot.yml) |
  • Main branch
  • weekly
| |
  • Create PRs for new dependency versions (will occur more often because p.ex. GitHub actions are pinned to commit hashes due to security best practices and not just to a version number).
| diff --git a/setup.py b/setup.py index 541085cf1b0..eaf8b65284f 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,8 @@ DEBUG = os.getenv("DEBUG", "0") == "1" COVERAGE = os.getenv("COVERAGE", "0") == "1" -# Customize the wheel plat-name, usually needed for MacOS builds. -# See usage in .github/workflows/release_mac.yml +# Customize the wheel plat-name; sometimes useful for MacOS builds. +# See https://github.com/onnx/onnx/pull/6117 ONNX_WHEEL_PLATFORM_NAME = os.getenv("ONNX_WHEEL_PLATFORM_NAME") ################################################################################