From b65b78a1828dba88575f11f74efaf814aca5ca44 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 26 May 2022 16:41:25 -0700 Subject: [PATCH 01/16] Add M1 wheels binary builds (#5948) --- .github/workflows/build-m1-binaries.yml | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/build-m1-binaries.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml new file mode 100644 index 00000000000..ecb07a8ed7d --- /dev/null +++ b/.github/workflows/build-m1-binaries.yml @@ -0,0 +1,48 @@ +name: Build on M1 +on: + pull_request: + paths: + - .github/workflows/build-m1-binaries.yml + push: + branches: + - nightly + workflow_dispatch: +jobs: + build_wheels: + name: "Build TorchVision M1 wheels" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Build TorchVision M1 wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + echo $PATH + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly torch + conda run -p ${ENV_NAME} python3 -mpip install delocate + conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + conda env remove -p ${ENV_NAME} + - name: Upload wheel to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1 + path: dist/ + - name: Upload wheel to S3 + shell: arch -arch arm64 bash {0} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} + CHANNEL: nightly + run: | + for pkg in dist/*; do + aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read + done From 99f36bc971361e4c5a656c16d7ba63e46d3af495 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Mon, 30 May 2022 23:05:48 -0700 Subject: [PATCH 02/16] [M1] Set build version and delocate wheels (#6110) This would package libpng and libjpeg.dylib into wheel files Add a very simple test step, copied from https://github.com/pytorch/pytorch.github.io/blob/1eaa33a3d3f1b83b64c5031c6dd04dbb238d6105/scripts/test_install.py#L78 Cherry-picked from https://github.com/pytorch/builder/blob/d0bc74cc363a9da5a8b6a40e883d40d25d050036/build_m1_domains.sh#L22 --- .github/workflows/build-m1-binaries.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index ecb07a8ed7d..8f6fcd3a6e1 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -23,13 +23,30 @@ jobs: ENV_NAME: conda-env-${{ github.run_id }} PY_VERS: ${{ matrix.py_vers }} run: | - echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex + export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly torch + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} + conda env remove -p ${ENV_NAME} + - name: Test wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl + # Test torch is importable, by changing cwd and running import commands + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub uses: actions/upload-artifact@v3 From f081cf951bbcefadafd0f983a007638c39a23c90 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 1 Jun 2022 21:15:11 -0700 Subject: [PATCH 03/16] [BE] Unify version computation (#6117) * [BE] Unify version computation Instead of hardcoding dev version in various script, use one from `version.txt` if `setup_build_version` is called without arguments Also, pass `--pre` option to M1 build/test pip install commands to build TorchVision against nightly pytorch * Pin torchvision dependency to a specific pytorch version --- .github/workflows/build-m1-binaries.yml | 8 +++++--- packaging/build_cmake.sh | 2 +- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 10 +++++++++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 8f6fcd3a6e1..09fb74ee273 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -25,12 +25,14 @@ jobs: run: | . ~/miniconda3/etc/profile.d/conda.sh set -ex - export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + . packaging/pkg_helpers.bash + setup_build_version WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel @@ -42,7 +44,7 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh index 0a4e4e948eb..35dfbc4a697 100755 --- a/packaging/build_cmake.sh +++ b/packaging/build_cmake.sh @@ -15,7 +15,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=conda -setup_env 0.13.0 +setup_env export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_plain_constraint diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 252bad295e3..4215c31016b 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=conda -setup_env 0.13.0 +setup_env export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 193eb386d13..3299d16ec92 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=wheel -setup_env 0.13.0 +setup_env setup_wheel_python pip_install numpy pyyaml future ninja pip_install --upgrade setuptools diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 1824b5b170c..7219920fdf1 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -100,7 +100,15 @@ setup_cuda() { # Usage: setup_build_version 0.2.0 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + if [[ -z "$1" ]]; then + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + else + _VERSION_BASE="$1" + fi + export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi From e02f37049d42decb2de368d7d9f929eb88a2ef56 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 2 Jun 2022 16:28:09 -0700 Subject: [PATCH 04/16] [M1] Install "jpeg<=9b" rather than OpenJpeg (#6122) Explicitly set PATH to point to `conda` binary, otherwise libjpeg detection logic does not work Pin libjpeg to the same version on x86 and m1 Add simple tests that jpeg can be decoded by a generated wheel --- .github/workflows/build-m1-binaries.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 09fb74ee273..b03cefbf25d 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -24,11 +24,13 @@ jobs: PY_VERS: ${{ matrix.py_vers }} run: | . ~/miniconda3/etc/profile.d/conda.sh + # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')` + export PATH=~/miniconda3/bin:$PATH set -ex . packaging/pkg_helpers.bash setup_build_version WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng "jpeg<=9b" wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel @@ -49,6 +51,7 @@ jobs: # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub uses: actions/upload-artifact@v3 From 6d6e4ceb44944d5834e8567e85ecc837269348a4 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 6 Jun 2022 13:06:53 +0100 Subject: [PATCH 05/16] Add unit-tests for M1 (#6132) * Add M1 testing job * libjpeg -> jpeg<=9b in test-m1.yml * Added export PATH=~/miniconda3/bin... from 6122 * Tests were OK, let's see if we can remove the pinning --- .github/workflows/build-m1-binaries.yml | 5 ++- .github/workflows/test-m1.yml | 45 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-m1.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index b03cefbf25d..780ace3bd5a 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -6,6 +6,7 @@ on: push: branches: - nightly + - main workflow_dispatch: jobs: build_wheels: @@ -30,7 +31,7 @@ jobs: . packaging/pkg_helpers.bash setup_build_version WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng "jpeg<=9b" wheel pkg-config + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel @@ -54,11 +55,13 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml new file mode 100644 index 00000000000..7ff3ade5820 --- /dev/null +++ b/.github/workflows/test-m1.yml @@ -0,0 +1,45 @@ +name: Unit-tests on M1 +on: + pull_request: + paths: + - .github/workflows/test-m1.yml + push: + branches: + - nightly + - main + workflow_dispatch: +jobs: + tests: + name: "Unit-tests on M1" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8"] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install TorchVision + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')` + export PATH=~/miniconda3/bin:$PATH + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 setup.py develop + conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av + - name: Run tests + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 + conda env remove -p ${ENV_NAME} From a02e3e5843eb9d4e1cc8a69ce863f4a4e7a38e25 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 8 Jun 2022 09:18:52 -0700 Subject: [PATCH 06/16] GH: Add M1 conda build workflows (#6135) Clean up Conda build folder before every run Enable artifact upload to GitHub for every workflow run, but upload to Conda/S3 only on nightly pushes Test plan: `conda install -c pytorch-nightly torchvision; python -c "import torchvision;print(torchvision.io.read_image('hummingbird.jpg').shape)"` --- .github/workflows/build-m1-binaries.yml | 50 ++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 780ace3bd5a..5b9555d6a8a 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -55,7 +55,6 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 @@ -71,3 +70,52 @@ jobs: for pkg in dist/*; do aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read done + build_conda: + name: "Build TorchVision M1 conda packages" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install conda-build and purge previous artifacts + shell: arch -arch arm64 bash {0} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda install -yq conda-build + conda build purge-all + + - name: Build TorchVision M1 conda package + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PYTHON_VERSION: ${{ matrix.py_vers }} + CU_VERSION: cpu + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + . packaging/pkg_helpers.bash + setup_build_version + setup_conda_pytorch_constraint + export SOURCE_ROOT_DIR=$(pwd) + conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision + mkdir -p dist + cp ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 dist/ + - name: Upload package to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda + path: dist/ + - name: Upload package to conda + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + shell: arch -arch arm64 bash {0} + env: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} + UPLOAD_CHANNEL: nightly + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda install -yq anaconda-client + set -x + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force From 6cf829d296c722286162b613489897b908cde325 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 8 Jun 2022 21:23:15 -0700 Subject: [PATCH 07/16] Fix `Test M1` workflow By passing `--pre` option to `pip install`, otherwise torchvision were always tested against last PyTorch release --- .github/workflows/test-m1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml index 7ff3ade5820..ed35d9325a3 100644 --- a/.github/workflows/test-m1.yml +++ b/.github/workflows/test-m1.yml @@ -30,7 +30,7 @@ jobs: export PATH=~/miniconda3/bin:$PATH set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 setup.py develop conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av - name: Run tests From b1c33ed1570958e082103890100be57f498d204b Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Thu, 9 Jun 2022 09:53:04 -0400 Subject: [PATCH 08/16] Adding tagged builds for M1 (#6140) * Adding tagged builds * Testing * Testing * Testing * Testing * Adding conda builds --- .github/workflows/build-m1-binaries.yml | 47 ++++++++++++++++++++----- packaging/pkg_helpers.bash | 23 +++++++----- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 5b9555d6a8a..eec54b2f113 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -7,7 +7,13 @@ on: branches: - nightly - main + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +env: + CHANNEL: "nightly" jobs: build_wheels: name: "Build TorchVision M1 wheels" @@ -18,6 +24,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -29,10 +42,16 @@ jobs: export PATH=~/miniconda3/bin:$PATH set -ex . packaging/pkg_helpers.bash - setup_build_version + # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" @@ -47,7 +66,7 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" @@ -60,12 +79,11 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (steps.extract_branch.outputs.branch == 'nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} - CHANNEL: nightly run: | for pkg in dist/*; do aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read @@ -79,7 +97,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | @@ -97,7 +121,13 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex . packaging/pkg_helpers.bash - setup_build_version + + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + setup_conda_pytorch_constraint export SOURCE_ROOT_DIR=$(pwd) conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision @@ -113,9 +143,8 @@ jobs: shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - UPLOAD_CHANNEL: nightly run: | . ~/miniconda3/etc/profile.d/conda.sh conda install -yq anaconda-client set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${CHANNEL}" --label main --no-progress --force diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 7219920fdf1..502e3c47f36 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -101,23 +101,30 @@ setup_cuda() { setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then if [[ -z "$1" ]]; then - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - # version.txt for some reason has `a` character after major.minor.rev - # command below yields 0.10.0 from version.txt containing 0.10.0a0 - _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + setup_base_build_version else - _VERSION_BASE="$1" + BUILD_VERSION="$1" fi - export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + BUILD_VERSION="$BUILD_VERSION.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else - export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" + BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi # Set build version based on tag if on tag if [[ -n "${CIRCLE_TAG}" ]]; then # Strip tag - export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}" + BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}" fi + + export BUILD_VERSION +} + +setup_base_build_version() { + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + BUILD_VERSION=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + export BUILD_VERSION } # Set some useful variables for OS X, if applicable From fd13011b6a94856670deb1980e0287ee24ab61ea Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 9 Jun 2022 22:02:29 -0700 Subject: [PATCH 09/16] Fix `if` condition for s3/conda uploads (#6146) Replace `steps.extract_branch.outputs.branch` (which were probably taken from https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions ) with straightforward `github.event.ref` Tested in https://github.com/malfet/deleteme/runs/6822647720?check_suite_focus=true and https://github.com/malfet/deleteme/runs/6822691158?check_suite_focus=true --- .github/workflows/build-m1-binaries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index eec54b2f113..fe695e4bf6e 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -79,7 +79,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (steps.extract_branch.outputs.branch == 'nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} @@ -139,7 +139,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda path: dist/ - name: Upload package to conda - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} From f2468e8305ae28a383e85670db9463aa62b8d772 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Mon, 13 Jun 2022 08:14:07 -0700 Subject: [PATCH 10/16] Fix typo in GHA nightly build condition (#6158) s#ref/heads/#refs/heads/# I should have noticed it while copy-n-pasting the condition. Unfortunately there are no way to test is other than in prod, but nightly builds are still not getting pushed, see https://github.com/pytorch/vision/runs/6860407007?check_suite_focus=true for example --- .github/workflows/build-m1-binaries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index fe695e4bf6e..6b93c17968d 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -79,7 +79,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} @@ -139,7 +139,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda path: dist/ - name: Upload package to conda - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} From fba2a5430323f4f990f4357633a2d60005f57d7f Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 14:44:53 -0700 Subject: [PATCH 11/16] Making sure we are building against release --- .github/workflows/build-m1-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 6b93c17968d..eb44d766af8 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - name: Set CHANNEL (only for tagged pushes) - if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + if: ${{ github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags/') || startsWith(github.event.ref, 'refs/heads/release')) }} run: | # reference ends with an RC suffix if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then From f78e5888e653c72f3b8020a3697577c80ec180ab Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 14:52:51 -0700 Subject: [PATCH 12/16] Testing Testing Testing Testing testing Testing Testing Testing Testing Testing Testing Testing --- .github/workflows/build-m1-binaries.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index eb44d766af8..b4ffbc6ac63 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -14,6 +14,8 @@ on: workflow_dispatch: env: CHANNEL: "nightly" + BASE_REF: ${{ github.base_ref }} + EVENT_NAME: ${{ github.event_name }} jobs: build_wheels: name: "Build TorchVision M1 wheels" @@ -25,12 +27,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - name: Set CHANNEL (only for tagged pushes) - if: ${{ github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags/') || startsWith(github.event.ref, 'refs/heads/release')) }} + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} run: | # reference ends with an RC suffix if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then echo "CHANNEL=test" >> "$GITHUB_ENV" fi + - name: Set Release CHANNEL (for release) + if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -104,6 +110,10 @@ jobs: if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then echo "CHANNEL=test" >> "$GITHUB_ENV" fi + - name: Set CHANNEL Release (for release) + if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | @@ -123,12 +133,19 @@ jobs: . packaging/pkg_helpers.bash if [[ $CHANNEL == "test" ]]; then + setup_cuda setup_base_build_version + setup_macos else + setup_cuda setup_build_version + setup_macos fi setup_conda_pytorch_constraint + setup_conda_cudatoolkit_constraint + setup_visual_studio_constraint + setup_junit_results_folder export SOURCE_ROOT_DIR=$(pwd) conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision mkdir -p dist From a6d43d9ac4c9a45aafd5654610b9dd6509e79ccc Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 15:48:15 -0700 Subject: [PATCH 13/16] Testing --- .github/workflows/test-m1.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml index ed35d9325a3..12292c38fb0 100644 --- a/.github/workflows/test-m1.yml +++ b/.github/workflows/test-m1.yml @@ -7,7 +7,13 @@ on: branches: - nightly - main + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +env: + CHANNEL: "nightly" jobs: tests: name: "Unit-tests on M1" @@ -19,6 +25,17 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi + - name: Set Release CHANNEL (for release) + if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install TorchVision shell: arch -arch arm64 bash {0} env: @@ -30,7 +47,7 @@ jobs: export PATH=~/miniconda3/bin:$PATH set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 setup.py develop conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av - name: Run tests From a1fd2177643e4a7b0a51a0e2653cda71b3b16e7b Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 16:06:14 -0700 Subject: [PATCH 14/16] Testing --- .github/workflows/build-m1-binaries.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index b4ffbc6ac63..fad916ac17e 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -14,8 +14,6 @@ on: workflow_dispatch: env: CHANNEL: "nightly" - BASE_REF: ${{ github.base_ref }} - EVENT_NAME: ${{ github.event_name }} jobs: build_wheels: name: "Build TorchVision M1 wheels" @@ -137,15 +135,10 @@ jobs: setup_base_build_version setup_macos else - setup_cuda setup_build_version - setup_macos fi setup_conda_pytorch_constraint - setup_conda_cudatoolkit_constraint - setup_visual_studio_constraint - setup_junit_results_folder export SOURCE_ROOT_DIR=$(pwd) conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision mkdir -p dist From 229f7cd99a7fd910d1f4e8f92ebb7c8a0fc020d4 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 16:10:49 -0700 Subject: [PATCH 15/16] Cleanup --- .github/workflows/build-m1-binaries.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index fad916ac17e..f57ef0a1941 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -131,9 +131,7 @@ jobs: . packaging/pkg_helpers.bash if [[ $CHANNEL == "test" ]]; then - setup_cuda setup_base_build_version - setup_macos else setup_build_version fi From a0f8082b08e2576c5a802fb511e0af99cb0a6ce5 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 14 Jun 2022 16:46:17 -0700 Subject: [PATCH 16/16] Refactoring logic --- .github/workflows/build-m1-binaries.yml | 5 +++-- .github/workflows/test-m1.yml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index f57ef0a1941..094bf3e6be9 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -7,6 +7,7 @@ on: branches: - nightly - main + - release/* tags: # NOTE: Binary build pipelines should only get triggered on release candidate builds # Release candidate tags look like: v1.11.0-rc1 @@ -32,7 +33,7 @@ jobs: echo "CHANNEL=test" >> "$GITHUB_ENV" fi - name: Set Release CHANNEL (for release) - if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }} + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }} run: | echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Build TorchVision M1 wheel @@ -109,7 +110,7 @@ jobs: echo "CHANNEL=test" >> "$GITHUB_ENV" fi - name: Set CHANNEL Release (for release) - if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release') }} + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }} run: | echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install conda-build and purge previous artifacts diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml index 12292c38fb0..6b05d67a196 100644 --- a/.github/workflows/test-m1.yml +++ b/.github/workflows/test-m1.yml @@ -7,6 +7,7 @@ on: branches: - nightly - main + - release/* tags: # NOTE: Binary build pipelines should only get triggered on release candidate builds # Release candidate tags look like: v1.11.0-rc1