From cd7639fb0a164ec0b02e616c9bc7052cf8dbc7fa Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 26 May 2022 16:41:25 -0700 Subject: [PATCH 01/14] Add M1 wheels binary builds (#5948) --- .github/workflows/build-m1-binaries.yml | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/build-m1-binaries.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml new file mode 100644 index 00000000000..ecb07a8ed7d --- /dev/null +++ b/.github/workflows/build-m1-binaries.yml @@ -0,0 +1,48 @@ +name: Build on M1 +on: + pull_request: + paths: + - .github/workflows/build-m1-binaries.yml + push: + branches: + - nightly + workflow_dispatch: +jobs: + build_wheels: + name: "Build TorchVision M1 wheels" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Build TorchVision M1 wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + echo $PATH + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly torch + conda run -p ${ENV_NAME} python3 -mpip install delocate + conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + conda env remove -p ${ENV_NAME} + - name: Upload wheel to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1 + path: dist/ + - name: Upload wheel to S3 + shell: arch -arch arm64 bash {0} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} + CHANNEL: nightly + run: | + for pkg in dist/*; do + aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read + done From a7aeb18f26a9bc1985cb6438d924f24e750fe279 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Mon, 30 May 2022 23:05:48 -0700 Subject: [PATCH 02/14] [M1] Set build version and delocate wheels (#6110) This would package libpng and libjpeg.dylib into wheel files Add a very simple test step, copied from https://github.com/pytorch/pytorch.github.io/blob/1eaa33a3d3f1b83b64c5031c6dd04dbb238d6105/scripts/test_install.py#L78 Cherry-picked from https://github.com/pytorch/builder/blob/d0bc74cc363a9da5a8b6a40e883d40d25d050036/build_m1_domains.sh#L22 --- .github/workflows/build-m1-binaries.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index ecb07a8ed7d..8f6fcd3a6e1 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -23,13 +23,30 @@ jobs: ENV_NAME: conda-env-${{ github.run_id }} PY_VERS: ${{ matrix.py_vers }} run: | - echo $PATH . ~/miniconda3/etc/profile.d/conda.sh set -ex + export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly torch + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} + conda env remove -p ${ENV_NAME} + - name: Test wheel + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl + # Test torch is importable, by changing cwd and running import commands + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub uses: actions/upload-artifact@v3 From 56eb2b6551274b837837780f5805e697720f0d92 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 1 Jun 2022 09:13:43 +0100 Subject: [PATCH 03/14] 0.14 -> 0.13 --- .github/workflows/build-m1-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 8f6fcd3a6e1..4595a983951 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -25,7 +25,7 @@ jobs: run: | . ~/miniconda3/etc/profile.d/conda.sh set -ex - export BUILD_VERSION=0.14.0.dev$(date "+%Y%m%d") + export BUILD_VERSION=0.13.0.dev$(date "+%Y%m%d") WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly From d3b1d3c0428997c524070358ec546444f267dccd Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 6 Jun 2022 13:26:15 +0100 Subject: [PATCH 04/14] Cherry pick #6122 and #6132 which supersedes #6111 --- .github/workflows/build-m1-binaries.yml | 16 ++++++--- .github/workflows/test-m1.yml | 45 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-m1.yml diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 4595a983951..780ace3bd5a 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -6,6 +6,7 @@ on: push: branches: - nightly + - main workflow_dispatch: jobs: build_wheels: @@ -24,13 +25,17 @@ jobs: PY_VERS: ${{ matrix.py_vers }} run: | . ~/miniconda3/etc/profile.d/conda.sh + # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')` + export PATH=~/miniconda3/bin:$PATH set -ex - export BUILD_VERSION=0.13.0.dev$(date "+%Y%m%d") + . packaging/pkg_helpers.bash + setup_build_version WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng openjpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel + export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" conda run -p ${ENV_NAME} DYLD_FALLBACK_LIBRARY_PATH="${ENV_NAME}/lib" delocate-wheel -v --ignore-missing-dependencies dist/${WHL_NAME} conda env remove -p ${ENV_NAME} - name: Test wheel @@ -42,18 +47,21 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torch;import torchvision;print('Is torchvision useable?', all(x is not None for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]))" + conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml new file mode 100644 index 00000000000..7ff3ade5820 --- /dev/null +++ b/.github/workflows/test-m1.yml @@ -0,0 +1,45 @@ +name: Unit-tests on M1 +on: + pull_request: + paths: + - .github/workflows/test-m1.yml + push: + branches: + - nightly + - main + workflow_dispatch: +jobs: + tests: + name: "Unit-tests on M1" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8"] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install TorchVision + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + # Needed for JPEG library detection as setup.py detects conda presence by running `shlex.which('conda')` + export PATH=~/miniconda3/bin:$PATH + set -ex + conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy + conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 setup.py develop + conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av + - name: Run tests + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PY_VERS: ${{ matrix.py_vers }} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 + conda env remove -p ${ENV_NAME} From 20fbda9cebe02ff520e891f77c8501a176010c7d Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 1 Jun 2022 21:15:11 -0700 Subject: [PATCH 05/14] [BE] Unify version computation (#6117) * [BE] Unify version computation Instead of hardcoding dev version in various script, use one from `version.txt` if `setup_build_version` is called without arguments Also, pass `--pre` option to M1 build/test pip install commands to build TorchVision against nightly pytorch * Pin torchvision dependency to a specific pytorch version --- packaging/build_cmake.sh | 2 +- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- packaging/pkg_helpers.bash | 10 +++++++++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh index 0a4e4e948eb..35dfbc4a697 100755 --- a/packaging/build_cmake.sh +++ b/packaging/build_cmake.sh @@ -15,7 +15,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=conda -setup_env 0.13.0 +setup_env export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_plain_constraint diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 252bad295e3..4215c31016b 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=conda -setup_env 0.13.0 +setup_env export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 193eb386d13..3299d16ec92 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -5,7 +5,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" export BUILD_TYPE=wheel -setup_env 0.13.0 +setup_env setup_wheel_python pip_install numpy pyyaml future ninja pip_install --upgrade setuptools diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 1824b5b170c..7219920fdf1 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -100,7 +100,15 @@ setup_cuda() { # Usage: setup_build_version 0.2.0 setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + if [[ -z "$1" ]]; then + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + else + _VERSION_BASE="$1" + fi + export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi From 489f03fd3b5a705e035409555a3da972dabe5080 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 8 Jun 2022 17:39:51 +0100 Subject: [PATCH 06/14] Update version.txt --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 19b4ae8e531..54d1a4f2a4a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.13.0a0 +0.13.0 From cbc850d29dad64af8fc69143c2e29ca0a02d9ec2 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 8 Jun 2022 09:18:52 -0700 Subject: [PATCH 07/14] GH: Add M1 conda build workflows (#6135) Clean up Conda build folder before every run Enable artifact upload to GitHub for every workflow run, but upload to Conda/S3 only on nightly pushes Test plan: `conda install -c pytorch-nightly torchvision; python -c "import torchvision;print(torchvision.io.read_image('hummingbird.jpg').shape)"` --- .github/workflows/build-m1-binaries.yml | 50 ++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 780ace3bd5a..5b9555d6a8a 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -55,7 +55,6 @@ jobs: conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print(torchvision.io.read_image('${PWD}/gallery/assets/dog1.jpg').shape)" conda env remove -p ${ENV_NAME} - name: Upload wheel to GitHub - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} uses: actions/upload-artifact@v3 with: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 @@ -71,3 +70,52 @@ jobs: for pkg in dist/*; do aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read done + build_conda: + name: "Build TorchVision M1 conda packages" + runs-on: macos-m1 + strategy: + matrix: + py_vers: [ "3.8", "3.9", "3.10" ] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install conda-build and purge previous artifacts + shell: arch -arch arm64 bash {0} + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda install -yq conda-build + conda build purge-all + + - name: Build TorchVision M1 conda package + shell: arch -arch arm64 bash {0} + env: + ENV_NAME: conda-env-${{ github.run_id }} + PYTHON_VERSION: ${{ matrix.py_vers }} + CU_VERSION: cpu + run: | + . ~/miniconda3/etc/profile.d/conda.sh + set -ex + . packaging/pkg_helpers.bash + setup_build_version + setup_conda_pytorch_constraint + export SOURCE_ROOT_DIR=$(pwd) + conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision + mkdir -p dist + cp ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 dist/ + - name: Upload package to GitHub + uses: actions/upload-artifact@v3 + with: + name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda + path: dist/ + - name: Upload package to conda + if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + shell: arch -arch arm64 bash {0} + env: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} + UPLOAD_CHANNEL: nightly + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda install -yq anaconda-client + set -x + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force From b46fd4794652d892e10a4bd9c1222f3556f9b630 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 8 Jun 2022 17:51:06 +0100 Subject: [PATCH 08/14] rely on test instead of nightly --- .github/workflows/build-m1-binaries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 5b9555d6a8a..a3928daacca 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -32,7 +32,7 @@ jobs: setup_build_version WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre -f https://download.pytorch.org/whl/test/cpu/torch_test.html conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" @@ -47,7 +47,7 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/nightly + conda run -p ${ENV_NAME} python3 -mpip install torch --pre -f https://download.pytorch.org/whl/test/cpu/torch_test.html conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" From 434e02a5f61651ec48f10248ebe5d49aa69608d4 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Thu, 9 Jun 2022 09:53:04 -0400 Subject: [PATCH 09/14] Adding tagged builds for M1 (#6140) * Adding tagged builds * Testing * Testing * Testing * Testing * Adding conda builds --- .github/workflows/build-m1-binaries.yml | 47 ++++++++++++++++++++----- packaging/pkg_helpers.bash | 23 +++++++----- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index a3928daacca..eec54b2f113 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -7,7 +7,13 @@ on: branches: - nightly - main + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +env: + CHANNEL: "nightly" jobs: build_wheels: name: "Build TorchVision M1 wheels" @@ -18,6 +24,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -29,10 +42,16 @@ jobs: export PATH=~/miniconda3/bin:$PATH set -ex . packaging/pkg_helpers.bash - setup_build_version + # if we are uploading to test channell, our version consist only of the base: 0.x.x - no date string or suffix added + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + WHL_NAME=torchvision-${BUILD_VERSION}-cp${PY_VERS/.}-cp${PY_VERS/.}-macosx_11_0_arm64.whl conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg wheel pkg-config - conda run -p ${ENV_NAME} python3 -mpip install torch --pre -f https://download.pytorch.org/whl/test/cpu/torch_test.html + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install delocate conda run -p ${ENV_NAME} python3 setup.py bdist_wheel export PYTORCH_VERSION="$(conda run -p ${ENV_NAME} python3 -mpip show torch | grep ^Version: | sed 's/Version: *//')" @@ -47,7 +66,7 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy - conda run -p ${ENV_NAME} python3 -mpip install torch --pre -f https://download.pytorch.org/whl/test/cpu/torch_test.html + conda run -p ${ENV_NAME} python3 -mpip install torch --pre --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} conda run -p ${ENV_NAME} python3 -mpip install dist/*.whl # Test torch is importable, by changing cwd and running import commands conda run --cwd /tmp -p ${ENV_NAME} python3 -c "import torchvision;print('torchvision version is ', torchvision.__version__)" @@ -60,12 +79,11 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (steps.extract_branch.outputs.branch == 'nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} - CHANNEL: nightly run: | for pkg in dist/*; do aws s3 cp "$pkg" "s3://pytorch/whl/${CHANNEL}/cpu/" --acl public-read @@ -79,7 +97,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - + - name: Set CHANNEL (only for tagged pushes) + if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + run: | + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | @@ -97,7 +121,13 @@ jobs: . ~/miniconda3/etc/profile.d/conda.sh set -ex . packaging/pkg_helpers.bash - setup_build_version + + if [[ $CHANNEL == "test" ]]; then + setup_base_build_version + else + setup_build_version + fi + setup_conda_pytorch_constraint export SOURCE_ROOT_DIR=$(pwd) conda build -c defaults $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision @@ -113,9 +143,8 @@ jobs: shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - UPLOAD_CHANNEL: nightly run: | . ~/miniconda3/etc/profile.d/conda.sh conda install -yq anaconda-client set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/miniconda3/conda-bld/osx-arm64/*.tar.bz2 -u "pytorch-${CHANNEL}" --label main --no-progress --force diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 7219920fdf1..502e3c47f36 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -101,23 +101,30 @@ setup_cuda() { setup_build_version() { if [[ -z "$BUILD_VERSION" ]]; then if [[ -z "$1" ]]; then - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - # version.txt for some reason has `a` character after major.minor.rev - # command below yields 0.10.0 from version.txt containing 0.10.0a0 - _VERSION_BASE=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + setup_base_build_version else - _VERSION_BASE="$1" + BUILD_VERSION="$1" fi - export BUILD_VERSION="$_VERSION_BASE.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + BUILD_VERSION="$BUILD_VERSION.dev$(date "+%Y%m%d")$VERSION_SUFFIX" else - export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" + BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" fi # Set build version based on tag if on tag if [[ -n "${CIRCLE_TAG}" ]]; then # Strip tag - export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}" + BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}" fi + + export BUILD_VERSION +} + +setup_base_build_version() { + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # version.txt for some reason has `a` character after major.minor.rev + # command below yields 0.10.0 from version.txt containing 0.10.0a0 + BUILD_VERSION=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" ) + export BUILD_VERSION } # Set some useful variables for OS X, if applicable From d14d8b90aa091ca1ea8e7b96d17aff3ccb577233 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 9 Jun 2022 22:02:29 -0700 Subject: [PATCH 10/14] Fix `if` condition for s3/conda uploads (#6146) Replace `steps.extract_branch.outputs.branch` (which were probably taken from https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions ) with straightforward `github.event.ref` Tested in https://github.com/malfet/deleteme/runs/6822647720?check_suite_focus=true and https://github.com/malfet/deleteme/runs/6822691158?check_suite_focus=true --- .github/workflows/build-m1-binaries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index eec54b2f113..fe695e4bf6e 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -79,7 +79,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (steps.extract_branch.outputs.branch == 'nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} @@ -139,7 +139,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda path: dist/ - name: Upload package to conda - if: ${{ github.event_name == 'push' && steps.extract_branch.outputs.branch == 'nightly' }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} From efc569fe1d068aeb7fccd729956bf8a8df64e089 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 13 Jun 2022 15:44:13 +0100 Subject: [PATCH 11/14] Force CHANNEL=test --- .github/workflows/build-m1-binaries.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index fe695e4bf6e..3cc392ac1ad 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -27,10 +27,7 @@ jobs: - name: Set CHANNEL (only for tagged pushes) if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} run: | - # reference ends with an RC suffix - if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then - echo "CHANNEL=test" >> "$GITHUB_ENV" - fi + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -100,10 +97,7 @@ jobs: - name: Set CHANNEL (only for tagged pushes) if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} run: | - # reference ends with an RC suffix - if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then - echo "CHANNEL=test" >> "$GITHUB_ENV" - fi + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | From 91233430d12f80179abe19f13e1f1ec8fd5c95bf Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 13 Jun 2022 15:45:55 +0100 Subject: [PATCH 12/14] Better way --- .github/workflows/build-m1-binaries.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 3cc392ac1ad..2cb9a24a1ba 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -13,7 +13,7 @@ on: - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: env: - CHANNEL: "nightly" + CHANNEL: "test" jobs: build_wheels: name: "Build TorchVision M1 wheels" @@ -27,7 +27,10 @@ jobs: - name: Set CHANNEL (only for tagged pushes) if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -97,7 +100,10 @@ jobs: - name: Set CHANNEL (only for tagged pushes) if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" + # reference ends with an RC suffix + if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then + echo "CHANNEL=test" >> "$GITHUB_ENV" + fi - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | From 98257ec4c2d55231166814534999cfbac802fc2d Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 13 Jun 2022 16:26:11 +0100 Subject: [PATCH 13/14] Cherry-picking #6158 Co-authored-by: Vasilis Vryniotis --- .github/workflows/build-m1-binaries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 2cb9a24a1ba..095a97bd312 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -79,7 +79,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1 path: dist/ - name: Upload wheel to S3 - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} @@ -139,7 +139,7 @@ jobs: name: torchvision-py${{ matrix.py_vers }}-macos11-m1-conda path: dist/ - name: Upload package to conda - if: ${{ github.event_name == 'push' && (github.event.ref == 'ref/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} + if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/')) }} shell: arch -arch arm64 bash {0} env: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} From fb9a761abac621f16b50ebb01204fc31fae1d3bd Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Tue, 14 Jun 2022 20:14:57 -0400 Subject: [PATCH 14/14] Make sure we are building against test channel for release (#6168) * Make sure we are building against test chanell for release * Cleanup * Cleanup * Refactoring logic * Remove tagged tests --- .github/workflows/build-m1-binaries.yml | 9 +++++++++ .github/workflows/test-m1.yml | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/build-m1-binaries.yml b/.github/workflows/build-m1-binaries.yml index 095a97bd312..c977fca16d6 100644 --- a/.github/workflows/build-m1-binaries.yml +++ b/.github/workflows/build-m1-binaries.yml @@ -7,6 +7,7 @@ on: branches: - nightly - main + - release/* tags: # NOTE: Binary build pipelines should only get triggered on release candidate builds # Release candidate tags look like: v1.11.0-rc1 @@ -31,6 +32,10 @@ jobs: if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then echo "CHANNEL=test" >> "$GITHUB_ENV" fi + - name: Set Release CHANNEL (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Build TorchVision M1 wheel shell: arch -arch arm64 bash {0} env: @@ -104,6 +109,10 @@ jobs: if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then echo "CHANNEL=test" >> "$GITHUB_ENV" fi + - name: Set CHANNEL Release (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install conda-build and purge previous artifacts shell: arch -arch arm64 bash {0} run: | diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml index 7ff3ade5820..13494105fad 100644 --- a/.github/workflows/test-m1.yml +++ b/.github/workflows/test-m1.yml @@ -7,7 +7,10 @@ on: branches: - nightly - main + - release/* workflow_dispatch: +env: + CHANNEL: "nightly" jobs: tests: name: "Unit-tests on M1" @@ -19,6 +22,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set Release CHANNEL (for release) + if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'release') }} + run: | + echo "CHANNEL=test" >> "$GITHUB_ENV" - name: Install TorchVision shell: arch -arch arm64 bash {0} env: @@ -30,7 +37,11 @@ jobs: export PATH=~/miniconda3/bin:$PATH set -ex conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy +<<<<<<< HEAD conda run -p ${ENV_NAME} python3 -mpip install torch --extra-index-url=https://download.pytorch.org/whl/nightly +======= + conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} +>>>>>>> 369317f45 (Make sure we are building against test channel for release (#6168)) conda run -p ${ENV_NAME} python3 setup.py develop conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock av - name: Run tests