From 1f56ec1d3782fc722d57e01f6a7540f57cc643ff Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 10:15:08 +0000 Subject: [PATCH 1/7] Added nightly build/release action --- .../workflows/binaries-nightly-release.yml | 52 +++++++++++++++++++ .github/workflows/docker-publish.yml | 2 +- .github/workflows/pytorch-version-tests.yml | 3 +- .github/workflows/unit-tests.yml | 2 - conda.recipe/build_and_upload.sh | 34 ++++++++++++ 5 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/binaries-nightly-release.yml create mode 100644 conda.recipe/build_and_upload.sh diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml new file mode 100644 index 000000000000..6ff47cc8d5c0 --- /dev/null +++ b/.github/workflows/binaries-nightly-release.yml @@ -0,0 +1,52 @@ +name: Nightly Releases + +on: pull_request +#on: +# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule +# schedule: +# # Run at 00:00 UTC Every Day +# - cron: '0 0 * * *' + + +jobs: + build-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1 + with: + miniconda-version: "latest" + python-version: "3.7" + + - name: Setup nightly version + run: | + sed -i "s/__version__ = \"\(.*\)\"/__version__ = \"\1.dev$(date -u +%Y%m%d)\"/g" ignite/__init__.py + + - name: Install dependencies + run: | + conda install pytorch torchvision cpuonly -c pytorch-nightly + pip install -r requirements-dev.txt + pip install --upgrade --no-cache-dir twine + python setup.py install + + - name: Build and Publish Conda binaries + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + #UPLOAD_USER: "pytorch-nightly" + UPLOAD_USER: "vfdev-5" + run: | + chmod +x ./conda.recipe/build_and_upload.sh + ./conda.recipe/build_and_upload.sh + + - name: Build and Publish PyPi binaries + env: + PYPI_USER: ${{ secrets.PYPI_USER }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine check dist/* + TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/* + + #TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --verbose dist/* diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1f00f0d5cd7c..a081aa01b718 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -43,7 +43,7 @@ jobs: run: docker images | grep pytorchignite - name: Push all PyTorch-Ignite Docker images - if: github.event_name == 'push' + if: github.event_name != 'pull_request' env: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 46b5f7f49c2c..75666ee5c163 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -1,4 +1,5 @@ -on: +on: + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule schedule: # Run at 00:00 UTC Every Day - cron: '0 0 * * *' diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ec606044ee3f..0b605f95bdad 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -54,8 +54,6 @@ jobs: run: | conda install pytorch torchvision cpuonly -c ${{ matrix.pytorch-channel }} pip install -r requirements-dev.txt - # Fixes #1153 - pip install --upgrade scipy==1.4.1 python setup.py install - name: Run Mypy diff --git a/conda.recipe/build_and_upload.sh b/conda.recipe/build_and_upload.sh new file mode 100644 index 000000000000..46727f6e40fe --- /dev/null +++ b/conda.recipe/build_and_upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +echo "Build and upload Conda binaries" + +# ANACONDA_TOKEN should be provided +# How to generate ANACONDA_TOKEN: https://docs.anaconda.com/anaconda-cloud/user-guide/tasks/work-with-accounts#creating-access-tokens +# https://conda.io/docs/user-guide/tasks/build-packages/install-conda-build.html + +if [ -z $ANACONDA_TOKEN ]; then + echo "Can not find ANACONDA_TOKEN env variable" + echo "Please, export ANACONDA_TOKEN= before calling this script" + exit 1 +fi + +if [ -z $UPLOAD_USER ]; then + echo "Can not find UPLOAD_USER env variable" + echo "Please, export UPLOAD_USER= before calling this script" + exit 1 +fi + +set -xeu + +conda install -y conda-build conda-verify anaconda-client +conda config --set anaconda_upload no + +conda build --quiet --no-test --output-folder conda_build conda.recipe -c pytorch + +# Convert to other platforms: OSX, WIN +conda convert --platform win-64 conda_build/linux-64/*.tar.bz2 -o conda_build/ +conda convert --platform osx-64 conda_build/linux-64/*.tar.bz2 -o conda_build/ + +# Upload to Anaconda +# We could use --all but too much platforms to uploaded +ls conda_build/*/*.tar.bz2 | xargs -I {} anaconda -v -t $ANACONDA_TOKEN upload -u $UPLOAD_USER {} \ No newline at end of file From 0339cf91af1efbe79d760adaa4a773555df995f9 Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 10:31:13 +0000 Subject: [PATCH 2/7] Updated yml --- .github/workflows/binaries-nightly-release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml index 6ff47cc8d5c0..abdfb747992b 100644 --- a/.github/workflows/binaries-nightly-release.yml +++ b/.github/workflows/binaries-nightly-release.yml @@ -18,20 +18,23 @@ jobs: uses: goanpeca/setup-miniconda@v1 with: miniconda-version: "latest" - python-version: "3.7" + python-version: 3.7 - name: Setup nightly version run: | sed -i "s/__version__ = \"\(.*\)\"/__version__ = \"\1.dev$(date -u +%Y%m%d)\"/g" ignite/__init__.py + cat ignite/__init__.py - name: Install dependencies + shell: bash -l {0} run: | - conda install pytorch torchvision cpuonly -c pytorch-nightly + conda install -y pytorch torchvision cpuonly -c pytorch-nightly pip install -r requirements-dev.txt pip install --upgrade --no-cache-dir twine python setup.py install - name: Build and Publish Conda binaries + shell: bash -l {0} env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} #UPLOAD_USER: "pytorch-nightly" @@ -41,6 +44,7 @@ jobs: ./conda.recipe/build_and_upload.sh - name: Build and Publish PyPi binaries + shell: bash -l {0} env: PYPI_USER: ${{ secrets.PYPI_USER }} PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} From 936d654785ba2189d01c80eb278ba2b2e6c6113a Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 10:51:46 +0000 Subject: [PATCH 3/7] Updated to conda-incubator/setup-miniconda@v2 --- .github/workflows/binaries-nightly-release.yml | 2 +- .github/workflows/hvd-tests.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml index abdfb747992b..61e764819d8a 100644 --- a/.github/workflows/binaries-nightly-release.yml +++ b/.github/workflows/binaries-nightly-release.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1 + uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" python-version: 3.7 diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml index b1fcae72381e..8348fb793dda 100644 --- a/.github/workflows/hvd-tests.yml +++ b/.github/workflows/hvd-tests.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1 + uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 0b605f95bdad..1a4a00d333e4 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1 + uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" python-version: ${{ matrix.python-version }} From 70893753b1aaf2c84e55689b2c2c43e73fef249b Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 11:39:42 +0000 Subject: [PATCH 4/7] Reverted modifications in other actions --- .github/workflows/docker-publish.yml | 2 +- .github/workflows/hvd-tests.yml | 2 +- .github/workflows/pytorch-version-tests.yml | 1 - .github/workflows/unit-tests.yml | 4 +++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a081aa01b718..1f00f0d5cd7c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -43,7 +43,7 @@ jobs: run: docker images | grep pytorchignite - name: Push all PyTorch-Ignite Docker images - if: github.event_name != 'pull_request' + if: github.event_name == 'push' env: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml index 8348fb793dda..b1fcae72381e 100644 --- a/.github/workflows/hvd-tests.yml +++ b/.github/workflows/hvd-tests.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: goanpeca/setup-miniconda@v1 with: miniconda-version: "latest" python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 75666ee5c163..1f94bf1a817c 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -1,5 +1,4 @@ on: - # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule schedule: # Run at 00:00 UTC Every Day - cron: '0 0 * * *' diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1a4a00d333e4..ec606044ee3f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: goanpeca/setup-miniconda@v1 with: miniconda-version: "latest" python-version: ${{ matrix.python-version }} @@ -54,6 +54,8 @@ jobs: run: | conda install pytorch torchvision cpuonly -c ${{ matrix.pytorch-channel }} pip install -r requirements-dev.txt + # Fixes #1153 + pip install --upgrade scipy==1.4.1 python setup.py install - name: Run Mypy From d650b91569ee82dcfb33f7e01b0800c5c2845a03 Mon Sep 17 00:00:00 2001 From: vfdev Date: Tue, 10 Nov 2020 12:46:34 +0100 Subject: [PATCH 5/7] Migrated nightly build/release to GitHub Actions (#1448) * Added nightly build/release action * Updated yml * Updated to conda-incubator/setup-miniconda@v2 * Reverted modifications in other actions --- .../workflows/binaries-nightly-release.yml | 56 +++++++++++++++++++ .github/workflows/pytorch-version-tests.yml | 2 +- conda.recipe/build_and_upload.sh | 34 +++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/binaries-nightly-release.yml create mode 100644 conda.recipe/build_and_upload.sh diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml new file mode 100644 index 000000000000..61e764819d8a --- /dev/null +++ b/.github/workflows/binaries-nightly-release.yml @@ -0,0 +1,56 @@ +name: Nightly Releases + +on: pull_request +#on: +# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule +# schedule: +# # Run at 00:00 UTC Every Day +# - cron: '0 0 * * *' + + +jobs: + build-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + python-version: 3.7 + + - name: Setup nightly version + run: | + sed -i "s/__version__ = \"\(.*\)\"/__version__ = \"\1.dev$(date -u +%Y%m%d)\"/g" ignite/__init__.py + cat ignite/__init__.py + + - name: Install dependencies + shell: bash -l {0} + run: | + conda install -y pytorch torchvision cpuonly -c pytorch-nightly + pip install -r requirements-dev.txt + pip install --upgrade --no-cache-dir twine + python setup.py install + + - name: Build and Publish Conda binaries + shell: bash -l {0} + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + #UPLOAD_USER: "pytorch-nightly" + UPLOAD_USER: "vfdev-5" + run: | + chmod +x ./conda.recipe/build_and_upload.sh + ./conda.recipe/build_and_upload.sh + + - name: Build and Publish PyPi binaries + shell: bash -l {0} + env: + PYPI_USER: ${{ secrets.PYPI_USER }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine check dist/* + TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/* + + #TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --verbose dist/* diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 46b5f7f49c2c..1f94bf1a817c 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -1,4 +1,4 @@ -on: +on: schedule: # Run at 00:00 UTC Every Day - cron: '0 0 * * *' diff --git a/conda.recipe/build_and_upload.sh b/conda.recipe/build_and_upload.sh new file mode 100644 index 000000000000..46727f6e40fe --- /dev/null +++ b/conda.recipe/build_and_upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +echo "Build and upload Conda binaries" + +# ANACONDA_TOKEN should be provided +# How to generate ANACONDA_TOKEN: https://docs.anaconda.com/anaconda-cloud/user-guide/tasks/work-with-accounts#creating-access-tokens +# https://conda.io/docs/user-guide/tasks/build-packages/install-conda-build.html + +if [ -z $ANACONDA_TOKEN ]; then + echo "Can not find ANACONDA_TOKEN env variable" + echo "Please, export ANACONDA_TOKEN= before calling this script" + exit 1 +fi + +if [ -z $UPLOAD_USER ]; then + echo "Can not find UPLOAD_USER env variable" + echo "Please, export UPLOAD_USER= before calling this script" + exit 1 +fi + +set -xeu + +conda install -y conda-build conda-verify anaconda-client +conda config --set anaconda_upload no + +conda build --quiet --no-test --output-folder conda_build conda.recipe -c pytorch + +# Convert to other platforms: OSX, WIN +conda convert --platform win-64 conda_build/linux-64/*.tar.bz2 -o conda_build/ +conda convert --platform osx-64 conda_build/linux-64/*.tar.bz2 -o conda_build/ + +# Upload to Anaconda +# We could use --all but too much platforms to uploaded +ls conda_build/*/*.tar.bz2 | xargs -I {} anaconda -v -t $ANACONDA_TOKEN upload -u $UPLOAD_USER {} \ No newline at end of file From 43f76438b6c28606847935b775d0cdaae533c0cc Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 12:34:49 +0000 Subject: [PATCH 6/7] Fix PyPi upload --- .github/workflows/binaries-nightly-release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml index 61e764819d8a..1859a3f37f98 100644 --- a/.github/workflows/binaries-nightly-release.yml +++ b/.github/workflows/binaries-nightly-release.yml @@ -41,16 +41,13 @@ jobs: UPLOAD_USER: "vfdev-5" run: | chmod +x ./conda.recipe/build_and_upload.sh - ./conda.recipe/build_and_upload.sh +# ./conda.recipe/build_and_upload.sh - name: Build and Publish PyPi binaries shell: bash -l {0} - env: - PYPI_USER: ${{ secrets.PYPI_USER }} - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} run: | python setup.py sdist bdist_wheel twine check dist/* - TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/* + TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/* - #TWINE_USERNAME="$PYPI_USER" TWINE_PASSWORD="$PYPI_TOKEN" twine upload --verbose dist/* + #TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --verbose dist/* From 621622ea416eb9ae8e8bf149f34c9d6989879a1a Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Tue, 10 Nov 2020 12:59:45 +0000 Subject: [PATCH 7/7] Finalized binaries-nightly-release.yml --- .../workflows/binaries-nightly-release.yml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/binaries-nightly-release.yml b/.github/workflows/binaries-nightly-release.yml index 1859a3f37f98..e919da4499e6 100644 --- a/.github/workflows/binaries-nightly-release.yml +++ b/.github/workflows/binaries-nightly-release.yml @@ -1,11 +1,10 @@ name: Nightly Releases -on: pull_request -#on: -# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule -# schedule: -# # Run at 00:00 UTC Every Day -# - cron: '0 0 * * *' +on: + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule + schedule: + # Run at 00:00 UTC Every Day + - cron: '0 0 * * *' jobs: @@ -37,17 +36,14 @@ jobs: shell: bash -l {0} env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - #UPLOAD_USER: "pytorch-nightly" - UPLOAD_USER: "vfdev-5" + UPLOAD_USER: "pytorch-nightly" run: | chmod +x ./conda.recipe/build_and_upload.sh -# ./conda.recipe/build_and_upload.sh + ./conda.recipe/build_and_upload.sh - name: Build and Publish PyPi binaries shell: bash -l {0} run: | python setup.py sdist bdist_wheel twine check dist/* - TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/* - - #TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --verbose dist/* + TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --verbose dist/*