release.yml

name: Publishing Release
on:
  schedule:
    - cron: '0 20 * * *'  # GMT 20 == GMT+8 04
  workflow_dispatch:
    # Manually trigger the release workflow, a version must be provided
    inputs:
      version:
        description: 'The version to release (e.g. v0.8.0), or leave it empty for nightly build'
        type: string
        required: false

env:
  PROD_PWD: ${{ secrets.PYPI_PWD_PROD }}
  NIGHT_PWD: ${{ secrets.TAICHI_PYPI_PWD }}
  NIGHT_USERNAME: ${{ secrets.TAICHI_PYPI_USERNAME }}
  METADATA_USERNAME: ${{ secrets.METADATA_USERNAME }}
  METADATA_PASSWORD: ${{ secrets.METADATA_PASSWORD }}
  METADATA_URL: ${{ secrets.METADATA_URL }}
  RELEASE_VERSION: ${{ github.event.inputs.version }}
  NIGHTLY: ${{ !github.event.inputs.version && '--nightly' || '' }}
  CI_IMAGE_VERSION: '202311231829'
  TI_CI: "1"
  TI_SKIP_VERSION_CHECK: 'ON'
  TI_USE_GIT_CACHE: ${{ vars.TI_USE_GIT_CACHE }}


jobs:
  add_version_to_database:
    name: Add version to database
    # Skip running release workflow on forks
    if: github.repository_owner == 'taichi-dev'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Save new version
        run: |
          if [ -z "$RELEASE_VERSION" ]; then
            echo "Not a production release"
            exit 0
          fi
          python3 -m pip install requests==2.26
          python3 misc/save_new_version.py

  # This job set environment matrix with respect to production release and nightly release.
  matrix_prep:
    runs-on: ubuntu-latest
    needs: add_version_to_database
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      matrix_osx: ${{ steps.set-matrix.outputs.matrix_osx }}
    steps:
      - id: set-matrix
        run: |
          if [ -n "$RELEASE_VERSION" ]; then
            # For production release, we run on five python versions.
            echo 'matrix={"include":[{"name":"taichi","python":"3.7"},{"name":"taichi","python":"3.8"},{"name":"taichi","python":"3.9"},{"name":"taichi","python":"3.10"},{"name":"taichi","python":"3.11"}]}"' >> $GITHUB_OUTPUT

            # M1 only supports 3.8, 3.9, and 3.10(conda), so change matrix.
            echo 'matrix_osx={"include":[{"name":"taichi","python":"3.8"},{"name":"taichi","python":"3.9"},{"name":"taichi","python":"3.10"},{"name":"taichi","python":"3.11"}]}"' >> $GITHUB_OUTPUT
          else
            # For nightly release, we run on three python versions.
            echo 'matrix={"include":[{"name":"taichi-nightly","python":"3.7"},{"name":"taichi-nightly","python":"3.8"},{"name":"taichi-nightly","python":"3.9"},{"name":"taichi-nightly","python":"3.10"},{"name":"taichi-nightly","python":"3.11"}]}"' >> $GITHUB_OUTPUT

            # M1 only supports 3.8 and 3.10(conda), so change matrix.
            echo 'matrix_osx={"include":[{"name":"taichi-nightly","python":"3.8"},{"name":"taichi-nightly","python":"3.9"},{"name":"taichi-nightly","python":"3.10"},{"name":"taichi-nightly","python":"3.11"}]}"' >> $GITHUB_OUTPUT
          fi

  build_and_test_linux:
    name: Build and Upload (linux only)
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
    runs-on: [self-hosted, cuda, vulkan, cn, release, sm70]
    env:
      PY: ${{ matrix.python }}
      PROJECT_NAME: ${{ matrix.name }}
      TAICHI_CMAKE_ARGS: >-
        -DTI_WITH_OPENGL:BOOL=ON
        -DTI_WITH_VULKAN:BOOL=ON
        -DTI_BUILD_TESTS:BOOL=ON

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: '0'

      - name: Prepare Environment
        run: |
          . .github/workflows/scripts/common-utils.sh
          prepare-build-cache
          echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV

      - name: Build
        run: |
          . .github/workflows/scripts/common-utils.sh

          ci-docker-run-gpu --name taichi-build \
            registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \
            /home/dev/taichi/.github/workflows/scripts/build.py $NIGHTLY

      - name: Archive Wheel Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist/*.whl
          retention-days: 20

      - name: Test
        run: |
          . .github/workflows/scripts/common-utils.sh

          ci-docker-run-gpu --name taichi-test \
             registry.botmaster.tgr/taichi-test-cuda:${{ env.CI_IMAGE_VERSION }} \
             /home/dev/taichi/.github/workflows/scripts/unix_test.sh
        env:
          PY: ${{ matrix.python }}

  build_and_test_manylinux2014:
    name: Build and Upload (manylinux2014)
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
    runs-on: [self-hosted, cuda, cn, release]
    env:
      PY: ${{ matrix.python }}
      PROJECT_NAME: ${{ matrix.name }}
      TAICHI_CMAKE_ARGS: >-
        -DTI_WITH_OPENGL:BOOL=OFF
        -DTI_WITH_VULKAN:BOOL=OFF
        -DTI_BUILD_TESTS:BOOL=ON

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: '0'

      - name: Prepare Environment
        run: |
          . .github/workflows/scripts/common-utils.sh
          prepare-build-cache
          echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV

      - name: Build
        run: |
          . .github/workflows/scripts/common-utils.sh

          ci-docker-run-gpu --name taichi-test \
             registry.botmaster.tgr/taichi-build-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \
             /home/dev/taichi/.github/workflows/scripts/build.py $NIGHTLY

      - name: Archive Wheel Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist/*.whl
          retention-days: 20

      - name: Test
        run: |
          . .github/workflows/scripts/common-utils.sh

          ci-docker-run-gpu --name taichi-test \
             registry.botmaster.tgr/taichi-test-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \
             /home/dev/taichi/.github/workflows/scripts/unix_test.sh
        env:
          PY: ${{ matrix.python }}
          TI_WANTED_ARCHS: "cuda,cpu"

  build_and_test_mac:
    name: Build and Upload (macOS only)
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
    runs-on: [self-hosted, macos-10.15]
    env:
      PY: ${{ matrix.python }}
      CXX: clang++
      TAICHI_CMAKE_ARGS: >-
        -DTI_WITH_VULKAN:BOOL=ON
        -DTI_WITH_OPENGL:BOOL=OFF
        -DTI_BUILD_TESTS:BOOL=ON
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: '0'

      - name: Prepare Environment
        run: |
          . .github/workflows/scripts/common-utils.sh
          prepare-build-cache

      - name: Create Python Wheel
        run: |
          # Use the molten-vk v1.1.10 downloaded from taichi assets
          brew uninstall molten-vk -f
          .github/workflows/scripts/build.py $NIGHTLY
        env:
          PROJECT_NAME: ${{ matrix.name }}

      - name: Archive Wheel Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist/*.whl
          retention-days: 20

      - name: Test
        run: .github/workflows/scripts/unix_test.sh
        env:
          TI_WANTED_ARCHS: 'cpu'
          TI_SKIP_CPP_TESTS: Disabled because Vulkan is supported but not working on buildbot4

  build_and_test_m1:
    name: Build and Upload (Apple M1)
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix_osx) }}
    runs-on: [self-hosted, m1, release]
    defaults:
      run:
        shell: '/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}'
    env:
      PY: ${{ matrix.python }}
      CXX: clang++
      PROJECT_NAME: ${{ matrix.name }}
      TAICHI_CMAKE_ARGS: >-
        -DTI_WITH_OPENGL:BOOL=OFF
        -DTI_WITH_CUDA:BOOL=OFF
        -DTI_WITH_VULKAN:BOOL=ON
        -DTI_BUILD_TESTS:BOOL=ON

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: '0'

      - name: Prepare Environment
        run: |
          . .github/workflows/scripts/common-utils.sh
          prepare-build-cache

      - name: Build
        run: |
          brew install molten-vk
          .github/workflows/scripts/build.py $NIGHTLY

      - name: Archive Wheel Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist/*.whl
          retention-days: 20

      - name: Test
        run: |
          .github/workflows/scripts/unix_test.sh
        env:
          TI_WANTED_ARCHS: 'metal,vulkan,cpu'
          PY: ${{ matrix.python }}
          GPU_TEST: ON

  build_and_test_windows:
    name: Build and Upload (Windows only)
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
    # FIXME: force running on sm70 for now. Should be refactored out
    runs-on: [self-hosted, windows, cuda, OpenGL, sm70]
    env:
      PY: ${{ matrix.python }}
      PROJECT_NAME: ${{ matrix.name }}
      TAICHI_CMAKE_ARGS: >-
        -DTI_WITH_OPENGL:BOOL=ON
        -DTI_WITH_VULKAN:BOOL=ON
        -DTI_WITH_DX11:BOOL=ON
        -DTI_WITH_DX12:BOOL=ON
        -DTI_BUILD_TESTS:BOOL=ON
        -DTI_WITH_C_API=ON

    steps:
      - name: Workaround checkout Needed single revision issue
        run: |
          git config --system core.longpaths true
          git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD'

      - uses: actions/checkout@v3
        with:
          submodules: 'recursive'
          fetch-depth: '0'

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}

      - name: Build
        shell: pwsh
        run: |
          $nightlyFlag = $null
          if ($env:NIGHTLY) { $nightlyFlag = "--nightly" }
          python .\.github\workflows\scripts\build.py $nightlyFlag

      - name: Archive Wheel Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist/*.whl
          retention-days: 20

      - name: Test
        shell: pwsh
        run: |
          .\.github\workflows\scripts\win_test.ps1
        env:
          PY: ${{ matrix.python }}
          TI_WANTED_ARCHS: "cpu,cuda,opengl,vulkan"
          TI_SKIP_VERSION_CHECK: ON

      - name: Cleanup Git Cache Configs
        shell: pwsh
        if: always()
        run: |
          . .github/workflows/scripts/common-utils.ps1
          UnsetGitCachingProxy
          exit 0

  upload_to_pypi:
    name: Upload release to PyPI
    needs: [build_and_test_linux, build_and_test_manylinux2014, build_and_test_mac, build_and_test_m1, build_and_test_windows]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Get dist files
        uses: actions/download-artifact@v3
        with:
          name: dist
          path: dist

      - name: Upload to PyPI
        run: |
          set -ex
          ls -l dist/
          if [ -z "$RELEASE_VERSION" ]; then
            export PROJECT_NAME="taichi-nightly"
          else
            export PROJECT_NAME="taichi"
            VER=$(echo $RELEASE_VERSION | sed 's/^v//')
            if [ $(ls dist/taichi-$VER-*.whl | wc -l) != $(ls dist/*.whl | wc -l) ]; then
              echo Version number mismatch
              exit 1
            fi
            pip install -f dist taichi==$VER
            if ! python -c 'import taichi' | grep "^.Taichi. version $VER,"; then
              echo Taichi init version number mismatch!!
              exit 1
            fi
          fi
          python -m pip install requests twine
          python misc/upload_release.py

  create_release:
    name: Create tag and publish release
    needs: upload_to_pypi
    runs-on: ubuntu-latest
    if: ${{ github.event.inputs.version }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Generate Changelog
        id: changelog
        run: |
          pip3 install gitpython
          python3 misc/make_changelog.py > CHANGELOG.md
          cat CHANGELOG.md

      - name: Create tag
        run: |
          git config user.email "taichigardener@gmail.com"
          git config user.name "Taichi Gardener"
          git tag -a ${RELEASE_VERSION} -m "Release ${RELEASE_VERSION}"
          git push origin --tags

      - name: Publish release
        uses: softprops/action-gh-release@v1
        with:
          body_path: CHANGELOG.md
          tag_name: ${{ github.event.inputs.version }}