release.yml

name: Release

on:
  push:
    tags:
      - "v*"
  schedule:
    - cron: "0 22 * * *"
  workflow_dispatch:
    inputs:
      tags:
        description: The tags to be released
        required: false
        type: string

permissions:
  id-token: write
  pull-requests: write
  checks: write
  statuses: write
  contents: write

env:
  BUILD_PROFILE: release

jobs:
  create_release:
    name: create release
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.generated-tag.outputs.tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Get latest tag
        id: get-latest-tag
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          echo "tag=`gh release list -L 1 | cut -f 1`" >> $GITHUB_OUTPUT
      - name: Bump version
        id: generated-tag
        uses: actions/github-script@v6
        with:
          script: |
            if (context.ref.startsWith("refs/tags/")) {
              let tag = context.ref.replace("refs/tags/", "");
              core.setOutput('tag', tag);
              console.log(`This event pushed a tag ${tag}, return directly.`)
              return
            }

            if ("${{ github.event.inputs.tags }}") {
              let tag = "${{ github.event.inputs.tags }}";
              core.setOutput('tag', tag);
              console.log(`This event triggered by workflow_dispatch with a tag ${tag}, return directly.`)
              return
            }

            let tag = "${{ steps.get-latest-tag.outputs.tag }}";
            let result = /v(\d+)\.(\d+)\.(\d+)/g.exec(tag);
            if (result === null) {
              throw `The previous tag ${{ steps.get-latest-tag.outputs.tag }} is invalid, ignoring`;
            }

            let major = result[1];
            let minor = result[2];
            let patch = (parseInt(result[3]) + 1).toString();
            let next_tag = `v${major}.${minor}.${patch}-nightly`;

            console.log(`This event is triggered, return generated ${next_tag}.`)
            core.setOutput('tag', next_tag)
      - name: Create github release if not exist
        # Only create release when the tag is not exist
        if: 'steps.generated-tag.outputs.tag != steps.get-latest-tag.outputs.tag'
        # Allow this action failure
        # continue-on-error: true
        # Reference: https://cli.github.com/manual/gh_release_create
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          echo "Create a release for ${{ steps.generated-tag.outputs.tag }}"
          gh release create ${{ steps.generated-tag.outputs.tag }} --generate-notes -p

  macos:
    runs-on: macos-11
    needs: create_release
    env:
      RUNNER_PROVIDER: github
    strategy:
      fail-fast: false
      matrix:
        arch:
          - x86_64
          - aarch64
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Get target
        id: target
        run: echo 'target=${{ matrix.arch }}-apple-darwin' >> $GITHUB_OUTPUT
      - name: Install coreutils for macOS sha256sum
        run: brew install coreutils
      - name: Build Release
        uses: ./.github/actions/build_macos
        with:
          target: ${{ steps.target.outputs.target }}
          artifacts: sqllogictests,metactl,meta,query
          upload: false
      - name: Basic Sqllogic Test
        if: matrix.arch == 'x86_64'
        shell: bash
        env:
          TEST_HANDLERS: http
        run: |
          mkdir -p target/release
          cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,metactl,query,sqllogictests} ./target/release/
          bash ./scripts/ci/ci-run-sqllogic-tests.sh base
      - name: Pack binaries
        run: |
          brew install gnu-tar
          sudo /usr/sbin/purge
          target=${{ steps.target.outputs.target }}
          version=${{ needs.create_release.outputs.version }}
          mkdir -p release/${target}/{bin,configs,scripts}
          cp ./target/${target}/release/databend-* release/${target}/bin/
          rm -f release/${target}/bin/*.d
          cp ./scripts/distribution/configs/databend-* release/${target}/configs/
          cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
          cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
          gtar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs scripts readme.txt
      - name: generate sha256sums
        run: |
          target=${{ steps.target.outputs.target }}
          version=${{ needs.create_release.outputs.version }}
          sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
      - name: post sha256
        uses: actions/upload-artifact@v3
        with:
          name: sha256sums
          path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
          retention-days: 1
      - name: Publish Binaries
        uses: ./.github/actions/publish_binary
        env:
          GH_TOKEN: ${{ github.token }}
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        with:
          version: ${{ needs.create_release.outputs.version }}
          target: ${{ steps.target.outputs.target }}

  linux:
    runs-on: [self-hosted, X64, Linux, 16c32g, gcp]
    needs: create_release
    env:
      RUNNER_PROVIDER: gcp
    strategy:
      fail-fast: false
      matrix:
        arch:
          - x86_64
          - aarch64
        libc:
          - gnu
          - musl
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Get the version
        id: get_version
        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
      - name: Get target
        id: target
        run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT
      - name: Build Release
        uses: ./.github/actions/build_linux
        with:
          sha: ${{ github.sha }}
          target: ${{ steps.target.outputs.target }}
          artifacts: sqllogictests,sqlsmith,metactl,meta,query
          upload: false
      - name: Basic Sqllogic Test
        if: matrix.arch == 'x86_64'
        shell: bash
        env:
          TEST_HANDLERS: http
        run: |
          mkdir -p target/release
          cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,query,sqllogictests} ./target/release/
          bash ./scripts/ci/ci-run-sqllogic-tests.sh base
      - name: Pack Binaries
        run: |
          target=${{ steps.target.outputs.target }}
          version=${{ needs.create_release.outputs.version }}
          mkdir -p release/${target}/{bin,configs,systemd,scripts}
          cp ./target/${target}/release/databend-{query,meta,metactl} release/${target}/bin/
          rm -f release/${target}/bin/*.d
          cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/
          cp ./scripts/distribution/configs/databend-* release/${target}/configs/
          cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
          cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
          cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
          tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
          sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
      - name: Pack Testsuite
        if: steps.target.outputs.target == 'x86_64-unknown-linux-gnu'
        run: |
          target=${{ steps.target.outputs.target }}
          version=${{ needs.create_release.outputs.version }}
          mkdir -p release/testsuite/bin
          cp -r ./tests/sqllogictests/suites ./release/testsuite/
          cp ./target/${target}/release/databend-{sqllogictests,sqlsmith} release/testsuite/bin/
          tar -C ./release/testsuite -czvf databend-testsuite-${version}-${target}.tar.gz bin suites
          sha256sum databend-testsuite-${version}-${target}.tar.gz >> sha256-testsuite-${version}-${target}.txt
      - name: post sha256
        uses: actions/upload-artifact@v3
        with:
          name: sha256sums
          path: sha256-*.txt
          retention-days: 1
      - name: Publish Binaries
        uses: ./.github/actions/publish_binary
        env:
          GH_TOKEN: ${{ github.token }}
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        with:
          version: ${{ needs.create_release.outputs.version }}
          target: ${{ steps.target.outputs.target }}
      - name: Publish Testsuite
        if: steps.target.outputs.target == 'x86_64-unknown-linux-gnu'
        uses: ./.github/actions/publish_binary
        env:
          GH_TOKEN: ${{ github.token }}
        with:
          version: ${{ needs.create_release.outputs.version }}
          target: ${{ steps.target.outputs.target }}
          category: testsuite

  hdfs:
    runs-on: [self-hosted, X64, Linux, 16c32g, gcp]
    needs: create_release
    env:
      RUNNER_PROVIDER: gcp
    strategy:
      fail-fast: false
      matrix:
        arch:
          - x86_64
        libc:
          - gnu
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Get the version
        id: get_version
        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
      - name: Get target
        id: target
        run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT
      - name: Setup Build Tool
        uses: ./.github/actions/setup_build_tool
        with:
          target: ${{ steps.target.outputs.target }}
      - name: Build Release
        uses: ./.github/actions/build_linux
        with:
          sha: ${{ github.sha }}
          target: ${{ steps.target.outputs.target }}
          artifacts: meta,query
          category: hdfs
          features: storage-hdfs
          upload: false
      - name: Pack binaries
        run: |
          target=${{ steps.target.outputs.target }}
          version=${{ needs.create_release.outputs.version }}
          mkdir -p release/${target}/{bin,configs,systemd,scripts}
          cp ./target/${target}/release/databend-* release/${target}/bin/
          rm -f release/${target}/bin/*.d
          cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/
          cp ./scripts/distribution/configs/databend-* release/${target}/configs/
          cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
          cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
          cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
          tar -C ./release/${target} -czvf databend-hdfs-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
          sha256sum databend-hdfs-${version}-${target}.tar.gz >> sha256-hdfs-${version}-${target}.txt
      - name: post sha256
        uses: actions/upload-artifact@v3
        with:
          name: sha256sums
          path: sha256-*.txt
          retention-days: 1
      - name: Publish Binaries
        uses: ./.github/actions/publish_binary
        env:
          GH_TOKEN: ${{ github.token }}
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        with:
          version: ${{ needs.create_release.outputs.version }}
          target: ${{ steps.target.outputs.target }}
          category: hdfs

  docker_combined:
    name: docker combined
    runs-on: ubuntu-latest
    needs: [create_release, linux]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Get the version
        id: get_version
        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
      - name: Download binaries for usage
        id: download_binaries
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          version="${{ needs.create_release.outputs.version }}"
          declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
          mkdir -p ./distro/
          for platform in ${!platform_targets[@]}; do
              target=${platform_targets[$platform]}
              gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
              mkdir -p ./target/${target}/release
              tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
              mkdir -p ./distro/linux/${platform}
              cp ./target/${target}/release/databend-* ./distro/linux/${platform}
          done
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - uses: ./.github/actions/setup_docker
        id: login
        with:
          repo: databend
          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v3
        with:
          push: true
          tags: |
            ${{ steps.login.outputs.dockerhub_repo }}:latest
            ${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}
            ${{ steps.login.outputs.ecr_repo }}:latest
            ${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}
          platforms: linux/amd64,linux/arm64
          context: .
          file: ./docker/Dockerfile
      - name: Update repo description
        continue-on-error: true
        uses: peter-evans/dockerhub-description@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
          repository: ${{ steps.login.outputs.dockerhub_repo }}
          short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com."
          readme-filepath: ./docker/README.md

  docker_separate:
    name: docker separate
    runs-on: ubuntu-latest
    needs: [create_release, linux]
    strategy:
      fail-fast: false
      matrix:
        service:
          - meta
          - query
        distro:
          - debian
          - distroless
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Get the version
        id: get_version
        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
      - name: Download binaries for usage
        id: download_binaries
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          version="${{ needs.create_release.outputs.version }}"
          declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
          mkdir -p ./distro/
          for platform in ${!platform_targets[@]}; do
              target=${platform_targets[$platform]}
              gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
              mkdir -p ./target/${target}/release
              tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
              mkdir -p ./distro/linux/${platform}
              cp ./target/${target}/release/databend-* ./distro/linux/${platform}
          done
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - uses: ./.github/actions/setup_docker
        id: login
        with:
          repo: databend-${{ matrix.service }}
          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: get image tags
        id: get_image_tags
        shell: bash
        run: |
          _tags="${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
          _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
          if [[ "${{ matrix.distro }}" == "debian" ]]; then
            _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}"
            _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}"
            _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:latest"
            _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest"
          fi
          echo "IMAGE_TAGS=${_tags}" >> $GITHUB_OUTPUT
      - name: push service image
        uses: docker/build-push-action@v3
        with:
          push: true
          tags: ${{ steps.get_image_tags.outputs.IMAGE_TAGS }}
          platforms: linux/amd64,linux/arm64
          context: .
          file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile

  distribution:
    runs-on: ubuntu-latest
    needs: [create_release, linux]
    strategy:
      matrix:
        arch:
          - x86_64
          - aarch64
        packager:
          - deb
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install nfpm@latest
        run: |
          curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
          tar xf nfpm.tar.gz
          sudo mv nfpm /usr/local/bin
          sudo chmod a+x /usr/local/bin/nfpm
          rm nfpm.tar.gz
      - name: Get target
        id: target
        run: |
          echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
      - name: Download binaries for usage
        id: download_binaries
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          target=${{ steps.target.outputs.target }}
          version="${{ needs.create_release.outputs.version }}"
          mkdir -p ./distro/
          gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
          tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz
      - name: Build Packages
        id: build_packages
        run: |
          export name="databend"
          export version="${{ needs.create_release.outputs.version }}"
          export path="distro"
          case "${{ matrix.arch }}" in
            x86_64)
              export arch="amd64"
              ;;
            aarch64)
              export arch="arm64"
              ;;
          esac
          nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
      - name: Update release to github
        shell: bash
        env:
          GH_TOKEN: ${{ github.token }}
        # Reference: https://cli.github.com/manual/gh_release_upload
        run: |
          version="${{ needs.create_release.outputs.version }}"
          # name looks like: `databend_0.8.144~nightly_amd64.deb`
          gh release upload ${version} databend_*.${{ matrix.packager }} --clobber

  deb:
    runs-on: ubuntu-latest
    needs: [create_release, distribution]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/publish_deb
        env:
          GH_TOKEN: ${{ github.token }}
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        with:
          version: ${{ needs.create_release.outputs.version }}
          gpg_signing_key: ${{ secrets.GPG_KEY_DEB }}

  sha256sums:
    needs: [create_release, linux, macos, distribution]
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
      - name: download sha256sums
        uses: actions/download-artifact@v3
        with:
          name: sha256sums
      - shell: bash
        run: |
          for file in *.txt
          do
            cat ${file} >> sha256sums.txt
          done
      - name: Upload checksums
        shell: bash
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          version="${{ needs.create_release.outputs.version }}"
          gh release upload ${version} sha256sums.txt --clobber

  benchmark:
    needs: [create_release, linux]
    uses: ./.github/workflows/reuse.benchmark.yml
    secrets: inherit
    with:
      sha: ${{ github.sha }}
      run_id: ${{ github.run_id }}
      source: release
      source_id: ${{ needs.create_release.outputs.version }}
      version: ${{ needs.create_release.outputs.version }}
      runner_provider: github

  sqlsmith:
    needs: [create_release, linux]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Download artifact for release
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          version=${{ needs.create_release.outputs.version }}
          target=x86_64-unknown-linux-gnu
          mkdir -p ./distro/
          mkdir -p ./target/release/
          gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
          gh release download ${version} --pattern "databend-testsuite-${version}-${target}.tar.gz" --dir distro/
          tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
          tar x -C ./target/release -f ./distro/databend-testsuite-${version}-${target}.tar.gz --strip-components 1 bin/
          chmod +x ./target/release/databend-*
      - name: Run sqlsmith
        timeout-minutes: 60
        shell: bash
        run: |
          bash ./scripts/ci/ci-run-sqlsmith-tests.sh
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-sqlsmith

  bindings_python:
    needs: create_release
    uses: ./.github/workflows/bindings.python.yml
    secrets: inherit
    with:
      tag: ${{ needs.create_release.outputs.version }}

  notify:
    runs-on: ubuntu-latest
    if: always()
    needs:
      - create_release
      - linux
      - docker_combined
      - docker_separate
      - distribution
      - deb
    steps:
      - uses: actions/checkout@v4
      - run: |
          status="${{ (contains(needs.*.result, 'failure') && 'failure') || (contains(needs.*.result, 'cancelled') && 'cancelled') || 'success' }}"
          jq -n -f .github/release-report.jq \
            --arg title "[Release] ${{ needs.create_release.outputs.version }}" \
            --arg content "Build result: ${status}" \
            --arg link "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            > /tmp/release-report.json
          curl -X POST "${{ secrets.RELEASE_REPORT_WEBHOOK }}" \
            -H 'Content-Type: application/json' \
            -H 'cache-control: no-cache' \
            -d @/tmp/release-report.json