diff --git a/.github/workflows/build-4.0.yml b/.github/workflows/build-4.0.yml new file mode 100644 index 00000000000..e41fde20444 --- /dev/null +++ b/.github/workflows/build-4.0.yml @@ -0,0 +1,476 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build (4.0) + +on: + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + schedule: + - cron: '*/30 * * * *' + +jobs: + prerelease: + name: Prerelease + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + outputs: + should_release: ${{ steps.check_diff.outputs.should_release }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'apache/doris' + ref: 'branch-4.0' + fetch-depth: 0 + + - name: Check Diff + id: check_diff + run: | + tag_name='automation-4.0' + title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + + if [[ -z "$(gh release list)" ]] || + ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then + gh release create -t "${title}" "${tag_name}" + fi + last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + current_version="$(git log -1 --format='%H')" + + echo "Last Version: ${last_version}" + echo "Current Version: ${current_version}" + + should_release=false + if [[ -z "${last_version}" ]]; then + echo "The first release was detected." + should_release=true + elif [[ "${last_version}" != "${current_version}" ]]; then + cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'" + echo "Execute: ${cmd}" + content="$(eval "${cmd}")" || true + if [[ -n "${content}" ]]; then + echo -e "Detect changes:\n${content}" + should_release=true + fi + fi + + if "${should_release}"; then + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + else + gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ + s/Update Time:.*/Update Time: *$(date)*/ + s/Doris Version:.*/Doris Version: *${current_version}*/ + }" >release_note.md + fi + gh release edit -F release_note.md "${tag_name}" + + echo "should_release=${should_release}" >> $GITHUB_OUTPUT + + - name: Download Source and Upload + if: steps.check_diff.outputs.should_release == 'true' + run: | + tag_name='automation-4.0' + + cd thirdparty + # Pre-download packages whose default mirror is blocked or gone on GH Actions runners: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is blocked from GH Actions + mkdir -p src + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - + + tar -zcvf doris-thirdparty-source.tgz src + + gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz + + build: + name: Build + needs: prerelease + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') + strategy: + matrix: + config: + - name: macOS-x86_64 + os: macos-13 + packages: >- + 'm4' + 'automake' + 'autoconf' + 'libtool' + 'pkg-config' + 'texinfo' + 'coreutils' + 'gnu-getopt' + 'python@3' + 'ninja' + 'ccache' + 'bison' + 'byacc' + 'gettext' + 'wget' + 'pcre' + 'openjdk@11' + 'maven' + 'node' + 'llvm@20' + + - name: macOS-arm64 + os: macos-14 + packages: >- + 'm4' + 'automake' + 'autoconf' + 'libtool' + 'pkg-config' + 'texinfo' + 'coreutils' + 'gnu-getopt' + 'python@3' + 'ninja' + 'ccache' + 'bison' + 'byacc' + 'gettext' + 'wget' + 'pcre' + 'openjdk@11' + 'maven' + 'node' + 'llvm@20' + + - name: Linux + os: ubuntu-22.04 + packages: >- + 'build-essential' + 'automake' + 'autoconf' + 'libtool-bin' + 'pkg-config' + 'cmake=3.22.1-1ubuntu1.22.04.2' + 'ninja-build' + 'ccache' + 'python-is-python3' + 'bison' + 'byacc' + 'flex' + 'binutils-dev' + 'libiberty-dev' + 'curl' + 'git' + 'zip' + 'unzip' + 'autopoint' + 'openjdk-8-jdk' + 'openjdk-8-jdk-headless' + 'maven' + + runs-on: ${{ matrix.config.os }} + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout easimon/maximize-build-space + if: ${{ matrix.config.name == 'Linux' }} + run: | + git clone -b v7 https://github.com/easimon/maximize-build-space + + #- name: Maximize build space + # if: ${{ matrix.config.name == 'Linux' }} + # uses: ./maximize-build-space + # with: + # root-reserve-mb: 4096 + # swap-size-mb: 8192 + # remove-dotnet: 'true' + # remove-android: 'true' + # remove-haskell: 'true' + # remove-codeql: 'true' + # remove-docker-images: 'true' + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'apache/doris' + ref: 'branch-4.0' + + - name: Download + run: | + tag_name='automation-4.0' + + cd thirdparty + curl -L "https://github.com/${{ github.repository }}/releases/download/${tag_name}/doris-thirdparty-source.tgz" \ + -o doris-thirdparty-source.tgz + tar -zxvf doris-thirdparty-source.tgz + + - name: Prepare for ${{ matrix.config.os }} + run: | + if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then + # Install packages except cmake + brew install ${{ matrix.config.packages }} || true + # Install specific version of cmake + brew unlink cmake || true + wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz + tar -xzf cmake-3.22.1-macos-universal.tar.gz + sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + cmake --version + else + export DEFAULT_DIR='/opt/doris' + export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" + + sudo apt update + sudo apt-cache policy cmake + sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }} + + mkdir -p "${DEFAULT_DIR}" + wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.25/ldb_toolchain_gen.sh \ + -q -O /tmp/ldb_toolchain_gen.sh + bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" + fi + + - name: Build and Upload + run: | + tag_name='automation-4.0' + + if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then + export DEFAULT_DIR='/opt/doris' + export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" + export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" + export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" + export DORIS_TOOLCHAIN=clang + else + export MACOSX_DEPLOYMENT_TARGET=12.0 + fi + + export CMAKE_POLICY_VERSION_MINIMUM=3.10 + export CUSTOM_CMAKE="/usr/local/bin/cmake" + cd thirdparty + # Fix packages that may be empty if the source tarball was built before the mirror fixes: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) blocked from GH Actions + if [[ ! -s src/lzo-2.10.tar.gz ]]; then + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + fi + if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + fi + ./build-thirdparty.sh -j "$(nproc)" + + kernel="$(uname -s | awk '{print tolower($0)}')" + arch="$(uname -m)" + rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # Delete existing asset first to avoid gh CLI --clobber race-condition bug + # (gh release upload --clobber sometimes returns "release not found" exit 1 + # when deleting an existing asset, even though the upload itself succeeds) + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" 2>/dev/null || true + gh release upload "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + + update-docker: + name: Update Docker Image + needs: [prerelease, build] + if: | + always() && ( + (needs.prerelease.outputs.should_release == 'true' && needs.build.result == 'success') || + (github.event_name == 'workflow_dispatch' && needs.build.result != 'failure') + ) + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout docker/setup-buildx-action + run: | + rm -rf ./.github/actions/setup-buildx-action + git clone https://github.com/docker/setup-buildx-action .github/actions/setup-buildx-action + + pushd .github/actions/setup-buildx-action &>/dev/null + git checkout 8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + popd &>/dev/null + + - name: Checkout docker/login-action + run: | + rm -rf ./.github/actions/login-action + git clone https://github.com/docker/login-action .github/actions/login-action + + pushd .github/actions/login-action &>/dev/null + git checkout c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + popd &>/dev/null + + - name: Checkout docker/build-push-action + run: | + rm -rf ./.github/actions/build-push-action + git clone https://github.com/docker/build-push-action .github/actions/build-push-action + + pushd .github/actions/build-push-action &>/dev/null + git checkout ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + popd &>/dev/null + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx-action + + - name: Login to Docker Hub + uses: ./.github/actions/login-action + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Prepare Dockerfiles + run: | + # Pull the upstream Dockerfile from apache/doris branch-4.0 to stay in sync with + # any toolchain / dependency changes they make. + curl -fsSL https://raw.githubusercontent.com/apache/doris/branch-4.0/docker/compilation/Dockerfile \ + -o Dockerfile.upstream + + python3 - << 'PYEOF' + import sys, re + + with open('Dockerfile.upstream') as f: + content = f.read() + + # Patch 1: fix epel metalink. + # The upstream RUN line installs epel-release then immediately runs yum + # install/clean/makecache, but epel.repo's metalink may be unreliable. + # Insert a sed fix between epel-release install and the next yum install. + old_epel = 'yum install epel-release -y && yum install https://packages.endpointdev.com' + new_epel = ('yum install epel-release -y \\\n' + ' && sed -i \\\n' + ' -e \'s/^metalink=/#metalink=/\' \\\n' + ' -e \'s|^#baseurl=http://download.fedoraproject.org/pub/epel/7|baseurl=https://mirrors.aliyun.com/epel/7|\' \\\n' + ' /etc/yum.repos.d/epel*.repo \\\n' + ' && yum install https://packages.endpointdev.com') + assert old_epel in content, f"Patch 1 failed: target string not found in upstream Dockerfile" + patched = content.replace(old_epel, new_epel, 1) + assert patched != content, "Patch 1 was a no-op: upstream Dockerfile may have changed" + + # Patch 2: replace "clone & build thirdparty" block with downloading + # our prebuilt artifact. The block starts with "# clone lastest source + # code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris". + prebuilt_block = ( + '# Download prebuilt thirdparty from GitHub Release (built by doris-thirdparty automation)\n' + 'ARG GITHUB_REPOSITORY\n' + 'RUN mkdir -p /var/local/thirdparty \\\n' + ' && wget -q "https://github.com/${GITHUB_REPOSITORY}/releases/download/automation-4.0/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \\\n' + ' -O /tmp/prebuilt.tar.xz \\\n' + ' && tar -xf /tmp/prebuilt.tar.xz -C /var/local/thirdparty \\\n' + ' && rm /tmp/prebuilt.tar.xz\n' + ) + patched_2 = re.sub( + r'# clone lastest source code.*?rm -rf \$\{DEFAULT_DIR\}/doris\n', + prebuilt_block, + patched, flags=re.DOTALL + ) + assert patched_2 != patched, "Patch 2 was a no-op: 'clone lastest source code' block not found in upstream Dockerfile" + patched = patched_2 + + # Write normal image Dockerfile + with open('Dockerfile.patched', 'w') as f: + f.write(patched) + + # Patch 3 (no-avx2): add USE_AVX2=0 to the ENV block in builder stage. + # The ENV block ends with PATH=... just before "# install ccache". + old_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH"\n # USE_AVX2=0' + new_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH" \\\n USE_AVX2=0' + assert old_avx2 in patched, "Patch 3 failed: '# USE_AVX2=0' comment not found; upstream Dockerfile may have changed" + noavx2 = patched.replace(old_avx2, new_avx2) + assert noavx2 != patched, "Patch 3 was a no-op: no-avx2 Dockerfile is identical to normal Dockerfile" + with open('Dockerfile.patched-noavx2', 'w') as f: + f.write(noavx2) + + print("=== normal: check ===") + for line in open('Dockerfile.patched'): + if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): + print(line, end='') + print("=== noavx2: check ===") + for line in open('Dockerfile.patched-noavx2'): + if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): + print(line, end='') + PYEOF + + - name: Build and Push Docker Image (normal) + uses: ./.github/actions/build-push-action + with: + context: . + file: Dockerfile.patched + build-args: | + GITHUB_REPOSITORY=${{ github.repository }} + push: true + tags: ${{ vars.DOCKER_TAGS_4_0 || 'apache/doris:build-env-ldb-toolchain-4.0-latest' }} + platforms: linux/amd64 + + - name: Build and Push Docker Image (no-avx2) + uses: ./.github/actions/build-push-action + with: + context: . + file: Dockerfile.patched-noavx2 + build-args: | + GITHUB_REPOSITORY=${{ github.repository }} + push: true + tags: ${{ vars.DOCKER_TAGS_NOAVX2_4_0 || 'apache/doris:build-env-ldb-toolchain-4.0-no-avx2-latest' }} + platforms: linux/amd64 + + success: + name: Success + needs: [prerelease, build, update-docker] + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + steps: + - name: Update Checksums + run: | + tag_name='automation-4.0' + + gh release download "${tag_name}" + + content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" + echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + gh release edit -F release_note.md "${tag_name}" + + failure: + name: Failure + needs: [build, update-docker] + if: failure() + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + steps: + - name: Update Checksums + run: | + tag_name='automation-4.0' + + gh release download "${tag_name}" + + echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b8f82f6ef2..7d896e6ece6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,11 @@ name: Build on: workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" schedule: - cron: '*/30 * * * *' @@ -92,7 +97,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -236,7 +241,7 @@ jobs: export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" - export DORIS_TOOLCHAIN=gcc + export DORIS_TOOLCHAIN=clang else export MACOSX_DEPLOYMENT_TARGET=12.0 fi @@ -398,7 +403,7 @@ jobs: success: name: Success needs: [prerelease, build, update-docker] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }}