From 41a320259f8d873365e1e6998e7e709fd160b86f Mon Sep 17 00:00:00 2001 From: Mikhail Bautin <552936+mbautin@users.noreply.github.com> Date: Thu, 2 Nov 2023 21:51:06 -0700 Subject: [PATCH 1/5] Disable uploads for build types that we are not using in CI or development --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++---- build_and_release.sh | 5 ++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8935460..581a1ca7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: ci +fname: ci on: push: @@ -54,6 +54,7 @@ jobs: build_thirdparty_args: >- --devtoolset=11 --expected-major-compiler-version=11 + upload: true # Clang 16 - name: centos7-x86_64-clang16 @@ -62,6 +63,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 + upload: true - name: centos7-x86_64-clang16-full-lto os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -70,6 +72,8 @@ jobs: --toolchain=llvm16 --expected-major-compiler-version=16 --lto=full + --skip-sanitizers + upload: true # Clang 17 - name: centos7-x86_64-clang17 @@ -78,6 +82,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 + upload: true - name: centos7-x86_64-clang17-full-lto os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -86,6 +91,8 @@ jobs: --toolchain=llvm17 --expected-major-compiler-version=17 --lto=full + --skip-sanitizers + upload: true # --------------------------------------------------------------------------------------- # Ubuntu 20.04 @@ -96,6 +103,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 + upload: true # --------------------------------------------------------------------------------------- # Ubuntu 22.04 @@ -109,6 +117,8 @@ jobs: --compiler-family=gcc --compiler-suffix=-11 --expected-major-compiler-version=11 + # We can use the CentOS 7 GCC 11 build on Ubuntu 22.04. + upload: false - name: ubuntu2204-x86_64-clang16 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -116,6 +126,8 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 + # We can use the CentOS 7 Clang 16 build on Ubuntu 22.04. + upload: false - name: ubuntu2204-x86_64-clang17 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -123,6 +135,8 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 + # We can use the CentOS 7 Clang 17 build on Ubuntu 22.04. + upload: false # --------------------------------------------------------------------------------------- # AlmaLinux 8 @@ -134,6 +148,8 @@ jobs: build_thirdparty_args: >- --devtoolset=11 --expected-major-compiler-version=11 + # We can use the CentOS 7 GCC 11 build on AlmaLinux 8. + upload: false # Clang/LLVM 16 - name: almalinux8-x86_64-clang16 @@ -142,6 +158,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 + upload: true # Clang/LLVM 17 - name: almalinux8-x86_64-clang17 @@ -150,6 +167,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 + upload: true # --------------------------------------------------------------------------------------- # Ubuntu 23.04 @@ -163,6 +181,7 @@ jobs: --compiler-suffix=-13 --compiler-family=gcc --expected-major-compiler-version=13 + upload: true # --------------------------------------------------------------------------------------- # AlmaLinux 9 @@ -174,6 +193,7 @@ jobs: build_thirdparty_args: >- --devtoolset=12 --expected-major-compiler-version=12 + upload: false - name: almalinux9-x86_64-clang16 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -181,6 +201,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 + upload: false - name: almalinux9-x86_64-clang17 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -188,6 +209,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 + upload: false # --------------------------------------------------------------------------------------- # macOS @@ -197,6 +219,7 @@ jobs: os: macos-12 docker_image: build_thirdparty_args: + upload: true steps: - name: Increase disk space @@ -216,11 +239,12 @@ jobs: - name: Build run: .github/workflows/github_actions_build.sh env: - YB_DOCKER_IMAGE: ${{ matrix.docker_image }} - YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX: ${{ matrix.name }} - YB_BUILD_THIRDPARTY_ARGS: ${{ matrix.build_thirdparty_args }} GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }} SNYK_TOKEN: ${{ secrets.SNYK_AUTH_TOKEN }} + YB_BUILD_THIRDPARTY_ARGS: ${{ matrix.build_thirdparty_args }} + YB_DOCKER_IMAGE: ${{ matrix.docker_image }} + YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX: ${{ matrix.name }} + YB_UPLOAD_THIRDPARTY_ARCHIVE: ${{ matrix.upload }} - uses: actions/upload-artifact@v2 with: diff --git a/build_and_release.sh b/build_and_release.sh index f07ea9fe..387cff3f 100755 --- a/build_and_release.sh +++ b/build_and_release.sh @@ -229,6 +229,7 @@ if [[ -n ${GITHUB_TOKEN:-} && ( set -x; hub issue -L 0 ) else log "GITHUB_TOKEN length is ${#GITHUB_TOKEN} characters (not 40), considering it as unset." + GITHUB_TOKEN="" fi # We intentionally don't escape variables here so they get split into multiple arguments. @@ -246,8 +247,10 @@ fi # shellcheck disable=SC2206 build_thirdparty_cmd_args=( $build_thirdparty_cmd_str ) -if [[ -z ${YB_SKIP_UPLOAD:-} ]]; then +if [[ -n $GITHUB_TOKEN && ${YB_UPLOAD_THIRDPARTY_ARCHIVE:-} == "true" ]]; then build_thirdparty_cmd_args+=( --upload-as-tag "$tag" ) +else + echo "To enable package upload, set YB_UPLOAD_THIRDPARTY_ARCHIVE to true and set GITHUB_TOKEN." fi ( From 4faecccb02a9dcfd8c1500036e180e496d1536ee Mon Sep 17 00:00:00 2001 From: Mikhail Bautin <552936+mbautin@users.noreply.github.com> Date: Thu, 2 Nov 2023 21:52:01 -0700 Subject: [PATCH 2/5] Fix typo --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 581a1ca7..7c91335e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -fname: ci +name: ci on: push: @@ -103,7 +103,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 - upload: true + upload: false # --------------------------------------------------------------------------------------- # Ubuntu 22.04 From 18b13a2091104ff5d65b53a2220ba8af1d8d9e58 Mon Sep 17 00:00:00 2001 From: Eric Sheng Date: Thu, 2 Nov 2023 19:25:17 -0700 Subject: [PATCH 3/5] Change OpenSSL version to 3.0.8 (#248) Moving to 3.0.8 for now due to it being FIPS 140-2 certified, per slack discussion. The previous 3.1.3 change hasn't been pulled into the main repo yet, so there should be no problems. Context: https://github.com/yugabyte/yugabyte-db/issues/16407 --- python/build_definitions/openssl.py | 2 +- thirdparty_src_checksums.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/build_definitions/openssl.py b/python/build_definitions/openssl.py index 175ba052..d85608c5 100644 --- a/python/build_definitions/openssl.py +++ b/python/build_definitions/openssl.py @@ -34,7 +34,7 @@ class OpenSSLDependency(Dependency): def __init__(self) -> None: super(OpenSSLDependency, self).__init__( name='openssl', - version='3.1.3', + version='3.0.8', url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz', build_group=BuildGroup.COMMON) self.copy_sources = True diff --git a/thirdparty_src_checksums.txt b/thirdparty_src_checksums.txt index 95481329..8ee630f0 100644 --- a/thirdparty_src_checksums.txt +++ b/thirdparty_src_checksums.txt @@ -112,6 +112,7 @@ c48450d27524c2e5856997133e059e3cf9909241110a6e21ad278890ac425afc lz4-r130.tar.g 9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f openssl-1.1.1o.tar.gz 8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b openssl-1.1.1t.tar.gz cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 openssl-1.1.1w.tar.gz +6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e openssl-3.0.8.tar.gz f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 openssl-3.1.3.tar.gz 0fdbefbdc2c154634728097e26de52a8210ed95cb032beb5f35da0a493cd5066 opentelemetry-cpp-1.9.0.tar.gz 464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be opentelemetry-proto-0.19.0.tar.gz From 644f46783f0b5377c8bbc26261d975f92e5b9e1e Mon Sep 17 00:00:00 2001 From: Eric Sheng Date: Mon, 6 Nov 2023 19:43:05 -0800 Subject: [PATCH 4/5] [skip ci] Manually install hub for macOS builds (#251) Fix `./build_and_release.sh: line 229: hub: command not found` error in macOS builds due to `hub` being removed from out-of-the-box macOS runners. --- .github/workflows/macos_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_build.sh b/.github/workflows/macos_build.sh index aeb16e33..20bdd719 100755 --- a/.github/workflows/macos_build.sh +++ b/.github/workflows/macos_build.sh @@ -2,7 +2,7 @@ set -euo pipefail -brew install autoconf automake pkg-config shellcheck +brew install autoconf automake pkg-config shellcheck hub dirs=( /opt/yb-build/{thirdparty,brew,tmp} ) sudo mkdir -p "${dirs[@]}" sudo chmod 777 "${dirs[@]}" From 92b12afa22c1453c52575ea3b0b94d2af26959e7 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin <552936+mbautin@users.noreply.github.com> Date: Mon, 6 Nov 2023 20:42:45 -0800 Subject: [PATCH 5/5] Fix upload flags --- .github/workflows/ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c91335e..040e378e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 - upload: false + upload: true # --------------------------------------------------------------------------------------- # Ubuntu 22.04 @@ -117,7 +117,7 @@ jobs: --compiler-family=gcc --compiler-suffix=-11 --expected-major-compiler-version=11 - # We can use the CentOS 7 GCC 11 build on Ubuntu 22.04. + # We can use the CentOS 7 GCC 11 build on Ubuntu 22.04, no need to upload. upload: false - name: ubuntu2204-x86_64-clang16 @@ -126,8 +126,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 - # We can use the CentOS 7 Clang 16 build on Ubuntu 22.04. - upload: false + upload: true - name: ubuntu2204-x86_64-clang17 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -135,8 +134,8 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 - # We can use the CentOS 7 Clang 17 build on Ubuntu 22.04. - upload: false + # We can use the CentOS 7 Clang 17 build on Ubuntu 22.04, no need to upload. + upload: true # --------------------------------------------------------------------------------------- # AlmaLinux 8 @@ -193,7 +192,7 @@ jobs: build_thirdparty_args: >- --devtoolset=12 --expected-major-compiler-version=12 - upload: false + upload: true - name: almalinux9-x86_64-clang16 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -201,7 +200,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm16 --expected-major-compiler-version=16 - upload: false + upload: true - name: almalinux9-x86_64-clang17 os: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. @@ -209,7 +208,7 @@ jobs: build_thirdparty_args: >- --toolchain=llvm17 --expected-major-compiler-version=17 - upload: false + upload: true # --------------------------------------------------------------------------------------- # macOS