From 74aa2f9dad0a0a624c9177395424695d271a03be Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 14:21:16 +0800 Subject: [PATCH 1/6] Use vcpkg to build alpine packages --- .../workflows/ci-build-binary-artifacts.yaml | 8 +- pkg/apk/APKBUILD | 32 ++++--- pkg/apk/Dockerfile | 92 +------------------ pkg/apk/docker-build-apk-arm64.sh | 2 +- 4 files changed, 30 insertions(+), 104 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 0a49c706..f91e117d 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -93,16 +93,16 @@ jobs: matrix: include: - name: 'Windows x64' - os: windows-2019 + os: windows-2022 triplet: x64-windows-static suffix: 'windows-win64' - generator: 'Visual Studio 16 2019' + generator: 'Visual Studio 17 2022' arch: '-A x64' - name: 'Windows x86' - os: windows-2019 + os: windows-2022 triplet: x86-windows-static suffix: 'windows-win32' - generator: 'Visual Studio 16 2019' + generator: 'Visual Studio 17 2022' arch: '-A Win32' steps: diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index 3c3cdc70..3220f0f1 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -38,19 +38,27 @@ build() { if [ "$CBUILD" != "$CHOST" ]; then CMAKE_CROSSOPTS="-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_HOST_SYSTEM_NAME=Linux" fi - cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DBUILD_SHARED_LIBS=True \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ - -DCMAKE_C_FLAGS="$CFLAGS" \ - -DBUILD_TESTS=OFF \ - -DLINK_STATIC=ON \ - ${CMAKE_CROSSOPTS} ${ROOT_DIR} - make -j6 + git clone https://github.com/microsoft/vcpkg.git + mv vcpkg $ROOT_DIR/ + if [[ $PLATFORM == "aarch64" ]]; then + export VCPKG_FORCE_SYSTEM_BINARIES=1 + fi + set +e + cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_MAKE_PROGRAM=make \ + -DCMAKE_INSTALL_PREFIX="$pkgdir" \ + -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \ + $ROOT_DIR + if [[ $? -ne 0 ]]; then + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-CMakeCache.txt.log + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-out.log + cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-err.log + exit 1 + fi + set -e + cmake --build build -j8 } package() { - make DESTDIR="$pkgdir" install + cmake --build build --target install } diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index db501f06..ca23cdb2 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM alpine:3.12 +FROM alpine:3.19 ARG PLATFORM @@ -31,91 +31,9 @@ RUN apk add \ g++ \ make \ cmake \ - python3 \ - py3-pip \ perl \ git \ - sudo - -RUN pip3 install pyyaml - -ADD .build/dependencies.yaml / -ADD .build/dep-version.py /usr/local/bin - -# Download and compile boost -RUN BOOST_VERSION=$(dep-version.py boost) && \ - curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \ - tar zxf boost-${BOOST_VERSION}.tar.gz && \ - cd boost-${BOOST_VERSION} && \ - ./bootstrap.sh --with-libraries=regex && \ - ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION} - -# Download and compile protobuf -RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ - cd protobuf-${PROTOBUF_VERSION}/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} - -# ZLib -RUN ZLIB_VERSION=$(dep-version.py zlib) && \ - curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ - tar xfz v${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - CFLAGS="-fPIC -O3" ./configure && \ - make -j8 && make install && \ - rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} - -# Zstandard -RUN ZSTD_VERSION=$(dep-version.py zstd) && \ - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ - tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ - cd zstd-${ZSTD_VERSION} && \ - CFLAGS="-fPIC -O3" make -j8 && \ - make install && \ - rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz - -# Snappy -RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ - curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ - tar xfz ${SNAPPY_VERSION}.tar.gz && \ - cd snappy-${SNAPPY_VERSION} && \ - CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ - make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz - -# Roaring -RUN ROARING_VERSION=$(dep-version.py roaring) && \ - curl -O -L https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v${ROARING_VERSION}.tar.gz && \ - tar xfz v${ROARING_VERSION}.tar.gz && \ - cd CRoaring-${ROARING_VERSION} && \ - mkdir build && cd build && CXXFLAGS="-fPIC -O3" cmake .. && \ - make -j8 && make install && \ - rm -rf /v${ROARING_VERSION}.tar.gz /CRoaring-${ROARING_VERSION} - -RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ - OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ - cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ - ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ - make -j8 && make install && \ - rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - -ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ - -# LibCurl -RUN CURL_VERSION=$(dep-version.py curl) && \ - CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ - curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ - tar xfz curl-${CURL_VERSION}.tar.gz && \ - cd curl-${CURL_VERSION} && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \ - make -j8 && make install && \ - rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} - - + ninja \ + zip \ + unzip \ + openssl-dev diff --git a/pkg/apk/docker-build-apk-arm64.sh b/pkg/apk/docker-build-apk-arm64.sh index 75631523..8480ee5f 100755 --- a/pkg/apk/docker-build-apk-arm64.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -25,7 +25,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ - --env PLATFORM=arm64 \ + --env PLATFORM=aarch64 \ $IMAGE_NAME \ /pulsar-client-cpp/pkg/apk/build-apk.sh From 58c3d0a90f55ef7da4232974938666be85cefc7c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 16:22:29 +0800 Subject: [PATCH 2/6] Fix --- pkg/apk/APKBUILD | 9 +-------- pkg/apk/Dockerfile | 3 ++- pkg/apk/build-apk.sh | 4 ++-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index 3220f0f1..91180c30 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -43,19 +43,12 @@ build() { if [[ $PLATFORM == "aarch64" ]]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 fi - set +e + set -e cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_MAKE_PROGRAM=make \ -DCMAKE_INSTALL_PREFIX="$pkgdir" \ -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \ $ROOT_DIR - if [[ $? -ne 0 ]]; then - cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-CMakeCache.txt.log - cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-out.log - cat /pulsar-client-cpp/vcpkg/buildtrees/detect_compiler/config-x64-linux-rel-err.log - exit 1 - fi - set -e cmake --build build -j8 } diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index ca23cdb2..1335174b 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -36,4 +36,5 @@ RUN apk add \ ninja \ zip \ unzip \ - openssl-dev + openssl-dev \ + sudo diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index 66cb7633..02703bb1 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -36,7 +36,7 @@ export VERSION=`echo $POM_VERSION | sed -E 's/\-[a-zA-Z]+//'` echo "VERSION: $VERSION" -abuild-keygen -a -i -n -abuild -F -c -r +sudo abuild-keygen -a -i -n +sudo abuild -F -c -r cp -r /root/packages/pkg ./build From 4429072b1b46c4d3eb2edb364c7f002a28cc8785 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 16:35:22 +0800 Subject: [PATCH 3/6] Fix --- pkg/apk/APKBUILD | 2 +- pkg/apk/build-apk.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index 91180c30..85511994 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -17,7 +17,7 @@ # under the License. # -# Maintainer: "Apache Pulsar " +# Maintainer: Apache Pulsar pkgname="apache-pulsar-client" pkgver="$VERSION" diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index 02703bb1..54360481 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -37,6 +37,6 @@ export VERSION=`echo $POM_VERSION | sed -E 's/\-[a-zA-Z]+//'` echo "VERSION: $VERSION" sudo abuild-keygen -a -i -n -sudo abuild -F -c -r +abuild -F -c -r cp -r /root/packages/pkg ./build From d76acf64e9c1c3e21a56eca47f6fa7139e62d5b8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 17:35:40 +0800 Subject: [PATCH 4/6] Build CMake from source --- pkg/apk/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index 1335174b..ca06ad7c 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -38,3 +38,10 @@ RUN apk add \ unzip \ openssl-dev \ sudo + +# Build latest CMake for vcpkg +RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.31.9/cmake-3.31.9.tar.gz \ + && tar zxf cmake-3.31.9.tar.gz \ + && cd cmake-3.31.9 \ + && ./configure && make -j8 && make install \ + && cd .. && rm -rf cmake-3.31.9 cmake-3.31.9.tar.gz From b23f2772a47ebc432636606647b7d1cfa0c91351 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 18:25:51 +0800 Subject: [PATCH 5/6] Force building vcpkg from source --- pkg/apk/APKBUILD | 4 +--- pkg/apk/Dockerfile | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index 85511994..ff70b44f 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -40,9 +40,7 @@ build() { fi git clone https://github.com/microsoft/vcpkg.git mv vcpkg $ROOT_DIR/ - if [[ $PLATFORM == "aarch64" ]]; then - export VCPKG_FORCE_SYSTEM_BINARIES=1 - fi + export VCPKG_FORCE_SYSTEM_BINARIES=1 set -e cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_MAKE_PROGRAM=make \ diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index ca06ad7c..1335174b 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -38,10 +38,3 @@ RUN apk add \ unzip \ openssl-dev \ sudo - -# Build latest CMake for vcpkg -RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.31.9/cmake-3.31.9.tar.gz \ - && tar zxf cmake-3.31.9.tar.gz \ - && cd cmake-3.31.9 \ - && ./configure && make -j8 && make install \ - && cd .. && rm -rf cmake-3.31.9 cmake-3.31.9.tar.gz From 1a5de0180490faca3e3859a2646763dfe24b0584 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 29 Sep 2025 18:30:22 +0800 Subject: [PATCH 6/6] cat error logs --- pkg/apk/APKBUILD | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index ff70b44f..1b476c2d 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -41,12 +41,17 @@ build() { git clone https://github.com/microsoft/vcpkg.git mv vcpkg $ROOT_DIR/ export VCPKG_FORCE_SYSTEM_BINARIES=1 - set -e + set +e cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_MAKE_PROGRAM=make \ -DCMAKE_INSTALL_PREFIX="$pkgdir" \ -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \ $ROOT_DIR + if [[ $? -ne 0 ]]; then + cat $ROOT_DIR/vcpkg/buildtrees/detect_compiler/*.log + exit 1 + fi + set -e cmake --build build -j8 }