From 6652ce35514432d602b65ef5611e63352f6caf11 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Thu, 11 Dec 2025 02:54:37 +1100 Subject: [PATCH 1/2] chore: build and release updates 1. remove default build for x86_64 apple 2. remove aarch64 kafka build for linux 3. add x86_64 kafka binary for linux 4. add aarch64 kafka binary for apple 5. remove default binary for x86_64 apple --- .github/workflows/build.yaml | 125 +++------------------------ .github/workflows/release.yml | 154 +++++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fd4a93069..46587904e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,10 +25,7 @@ jobs: target: aarch64-unknown-linux-gnu use_cross: true - # macOS builds - both native on macos-latest (M1) - - os: macos-latest - target: x86_64-apple-darwin - use_cross: false + # macOS build - aarch64 only (M1/M2) - os: macos-latest target: aarch64-apple-darwin use_cross: false @@ -78,43 +75,14 @@ jobs: CXX: ${{ runner.os == 'Linux' && 'g++-11' || '' }} run: cargo build --target ${{ matrix.target }} --release - # Kafka build for supported platforms + # Kafka build for x86_64 Linux only build-kafka: - name: Build Kafka ${{matrix.target}} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - # Linux x86_64 - native build - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - use_cross: false - - # Linux aarch64 - cross-compile - - os: ubuntu-latest - target: aarch64-unknown-linux-gnu - use_cross: true - - # macOS aarch64 - native on M1 - - os: macos-latest - target: aarch64-apple-darwin - use_cross: false - + name: Build Kafka x86_64-unknown-linux-gnu + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Docker Buildx - if: matrix.use_cross - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU for cross-arch builds - if: matrix.use_cross - uses: docker/setup-qemu-action@v3 - - # Linux-specific dependencies for native x86_64 build - - name: Install Linux dependencies (x86_64) - if: runner.os == 'Linux' && matrix.target == 'x86_64-unknown-linux-gnu' + - name: Install Linux dependencies run: | sudo apt-get update sudo apt-get install -y \ @@ -131,27 +99,10 @@ jobs: libsasl2-dev \ python3 - # macOS-specific dependencies - - name: Install macOS dependencies - if: runner.os == 'macOS' - run: | - brew install \ - llvm \ - pkg-config \ - zstd \ - lz4 \ - openssl@3.0 \ - cyrus-sasl \ - python3 - - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@nightly with: - targets: ${{ matrix.target }} - - - name: Install cross for aarch64 - if: matrix.use_cross - run: cargo install cross --git https://github.com/cross-rs/cross + targets: x86_64-unknown-linux-gnu - name: Cache dependencies uses: actions/cache@v4 @@ -160,22 +111,17 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-cargo-${{ matrix.target }}-kafka-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-cargo-x86_64-unknown-linux-gnu-kafka-${{ hashFiles('**/Cargo.lock') }} - - name: Find and fix librdkafka CMakeLists.txt for Linux - if: runner.os == 'Linux' && !matrix.use_cross + - name: Find and fix librdkafka CMakeLists.txt run: | cargo fetch - # Find the rdkafka-sys package directory RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" - # Find the librdkafka CMakeLists.txt file CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" if [ -f "$CMAKE_FILE" ]; then echo "Found CMakeLists.txt at: $CMAKE_FILE" - # Make a backup of the original file cp "$CMAKE_FILE" "$CMAKE_FILE.bak" - # Replace the minimum required version sed -i 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" echo "Modified CMakeLists.txt - before and after comparison:" diff "$CMAKE_FILE.bak" "$CMAKE_FILE" || true @@ -184,56 +130,9 @@ jobs: exit 1 fi - - name: Find and fix librdkafka CMakeLists.txt for macOS - if: runner.os == 'macOS' - run: | - cargo fetch - # Find the rdkafka-sys package directory - RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) - echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" - # Find the librdkafka CMakeLists.txt file - CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" - if [ -f "$CMAKE_FILE" ]; then - echo "Found CMakeLists.txt at: $CMAKE_FILE" - # Make a backup of the original file - cp "$CMAKE_FILE" "$CMAKE_FILE.bak" - # Replace the minimum required version - macOS requires '' after -i - sed -i '' 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" - echo "Modified CMakeLists.txt - before and after comparison:" - diff "$CMAKE_FILE.bak" "$CMAKE_FILE" || true - else - echo "Could not find librdkafka CMakeLists.txt file!" - exit 1 - fi - - - name: Build with Kafka (cross) - FIXED - if: matrix.use_cross - env: - LIBRDKAFKA_SSL_VENDORED: "1" - PKG_CONFIG_ALLOW_CROSS: "1" - CROSS_NO_WARNINGS: "0" - RUST_BACKTRACE: "1" - - # Standard Debian multiarch paths for aarch64 - ZLIB_INCLUDE_DIR: "/usr/include" - ZLIB_LIBRARY: "/usr/lib/aarch64-linux-gnu/libz.so" - - OPENSSL_ROOT_DIR: "/usr" - OPENSSL_INCLUDE_DIR: "/usr/include" - OPENSSL_CRYPTO_LIBRARY: "/usr/lib/aarch64-linux-gnu/libcrypto.so" - OPENSSL_SSL_LIBRARY: "/usr/lib/aarch64-linux-gnu/libssl.so" - - PKG_CONFIG_PATH: "/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig" - run: | - echo "=== HOST ENV DEBUG ===" - env | grep -E 'ZLIB_|OPENSSL_|PKG_CONFIG|LIBRDKAFKA' - echo "=== NOW RUNNING CROSS BUILD (pre-build will show zlib paths) ===" - cross build --target ${{ matrix.target }} --features kafka --release --verbose - - - name: Build with Kafka (native) - if: ${{ !matrix.use_cross }} + - name: Build with Kafka env: - CC: ${{ runner.os == 'Linux' && 'gcc-11' || '' }} - CXX: ${{ runner.os == 'Linux' && 'g++-11' || '' }} + CC: gcc-11 + CXX: g++-11 LIBRDKAFKA_SSL_VENDORED: "1" - run: cargo build --target ${{ matrix.target }} --features kafka --release + run: cargo build --target x86_64-unknown-linux-gnu --features kafka --release \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9a2c71e2..5ddc0fd6f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -125,7 +125,6 @@ jobs: matrix: target: - aarch64-apple-darwin - - x86_64-apple-darwin steps: - name: Checkout @@ -169,9 +168,160 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build-kafka-linux: + name: Build Kafka for x86_64-unknown-linux-gnu + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + attestations: write + steps: + - uses: actions/checkout@v2 + + - name: Install Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + gcc-11 \ + g++-11 \ + build-essential \ + pkg-config \ + cmake \ + clang \ + zlib1g-dev \ + libzstd-dev \ + liblz4-dev \ + libssl-dev \ + python3 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-unknown-linux-gnu + + - name: Find and fix librdkafka CMakeLists.txt + run: | + cargo fetch + RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) + echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" + CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" + if [ -f "$CMAKE_FILE" ]; then + echo "Found CMakeLists.txt at: $CMAKE_FILE" + cp "$CMAKE_FILE" "$CMAKE_FILE.bak" + sed -i 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" + echo "Modified CMakeLists.txt" + else + echo "Could not find librdkafka CMakeLists.txt file!" + exit 1 + fi + + - name: Build with Kafka + env: + CC: gcc-11 + CXX: g++-11 + LIBRDKAFKA_SSL_VENDORED: "1" + run: cargo build --profile release-lto --target x86_64-unknown-linux-gnu --features kafka + + - name: Rename binary + run: | + mv target/x86_64-unknown-linux-gnu/release-lto/parseable Parseable_OSS_x86_64-unknown-linux-gnu-kafka + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: Parseable_OSS_x86_64-unknown-linux-gnu-kafka + + - name: Create Artifact + uses: actions/upload-artifact@v4 + with: + name: Parseable_OSS_x86_64-unknown-linux-gnu-kafka + path: Parseable_OSS_x86_64-unknown-linux-gnu-kafka + + - name: Publish Archive to Release Page + uses: softprops/action-gh-release@v0.1.15 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + draft: false + files: Parseable_OSS_x86_64-unknown-linux-gnu-kafka + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-kafka-mac: + name: Build Kafka for aarch64-apple-darwin + runs-on: macos-latest + permissions: + id-token: write + contents: write + attestations: write + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Install macOS dependencies + run: | + brew install \ + llvm \ + pkg-config \ + zstd \ + lz4 \ + openssl@3.0 \ + python3 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-apple-darwin + + - name: Find and fix librdkafka CMakeLists.txt + run: | + cargo fetch + RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) + echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" + CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" + if [ -f "$CMAKE_FILE" ]; then + echo "Found CMakeLists.txt at: $CMAKE_FILE" + cp "$CMAKE_FILE" "$CMAKE_FILE.bak" + sed -i '' 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" + echo "Modified CMakeLists.txt" + else + echo "Could not find librdkafka CMakeLists.txt file!" + exit 1 + fi + + - name: Build with Kafka + env: + LIBRDKAFKA_SSL_VENDORED: "1" + run: | + cargo build --profile release-lto --target aarch64-apple-darwin --features kafka + strip target/aarch64-apple-darwin/release-lto/Parseable + + - name: Rename binary + run: | + mv target/aarch64-apple-darwin/release-lto/Parseable Parseable_OSS_aarch64-apple-darwin-kafka + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: Parseable_OSS_aarch64-apple-darwin-kafka + + - name: Create artifact + uses: actions/upload-artifact@v4 + with: + name: Parseable_OSS_aarch64-apple-darwin-kafka + path: Parseable_OSS_aarch64-apple-darwin-kafka + + - name: Publish Archive to Release Page + uses: softprops/action-gh-release@v0.1.15 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + draft: false + files: Parseable_OSS_aarch64-apple-darwin-kafka + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + create-checksum: runs-on: ubuntu-latest - needs: [build-linux, build-windows, build-mac] + needs: [build-linux, build-windows, build-mac, build-kafka-linux, build-kafka-mac] steps: - name: Download artifacts created uses: actions/download-artifact@v4.1.7 From 7772d4f7102a7b984bc8a2ebd20fc09de4f67a10 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha Date: Thu, 11 Dec 2025 03:13:39 +1100 Subject: [PATCH 2/2] checkout version fix, aarch64 apple kafka build --- .github/workflows/build.yaml | 56 ++++++++++++++++++++++++++++++++++- .github/workflows/release.yml | 10 +++---- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 46587904e..0ad87f0aa 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -135,4 +135,58 @@ jobs: CC: gcc-11 CXX: g++-11 LIBRDKAFKA_SSL_VENDORED: "1" - run: cargo build --target x86_64-unknown-linux-gnu --features kafka --release \ No newline at end of file + run: cargo build --target x86_64-unknown-linux-gnu --features kafka --release + + # Kafka build for aarch64 macOS + build-kafka-mac: + name: Build Kafka aarch64-apple-darwin + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Install macOS dependencies + run: | + brew install \ + llvm \ + pkg-config \ + zstd \ + lz4 \ + openssl@3.0 \ + cyrus-sasl \ + python3 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@nightly + with: + targets: aarch64-apple-darwin + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-aarch64-apple-darwin-kafka-${{ hashFiles('**/Cargo.lock') }} + + - name: Find and fix librdkafka CMakeLists.txt + run: | + cargo fetch + RDKAFKA_SYS_DIR=$(find ~/.cargo/registry/src -name "rdkafka-sys-*" -type d | head -n 1) + echo "Found rdkafka-sys at: $RDKAFKA_SYS_DIR" + CMAKE_FILE="$RDKAFKA_SYS_DIR/librdkafka/CMakeLists.txt" + if [ -f "$CMAKE_FILE" ]; then + echo "Found CMakeLists.txt at: $CMAKE_FILE" + cp "$CMAKE_FILE" "$CMAKE_FILE.bak" + sed -i '' 's/cmake_minimum_required(VERSION 3.2)/cmake_minimum_required(VERSION 3.5)/' "$CMAKE_FILE" + echo "Modified CMakeLists.txt - before and after comparison:" + diff "$CMAKE_FILE.bak" "$CMAKE_FILE" || true + else + echo "Could not find librdkafka CMakeLists.txt file!" + exit 1 + fi + + - name: Build with Kafka + env: + LIBRDKAFKA_SSL_VENDORED: "1" + run: cargo build --target aarch64-apple-darwin --features kafka --release \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ddc0fd6f..cc50f6f1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: - aarch64-unknown-linux-gnu # linux(arm) - x86_64-unknown-linux-gnu # linux(64 bit) steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -78,7 +78,7 @@ jobs: attestations: write steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Install latest rust toolchain uses: actions-rs/toolchain@v1 @@ -128,7 +128,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Install latest rust toolchain uses: actions-rs/toolchain@v1 @@ -176,7 +176,7 @@ jobs: contents: write attestations: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install Linux dependencies run: | @@ -255,7 +255,7 @@ jobs: attestations: write steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Install macOS dependencies run: |