From 7549cc840185a933eccebbeb89dbe834dc178cb6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:26:12 -0600 Subject: [PATCH 1/8] chore(release): add Dockerfile for linux native lib builds --- dev/release/datafusion-java-rm/Dockerfile | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 dev/release/datafusion-java-rm/Dockerfile diff --git a/dev/release/datafusion-java-rm/Dockerfile b/dev/release/datafusion-java-rm/Dockerfile new file mode 100644 index 0000000..48b0266 --- /dev/null +++ b/dev/release/datafusion-java-rm/Dockerfile @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM ubuntu:20.04 + +USER root + +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NONINTERACTIVE_SEEN=true +ENV LC_ALL=C + +RUN export LC_ALL=C \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + ca-certificates \ + build-essential \ + curl \ + wget \ + git \ + llvm \ + clang \ + libssl-dev \ + cmake \ + cpio \ + libxml2-dev \ + patch \ + bzip2 \ + libbz2-dev \ + zlib1g-dev \ + default-jdk \ + unzip + +RUN apt install -y gcc-10 g++-10 cpp-10 +ENV CC="gcc-10" +ENV CXX="g++-10" + +# protoc — picks the host arch automatically by reading $(uname -m) +RUN PB_REL="https://github.com/protocolbuffers/protobuf/releases" \ + && ARCH="$(uname -m)" \ + && case "$ARCH" in \ + x86_64) PB_ARCH="x86_64" ;; \ + aarch64) PB_ARCH="aarch_64" ;; \ + *) echo "Unsupported arch: $ARCH" >&2; exit 1 ;; \ + esac \ + && curl -LO "$PB_REL/download/v30.2/protoc-30.2-linux-${PB_ARCH}.zip" \ + && unzip "protoc-30.2-linux-${PB_ARCH}.zip" -d /root/.local \ + && rm "protoc-30.2-linux-${PB_ARCH}.zip" +ENV PATH="$PATH:/root/.local/bin" + +# Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +COPY build-native-libs.sh /opt/datafusion-java-rm/build-native-libs.sh +WORKDIR /opt/datafusion-java-rm + +ENTRYPOINT ["/opt/datafusion-java-rm/build-native-libs.sh"] From 28242f5d597136c0f43def944665f8202f46a77e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:27:42 -0600 Subject: [PATCH 2/8] chore(release): add in-container build script for native libs --- .../datafusion-java-rm/build-native-libs.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 dev/release/datafusion-java-rm/build-native-libs.sh diff --git a/dev/release/datafusion-java-rm/build-native-libs.sh b/dev/release/datafusion-java-rm/build-native-libs.sh new file mode 100755 index 0000000..5f273cc --- /dev/null +++ b/dev/release/datafusion-java-rm/build-native-libs.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Build the datafusion_jni release native lib inside a Linux container. +# The container's --platform determines the target arch; we just build. + +set -euo pipefail + +REPO=${1:-} +BRANCH=${2:-} + +if [ -z "$REPO" ] || [ -z "$BRANCH" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +echo "Building datafusion_jni for $(uname -m) from ${REPO}/${BRANCH}" + +rm -rf datafusion-java +git clone "$REPO" datafusion-java +cd datafusion-java +git checkout "$BRANCH" + +cd native +cargo build --release + +echo "Built $(pwd)/target/release/libdatafusion_jni.so" +ls -l target/release/libdatafusion_jni.so From 1e57d30473b67270a309cda174c0fdfbeb68bd98 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:29:55 -0600 Subject: [PATCH 3/8] chore(release): add multi-platform JAR build orchestrator --- dev/release/build-release.sh | 169 +++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100755 dev/release/build-release.sh diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh new file mode 100755 index 0000000..bb59b8d --- /dev/null +++ b/dev/release/build-release.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Build a multi-platform datafusion-java JAR bundling native libs for +# linux/amd64, linux/aarch64, darwin/x86_64, and darwin/aarch64. The +# resulting JAR is installed into a temporary local Maven repository +# whose path is printed at the end. This script must run on a macOS host. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +PROJECT_HOME="$(cd "${SCRIPT_DIR}/../.." >/dev/null && pwd)" + +REPO="https://github.com/apache/datafusion-java.git" +BRANCH="main" +IMGTAG="latest" + +function usage { + cat <&2 + exit 1 +fi + +JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{print $1}') +if [ -z "$JAVA_VERSION" ] || [ "$JAVA_VERSION" -lt 17 ]; then + echo "Java 17+ is required. Found: $(java -version 2>&1 | head -n 1)" >&2 + exit 1 +fi + +HOST_ARCH="$(uname -m)" # arm64 (Apple Silicon) or x86_64 (Intel) + +case "$HOST_ARCH" in + arm64) + HOST_DARWIN_DIR="aarch64" + OTHER_DARWIN_TARGET="x86_64-apple-darwin" + OTHER_DARWIN_DIR="x86_64" + ;; + x86_64) + HOST_DARWIN_DIR="x86_64" + OTHER_DARWIN_TARGET="aarch64-apple-darwin" + OTHER_DARWIN_DIR="aarch64" + ;; + *) + echo "Unsupported macOS arch: $HOST_ARCH" >&2 + exit 1 + ;; +esac + +CONTAINER_AMD64="datafusion-java-amd64-builder-container" +CONTAINER_ARM64="datafusion-java-arm64-builder-container" +IMAGE_AMD64="datafusion-java-rm-amd64:${IMGTAG}" +IMAGE_ARM64="datafusion-java-rm-arm64:${IMGTAG}" + +CLEANUP=1 +cleanup() { + [ "$CLEANUP" != "0" ] || return 0 + echo "Cleaning up build containers..." + docker rm -f "$CONTAINER_AMD64" "$CONTAINER_ARM64" >/dev/null 2>&1 || true + CLEANUP=0 +} +trap cleanup SIGINT SIGTERM EXIT + +echo "Cleaning previous Java and native build output" +(cd "$PROJECT_HOME" && ./mvnw -q clean) +(cd "$PROJECT_HOME/native" && cargo clean) + +echo "Building amd64 builder image" +docker build --no-cache \ + --platform=linux/amd64 \ + -t "$IMAGE_AMD64" \ + "$SCRIPT_DIR/datafusion-java-rm" + +echo "Building arm64 builder image" +docker build --no-cache \ + --platform=linux/arm64 \ + -t "$IMAGE_ARM64" \ + "$SCRIPT_DIR/datafusion-java-rm" + +echo "Building linux/amd64 native lib" +docker run --name "$CONTAINER_AMD64" \ + --platform=linux/amd64 \ + "$IMAGE_AMD64" "$REPO" "$BRANCH" + +echo "Building linux/aarch64 native lib" +docker run --name "$CONTAINER_ARM64" \ + --platform=linux/arm64 \ + "$IMAGE_ARM64" "$REPO" "$BRANCH" + +JVM_TARGET_DIR="$PROJECT_HOME/core/target/classes/org/apache/datafusion" + +mkdir -p "$JVM_TARGET_DIR/linux/amd64" +docker cp \ + "$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \ + "$JVM_TARGET_DIR/linux/amd64/" + +mkdir -p "$JVM_TARGET_DIR/linux/aarch64" +docker cp \ + "$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \ + "$JVM_TARGET_DIR/linux/aarch64/" + +echo "Building macOS native libs on the host (host=$HOST_ARCH)" +rustup target add "$OTHER_DARWIN_TARGET" + +(cd "$PROJECT_HOME/native" && cargo build --release) +(cd "$PROJECT_HOME/native" && cargo build --release --target "$OTHER_DARWIN_TARGET") + +mkdir -p "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR" +cp "$PROJECT_HOME/native/target/release/libdatafusion_jni.dylib" \ + "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR/" + +mkdir -p "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR" +cp "$PROJECT_HOME/native/target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \ + "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR/" + +echo "Installing JAR into local Maven repo" +LOCAL_REPO=$(mktemp -d /tmp/datafusion-java-staging-repo-XXXXXX) +(cd "$PROJECT_HOME" && ./mvnw "-Dmaven.repo.local=$LOCAL_REPO" -DskipTests install) + +echo "" +echo "====================================================================" +echo "Multi-platform JAR installed to local Maven repo: $LOCAL_REPO" +JAR_PATH=$(find "$LOCAL_REPO/org/apache/datafusion/datafusion-java" -name 'datafusion-java-*.jar' \ + -not -name '*-sources.jar' -not -name '*-javadoc.jar' | head -n 1) +echo "JAR: $JAR_PATH" +echo "Bundled native libraries:" +unzip -l "$JAR_PATH" | grep -E 'libdatafusion_jni\.(so|dylib)$' || true +echo "====================================================================" From 940fc31d10ef9399cc7e3670584bd6ac6a01a0f8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:32:39 -0600 Subject: [PATCH 4/8] chore(release): add publish-to-maven script for binary artifacts --- dev/release/publish-to-maven.sh | 138 ++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 dev/release/publish-to-maven.sh diff --git a/dev/release/publish-to-maven.sh b/dev/release/publish-to-maven.sh new file mode 100755 index 0000000..68d8636 --- /dev/null +++ b/dev/release/publish-to-maven.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### +# Based on Apache DataFusion Comet's publish-to-maven.sh (itself adapted +# from Spark's release-build.sh). +### + +function usage { + local NAME=$(basename "$0") + cat << EOF +usage: $NAME options + +Publish signed artifacts to Apache Nexus staging. + +Options: + -u ASF_USERNAME Username of ASF committer account + -r LOCAL_REPO Path to the local Maven repo created by build-release.sh + +The following will be prompted for: + ASF_PASSWORD Password of ASF committer account + GPG_KEY Optional: specific GPG key id to sign with + GPG_PASSPHRASE Passphrase for the GPG signing key +EOF + exit 1 +} + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +PROJECT_HOME="$(cd "${SCRIPT_DIR}/../.." >/dev/null && pwd)" + +ASF_USERNAME="" +LOCAL_REPO="" + +NEXUS_ROOT=https://repository.apache.org/service/local/staging +NEXUS_PROFILE=789e15c00fd47 + +while getopts "u:r:h" opt; do + case "$opt" in + u) ASF_USERNAME="$OPTARG" ;; + r) LOCAL_REPO="$OPTARG" ;; + h|*) usage ;; + esac +done + +if [ -z "$LOCAL_REPO" ]; then + echo "Please provide the local Maven repo path (-r), as produced by build-release.sh." >&2 + usage +fi + +if [ -z "$ASF_USERNAME" ]; then + read -p "ASF Username : " ASF_USERNAME && echo "" +fi +read -s -p "ASF Password : " ASF_PASSWORD && echo "" +read -s -p "GPG Key (optional) : " GPG_KEY && echo "" +read -s -p "GPG Passphrase : " GPG_PASSPHRASE && echo "" + +if [ -z "$ASF_USERNAME" ] || [ -z "$ASF_PASSWORD" ] || [ -z "$GPG_PASSPHRASE" ]; then + echo "Missing credentials" >&2 + exit 1 +fi + +GPG="gpg --pinentry-mode loopback" +if [ -n "$GPG_KEY" ]; then + GPG="$GPG -u $GPG_KEY" +fi + +SHA1SUM=$(which sha1sum || which shasum) + +GIT_HASH=$(cd "$PROJECT_HOME" && git rev-parse --short HEAD) + +echo "Creating Nexus staging repository" +REPO_REQUEST="Apache DataFusion Java (commit $GIT_HASH)" +REPO_REQUEST_RESPONSE=$(curl -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \ + -H "Content-Type:application/xml" \ + "$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start") + +STAGED_REPO_ID=$(echo "$REPO_REQUEST_RESPONSE" | xmllint --xpath "//stagedRepositoryId/text()" -) +echo "Created Nexus staging repository: $STAGED_REPO_ID" + +if [ -z "$STAGED_REPO_ID" ]; then + echo "Error creating staged repository" >&2 + echo "$REPO_REQUEST_RESPONSE" >&2 + exit 1 +fi + +echo "Deploying artifacts from $LOCAL_REPO" +pushd "$LOCAL_REPO/org/apache/datafusion" >/dev/null + +# Remove any extra files that mvn install might have written alongside +# the jar/pom (e.g. -lastUpdated metadata files). +find . -type f | grep -v '\.jar$' | grep -v '\.pom$' | xargs rm -f || true + +echo "Creating hash and signature files" +for file in $(find . -type f); do + echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --output "$file.asc" \ + --detach-sig --armour "$file" + if command -v md5 >/dev/null; then + md5 -q "$file" > "$file.md5" + else + md5sum "$file" | cut -f1 -d' ' > "$file.md5" + fi + $SHA1SUM "$file" | cut -f1 -d' ' > "$file.sha1" +done + +NEXUS_UPLOAD=$NEXUS_ROOT/deployByRepositoryId/$STAGED_REPO_ID +echo "Uploading files to $NEXUS_UPLOAD" +for file in $(find . -type f); do + FILE_SHORT=$(echo "$file" | sed -e "s/\.\///") + DEST_URL="$NEXUS_UPLOAD/org/apache/datafusion/$FILE_SHORT" + echo " Uploading $FILE_SHORT" + curl -u "$ASF_USERNAME:$ASF_PASSWORD" --upload-file "$FILE_SHORT" "$DEST_URL" +done + +echo "Closing nexus staging repository" +REPO_REQUEST="$STAGED_REPO_IDApache DataFusion Java (commit $GIT_HASH)" +curl -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \ + -H "Content-Type:application/xml" -v \ + "$NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish" +echo "Closed Nexus staging repository: $STAGED_REPO_ID" + +popd >/dev/null From b8b47360c6a63fa80a22d5bb84c077ab1ed0424a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:34:24 -0600 Subject: [PATCH 5/8] docs(release): document multi-platform JAR build and publish flow --- dev/release/README.md | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/dev/release/README.md b/dev/release/README.md index 16c985d..5557a47 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -158,3 +158,52 @@ svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-j svn delete -m "delete old DataFusion Java release" \ https://dist.apache.org/repos/dist/release/datafusion/datafusion-java-0.1.0 ``` + +## Binary Release: Multi-Platform JAR + +Source tarballs are the official Apache release artifact, but consumers +also expect a published JAR on Maven Central that bundles native libs +for the common platforms. This section covers building that JAR. + +### Prerequisites (release manager machine) + +- macOS host (Apple Silicon or Intel) +- Docker Desktop running with BuildKit enabled +- Java 17+ +- Rust toolchain via rustup +- `gpg` configured with a key listed in the ASF KEYS file + +### Build the multi-platform JAR + +`build-release.sh` clones the repo into two Linux Docker containers +(one for `linux/amd64`, one for `linux/arm64`), builds the native +`.so` libraries inside each, then builds the two macOS `.dylib` +libraries directly on the host. All four libraries are placed in the +JAR's resource tree at +`org/apache/datafusion///lib.`, and the +JAR is installed into a temporary local Maven repository. + +```shell +./dev/release/build-release.sh +``` + +The script prints the local Maven repo path at the end. Inspect the JAR +to verify all four native libraries are bundled: + +```shell +unzip -l "$JAR" | grep org/apache/datafusion/ +``` + +### Publish to Apache Nexus staging + +Once the local Maven repo from `build-release.sh` looks correct, sign +and upload to Apache Nexus staging using `publish-to-maven.sh`: + +```shell +./dev/release/publish-to-maven.sh -u -r +``` + +The script prompts for the ASF password and GPG passphrase, creates a +staging repository on `repository.apache.org`, signs every artifact, +uploads it, and closes the staging repository. Verify in the Nexus UI +that the staged artifacts look correct before promoting to release. From d0f8270f92798d8291944e8aefde883803a8f871 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:54:31 -0600 Subject: [PATCH 6/8] fix(build): exempt .github/** and rat_exclude_files.txt from RAT check The apache-rat-plugin runs at the `verify` phase, which surfaces during `mvn install` (the binary release flow) but not during the `make test` lifecycle. Without these exclusions, the PR / issue templates and the source-tarball exclude list itself fail the license-header check. The source-tarball flow already considers these files exempt via dev/release/rat_exclude_files.txt; this brings the pom's plugin configuration into alignment. --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 0a92f4b..8827f31 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,8 @@ under the License. .vscode/** **/*.iml **/.DS_Store + + .github/** mvnw mvnw.cmd @@ -169,6 +171,8 @@ under the License. tpch-data/** native/Cargo.lock + + dev/release/rat_exclude_files.txt From aa44d10a805263d91a17705a22b044826c2cfdc9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 09:54:34 -0600 Subject: [PATCH 7/8] fix(release): build with the release native profile in build-release.sh `core/pom.xml` defaults `datafusion.native.profile=debug`, so the antrun copy-native-lib step expects a debug-mode dylib at `native/target/debug/...`. The release orchestrator builds with `cargo build --release`, which writes to `native/target/release/...`. Pass `-Ddatafusion.native.profile=release` so the antrun check looks at the right path and the produced JAR bundles the release native libraries. --- dev/release/build-release.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh index bb59b8d..6539430 100755 --- a/dev/release/build-release.sh +++ b/dev/release/build-release.sh @@ -156,7 +156,10 @@ cp "$PROJECT_HOME/native/target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.d echo "Installing JAR into local Maven repo" LOCAL_REPO=$(mktemp -d /tmp/datafusion-java-staging-repo-XXXXXX) -(cd "$PROJECT_HOME" && ./mvnw "-Dmaven.repo.local=$LOCAL_REPO" -DskipTests install) +(cd "$PROJECT_HOME" && ./mvnw \ + "-Dmaven.repo.local=$LOCAL_REPO" \ + "-Ddatafusion.native.profile=release" \ + -DskipTests install) echo "" echo "====================================================================" From 3da6f7b00aad7d9ff361458ad77d633698f489ca Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 25 May 2026 10:02:05 -0600 Subject: [PATCH 8/8] chore(release): tighten release scripts after review Address review feedback on the multi-platform JAR pipeline: - build-release.sh: pre-clean leftover builder containers before the first `docker run`. Without this, a prior SIGKILL'd run leaves named containers behind and the next invocation fails immediately on `docker run --name` with a name conflict. - publish-to-maven.sh: pass `--fail-with-body` to the upload-loop and close-staging curl calls. curl exits 0 on HTTP 4xx/5xx by default, which would silently let an unauthorised or rejected upload run to completion and corrupt the staging repo. - Dockerfile: fold gcc-10 / g++-10 / cpp-10 into the main `apt-get install --no-install-recommends -y` layer so the apt cache stays consistent and the layer count is reduced. - README: document `xmllint` as a release-manager prerequisite (used by publish-to-maven.sh to parse Nexus staging responses). --- dev/release/README.md | 1 + dev/release/build-release.sh | 3 +++ dev/release/datafusion-java-rm/Dockerfile | 7 ++++--- dev/release/publish-to-maven.sh | 8 +++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 5557a47..ca637fa 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -172,6 +172,7 @@ for the common platforms. This section covers building that JAR. - Java 17+ - Rust toolchain via rustup - `gpg` configured with a key listed in the ASF KEYS file +- `xmllint` on `PATH` (pre-installed on macOS; `libxml2-utils` on Debian/Ubuntu) ### Build the multi-platform JAR diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh index 6539430..2b033bb 100755 --- a/dev/release/build-release.sh +++ b/dev/release/build-release.sh @@ -102,6 +102,9 @@ cleanup() { } trap cleanup SIGINT SIGTERM EXIT +echo "Cleaning leftover builder containers from any prior interrupted run" +docker rm -f "$CONTAINER_AMD64" "$CONTAINER_ARM64" >/dev/null 2>&1 || true + echo "Cleaning previous Java and native build output" (cd "$PROJECT_HOME" && ./mvnw -q clean) (cd "$PROJECT_HOME/native" && cargo clean) diff --git a/dev/release/datafusion-java-rm/Dockerfile b/dev/release/datafusion-java-rm/Dockerfile index 48b0266..0583167 100644 --- a/dev/release/datafusion-java-rm/Dockerfile +++ b/dev/release/datafusion-java-rm/Dockerfile @@ -41,9 +41,10 @@ RUN export LC_ALL=C \ libbz2-dev \ zlib1g-dev \ default-jdk \ - unzip - -RUN apt install -y gcc-10 g++-10 cpp-10 + unzip \ + gcc-10 \ + g++-10 \ + cpp-10 ENV CC="gcc-10" ENV CXX="g++-10" diff --git a/dev/release/publish-to-maven.sh b/dev/release/publish-to-maven.sh index 68d8636..f9ab966 100755 --- a/dev/release/publish-to-maven.sh +++ b/dev/release/publish-to-maven.sh @@ -125,13 +125,15 @@ for file in $(find . -type f); do FILE_SHORT=$(echo "$file" | sed -e "s/\.\///") DEST_URL="$NEXUS_UPLOAD/org/apache/datafusion/$FILE_SHORT" echo " Uploading $FILE_SHORT" - curl -u "$ASF_USERNAME:$ASF_PASSWORD" --upload-file "$FILE_SHORT" "$DEST_URL" + curl --fail-with-body \ + -u "$ASF_USERNAME:$ASF_PASSWORD" \ + --upload-file "$FILE_SHORT" "$DEST_URL" done echo "Closing nexus staging repository" REPO_REQUEST="$STAGED_REPO_IDApache DataFusion Java (commit $GIT_HASH)" -curl -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \ - -H "Content-Type:application/xml" -v \ +curl --fail-with-body -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \ + -H "Content-Type:application/xml" \ "$NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish" echo "Closed Nexus staging repository: $STAGED_REPO_ID"