[chore](tools) Add release helper scripts for cutting a source release candidate

.asf.yaml

@@ -25,7 +25,7 @@ github:
     - lakehouse
     - ai
     - agent
-    - elt
+    - observability
     - sql
     - snowflake
     - redshift

.gitleaks.toml

@@ -27,6 +27,13 @@ regexTarget = "line"
 paths = ['''^fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageTest\.java$''']
 regexes = ['''(access_key|secret_key).*"(ak-|sk-|canonical|sess-tok)''']
 
+[[rules.allowlists]]
+description = "Ignore the GPG signing-key fingerprint in the release-tools config template (a public key id, not a secret)"
+condition = "AND"
+regexTarget = "line"
+paths = ['''^tools/release-tools/release\.env$''']
+regexes = ['''SIGNING_KEY="[A-F0-9]{40}"''']
+
 [[rules]]
 id = "private-key"
 

tools/release-tools/01-check-env.sh

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Step 01 - prepare / check the signing environment.
+# Read-mostly. The only state-changing paths (import key / generate key /
+# publish KEYS / edit gpg.conf) are opt-in and prompt before acting.
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=release.env
+source "${HERE}/release.env"
+
+ok()   { echo "[ OK ] $*"; }
+warn() { echo "[WARN] $*"; }
+err()  { echo "[FAIL] $*"; }
+die()  { err "$*"; exit 1; }
+confirm() { local a; read -r -p "$1 [y/N] " a; [[ "$a" == y || "$a" == Y ]]; }
+
+# does $1 (fingerprint) appear in a KEYS stream piped on stdin?
+key_in_keys() {
+  local fpr="$1" kr ret=1; kr="$(mktemp -d)"
+  if gpg --homedir "$kr" --import >/dev/null 2>&1 && gpg --homedir "$kr" --list-keys "$fpr" >/dev/null 2>&1; then ret=0; fi
+  rm -rf "$kr"; return "$ret"
+}
+
+# svn auth args (only added if exported)
+svn_auth=(--non-interactive --no-auth-cache)
+[[ -n "${ASF_USERNAME:-}" ]] && svn_auth+=(--username "$ASF_USERNAME")
+[[ -n "${ASF_PASSWORD:-}" ]] && svn_auth+=(--password "$ASF_PASSWORD")
+
+problems=0
+echo "== Apache Doris ${TAG} - signing environment check =="
+
+# 1. required tools
+for t in git gpg svn sha512sum curl gzip; do
+  if command -v "$t" >/dev/null 2>&1; then ok "tool: $t"; else err "missing tool: $t"; problems=$((problems+1)); fi
+done
+
+# 2. GPG_TTY (needed so the passphrase prompt can appear)
+export GPG_TTY="$(tty || true)"
+ok "GPG_TTY=${GPG_TTY:-<none>}"
+
+# 3. gpg.conf SHA512 preference (Apache recommendation)
+gpgconf_file="${GNUPGHOME:-$HOME/.gnupg}/gpg.conf"
+if grep -q "cert-digest-algo SHA512" "$gpgconf_file" 2>/dev/null; then
+  ok "gpg.conf has SHA512 digest preferences"
+else
+  warn "gpg.conf missing SHA512 preferences ($gpgconf_file)"
+  if confirm "Append recommended SHA512 lines to gpg.conf?"; then
+    {
+      echo "personal-digest-preferences SHA512"
+      echo "cert-digest-algo SHA512"
+      echo "default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed"
+    } >> "$gpgconf_file"
+    ok "updated $gpgconf_file"
+  fi
+fi
+
+# 4. resolve a usable secret key (one primary fpr per secret key)
+list_secret_fprs() {
+  gpg --list-secret-keys --with-colons 2>/dev/null \
+    | awk -F: '$1=="sec"{w=1} $1=="fpr"&&w{print $10; w=0}'
+}
+resolve_secret_key() {
+  if [[ -n "${SIGNING_KEY}" ]]; then
+    gpg --list-secret-keys "${SIGNING_KEY}" >/dev/null 2>&1 && { echo "${SIGNING_KEY}"; return 0; }
+    return 1
+  fi
+  local fprs n; fprs="$(list_secret_fprs)"; n="$(printf '%s\n' "$fprs" | grep -c . || true)"
+  if   [[ "$n" -eq 1 ]]; then printf '%s\n' "$fprs"; return 0
+  elif [[ "$n" -gt 1 ]]; then return 2
+  fi
+  return 1
+}
+
+import_key_from_file() {
+  local p; read -r -p "Path to exported secret key file (.asc/.gpg): " p
+  [[ -f "$p" ]] || die "no such file: $p"
+  gpg --import "$p"
+}
+
+gen_key() {
+  local rn em
+  read -r -p "Real name (>=5 chars, e.g. morningman): " rn
+  read -r -p "Apache email (xxx@apache.org): " em
+  echo "Generating RSA-4096 sign key, no expiry. You'll be asked for a passphrase - REMEMBER IT."
+  gpg --quick-generate-key "${rn} (CODE SIGNING KEY) <${em}>" rsa4096 sign never
+}
+
+# append the new public key to dev + release KEYS and commit (so voters can verify)
+# append the key to the dev + release KEYS files and commit (append-only).
+# idempotent: skips a repo whose KEYS already contains the key, so re-running
+# after a partial failure won't duplicate the entry.
+# caller is responsible for asking the user first.
+publish_key_to_keys() {
+  local fpr="$1" spec name base wc
+  mkdir -p "${WORK_DIR}"
+  for spec in "dev=${DEV_SVN_BASE}" "release=${RELEASE_SVN_BASE}"; do
+    name="${spec%%=*}"; base="${spec#*=}"
+    wc="${WORK_DIR}/keys-${name}-$$"
+    rm -rf "$wc"                       # avoid stale working-copy collisions
+    echo ">> ${base}/KEYS"
+    svn checkout --depth files "${svn_auth[@]}" "$base" "$wc"
+    if key_in_keys "$fpr" < "$wc/KEYS"; then
+      ok "key already in ${name} KEYS - skip"
+      continue
+    fi
+    # append only - never rewrite existing KEYS content
+    { echo; gpg --list-sigs "$fpr"; gpg --armor --export "$fpr"; } >> "$wc/KEYS"
+    echo "--- appended to ${name}/KEYS (tail) ---"; tail -n 18 "$wc/KEYS"
+    svn commit "${svn_auth[@]}" -m "Add KEY for ${APACHE_ID}" "$wc"
+    ok "committed KEYS to ${base}"
+  done
+  warn "MANUAL: paste this fingerprint into https://id.apache.org (OpenPGP field):"
+  gpg --fingerprint "$fpr" | sed -n '2p'
+}
+
+SIGNER=""
+if SIGNER="$(resolve_secret_key)"; then
+  ok "signing key resolved: ${SIGNER}"
+else
+  rc=$?
+  if [[ "$rc" -eq 2 ]]; then
+    err "multiple secret keys found - set SIGNING_KEY in release.env to choose one:"
+    gpg --list-secret-keys --keyid-format=long
+    problems=$((problems+1))
+  else
+    warn "no usable secret key in this environment. Options:"
+    echo "  1) import an existing exported secret key from a file"
+    echo "  2) generate a NEW key here and publish it to the Doris KEYS files"
+    echo "  3) skip (you will sign on another machine)"
+    read -r -p "choose [1/2/3]: " choice
+    case "$choice" in
+      1) import_key_from_file; SIGNER="$(resolve_secret_key || true)";;
+      2) gen_key; SIGNER="$(resolve_secret_key || true)"
+         if [[ -n "$SIGNER" ]] && confirm "Publish this new key to dev+release KEYS now?"; then
+           publish_key_to_keys "$SIGNER"
+         fi;;
+      3) warn "skipping local key; sign on another machine.";;
+      *) warn "no choice made.";;
+    esac
+    if [[ -n "$SIGNER" ]]; then ok "signing key resolved: ${SIGNER}"; else warn "still no signing key"; problems=$((problems+1)); fi
+  fi
+fi
+
+# 5. is the signing key published in the live KEYS? + 6. test signature
+if [[ -n "$SIGNER" ]]; then
+  fpr="$(gpg --list-keys --with-colons "$SIGNER" | awk -F: '/^fpr:/{print $10; exit}')"
+  ok "fingerprint: ${fpr}"
+
+  if curl -fsSL "$KEYS_URL" 2>/dev/null | key_in_keys "$fpr"; then
+    ok "key is present in published KEYS"
+  else
+    warn "key NOT found in published KEYS ($KEYS_URL) - voters can't verify until it's published"
+    if confirm "Append this key to the Doris dev+release KEYS and commit now?"; then
+      publish_key_to_keys "$fpr"
+      if svn cat "${svn_auth[@]}" "${RELEASE_SVN_BASE}/KEYS" 2>/dev/null | key_in_keys "$fpr"; then
+        ok "key now in release SVN KEYS (downloads.apache.org mirror syncs in a few min)"
+      else
+        err "key still not in release SVN KEYS after commit - check output above"
+        problems=$((problems+1))
+      fi
+    else
+      warn "skipped KEYS publish; key MUST be in published KEYS before the vote"
+      problems=$((problems+1))
+    fi
+  fi
+
+  tf="$(mktemp)"; echo "doris ${TAG} signing test" > "$tf"
+  if gpg -u "$SIGNER" --armor --detach-sign -o "$tf.asc" "$tf" >/dev/null 2>&1 \
+     && gpg --verify "$tf.asc" "$tf" >/dev/null 2>&1; then
+    ok "test sign + verify succeeded (passphrase/agent working)"
+  else
+    err "test signing failed - check passphrase / GPG_TTY / pinentry"; problems=$((problems+1))
+  fi
+  rm -f "$tf" "$tf.asc"
+fi
+
+# 7. ASF SVN credentials
+if [[ -n "${ASF_USERNAME:-}" && -n "${ASF_PASSWORD:-}" ]]; then
+  ok "ASF_USERNAME/ASF_PASSWORD present in env"
+else
+  warn "ASF_USERNAME/ASF_PASSWORD not both set - needed for SVN upload (step 02) and KEYS publish"
+fi
+
+echo
+if [[ "$problems" -eq 0 ]]; then
+  ok "environment looks READY for ${TAG}"
+else
+  err "${problems} problem(s) above - resolve before packaging"; exit 1
+fi

tools/release-tools/02-package-sign-upload.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Step 02 - package the source tarball, sign it, and upload to the Apache dev SVN.
+# The SVN commit is outward-facing: it pauses twice for confirmation before commit.
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=release.env
+source "${HERE}/release.env"
+
+ok()   { echo "[ OK ] $*"; }
+warn() { echo "[WARN] $*"; }
+die()  { echo "[FAIL] $*" >&2; exit 1; }
+confirm() { local a; read -r -p "$1 [y/N] " a; [[ "$a" == y || "$a" == Y ]]; }
+export GPG_TTY="$(tty || true)"
+
+svn_auth=(--non-interactive --no-auth-cache)
+[[ -n "${ASF_USERNAME:-}" ]] && svn_auth+=(--username "$ASF_USERNAME")
+[[ -n "${ASF_PASSWORD:-}" ]] && svn_auth+=(--password "$ASF_PASSWORD")
+
+# resolve signer
+if [[ -n "${SIGNING_KEY}" ]]; then
+  SIGNER="${SIGNING_KEY}"
+else
+  SIGNER="$(gpg --list-secret-keys --with-colons 2>/dev/null | awk -F: '$1=="sec"{w=1} $1=="fpr"&&w{print $10; exit}')"
+fi
+[[ -n "$SIGNER" ]] || die "no signing key found; run ./01-check-env.sh first"
+ok "signer: $SIGNER"
+
+# 1. verify the tag matches the apache remote (don't release a local-only tag)
+cd "$REPO_DIR"
+git rev-parse "$TAG" >/dev/null 2>&1 || die "local tag $TAG not found in $REPO_DIR"
+local_tag="$(git rev-parse "$TAG")"
+remote_tag="$(git ls-remote --tags "$GIT_REMOTE" "refs/tags/$TAG" | awk -v t="refs/tags/$TAG" '$2==t{print $1}')"
+[[ -n "$remote_tag" ]] || die "tag $TAG not found on remote $GIT_REMOTE"
+[[ "$local_tag" == "$remote_tag" ]] || die "tag mismatch: local=$local_tag $GIT_REMOTE=$remote_tag"
+ok "tag $TAG matches $GIT_REMOTE"
+
+# 2. package via git archive from the tag
+mkdir -p "$WORK_DIR"
+art="$WORK_DIR/${PKG_BASE}.tar.gz"
+echo "archiving $TAG -> $art"
+git archive --format=tar --prefix="$ARCHIVE_PREFIX" "$TAG" | gzip > "$art"
+ok "source tarball: $(du -h "$art" | cut -f1)  $art"
+
+# 3. sign + checksum (inside WORK_DIR so the files reference bare basenames)
+cd "$WORK_DIR"
+f="${PKG_BASE}.tar.gz"
+rm -f "$f.asc" "$f.sha512"
+gpg -u "$SIGNER" --armor --output "$f.asc" --detach-sign "$f"
+gpg --verify "$f.asc" "$f"
+ok "signature ok: $f.asc"
+sha512sum "$f" > "$f.sha512"
+sha512sum --check "$f.sha512"
+ok "sha512 ok: $f.sha512"
+echo; ls -l "$f" "$f.asc" "$f.sha512"; echo
+
+# 4. sign + checksum the prebuilt binary tarballs. Sidecar files are written NEXT TO
+#    each binary (not in WORK_DIR) and are NOT uploaded here -- you upload them manually.
+bin_count="${#BIN_FILES[@]}"
+if [[ "$bin_count" -gt 0 ]]; then
+  echo "signing $bin_count binary artifact(s)..."
+  for bin in "${BIN_FILES[@]}"; do
+    [[ -f "$bin" ]] || die "binary artifact not found: $bin (check BIN_FILES in release.env)"
+    bdir="$(cd "$(dirname "$bin")" && pwd)"
+    bname="$(basename "$bin")"
+    (
+      cd "$bdir"
+      rm -f "$bname.asc" "$bname.sha512"
+      gpg -u "$SIGNER" --armor --output "$bname.asc" --detach-sign "$bname"
+      gpg --verify "$bname.asc" "$bname"
+      sha512sum "$bname" > "$bname.sha512"
+      sha512sum --check "$bname.sha512"
+    )
+    ok "binary signed: $bdir/$bname (+ .asc, .sha512)"
+  done
+  echo
+fi
+
+# 5. upload to dev SVN (two confirmations; nothing public happens before them)
+echo "Target dev SVN folder: ${DEV_SVN_DIR}/"
+confirm "Checkout + add these 3 files for the above SVN URL?" || { warn "stopping before SVN."; exit 0; }
+wc="$WORK_DIR/dev-svn"
+rm -rf "$wc"
+svn checkout --depth empty "${svn_auth[@]}" "$DEV_SVN_BASE" "$wc"
+mkdir -p "$wc/$TAG"
+cp "$f" "$f.asc" "$f.sha512" "$wc/$TAG/"
+svn add "$wc/$TAG"
+echo "--- svn status ---"; svn status "$wc"; echo
+echo "Will commit the above to: ${DEV_SVN_DIR}/"
+confirm "FINAL confirm - svn commit now?" || { warn "left staged (uncommitted) at $wc"; exit 0; }
+svn commit "${svn_auth[@]}" -m "Add ${TAG}" "$wc"
+ok "committed. Vote artifacts now at: ${DEV_SVN_DIR}/"
+echo "Next: ./03-vote-mail.sh"