HBASE-27359 Publish binaries and maven artifacts for both hadoop2 and…

… hadoop3
apache · Oct 31, 2022 · 1c9bd3c · 1c9bd3c
1 parent dc4fa05
commit 1c9bd3c
Show file tree

Hide file tree

Showing 3 changed files with 129 additions and 35 deletions.
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
@@ -302,15 +302,31 @@ if [[ "$1" == "publish-release" ]]; then
   mvn_log="${BASE_DIR}/mvn_deploy_release.log"
   log "Staging release in nexus"
   maven_deploy release "$mvn_log"
-  declare staged_repo_id="dryrun-no-repo"
+  declare staged_repo_id
+  declare hadoop3_staged_repo_id
   if ! is_dry_run; then
-    staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \
-        | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/')
-    log "Release artifacts successfully published to repo ${staged_repo_id}"
+    staged_repo_ids=($(grep -o "Closing staging repository with ID .*" "$mvn_log" \
+        | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/'))
+    log "Release artifacts successfully published to repo: " "${staged_repo_ids[@]}"
+    repo_count="${#staged_repo_ids[@]}"
+    if [[ "${repo_count}" == "2" ]]; then
+      staged_repo_id=${staged_repo_ids[0]}
+      hadoop3_staged_repo_id=${staged_repo_ids[1]}
+    elif [[ "${repo_count}" == "1" ]]; then
+      staged_repo_id=${staged_repo_ids[0]}
+      hadoop3_staged_repo_id="not-applicable"
+    else
+      staged_repo_id="not-applicable"
+      hadoop3_staged_repo_id="not-applicable"
+    fi
     rm "$mvn_log"
   else
     log "Dry run: Release artifacts successfully built, but not published due to dry run."
+    staged_repo_id="dryrun-no-repo"
+    hadoop3_staged_repo_id="dryrun-no-repo"
   fi
+  export staged_repo_id
+  export hadoop3_staged_repo_id
   # Dump out email to send. Where we find vote.tmpl depends
   # on where this script is run from
   PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g'

diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
@@ -75,7 +75,7 @@ function banner {
 }
 
 function log {
-  echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ${1}"
+  echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") $*"
 }
 
 # current number of seconds since epoch
@@ -298,12 +298,11 @@ EOF
   fi
   GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}")
 
-  if ! is_dry_run; then
-    if [ -z "$ASF_PASSWORD" ]; then
-      stty -echo && printf "ASF_PASSWORD: " && read -r ASF_PASSWORD && printf '\n' && stty echo
-    fi
-  else
-    ASF_PASSWORD="***INVALID***"
+  # The nexus staging plugin needs the password to contact to remote server even if
+  # skipRemoteStaging is set to true, not sure why so here we need the password even
+  # if this is a dry run
+  if [ -z "$ASF_PASSWORD" ]; then
+    stty -echo && printf "ASF_PASSWORD: " && read -r ASF_PASSWORD && printf '\n' && stty echo
   fi
 
   export ASF_PASSWORD
@@ -645,35 +644,26 @@ make_src_release() {
   stop_step "${timing_token}"
 }
 
-# Make binary release.
-# Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
-# -- and then the version string. Expects to find checkout adjacent to this script
-# named for 'project', the first arg passed.
-# Expects the following three defines in the environment:
-# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
-# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
-# - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE".
-# For example:
-# $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
-make_binary_release() {
+build_release_binary() {
   local project="${1}"
   local version="${2}"
   local base_name="${project}-${version}"
-  local timing_token
-  timing_token="$(start_step)"
-  rm -rf "${base_name}"-bin*
-  cd "$project" || exit
+  local extra_flags=()
+  if [[ "${version}" = *-hadoop3 ]] || [[ "${version}" = *-hadoop3-SNAPSHOT ]]; then
+    extra_flags=("-Drevision=${version}" "-Dhadoop.profile=3.0")
+  fi
 
+  cd "$project" || exit
   git clean -d -f -x
   # Three invocations of maven. This seems to work. One to
   # populate the repo, another to build the site, and then
   # a third to assemble the binary artifact. Trying to do
   # all in the one invocation fails; a problem in our
   # assembly spec to in maven. TODO. Meantime, three invocations.
-  "${MVN[@]}" clean install -DskipTests
-  "${MVN[@]}" site -DskipTests
+  "${MVN[@]}" "${extra_flags[@]}" clean install -DskipTests
+  "${MVN[@]}" "${extra_flags[@]}" site -DskipTests
   kick_gpg_agent
-  "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}"
+  "${MVN[@]}" "${extra_flags[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}"
 
   # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty.
   local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}"
@@ -688,6 +678,30 @@ make_binary_release() {
     cd .. || exit
     log "No ${f_bin_prefix}*-bin.tar.gz product; expected?"
   fi
+}
+
+# Make binary release.
+# Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
+# -- and then the version string. Expects to find checkout adjacent to this script
+# named for 'project', the first arg passed.
+# Expects the following three defines in the environment:
+# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
+# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
+# - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE".
+# For example:
+# $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
+make_binary_release() {
+  local project="${1}"
+  local version="${2}"
+  local base_name="${project}-${version}"
+  local timing_token
+  timing_token="$(start_step)"
+  rm -rf "${base_name}"-bin*
+
+  build_release_binary "${project}" "${version}"
+  if should_build_with_hadoop3 "$project/pom.xml"; then
+    build_release_binary "${project}" "$(get_hadoop3_version "${version}")"
+  fi
 
   stop_step "${timing_token}"
 }
@@ -740,6 +754,7 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
   # Invoke with cwd=$PROJECT
   local deploy_type="$1"
   local mvn_log_file="$2" #secondary log file used later to extract staged_repo_id
+  local staging_dir
   if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then
     error "unrecognized deploy type, must be 'snapshot'|'release'"
   fi
@@ -753,6 +768,8 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
   elif [[ "$deploy_type" == "release" ]] && [[ "$RELEASE_VERSION" =~ SNAPSHOT ]]; then
     error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$RELEASE_VERSION'"
   fi
+  # Just output to parent directory, the staging directory has a staging prefix already
+  staging_dir="$(dirname "$(pwd)")-local-staged"
   # Publish ${PROJECT} to Maven repo
   # shellcheck disable=SC2154
   log "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)"
@@ -761,20 +778,44 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
   maven_set_version "$RELEASE_VERSION"
   # Prepare for signing
   kick_gpg_agent
-  declare -a mvn_goals=(clean)
-  if ! is_dry_run; then
-    mvn_goals=("${mvn_goals[@]}" deploy)
+  declare -a mvn_extra_flags=()
+  if is_dry_run; then
+    # In dry run mode, skip deploying to remote repo
+    mvn_extra_flags=("${mvn_extra_flags[@]}" -DskipRemoteStaging)
   fi
-  log "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" "${mvn_goals[@]}"
+  log "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true \
+    -DaltStagingDirectory="${staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}"
   log "Logging to ${mvn_log_file}.  This will take a while..."
   rm -f "$mvn_log_file"
   # The tortuous redirect in the next command allows mvn's stdout and stderr to go to mvn_log_file,
   # while also sending stderr back to the caller.
   # shellcheck disable=SC2094
-  if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \
-      "${mvn_goals[@]}" 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then
+  if ! "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \
+      -DaltStagingDirectory="${staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" \
+      1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then
     error "Deploy build failed, for details see log at '$mvn_log_file'."
   fi
+  local hadoop3_version
+  if should_build_with_hadoop3 pom.xml; then
+    hadoop3_version="$(get_hadoop3_version "${RELEASE_VERSION}")"
+    hadoop3_staging_dir="${staging_dir}-hadoop3"
+    log "Deploying artifacts for hadoop3..."
+    log "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true \
+      -Drevision="${hadoop3_version}" -Dhadoop.profile=3.0 \
+      -DaltStagingDirectory="${hadoop3_staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}"
+    {
+      echo "========================================================================"
+      echo "Deploy build for hadoop3"
+      echo "========================================================================"
+    } >> "$mvn_log_file"
+    # shellcheck disable=SC2094
+    if ! "${MVN[@]}" clean deploy -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \
+        -Drevision="${hadoop3_version}" -Dhadoop.profile=3.0 \
+        -DaltStagingDirectory="${hadoop3_staging_dir}" "${PUBLISH_PROFILES[@]}" "${mvn_extra_flags[@]}" \
+        1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then
+      error "Deploy build failed, for details see log at '$mvn_log_file'."
+    fi
+  fi
   log "BUILD SUCCESS."
   stop_step "${timing_token}"
   return 0
@@ -792,3 +833,36 @@ function is_tracked() {
   git ls-files --error-unmatch "$file" &>/dev/null
   return $?
 }
+
+# When we have all the below conditions matched, we will build hadoop3 binaries
+# 1. Use $revision place holder as version in pom
+# 2. Has a hadoop-2.0 profile
+# 3. Has a hadoop-3.0 profile
+function should_build_with_hadoop3() {
+  local pom="$1"
+  maven_version="$(parse_version < "${pom}")"
+  # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+  # If we use ${revision} as placeholder, the way to bump maven version will be different
+  # shellcheck disable=SC2016
+  if [[ "${maven_version}" != '${revision}' ]]; then
+    return 1
+  fi
+  if ! xmllint --xpath "//*[local-name()='project']/*[local-name()='profiles']/*[local-name()='profile']/*[local-name()='id']/text()" "${pom}" \
+    | grep -q ^hadoop-2.0$; then
+    return 1
+  fi
+  if ! xmllint --xpath "//*[local-name()='project']/*[local-name()='profiles']/*[local-name()='profile']/*[local-name()='id']/text()" "${pom}" \
+    | grep -q ^hadoop-3.0$; then
+    return 1
+  fi
+  return 0
+}
+
+function get_hadoop3_version() {
+  local version="$1"
+  if [[ "${version}" =~ -SNAPSHOT$ ]]; then
+    echo "${version/-SNAPSHOT/-hadoop3-SNAPSHOT}"
+  else
+    echo "${version}-hadoop3"
+  fi
+}
diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl
@@ -23,6 +23,10 @@ Maven artifacts are available in a staging repository at:
 
   https://repository.apache.org/content/repositories/${staged_repo_id}/
 
+Maven artifacts for hadoop3 are available in a staging repository at:
+
+  https://repository.apache.org/content/repositories/${hadoop3_staged_repo_id}/
+
 Artifacts were signed with the ${GPG_KEY} key which can be found in:
 
   https://downloads.apache.org/hbase/KEYS