HBASE-27449 Use xmllint to parse version and revision in release scripts

Also upgrade our Dockerfile for release scripts to use ubuntu 22.04,
maven 3.8.6 and yetus 0.14.1.
Use python3 as much as possible.

dev-support/create-release/hbase-rm/Dockerfile

@@ -15,11 +15,11 @@
 # limitations under the License.
 #
 
-# Image for building HBase releases. Based on Ubuntu 18.04.
+# Image for building HBase releases. Based on Ubuntu 22.04.
 #
 # Includes:
 # * Java 8
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 
 # Install extra needed repos and refresh.
@@ -28,40 +28,47 @@ FROM ubuntu:18.04
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
   && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-    curl='7.58.0-*' \
-    git='1:2.17.1-*' \
-    gnupg='2.2.4-*' \
-    libcurl4-openssl-dev='7.58.0-*' \
-    libxml2-dev='2.9.4+dfsg1-*' \
-    lsof='4.89+dfsg-*' \
+    curl='7.81.0-*' \
+    git='1:2.34.1-*' \
+    gnupg='2.2.27-*' \
+    libcurl4-openssl-dev='7.81.0-*' \
+    libxml2-dev='2.9.13+dfsg-*' \
+    libxml2-utils='2.9.13+dfsg-*' \
+    lsof='4.93.2+dfsg-*' \
     openjdk-8-jdk='8u*' \
-    python-pip='9.0.1-*' \
-    subversion='1.9.7-*' \
-    wget='1.19.4-*' \
+    python3-pip='22.0.2+dfsg-*' \
+    python2='2.7.18-*' \
+    subversion='1.14.1-*' \
+    wget='1.21.2-*' \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
-  && pip install \
-    python-dateutil==2.8.1
+  && pip3 install \
+    python-dateutil==2.8.2
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Install mvn 3.6.3.
-ARG MAVEN_VERSION=3.6.3
-ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0
-ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+# Install maven
+ENV MAVEN_VERSION=3.8.6
+ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
+ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26"
 RUN mkdir -p /opt/maven \
-  && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
-  && echo "${SHA}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
+  && curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \
+  && echo "${MAVEN_SHA512}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
   && tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \
   && rm -f /tmp/apache-maven.tar.gz \
   && ln -s /opt/maven/bin/mvn /usr/bin/mvn
 
 # Install Apache Yetus
-ENV YETUS_VERSION 0.12.0
-RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
-        tar xvz -C /opt
-ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
+ENV YETUS_VERSION 0.14.1
+ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz"
+ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109"
+RUN mkdir -p /opt/yetus \
+  && curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \
+  && echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \
+  && tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \
+  && rm -f /tmp/apache-yetus.tar.gz
+ENV YETUS_HOME /opt/yetus
 
 ARG UID
 ARG RM_USER

dev-support/create-release/release-build.sh

@@ -149,7 +149,7 @@ if [[ "$1" == "tag" ]]; then
   # check their position when generating release data. We can not put them under the source code
   # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
   # generating src release tarball, we will reset the git repo
-  if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
+  if is_tracked "CHANGES.md"; then
     git add RELEASENOTES.md CHANGES.md
     git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
     cp CHANGES.md ../

dev-support/create-release/release-util.sh

@@ -60,8 +60,11 @@ function read_config {
 }
 
 function parse_version {
-  grep -e '<version>.*</version>' | \
-    head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
+  xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" -
+}
+
+function parse_revision {
+  xmllint --xpath "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='revision']/text()" -
 }
 
 function banner {
@@ -164,6 +167,14 @@ function get_release_info {
   local version
   version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
     parse_version)"
+  # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+  # If we use ${revision} as placeholder, we need to parse the revision property to
+  # get maven version
+  # shellcheck disable=SC2016
+  if [[ "${version}" == '${revision}' ]]; then
+    version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
+      parse_revision)"
+  fi
   log "Current branch VERSION is $version."
 
   NEXT_VERSION="$version"
@@ -356,9 +367,13 @@ function init_java {
 
 function init_python {
   if ! [ -x "$(command -v python2)"  ]; then
-    error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2'
+    error 'python2 needed by api report. Install or add link?'
   fi
-  log "python version: $(python2 --version)"
+  log "python2 version: $(python2 --version)"
+  if ! [ -x "$(command -v python3)"  ]; then
+    error 'python3 needed by yetus. Install or add link?'
+  fi
+  log "python3 version: $(python3 --version)"
 }
 
 # Set MVN
@@ -440,8 +455,8 @@ function git_clone_overwrite {
     log "Clone will be of the gitbox repo for ${PROJECT}."
     if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
       # Ugly!
-      encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')")
-      encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')")
+      encoded_username=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_USERNAME''', ''))")
+      encoded_password=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_PASSWORD''', ''))")
       GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
     else
       GIT_REPO="https://${asf_repo}"
@@ -533,7 +548,7 @@ function get_jira_name {
 # Update the CHANGES.md
 # DOES NOT DO COMMITS! Caller should do that.
 # requires yetus to have a defined home already.
-# yetus requires python2 to be on the path.
+# yetus requires python3 to be on the path.
 function update_releasenotes {
   local project_dir="$1"
   local jira_fix_version="$2"
@@ -686,28 +701,13 @@ function kick_gpg_agent {
 function maven_set_version { #input: <version_to_set>
   local this_version="$1"
   local use_revision='false'
-  local -a version_splits=()
-  IFS='.' read -ar version_splits <<< "$(maven_get_version)"
-
-  # Do the right thing based on project and release line.
-  if [ "${PROJECT}" = 'hbase' ] ; then
-    if [ "${version_splits[0]}" -le 1 ] ; then
-      use_revision='false'
-    elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -le 4 ] ; then
-      use_revision='false'
-    elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -ge 5 ] ; then
-      use_revision='true'
-    elif [ "${version_splits[0]}" -ge 3 ] ; then
-      use_revision='true'
-    fi
-  elif [ "${PROJECT}" = 'hbase-thirdparty' ] ; then
-    use_revision='false'
-  elif [ "${PROJECT}" = 'hbase-connectors' ] ; then
+  local maven_version
+  maven_version="$(parse_version < pom.xml)"
+  # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+  # If we use ${revision} as placeholder, the way to bump maven version will be different
+  # shellcheck disable=SC2016
+  if [[ "${maven_version}" == '${revision}' ]]; then
     use_revision='true'
-  elif [ "${PROJECT}" = 'hbase-filesystem' ] ; then
-    use_revision='false'
-  elif [ "${PROJECT}" = 'hbase-operator-tools' ] ; then
-    use_revision='false'
   fi
 
   if [ "${use_revision}" = 'false' ] ; then