Cherry-pick necessary mirror PRs from trunk branch to Branch-3.4 branch

BUILDING.txt

@@ -29,7 +29,11 @@ This requires a recent version of docker (1.4.1 and higher are known to work).
 On Linux / Mac:
     Install Docker and run this command:
 
-    $ ./start-build-env.sh
+    $ ./start-build-env.sh [OS platform]
+
+   - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10].
+        Default is 'ubuntu_20'.
+        Note: Currently only default ('ubuntu_20') is supported on arm machine
 
 The prompt which is then presented is located at a mounted version of the source tree
 and all required tools for testing and building have been installed and configured.
@@ -457,7 +461,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
 
 ----------------------------------------------------------------------------------
 
-Building on CentOS 8
+Building on Rocky Linux 8
 
 ----------------------------------------------------------------------------------
 
@@ -478,7 +482,7 @@ Building on CentOS 8
   $ sudo make install
   $ cd ..
 
-* Install libraries provided by CentOS 8.
+* Install libraries provided by Rocky Linux 8.
   $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
 
 * Install GCC 9.3.0

dev-support/Jenkinsfile

@@ -83,24 +83,18 @@ pipeline {
         // optional stages after the first one.
         stage ('setup sources') {
             steps {
-                dir("${WORKSPACE}/centos-7") {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7
-                    '''
-                }
 
-                dir("${WORKSPACE}/centos-8") {
+                dir("${WORKSPACE}/rockylinux-8") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
                     '''
                 }
 
-                dir("${WORKSPACE}/debian-10") {
+                dir("${WORKSPACE}/debian-11") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11
                     '''
                 }
 
@@ -117,58 +111,12 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 7.
-        stage ('precommit-run Centos 7') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/centos-7/src"
-                PATCHDIR = "${WORKSPACE}/centos-7/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(getGithubCreds()) {
-                    sh '''#!/usr/bin/env bash
-
-                    chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                    "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                    '''
-                }
-            }
-
-            post {
-                // Since this is an optional platform, we want to copy the artifacts
-                // and archive it only if the build fails, to help with debugging.
-                failure {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}"
-                    '''
-                    archiveArtifacts "out/**"
-                }
-
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
+        // break the Hadoop build on Rocky Linux 8.
+        stage ('precommit-run Rocky Linux 8') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/centos-8/src"
-                PATCHDIR = "${WORKSPACE}/centos-8/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+                SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
+                PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8"
                 IS_OPTIONAL = 1
             }
 
@@ -188,7 +136,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }
@@ -209,12 +157,12 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
+        // break the Hadoop build on Debian 11.
+        stage ('precommit-run Debian 11') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/debian-10/src"
-                PATCHDIR = "${WORKSPACE}/debian-10/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+                SOURCEDIR = "${WORKSPACE}/debian-11/src"
+                PATCHDIR = "${WORKSPACE}/debian-11/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11"
                 IS_OPTIONAL = 1
             }
 
@@ -234,7 +182,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }

dev-support/docker/Dockerfile

@@ -44,9 +44,14 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
 ######
 # hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \
     && apt-get -q install -y --no-install-recommends python3 \
+    && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
+    && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc  \
+    && apt-get -q update \
     && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
     && apt-get clean \
+    && update-java-alternatives -s java-1.8.0-openjdk-amd64 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN locale-gen en_US.UTF-8

dev-support/docker/Dockerfile_aarch64

@@ -44,10 +44,15 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
 ######
 # hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \
     && apt-get -q install -y --no-install-recommends python3 \
     && apt-get -q install -y --no-install-recommends phantomjs \
+    && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
+    && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc  \
+    && apt-get -q update \
     && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
     && apt-get clean \
+    && update-java-alternatives -s java-1.8.0-openjdk-arm64 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN locale-gen en_US.UTF-8

dev-support/docker/Dockerfile_debian_10 → dev-support/docker/Dockerfile_debian_11

@@ -17,7 +17,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM debian:10
+FROM debian:11
 
 WORKDIR /root
 
@@ -29,65 +29,63 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
 RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_TERSE=true
 
 ######
 # Platform package dependency resolver
 ######
 COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/install-pkg-resolver.sh
-RUN pkg-resolver/install-pkg-resolver.sh debian:10
+RUN pkg-resolver/install-pkg-resolver.sh debian:11
 
 ######
 # Install packages from apt
 ######
 # hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \
-    && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \
-    && apt-get -q update \
-    && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \
     && apt-get clean \
+    && update-java-alternatives -s java-1.11.0-openjdk-amd64 \
     && rm -rf /var/lib/apt/lists/*
 
 # TODO : Set locale
 
 ######
 # Set env vars required to build Hadoop
 ######
-ENV MAVEN_HOME /usr
+ENV MAVEN_HOME=/usr
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
 
 #######
 # Set env vars for SpotBugs 4.2.2
 #######
-ENV SPOTBUGS_HOME /opt/spotbugs
+ENV SPOTBUGS_HOME=/opt/spotbugs
 
 #######
 # Set env vars for Google Protobuf 3.21.12
 #######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
+ENV PROTOBUF_HOME=/opt/protobuf
+ENV PATH="${PATH}:/opt/protobuf/bin"
 
 ###
 # Avoid out of memory errors in builds
 ###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
+ENV MAVEN_OPTS="-Xms256m -Xmx3072m"
 
 # Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
+ENV HADOOP_SKIP_YETUS_VERIFICATION=true
 
 ####
 # Install packages
 ####
-RUN pkg-resolver/install-cmake.sh debian:10
-RUN pkg-resolver/install-spotbugs.sh debian:10
-RUN pkg-resolver/install-boost.sh debian:10
-RUN pkg-resolver/install-protobuf.sh debian:10
-RUN pkg-resolver/install-hadolint.sh debian:10
-RUN pkg-resolver/install-intel-isa-l.sh debian:10
+RUN pkg-resolver/install-cmake.sh debian:11
+RUN pkg-resolver/install-spotbugs.sh debian:11
+RUN pkg-resolver/install-boost.sh debian:11
+RUN pkg-resolver/install-protobuf.sh debian:11
+RUN pkg-resolver/install-hadolint.sh debian:11
+RUN pkg-resolver/install-intel-isa-l.sh debian:11
 
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.