Build Hadoop from source

CHANGELOG.md

@@ -14,11 +14,13 @@ All notable changes to this project will be documented in this file.
 - stackable-base: bump ubi8-minimal image to latest 8.9 ([#514]).
 - Bump ubi8-rust-builder toolchain to `1.74.0` ([#517]).
 - GH workflows: make preflight an independent manual workflow and update to version 1.7.2 ([#519]).
+- hadoop: Build from source ([#526]).
 
 [#493]: https://github.com/stackabletech/docker-images/pull/493
 [#514]: https://github.com/stackabletech/docker-images/pull/514
 [#517]: https://github.com/stackabletech/docker-images/pull/517
 [#519]: https://github.com/stackabletech/docker-images/pull/519
+[#526]: https://github.com/stackabletech/docker-images/pull/526
 
 ## [23.11.0] - 2023-11-30
 

conf.py

@@ -63,27 +63,31 @@
                 "java-base": "11",
                 "java": "11",
                 "jmx_exporter": "0.20.0",
+                "protobuf": "2.5.0",
                 "topology_provider": "0.1.0"
             },
             {
                 "product": "3.2.4",
                 "java-base": "11",
                 "java": "11",
                 "jmx_exporter": "0.20.0",
+                "protobuf": "2.5.0",
                 "topology_provider": "0.1.0"
             },
             {
                 "product": "3.3.4",
                 "java-base": "11",
                 "java": "11",
                 "jmx_exporter": "0.20.0",
+                "protobuf": "3.7.1",
                 "topology_provider": "0.1.0"
             },
             {
                 "product": "3.3.6",
                 "java-base": "11",
                 "java": "11",
                 "jmx_exporter": "0.20.0",
+                "protobuf": "3.7.1",
                 "topology_provider": "0.1.0"
             },
         ],

hadoop/Dockerfile

@@ -4,40 +4,27 @@ FROM stackable/image/java-base AS builder
 ARG PRODUCT
 ARG JAVA
 ARG JMX_EXPORTER
+ARG PROTOBUF
 ARG TOPOLOGY_PROVIDER
 
 # https://github.com/hadolint/hadolint/wiki/DL4006
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# unzip & zip are required for log4shell.sh
 # All others are required for the FUSE build
 RUN microdnf update && \
     microdnf install \
-    cmake \
-    cyrus-sasl-devel \
-    fuse-devel \
-    gcc \
-    gcc-c++ \
-    java-${JAVA}-openjdk-devel \
-    maven \
-    openssl-devel \
-    tar \
-    unzip \
-    zip && \
+    # Required for Hadoop build
+    cmake cyrus-sasl-devel fuse-devel gcc gcc-c++ java-11-openjdk-devel maven openssl-devel tar xz git \
+    # Required for log4shell.sh
+    unzip zip && \
     microdnf clean all
 
 WORKDIR /stackable
 
-# This is needed here because it creates the JMX directory, we could create it any other way but this works
 COPY hadoop/stackable /stackable
 
-# The source is needed to build FUSE. The rest of the src package will not make it into the final image.
-# Both the src and binary variants extract into different root folders
-RUN curl --fail -L "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC . && \
-    curl --fail -L "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}.tar.gz" | tar -xzC . && \
-    ln -s "/stackable/hadoop-${PRODUCT}" /stackable/hadoop && \
-    rm -rf /stackable/hadoop/lib/native/examples && \
-    rm -rf /stackable/hadoop/share/doc
+# Build from source to enable FUSE module, and to apply custom patches.
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC .
 
 # The symlink from JMX Exporter 0.16.1 to the versionless link exists because old HDFS Operators (up until and including 23.7) used to hardcode
 # the version of JMX Exporter like this: "-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar"
@@ -50,6 +37,28 @@ RUN curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jm
     ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent.jar /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar
 
+# This Protobuf version is the exact version as used in the Hadoop Dockerfile
+# See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
+# (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
+WORKDIR /opt/protobuf-src
+RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz && \
+    tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner && \
+    ./configure --prefix=/opt/protobuf && \
+    make "-j$(nproc)" && \
+    make install
+
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+WORKDIR /stackable
+RUN patches/apply_patches.sh ${PRODUCT}
+
+WORKDIR /stackable/hadoop-${PRODUCT}-src
+# Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module
+RUN mvn clean package -Pdist,native -pl '!hadoop-tools/hadoop-pipes' -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true && \
+    cp -r hadoop-dist/target/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT} && \
+    # HDFS fuse-dfs is not part of the regular dist output, so we need to copy it in ourselves
+    cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop-${PRODUCT}/bin
 
 # ===
 # Mitigation for CVE-2021-44228 (Log4Shell)
@@ -74,33 +83,6 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner
 RUN /bin/log4shell_scanner s "/stackable/hadoop-${PRODUCT}"
 # ===
 
-
-# This Protobuf version is the exact version as used in the Hadoop Dockerfile
-# See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
-# (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
-# For now all versions of Hadoop we support use Protobuf 3.7.1 so we can hardcode it here.
-# Should it ever differ between versions we'll need to make this a variable as well.
-RUN mkdir -p /opt/protobuf-src && \
-    curl --fail -L -s -S https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-3.7.1.tar.gz -o /opt/protobuf.tar.gz && \
-    tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src --no-same-owner && \
-    cd /opt/protobuf-src && \
-    ./configure --prefix=/opt/protobuf && \
-    make "-j$(nproc)" && \
-    make install && \
-    cd /root && \
-    rm -rf /opt/protobuf-src
-
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-WORKDIR /stackable/hadoop-${PRODUCT}-src/hadoop-hdfs-project/hadoop-hdfs-native-client
-
-# This command comes from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
-RUN mvn clean package -Pnative -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true && \
-    cp target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop/bin && \
-    rm -rf /stackable/hadoop-${PRODUCT}-src
-
-
 # Final Image
 FROM stackable/image/java-base
 

hadoop/stackable/patches/3.2.2/001-HADOOP-15767-3.2.2.patch

@@ -0,0 +1,180 @@
+diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
+index fa4a838babb2..708228c90daa 100644
+--- a/hadoop-common-project/hadoop-common/pom.xml
++++ b/hadoop-common-project/hadoop-common/pom.xml
+@@ -612,48 +612,6 @@
+               </execution>
+             </executions>
+           </plugin>
+-          <plugin>
+-            <groupId>org.codehaus.mojo</groupId>
+-            <artifactId>native-maven-plugin</artifactId>
+-            <executions>
+-              <execution>
+-                <phase>compile</phase>
+-                <goals>
+-                  <goal>javah</goal>
+-                </goals>
+-                <configuration>
+-                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+-                  <javahClassNames>
+-                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Compressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory</javahClassName>
+-                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
+-                    <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName>
+-                    <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+-                    <javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
+-                    <javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
+-                  </javahClassNames>
+-                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+-                </configuration>
+-              </execution>
+-            </executions>
+-          </plugin>
+           <plugin>
+             <groupId>org.apache.hadoop</groupId>
+             <artifactId>hadoop-maven-plugins</artifactId>
+@@ -770,43 +728,6 @@
+               </execution>
+             </executions>
+           </plugin>
+-          <plugin>
+-            <groupId>org.codehaus.mojo</groupId>
+-            <artifactId>native-maven-plugin</artifactId>
+-            <executions>
+-              <execution>
+-                <phase>compile</phase>
+-                <goals>
+-                  <goal>javah</goal>
+-                </goals>
+-                <configuration>
+-                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+-                  <javahClassNames>
+-                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
+-                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardCompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.zstd.ZStandardDecompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder</javahClassName>
+-                    <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
+-                    <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName>
+-                    <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+-                  </javahClassNames>
+-                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+-                </configuration>
+-              </execution>
+-            </executions>
+-          </plugin>
+           <plugin>
+             <groupId>org.codehaus.mojo</groupId>
+             <artifactId>exec-maven-plugin</artifactId>
+diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+index 1c9f5ee67ea7..c16a798ad956 100644
+--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
++++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+@@ -139,26 +139,6 @@
+               </execution>
+             </executions>
+           </plugin>
+-          <plugin>
+-            <groupId>org.codehaus.mojo</groupId>
+-            <artifactId>native-maven-plugin</artifactId>
+-            <executions>
+-              <execution>
+-                <phase>compile</phase>
+-                <goals>
+-                  <goal>javah</goal>
+-                </goals>
+-                <configuration>
+-                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+-                  <javahClassNames>
+-                    <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName>
+-                    <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName>
+-                  </javahClassNames>
+-                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+-                </configuration>
+-              </execution>
+-            </executions>
+-          </plugin>
+           <plugin>
+             <groupId>org.apache.maven.plugins</groupId>
+             <artifactId>maven-antrun-plugin</artifactId>
+diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
+index e76347962b29..e1ee4b117682 100644
+--- a/hadoop-project/pom.xml
++++ b/hadoop-project/pom.xml
+@@ -171,7 +171,6 @@
+     <build-helper-maven-plugin.version>1.9</build-helper-maven-plugin.version>
+     <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
+     <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
+-    <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
+     <surefire.fork.timeout>900</surefire.fork.timeout>
+     <aws-java-sdk.version>1.11.563</aws-java-sdk.version>
+     <hsqldb.version>2.3.4</hsqldb.version>
+@@ -1609,11 +1608,6 @@
+             </dependency>
+           </dependencies>
+         </plugin>
+-        <plugin>
+-          <groupId>org.codehaus.mojo</groupId>
+-          <artifactId>native-maven-plugin</artifactId>
+-          <version>${native-maven-plugin.version}</version>
+-        </plugin>
+         <plugin>
+           <groupId>org.codehaus.mojo</groupId>
+           <artifactId>make-maven-plugin</artifactId>
+@@ -2079,6 +2073,27 @@
+         </dependencies>
+       </dependencyManagement>
+     </profile>
++    <profile>
++      <id>native</id>
++      <activation>
++        <activeByDefault>false</activeByDefault>
++      </activation>
++      <build>
++        <plugins>
++          <plugin>
++            <groupId>org.apache.maven.plugins</groupId>
++            <artifactId>maven-compiler-plugin</artifactId>
++            <configuration>
++              <!-- add flags to generate native headers -->
++              <compilerArgs combine.children="append">
++                <arg>-h</arg>
++                <arg>${project.build.directory}/native/javah/</arg>
++              </compilerArgs>
++            </configuration>
++          </plugin>
++        </plugins>
++      </build>
++    </profile>
+   </profiles>
+
+   <repositories>