Add async-profiler to Hadoop and HBase

CHANGELOG.md

@@ -8,6 +8,15 @@ All notable changes to this project will be documented in this file.
 
 - omid: init at 1.1.0 ([#493]).
 - hadoop: Allow datanodes to override their registration addresses ([#506], [#544]).
+- hadoop: Add async-profiler and backport HADOOP-18055 and HADOOP-18077
+  to support it ([#540]).
+- hadoop: Add `tar` package, so that `kubectl cp` can be used to copy
+  log files and profiler flamegraphs ([#540]).
+- hbase: Add async-profiler and backport HBASE-28242 to support it
+  ([#540]).
+- hbase: Allow multiple certificates in the KeyStores which is required for
+  rotating CA certificates. Because of this, HBASE-27027 was backported to
+  HBase version 2.4.12 ([#540]).
 - nifi: Add Apache Iceberg extensions ([#529]).
 - testing-tools: Add krb5-user library for Kerberos tests ([#531]).
 - testing-tools: Add the Python library Beautiful Soup 4 ([#536]).
@@ -23,6 +32,10 @@ All notable changes to this project will be documented in this file.
 - hadoop: Build from source ([#526]).
 - superset: Add patch that fixes saved queries export ([#539]).
 
+### Removed
+
+- hadoop: Remove support for version 3.2.2 ([#540]).
+
 [#493]: https://github.com/stackabletech/docker-images/pull/493
 [#506]: https://github.com/stackabletech/docker-images/pull/506
 [#514]: https://github.com/stackabletech/docker-images/pull/514
@@ -35,6 +48,7 @@ All notable changes to this project will be documented in this file.
 [#537]: https://github.com/stackabletech/docker-images/pull/537
 [#538]: https://github.com/stackabletech/docker-images/pull/538
 [#539]: https://github.com/stackabletech/docker-images/pull/539
+[#540]: https://github.com/stackabletech/docker-images/pull/540
 [#542]: https://github.com/stackabletech/docker-images/pull/542
 [#544]: https://github.com/stackabletech/docker-images/pull/544
 

conf.py

@@ -58,30 +58,26 @@
     {
         "name": "hadoop",
         "versions": [
-            {
-                "product": "3.2.2",
-                "java-base": "11",
-                "jmx_exporter": "0.20.0",
-                "protobuf": "2.5.0",
-                "topology_provider": "0.1.0"
-            },
             {
                 "product": "3.2.4",
                 "java-base": "11",
+                "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "2.5.0",
                 "topology_provider": "0.1.0"
             },
             {
                 "product": "3.3.4",
                 "java-base": "11",
+                "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "3.7.1",
                 "topology_provider": "0.1.0"
             },
             {
                 "product": "3.3.6",
                 "java-base": "11",
+                "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "3.7.1",
                 "topology_provider": "0.1.0"
@@ -100,6 +96,7 @@
                  "hbase_thirdparty": "3.5.1",
                  "hbase_operator_tools": "1.2.0",
                  "java-base": "11",
+                 "async_profiler": "2.9",
                  "phoenix": "2.4-5.1.2",
                  "hadoop_m2": "3.3.6",
                  "jmx_exporter": "0.20.0",
@@ -109,6 +106,7 @@
                  "hbase_thirdparty": "4.1.4",
                  "hbase_operator_tools": "1.2.0",
                  "java-base": "11",
+                 "async_profiler": "2.9",
                  "phoenix": "2.4-5.1.3",
                  "hadoop_m2": "3.3.6",
                  "jmx_exporter": "0.20.0",

hadoop/Dockerfile

@@ -2,9 +2,12 @@
 FROM stackable/image/java-base AS builder
 
 ARG PRODUCT
+ARG ASYNC_PROFILER
 ARG JMX_EXPORTER
 ARG PROTOBUF
 ARG TOPOLOGY_PROVIDER
+ARG TARGETARCH
+ARG TARGETOS
 
 # https://github.com/hadolint/hadolint/wiki/DL4006
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -36,6 +39,10 @@ RUN curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jm
     ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent.jar /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar
 
+RUN ARCH="${TARGETARCH/amd64/x64}" && \
+    curl --fail -L "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz"  | tar -xzC . && \
+    ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler
+
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
 # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github)
@@ -107,7 +114,10 @@ RUN microdnf update && \
     microdnf install \
     fuse \
     fuse-libs \
-    krb5-workstation && \
+    krb5-workstation \
+    # tar is required for `kubectl cp` which can be used to copy the log files
+    # or profiler flamegraph from the Pod
+    tar && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
@@ -122,6 +132,7 @@ WORKDIR /stackable
 
 COPY --chown=stackable:stackable --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/
 COPY --chown=stackable:stackable --from=builder /stackable/jmx /stackable/jmx/
+COPY --chown=stackable:stackable --from=builder /stackable/async-profiler /stackable/async-profiler/
 
 # The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes
 # labels to build a rackID from
@@ -136,6 +147,7 @@ ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server
 ENV PATH="${PATH}":/stackable/hadoop/bin
 ENV HADOOP_HOME=/stackable/hadoop
 ENV HADOOP_CONF_DIR=/stackable/config
+ENV ASYNC_PROFILER_HOME=/stackable/async-profiler
 
 WORKDIR /stackable/hadoop
 CMD ["echo", "This image is not meant to be 'run' directly."]