diff --git a/CHANGELOG.md b/CHANGELOG.md index c39c277c5..e0bfdc298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- hadoop: Added Stackable topology provider jar to enable k8s-based rack awareness ([#509]) - hadoop: Add all necessary components to the image to mount HDFS using FUSE ([#400]) - hbase: Add hbase-operator-tools ([#497], [#498]). - java-base: Add needed tzdata-java package ([#425]). @@ -109,6 +110,7 @@ All notable changes to this project will be documented in this file. [#498]: https://github.com/stackabletech/docker-images/pull/498 [#499]: https://github.com/stackabletech/docker-images/pull/499 [#505]: https://github.com/stackabletech/docker-images/pull/505 +[#509]: https://github.com/stackabletech/docker-images/pull/509 ## [23.7.0] - 2023-07-14 diff --git a/conf.py b/conf.py index 510312922..fd19323b8 100644 --- a/conf.py +++ b/conf.py @@ -62,25 +62,29 @@ "product": "3.2.2", "java-base": "11", "java": "11", - "jmx_exporter": "0.20.0" + "jmx_exporter": "0.20.0", + "topology_provider": "0.1.0" }, { "product": "3.2.4", "java-base": "11", "java": "11", - "jmx_exporter": "0.20.0" + "jmx_exporter": "0.20.0", + "topology_provider": "0.1.0" }, { "product": "3.3.4", "java-base": "11", "java": "11", - "jmx_exporter": "0.20.0" + "jmx_exporter": "0.20.0", + "topology_provider": "0.1.0" }, { "product": "3.3.6", "java-base": "11", "java": "11", - "jmx_exporter": "0.20.0" + "jmx_exporter": "0.20.0", + "topology_provider": "0.1.0" }, ], }, diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 69a6ae740..2f6ee210b 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -4,6 +4,7 @@ FROM stackable/image/java-base AS builder ARG PRODUCT ARG JAVA ARG JMX_EXPORTER +ARG TOPOLOGY_PROVIDER # https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -139,6 +140,12 @@ WORKDIR /stackable COPY --chown=stackable:stackable --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ COPY --chown=stackable:stackable --from=builder /stackable/jmx /stackable/jmx/ + +# The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes +# labels to build a rackID from +# source code is at: https://github.com/stackabletech/hdfs-topology-provider +RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-topology-provider/topology-provider-${TOPOLOGY_PROVIDER}.jar -o /stackable/hadoop/share/hadoop/tools/lib/ --chown=stackable:stackable + RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin