From 342361c0ffb9b10c4fb0d3d510c18ae8c1999a39 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 16 Feb 2024 11:46:04 +0100 Subject: [PATCH 1/5] feat(hadoop): Add hdfs-utils --- CHANGELOG.md | 1 + conf.py | 9 ++++++--- hadoop/Dockerfile | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70976cd30..de9bb91dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file. - trino: Add version `438` ([#547]). - vector: Switch from version `0.33.0` to `0.35.0` ([#547], [#549]). - zookeeper: Add version `3.9.1` ([#551]). +- hadoop: Add hdfs-utils ([#XXX]). ### Changed diff --git a/conf.py b/conf.py index 8f8ea66d7..c9bb55d72 100644 --- a/conf.py +++ b/conf.py @@ -65,7 +65,7 @@ "product": "28.0.1", # Java 17 should be fully supported as of 27.0.0 https://github.com/apache/druid/releases#27.0.0-highlights-java-17-support # Did not work in a quick test due to reflection error: - # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class + # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError "java-base": "11", "jackson_dataformat_xml": "2.12.7", # from https://github.com/apache/druid/blob/b8201e31aa6b124049a61764309145baaad78db7/pom.xml#L100 @@ -84,6 +84,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "2.5.0", + # no HDFS utils, as e.g. patches from https://github.com/apache/hadoop/pull/6553 are missing. 3.2.x will probably removed shortly anyway. "topology_provider": "0.2.0" }, { @@ -92,6 +93,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "3.7.1", + "hdfs_utils": "0.1.0-SNAPSHOT", "topology_provider": "0.2.0" }, { @@ -100,6 +102,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "3.7.1", + "hdfs_utils": "0.1.0-SNAPSHOT", "topology_provider": "0.2.0" }, ], @@ -356,8 +359,8 @@ { "product": "414", "java-base": "17", - "opa_authorizer": "stackable0.2.0", - "jmx_exporter": "0.20.0", + "opa_authorizer": "stackable0.2.0", + "jmx_exporter": "0.20.0", "storage_connector": "414" }, { diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index e492605ef..df513d80d 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -6,6 +6,7 @@ ARG ASYNC_PROFILER ARG JMX_EXPORTER ARG PROTOBUF ARG TOPOLOGY_PROVIDER +ARG HDFS_UTILS ARG TARGETARCH ARG TARGETOS @@ -140,6 +141,11 @@ COPY --chown=stackable:stackable --from=builder /stackable/async-profiler /stack # N.B. the artifact name changed from 0.2.0 onwards i.e. from topology-provider-0.1.0.jar to hdfs-topology-provider-0.2.0.jar RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-topology-provider/hdfs-topology-provider-${TOPOLOGY_PROVIDER}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-topology-provider-${TOPOLOGY_PROVIDER}.jar +# The Stackable HDFS utils contain an OPA authorizer and group mapper +RUN if [[ -n ${HDFS_UTILS} ]] ; then \ + curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-utils/hdfs-utils-${HDFS_UTILS}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-utils-${HDFS_UTILS}.jar \ + fi + RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin From 28ba4010a5e56371ff6b1b2f0aeb9332e6ee75bd Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 16 Feb 2024 11:47:30 +0100 Subject: [PATCH 2/5] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9bb91dd..181108ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ All notable changes to this project will be documented in this file. - trino: Add version `438` ([#547]). - vector: Switch from version `0.33.0` to `0.35.0` ([#547], [#549]). - zookeeper: Add version `3.9.1` ([#551]). -- hadoop: Add hdfs-utils ([#XXX]). +- hadoop: Add hdfs-utils ([#566]). ### Changed @@ -89,6 +89,7 @@ All notable changes to this project will be documented in this file. [#563]: https://github.com/stackabletech/docker-images/pull/563 [#564]: https://github.com/stackabletech/docker-images/pull/564 [#565]: https://github.com/stackabletech/docker-images/pull/565 +[#566]: https://github.com/stackabletech/docker-images/pull/566 ## [23.11.0] - 2023-11-30 From 52e48d35040e7195b5f26bc57ba12e899546f3a7 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 16 Feb 2024 15:46:00 +0100 Subject: [PATCH 3/5] update hdfs-utils version --- conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index 30250b185..a5e16150c 100644 --- a/conf.py +++ b/conf.py @@ -93,7 +93,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "3.7.1", - "hdfs_utils": "0.1.0-SNAPSHOT", + "hdfs_utils": "0.1.0", "topology_provider": "0.2.0" }, { @@ -102,7 +102,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "3.7.1", - "hdfs_utils": "0.1.0-SNAPSHOT", + "hdfs_utils": "0.1.0", "topology_provider": "0.2.0" }, ], From b9db85f7d475f138070443f268631f8b2be4e971 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 16 Feb 2024 16:03:33 +0100 Subject: [PATCH 4/5] fix syntax error --- hadoop/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index df513d80d..ce9f1f363 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -143,7 +143,7 @@ RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-to # The Stackable HDFS utils contain an OPA authorizer and group mapper RUN if [[ -n ${HDFS_UTILS} ]] ; then \ - curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-utils/hdfs-utils-${HDFS_UTILS}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-utils-${HDFS_UTILS}.jar \ + curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-utils/hdfs-utils-${HDFS_UTILS}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-utils-${HDFS_UTILS}.jar ; \ fi RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop From 66af6774b05842cb9e4bdbd24dbb8194c84dec51 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 19 Feb 2024 10:28:51 +0100 Subject: [PATCH 5/5] Update conf.py Co-authored-by: Andrew Kenworthy --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index a5e16150c..2a94daac3 100644 --- a/conf.py +++ b/conf.py @@ -84,7 +84,7 @@ "async_profiler": "2.9", "jmx_exporter": "0.20.0", "protobuf": "2.5.0", - # no HDFS utils, as e.g. patches from https://github.com/apache/hadoop/pull/6553 are missing. 3.2.x will probably removed shortly anyway. + # no HDFS utils, as e.g. patches from https://github.com/apache/hadoop/pull/6553 are missing. 3.2.x will probably removed in version 24.3 anyway. "topology_provider": "0.2.0" }, {