Rework Kafka image to build from source

kafka-testing-tools/licenses/KCAT_LICENSES

@@ -0,0 +1,123 @@
+librdkafka - Apache Kafka C driver library
+
+Copyright (c) 2012, Magnus Edenhill
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+----
+
+getdelim.c from newlib with Red Hat's copyright and the following license:
+(1) Red Hat Incorporated
+
+Copyright (c) 1994-2009  Red Hat, Inc. All rights reserved.
+
+This copyrighted material is made available to anyone wishing to use,
+modify, copy, or redistribute it subject to the terms and conditions
+of the BSD License.   This program is distributed in the hope that
+it will be useful, but WITHOUT ANY WARRANTY expressed or implied,
+including the implied warranties of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE.  A copy of this license is available at
+http://www.opensource.org/licenses. Any Red Hat trademarks that are
+incorporated in the source code or documentation are not subject to
+the BSD License and may only be used or replicated with the express
+permission of Red Hat, Inc.
+
+(2) University of California, Berkeley
+
+Copyright (c) 1981-2000 The Regents of the University of California.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the University nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+  OF SUCH DAMAGE.
+
+---
+
+For the files wingetopt.c wingetopt.h downloaded from https://github.com/alex85k/wingetopt
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Sponsored in part by the Defense Advanced Research Projects
+ * Agency (DARPA) and Air Force Research Laboratory, Air Force
+ * Materiel Command, USAF, under agreement number F39502-99-1-0512.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */

kafka/Dockerfile

@@ -4,11 +4,75 @@
 # Ignoring DL4006 globally because we inherit the SHELL from our base image
 # hadolint global ignore=DL3038,DL4006
 
-FROM stackable/image/java-base AS builder
+# Not tagging base image because it is built as part of the same process
+# hadolint ignore=DL3006
+FROM stackable/image/java-base as kafka-builder
+
+ARG PRODUCT
+ARG SCALA
+ARG OPA_AUTHORIZER
+ARG JMX_EXPORTER
+
+RUN microdnf update && \
+    microdnf install \
+    # Required by log4shell mitigation script
+    zip && \
+    microdnf clean all && \
+    rm -rf /var/cache/yum
+
+USER stackable
+WORKDIR /stackable
+
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
+    cd kafka-${PRODUCT}-src && \
+    # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
+    # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
+    # it's a good idea to run the tests in this case.
+    ./gradlew clean releaseTarGz && \
+    tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable && \
+    rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/ && \
+    rm -rf /stackable/kafka-${PRODUCT}-src
+
+RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
+    -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
+
+RUN mkdir -p /stackable/jmx/ && \
+    curl --fail -L https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
+    -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
+    chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
+    ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
+
+# ===
+# Mitigation for CVE-2021-44228 (Log4Shell)
+# This variable is supported as of Log4j version 2.10 and
+# disables the vulnerable feature
+ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
+
+# For earlier versions this script removes the .class file that contains the
+# vulnerable code.
+# TODO: This can be restricted to target only versions which do not honor the environment
+#   varible that has been set above but this has not currently been implemented
+COPY shared/log4shell.sh /bin
+RUN /bin/log4shell.sh /stackable/kafka_${SCALA}-${PRODUCT}
+
+# Ensure no vulnerable files are left over
+# This will currently report vulnerable files being present, as it also alerts on
+# SocketNode.class, which we do not remove with our scripts.
+# Further investigation will be needed whether this should also be removed.
+COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
+COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
+COPY shared/log4shell_scanner /bin/log4shell_scanner
+RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT}
+# ===
+
+# Normally we would use stackable/image/stackable-base here, *but* we fail to link kcat at the end with error messages
+# shown in Snippet 1, which we were not able to solve.
+FROM stackable/image/java-base AS kcat-builder
 
 ARG KCAT
 
-RUN microdnf install -y \
+RUN microdnf update && \
+    microdnf install \
     cmake \
     cyrus-sasl-devel \
     gcc-c++ \
@@ -19,6 +83,7 @@ RUN microdnf install -y \
     tar \
     wget \
     which \
+    # Required to build kcat
     zlib \
     zlib-devel && \
     microdnf clean all && \
@@ -30,14 +95,14 @@ RUN curl --fail -L -O https://repo.stackable.tech/repository/packages/kcat/kcat-
     && cd kcat-${KCAT} \
     && ./bootstrap.sh
 
-FROM stackable/image/java-base
+# Not tagging base image because it is built as part of the same process
+# hadolint ignore=DL3006
+FROM stackable/image/java-base AS final
 
+ARG RELEASE
 ARG PRODUCT
 ARG SCALA
 ARG KCAT
-ARG OPA_AUTHORIZER
-ARG JMX_EXPORTER
-ARG RELEASE
 
 LABEL name="Apache Kafka" \
       maintainer="info@stackable.tech" \
@@ -47,60 +112,56 @@ LABEL name="Apache Kafka" \
       summary="The Stackable image for Apache Kafka." \
       description="This image is deployed by the Stackable Operator for Apache Kafka."
 
+# This is needed for kubectl
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
 RUN microdnf update && \
     microdnf install \
-    gzip \
     # Can be removed once listener-operator integration is used
-    kubectl \
-    tar \
-    zip && \
+    kubectl && \
     microdnf clean all && \
+    rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt && \
     rm -rf /var/cache/yum
 
 USER stackable
 WORKDIR /stackable
 
-COPY --chown=stackable:stackable kafka/stackable /stackable
 COPY --chown=stackable:stackable kafka/licenses /licenses
 
-RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \
-    ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
-
-RUN curl --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
-    -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
-    chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
-    ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
+# We copy opa-authorizer.jar and jmx-exporter through the builder image to have an absolutely minimal final image
+# (e.g. we don't even need curl in it).
+COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
+COPY --chown=stackable:stackable --from=kafka-builder /stackable/jmx/ /stackable/jmx/
+COPY --chown=stackable:stackable --from=kcat-builder /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
 
-RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
-    -o /stackable/kafka/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
-
-# ===
-# Mitigation for CVE-2021-44228 (Log4Shell)
-# This variable is supported as of Log4j version 2.10 and
-# disables the vulnerable feature
-ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-
-# For earlier versions this script removes the .class file that contains the
-# vulnerable code.
-# TODO: This can be restricted to target only versions which do not honor the environment
-#   varible that has been set above but this has not currently been implemented
-COPY shared/log4shell.sh /bin
-RUN /bin/log4shell.sh /stackable/kafka_${SCALA}-${PRODUCT}
-
-# Ensure no vulnerable files are left over
-# This will currently report vulnerable files being present, as it also alerts on
-# SocketNode.class, which we do not remove with our scripts.
-# Further investigation will be needed whether this should also be removed.
-COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
-COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
-COPY shared/log4shell_scanner /bin/log4shell_scanner
-RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT}
-# ===
+RUN ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat && \
+    ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
 
-# Store kcat version with binary name and add softlink
-COPY --chown=stackable:stackable --from=builder /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
-RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat
+ENV PATH="${PATH}:/stackable/bin:/stackable/kafka/bin"
 
 WORKDIR /stackable/kafka
-CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"]
+CMD ["kafka-server-start.sh", "/stackable/kafka/config/server.properties"]
+
+# SNIPPET 1
+# 145.2 gcc  -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -g -O2 -Wall -Wsign-compare -Wfloat-equal -Wpointer-arith -Wcast-align -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib kcat.o format.o tools.o input.o json.o avro.o -o kcat  -lm -ldl -lpthread -lrt -lpthread -lrt  -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib  /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libserdes.a -Wl,-Bstatic -lavro -Wl,-Bdynamic /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libyajl_s.a -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib //stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/librdkafka.a -lm -ldl -lpthread -lrt -lz -lcrypto -lssl -lsasl2   -lm -ldl -lpthread -lrt -lpthread -lrt  -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib  /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl
+# 145.3 /usr/bin/ld: //stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/librdkafka.a(rdkafka_zstd.o): in function `rd_kafka_zstd_decompress':
+# 145.3 /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:44: undefined reference to `ZSTD_getFrameContentSize'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:91: undefined reference to `ZSTD_getErrorCode'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:80: undefined reference to `ZSTD_decompress'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:82: undefined reference to `ZSTD_isError'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:100: undefined reference to `ZSTD_getErrorName'
+# 145.3 /usr/bin/ld: //stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/librdkafka.a(rdkafka_zstd.o): in function `rd_kafka_zstd_compress':
+# 145.3 /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:130: undefined reference to `ZSTD_compressBound'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:141: undefined reference to `ZSTD_createCStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:155: undefined reference to `ZSTD_initCStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:157: undefined reference to `ZSTD_isError'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:213: undefined reference to `ZSTD_freeCStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:168: undefined reference to `ZSTD_compressStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:169: undefined reference to `ZSTD_isError'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:170: undefined reference to `ZSTD_getErrorName'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:158: undefined reference to `ZSTD_getErrorName'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:198: undefined reference to `ZSTD_endStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:199: undefined reference to `ZSTD_isError'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:213: undefined reference to `ZSTD_freeCStream'
+# 145.3 /usr/bin/ld: /stackable/kcat-1.7.0/tmp-bootstrap/librdkafka/src/rdkafka_zstd.c:200: undefined reference to `ZSTD_getErrorName'
+# 145.3 collect2: error: ld returned 1 exit status
+# 145.3 make: *** [mklove/Makefile.base:225: kcat] Error 1