New kafka versions 3.4.1, 3.5.1

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 
 - hadoop: added support for 3.2.4, 3.3.6 ([#478]).
 - hive: added upload new version script ([#472]).
+- kafka: add support for versions 3.4.1, 3.5.1 ([#476]).
 - nifi: added support for version 1.23.2 ([#473]).
 - opa: add version 0.57.0 ([#471]).
 - opa: add new version upload script ([#471]).
@@ -47,6 +48,7 @@ All notable changes to this project will be documented in this file.
 
 - hadoop: removed support for 3.3.1, 3.3.3 ([#478]).
 - hive: remove version 2.3.9 ([#472]).
+- kafka: removed support for versions 2.7.1, 3.1.0, 3.2.0, 3.3.1 ([#476]).
 - nifi: removed support for version 1.15.x, 1.16.x, 1.18.x, 1.20.x ([#473]).
 - nifi: removed openssl from image ([#473]).
 - opa: removed versions 0.27.1, 0.28.0, 0.37.2, 0.41.0, 0.45.0 ([#471]).
@@ -71,6 +73,7 @@ All notable changes to this project will be documented in this file.
 [#471]: https://github.com/stackabletech/docker-images/pull/471
 [#472]: https://github.com/stackabletech/docker-images/pull/472
 [#473]: https://github.com/stackabletech/docker-images/pull/473
+[#476]: https://github.com/stackabletech/docker-images/pull/476
 [#478]: https://github.com/stackabletech/docker-images/pull/478
 [#479]: https://github.com/stackabletech/docker-images/pull/479
 

conf.py

@@ -138,47 +138,45 @@
     {
         "name": "kafka",
         "versions": [
-            {
-                "product": "2.7.1",
-                "java-base": "11",
-                "scala": "2.13",
-                "opa_authorizer": "1.4.0",
-                "jmx_exporter": "0.16.1",
-            },
             {
                 "product": "2.8.1",
                 "java-base": "11",
                 "scala": "2.13",
+                "kcat": "1.7.0",
                 "opa_authorizer": "1.4.0",
-                "jmx_exporter": "0.16.1",
+                "jmx_exporter": "0.20.0",
             },
             {
-                "product": "3.1.0",
+                "product": "2.8.2",
                 "java-base": "11",
                 "scala": "2.13",
+                "kcat": "1.7.0",
                 "opa_authorizer": "1.4.0",
-                "jmx_exporter": "0.16.1",
+                "jmx_exporter": "0.20.0",
             },
             {
-                "product": "3.2.0",
+                "product": "3.4.0",
                 "java-base": "11",
                 "scala": "2.13",
-                "opa_authorizer": "1.4.0",
-                "jmx_exporter": "0.16.1",
+                "kcat": "1.7.0",
+                "opa_authorizer": "1.5.1",
+                "jmx_exporter": "0.20.0",
             },
             {
-                "product": "3.3.1",
+                "product": "3.4.1",
                 "java-base": "11",
                 "scala": "2.13",
-                "opa_authorizer": "1.4.0",
-                "jmx_exporter": "0.16.1",
+                "kcat": "1.7.0",
+                "opa_authorizer": "1.5.1",
+                "jmx_exporter": "0.20.0",
             },
             {
-                "product": "3.4.0",
+                "product": "3.5.1",
                 "java-base": "11",
                 "scala": "2.13",
+                "kcat": "1.7.0",
                 "opa_authorizer": "1.5.1",
-                "jmx_exporter": "0.18.0",
+                "jmx_exporter": "0.20.0",
             },
         ],
     },

kafka/Dockerfile

@@ -1,20 +1,23 @@
 # syntax=docker/dockerfile:1
 FROM stackable/image/java-base AS builder
 
+ARG KCAT
+
 RUN microdnf install -y zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel && \
     microdnf install -y tar which wget zlib gcc-c++ make cmake && \
     microdnf clean all
 
 WORKDIR /stackable
-RUN curl -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \
-    && tar xvfz 1.7.0.tar.gz \
-    && cd kcat-1.7.0 \
+RUN curl --fail -L -O https://github.com/edenhill/kcat/archive/refs/tags/${KCAT}.tar.gz \
+    && tar xvfz ${KCAT}.tar.gz \
+    && cd kcat-${KCAT} \
     && ./bootstrap.sh
 
 FROM stackable/image/java-base
 
 ARG PRODUCT
 ARG SCALA
+ARG KCAT
 ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
 ARG RELEASE
@@ -32,7 +35,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
 RUN microdnf update && \
-    microdnf install tar gzip zip openssl kubectl && \
+    microdnf install tar gzip zip kubectl && \
     microdnf clean all
 
 USER stackable
@@ -41,15 +44,15 @@ WORKDIR /stackable
 COPY --chown=stackable:stackable kafka/stackable /stackable
 COPY --chown=stackable:stackable kafka/licenses /licenses
 
-RUN curl -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \
+RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \
     ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
 
-RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
+RUN curl --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
     -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
 
-RUN curl -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
+RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
     -o /stackable/kafka/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
 
 # ===
@@ -75,7 +78,9 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner
 RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT}
 # ===
 
-COPY --chown=stackable:stackable --from=builder /stackable/kcat-1.7.0/kcat /stackable/kcat
+# Store kcat version with binary name and add softlink
+COPY --chown=stackable:stackable --from=builder /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
+RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat
 
 WORKDIR /stackable/kafka
 CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"]

kafka/upload_new_kafka_version.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+VERSION=${1:?"Missing version number argument (arg 1)"}
+NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
+
+read -r -s -p "Nexus Password: " NEXUS_PASSWORD
+echo ""
+
+# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory
+# Find the directory name of the script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# the temp directory used, within $DIR
+WORK_DIR=$(mktemp -d -p "$DIR")
+
+# check if tmp dir was created
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+  echo "Could not create temp dir"
+  exit 1
+fi
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+cd "$WORK_DIR" || exit
+
+bin_file="kafka_2.13-${VERSION}.tgz"
+
+echo "Downloading Kafka (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
+curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}"
+curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.asc"
+curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.sha512"
+
+# It is probably redundant to check both the checksum and the signature but it's cheap and why not
+echo "Validating SHA512 Checksum"
+if ! (gpg --print-md SHA512 "${bin_file}" | diff - "${bin_file}.sha512" ); then
+  echo "ERROR: The SHA512 sum does not match"
+  exit 1
+fi
+
+echo "Validating signature"
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html'
+
+if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then
+  echo "ERROR: The signature could not be verified"
+  exit 1
+fi
+
+echo "Uploading everything to Nexus"
+EXIT_STATUS=0
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$?
+
+if [ $EXIT_STATUS -ne 0 ]; then
+  echo "ERROR: Upload failed"
+  exit 1
+fi
+
+echo "Successfully uploaded version ${VERSION} of Kafka to Nexus"
+echo "https://repo.stackable.tech/service/rest/repository/browse/packages/kafka/"