From 7bacc1a64c639d2a9e95bf7288a5346ae89a3cf1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 15:45:02 +0200 Subject: [PATCH 1/8] remove openssl --- kafka/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 670d2770c..0c39f2e7f 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -32,7 +32,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo RUN microdnf update && \ - microdnf install tar gzip zip openssl kubectl && \ + microdnf install tar gzip zip kubectl && \ microdnf clean all USER stackable From 8b80e75ab144fbf29ea0dcaf94abb33b765f8cb4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 17:00:58 +0200 Subject: [PATCH 2/8] add kafka upload script --- kafka/upload_new_kafka_version.sh | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 kafka/upload_new_kafka_version.sh diff --git a/kafka/upload_new_kafka_version.sh b/kafka/upload_new_kafka_version.sh new file mode 100755 index 000000000..ec79f924b --- /dev/null +++ b/kafka/upload_new_kafka_version.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +VERSION=${1:?"Missing version number argument (arg 1)"} +NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} + +read -r -s -p "Nexus Password: " NEXUS_PASSWORD +echo "" + +# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory +# Find the directory name of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# the temp directory used, within $DIR +WORK_DIR=$(mktemp -d -p "$DIR") + +# check if tmp dir was created +if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then + echo "Could not create temp dir" + exit 1 +fi + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +cd "$WORK_DIR" || exit + +bin_file="kafka_2.13-${VERSION}.tgz" + +echo "Downloading Kafka (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.asc" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.sha512" + +# It is probably redundant to check both the checksum and the signature but it's cheap and why not +echo "Validating SHA512 Checksum" +if ! (gpg --print-md SHA512 "${bin_file}" | diff - "${bin_file}.sha512" ); then + echo "ERROR: The SHA512 sum does not match" + exit 1 +fi + +echo "Validating signature" +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html' + +if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then + echo "ERROR: The signature could not be verified" + exit 1 +fi + +echo "Uploading everything to Nexus" +EXIT_STATUS=0 +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? + +if [ $EXIT_STATUS -ne 0 ]; then + echo "ERROR: Upload failed" + exit 1 +fi + +echo "Successfully uploaded version ${VERSION} of Kafka to Nexus" +echo "https://repo.stackable.tech/service/rest/repository/browse/packages/kafka/" From 882132e60a2231e5dea3664a315b1a110ba317c8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 17:01:20 +0200 Subject: [PATCH 3/8] add 3.4.1, 3.5.0, remove old ones, bump to jmx exporter 0.20.0 --- conf.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/conf.py b/conf.py index 8aa3ae2b4..715c9fcba 100644 --- a/conf.py +++ b/conf.py @@ -146,47 +146,40 @@ { "name": "kafka", "versions": [ - { - "product": "2.7.1", - "java-base": "11", - "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", - }, { "product": "2.8.1", "java-base": "11", "scala": "2.13", "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.1.0", + "product": "2.8.2", "java-base": "11", "scala": "2.13", "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.2.0", + "product": "3.4.0", "java-base": "11", "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "opa_authorizer": "1.5.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.3.1", + "product": "3.4.1", "java-base": "11", "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "opa_authorizer": "1.5.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.4.0", + "product": "3.5.1", "java-base": "11", "scala": "2.13", "opa_authorizer": "1.5.1", - "jmx_exporter": "0.18.0", + "jmx_exporter": "0.20.0", }, ], }, From cac777afd6763e3fe142224ee76cdffeaca80c71 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 17:01:33 +0200 Subject: [PATCH 4/8] add fail to curl commands --- kafka/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 0c39f2e7f..e7f04ae6b 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -6,7 +6,7 @@ RUN microdnf install -y zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel microdnf clean all WORKDIR /stackable -RUN curl -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ +RUN curl --fail -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ && tar xvfz 1.7.0.tar.gz \ && cd kcat-1.7.0 \ && ./bootstrap.sh @@ -41,15 +41,15 @@ WORKDIR /stackable COPY --chown=stackable:stackable kafka/stackable /stackable COPY --chown=stackable:stackable kafka/licenses /licenses -RUN curl -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \ +RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \ ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka -RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \ +RUN curl --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \ -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar -RUN curl -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \ +RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \ -o /stackable/kafka/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar # === From 8e7d8df3014e4b3b4160822e360298619ebb4ea0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 10 Oct 2023 17:38:18 +0200 Subject: [PATCH 5/8] adapted changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64550625c..930556da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ All notable changes to this project will be documented in this file. - airflow: Updated statsd-exporter to 0.24, this was accidentally moved to a very old version previously (0.3.0) ([#431]). - airflow: Added wrapper script to allow the triggering of pre/post hook actions ([#435]). +- kafka: add support for versions 3.4.1, 3.5.0 ([#476]). + ### Removed - airflow: Remove unused environment variable `AIRFLOW_UID` ([#429]). @@ -34,6 +36,8 @@ All notable changes to this project will be documented in this file. - pyspark-k8s: The PySpark image has been removed completely. Python is now installed with the Spark image ([#436]) - Removed all product specific changelogs and updated the root file ([#440]) +- kafka: removed support for versions 2.7.1, 3.1.0, 3.2.0, 3.3.1 ([#476]). + [#400]: https://github.com/stackabletech/docker-images/pull/400 [#419]: https://github.com/stackabletech/docker-images/pull/419 [#425]: https://github.com/stackabletech/docker-images/pull/425 @@ -49,6 +53,7 @@ All notable changes to this project will be documented in this file. [#464]: https://github.com/stackabletech/docker-images/pull/464 [#465]: https://github.com/stackabletech/docker-images/pull/465 [#466]: https://github.com/stackabletech/docker-images/pull/466 +[#476]: https://github.com/stackabletech/docker-images/pull/476 ## [23.7.0] - 2023-07-14 From db3a89435ae38bd727a316eba050b5a81782737f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 17:06:08 +0200 Subject: [PATCH 6/8] fix changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 930556da3..4426f3bd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file. - java-base: Add needed tzdata-java package ([#425]). - testing-tools: Add java, tzdata-java, unzip ([#464], [#465], [#466]). +- kafka: add support for versions 3.4.1, 3.5.1 ([#476]). + ### Changed - Extract image tools their own [repository](https://github.com/stackabletech/image-tools) ([#437]) @@ -27,8 +29,6 @@ All notable changes to this project will be documented in this file. - airflow: Updated statsd-exporter to 0.24, this was accidentally moved to a very old version previously (0.3.0) ([#431]). - airflow: Added wrapper script to allow the triggering of pre/post hook actions ([#435]). -- kafka: add support for versions 3.4.1, 3.5.0 ([#476]). - ### Removed - airflow: Remove unused environment variable `AIRFLOW_UID` ([#429]). From 1956873696c98af9b856562f0f1061cef8d4125f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 17:50:36 +0200 Subject: [PATCH 7/8] extract kcat version --- conf.py | 47 ++++++++++++++++++++++++++--------------------- kafka/Dockerfile | 13 +++++++++---- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/conf.py b/conf.py index 579f84ca9..c9559c43a 100644 --- a/conf.py +++ b/conf.py @@ -138,31 +138,35 @@ { "name": "kafka", "versions": [ - { - "product": "2.8.1", - "java-base": "11", - "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.20.0", - }, - { - "product": "2.8.2", - "java-base": "11", - "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.20.0", - }, - { - "product": "3.4.0", - "java-base": "11", - "scala": "2.13", - "opa_authorizer": "1.5.1", - "jmx_exporter": "0.20.0", - }, + # { + # "product": "2.8.1", + # "java-base": "11", + # "scala": "2.13", + # "kcat": "1.7.0", + # "opa_authorizer": "1.4.0", + # "jmx_exporter": "0.20.0", + # }, + # { + # "product": "2.8.2", + # "java-base": "11", + # "scala": "2.13", + # "kcat": "1.7.0", + # "opa_authorizer": "1.4.0", + # "jmx_exporter": "0.20.0", + # }, + # { + # "product": "3.4.0", + # "java-base": "11", + # "scala": "2.13", + # "kcat": "1.7.0", + # "opa_authorizer": "1.5.1", + # "jmx_exporter": "0.20.0", + # }, { "product": "3.4.1", "java-base": "11", "scala": "2.13", + "kcat": "1.7.0", "opa_authorizer": "1.5.1", "jmx_exporter": "0.20.0", }, @@ -170,6 +174,7 @@ "product": "3.5.1", "java-base": "11", "scala": "2.13", + "kcat": "1.7.0", "opa_authorizer": "1.5.1", "jmx_exporter": "0.20.0", }, diff --git a/kafka/Dockerfile b/kafka/Dockerfile index e7f04ae6b..4a46ee6fe 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,20 +1,23 @@ # syntax=docker/dockerfile:1 FROM stackable/image/java-base AS builder +ARG KCAT + RUN microdnf install -y zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel && \ microdnf install -y tar which wget zlib gcc-c++ make cmake && \ microdnf clean all WORKDIR /stackable -RUN curl --fail -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ - && tar xvfz 1.7.0.tar.gz \ - && cd kcat-1.7.0 \ +RUN curl --fail -L -O https://github.com/edenhill/kcat/archive/refs/tags/${KCAT}.tar.gz \ + && tar xvfz ${KCAT}.tar.gz \ + && cd kcat-${KCAT} \ && ./bootstrap.sh FROM stackable/image/java-base ARG PRODUCT ARG SCALA +ARG KCAT ARG OPA_AUTHORIZER ARG JMX_EXPORTER ARG RELEASE @@ -75,7 +78,9 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT} # === -COPY --chown=stackable:stackable --from=builder /stackable/kcat-1.7.0/kcat /stackable/kcat +# Store kcat version with binary name and add softlink +COPY --chown=stackable:stackable --from=builder /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT} +RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat WORKDIR /stackable/kafka CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"] From 53af47ea1e07912ff97e0f3055f317d9e1333386 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 17:53:39 +0200 Subject: [PATCH 8/8] enable all product versions --- conf.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/conf.py b/conf.py index c9559c43a..68e47702d 100644 --- a/conf.py +++ b/conf.py @@ -138,30 +138,30 @@ { "name": "kafka", "versions": [ - # { - # "product": "2.8.1", - # "java-base": "11", - # "scala": "2.13", - # "kcat": "1.7.0", - # "opa_authorizer": "1.4.0", - # "jmx_exporter": "0.20.0", - # }, - # { - # "product": "2.8.2", - # "java-base": "11", - # "scala": "2.13", - # "kcat": "1.7.0", - # "opa_authorizer": "1.4.0", - # "jmx_exporter": "0.20.0", - # }, - # { - # "product": "3.4.0", - # "java-base": "11", - # "scala": "2.13", - # "kcat": "1.7.0", - # "opa_authorizer": "1.5.1", - # "jmx_exporter": "0.20.0", - # }, + { + "product": "2.8.1", + "java-base": "11", + "scala": "2.13", + "kcat": "1.7.0", + "opa_authorizer": "1.4.0", + "jmx_exporter": "0.20.0", + }, + { + "product": "2.8.2", + "java-base": "11", + "scala": "2.13", + "kcat": "1.7.0", + "opa_authorizer": "1.4.0", + "jmx_exporter": "0.20.0", + }, + { + "product": "3.4.0", + "java-base": "11", + "scala": "2.13", + "kcat": "1.7.0", + "opa_authorizer": "1.5.1", + "jmx_exporter": "0.20.0", + }, { "product": "3.4.1", "java-base": "11",