diff --git a/CHANGELOG.md b/CHANGELOG.md index 92b9b950a..3858a092d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - hadoop: added support for 3.2.4, 3.3.6 ([#478]). - hive: added upload new version script ([#472]). +- kafka: add support for versions 3.4.1, 3.5.1 ([#476]). - nifi: added support for version 1.23.2 ([#473]). - opa: add version 0.57.0 ([#471]). - opa: add new version upload script ([#471]). @@ -47,6 +48,7 @@ All notable changes to this project will be documented in this file. - hadoop: removed support for 3.3.1, 3.3.3 ([#478]). - hive: remove version 2.3.9 ([#472]). +- kafka: removed support for versions 2.7.1, 3.1.0, 3.2.0, 3.3.1 ([#476]). - nifi: removed support for version 1.15.x, 1.16.x, 1.18.x, 1.20.x ([#473]). - nifi: removed openssl from image ([#473]). - opa: removed versions 0.27.1, 0.28.0, 0.37.2, 0.41.0, 0.45.0 ([#471]). @@ -71,6 +73,7 @@ All notable changes to this project will be documented in this file. [#471]: https://github.com/stackabletech/docker-images/pull/471 [#472]: https://github.com/stackabletech/docker-images/pull/472 [#473]: https://github.com/stackabletech/docker-images/pull/473 +[#476]: https://github.com/stackabletech/docker-images/pull/476 [#478]: https://github.com/stackabletech/docker-images/pull/478 [#479]: https://github.com/stackabletech/docker-images/pull/479 diff --git a/conf.py b/conf.py index d7e025e7a..68e47702d 100644 --- a/conf.py +++ b/conf.py @@ -138,47 +138,45 @@ { "name": "kafka", "versions": [ - { - "product": "2.7.1", - "java-base": "11", - "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", - }, { "product": "2.8.1", "java-base": "11", "scala": "2.13", + "kcat": "1.7.0", "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.1.0", + "product": "2.8.2", "java-base": "11", "scala": "2.13", + "kcat": "1.7.0", "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.2.0", + "product": "3.4.0", "java-base": "11", "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "kcat": "1.7.0", + "opa_authorizer": "1.5.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.3.1", + "product": "3.4.1", "java-base": "11", "scala": "2.13", - "opa_authorizer": "1.4.0", - "jmx_exporter": "0.16.1", + "kcat": "1.7.0", + "opa_authorizer": "1.5.1", + "jmx_exporter": "0.20.0", }, { - "product": "3.4.0", + "product": "3.5.1", "java-base": "11", "scala": "2.13", + "kcat": "1.7.0", "opa_authorizer": "1.5.1", - "jmx_exporter": "0.18.0", + "jmx_exporter": "0.20.0", }, ], }, diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 670d2770c..4a46ee6fe 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,20 +1,23 @@ # syntax=docker/dockerfile:1 FROM stackable/image/java-base AS builder +ARG KCAT + RUN microdnf install -y zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel && \ microdnf install -y tar which wget zlib gcc-c++ make cmake && \ microdnf clean all WORKDIR /stackable -RUN curl -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ - && tar xvfz 1.7.0.tar.gz \ - && cd kcat-1.7.0 \ +RUN curl --fail -L -O https://github.com/edenhill/kcat/archive/refs/tags/${KCAT}.tar.gz \ + && tar xvfz ${KCAT}.tar.gz \ + && cd kcat-${KCAT} \ && ./bootstrap.sh FROM stackable/image/java-base ARG PRODUCT ARG SCALA +ARG KCAT ARG OPA_AUTHORIZER ARG JMX_EXPORTER ARG RELEASE @@ -32,7 +35,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo RUN microdnf update && \ - microdnf install tar gzip zip openssl kubectl && \ + microdnf install tar gzip zip kubectl && \ microdnf clean all USER stackable @@ -41,15 +44,15 @@ WORKDIR /stackable COPY --chown=stackable:stackable kafka/stackable /stackable COPY --chown=stackable:stackable kafka/licenses /licenses -RUN curl -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \ +RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \ ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka -RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \ +RUN curl --fail https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \ -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar -RUN curl -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \ +RUN curl --fail -L https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \ -o /stackable/kafka/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar # === @@ -75,7 +78,9 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT} # === -COPY --chown=stackable:stackable --from=builder /stackable/kcat-1.7.0/kcat /stackable/kcat +# Store kcat version with binary name and add softlink +COPY --chown=stackable:stackable --from=builder /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT} +RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat WORKDIR /stackable/kafka CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"] diff --git a/kafka/upload_new_kafka_version.sh b/kafka/upload_new_kafka_version.sh new file mode 100755 index 000000000..ec79f924b --- /dev/null +++ b/kafka/upload_new_kafka_version.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +VERSION=${1:?"Missing version number argument (arg 1)"} +NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} + +read -r -s -p "Nexus Password: " NEXUS_PASSWORD +echo "" + +# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory +# Find the directory name of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# the temp directory used, within $DIR +WORK_DIR=$(mktemp -d -p "$DIR") + +# check if tmp dir was created +if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then + echo "Could not create temp dir" + exit 1 +fi + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +cd "$WORK_DIR" || exit + +bin_file="kafka_2.13-${VERSION}.tgz" + +echo "Downloading Kafka (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.asc" +curl --fail -LOs "https://downloads.apache.org/kafka/${VERSION}/${bin_file}.sha512" + +# It is probably redundant to check both the checksum and the signature but it's cheap and why not +echo "Validating SHA512 Checksum" +if ! (gpg --print-md SHA512 "${bin_file}" | diff - "${bin_file}.sha512" ); then + echo "ERROR: The SHA512 sum does not match" + exit 1 +fi + +echo "Validating signature" +echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html' + +if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then + echo "ERROR: The signature could not be verified" + exit 1 +fi + +echo "Uploading everything to Nexus" +EXIT_STATUS=0 +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? + +if [ $EXIT_STATUS -ne 0 ]; then + echo "ERROR: Upload failed" + exit 1 +fi + +echo "Successfully uploaded version ${VERSION} of Kafka to Nexus" +echo "https://repo.stackable.tech/service/rest/repository/browse/packages/kafka/"