Update kafka confluent

.github/workflows/docker.yml

@@ -37,11 +37,11 @@ jobs:
           source ./tool.sh
           build_image elasticsearch latest   docker_elasticsearch/Dockerfile && push_image
 
-  qpod_kafka:
-    name: qpod/kafka
+  qpod_kafka_confluent:
+    name: qpod/kafka_confluent
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - run: |
           source ./tool.sh
-          build_image kafka latest   docker_kafka/Dockerfile && push_image
+          build_image kafka latest          docker_kafka_confluent/Dockerfile && push_image

docker_kafka/Dockerfile → docker_kafka_confluent/Dockerfile

@@ -7,20 +7,22 @@ ARG KAFKA_VERSION="7.3.3"
 
 LABEL maintainer="haobibo@gmail.com"
 
+# ref: https://docs.confluent.io/platform/current/installation/docker/image-reference.html
 # ref: https://github.com/confluentinc/kafka-images/blob/master/kafka/Dockerfile.ubi8
 
 COPY work /opt/utils/
 
-ENV KAFKA_HOME=/opt/kafka \
+ENV COMPONENT=kafka \
     KAFKA_VERSION="${KAFKA_VERSION}" \
-    COMPONENT=kafka
+    KAFKA_HOME=/opt/kafka
 
 RUN source /opt/utils/script-confluent-kafka.sh \
  && echo "Install confluent-kafka:" && setup_confluent_kafka \
- && echo "Setup confluent kafka to use KRaft" && setup_confluent_kafka_kraft \
- && pip install -U confluent-kafka \
+ && echo "Setup confluent Kafka to run in KRaft mode" && setup_confluent_kafka_kraft \
  && echo "Clean up" && list_installed_packages && install__clean
 
+ENV PATH=$PATH:$KAFKA_HOME/bin \
+    CLUSTER_ID="pUyrmY_RQHClQc9LPBJqTw"
 EXPOSE 9092
 VOLUME ["/var/lib/${COMPONENT}/data", "/etc/${COMPONENT}/secrets"]
 CMD ["/etc/confluent/docker/run"]

docker_kafka_confluent/README.md

@@ -0,0 +1,76 @@
+# Confluent Kafka in KRaft mode
+
+In the KRaft mode, zookeeper is not need.
+To start a standalone mode KRaft kafka, use the `docker-compose.yml` file in this folder.
+
+## Development - debug inside docker
+
+```bash
+docker run -it \
+    --name=cp-ckafka \
+    -h=broker \
+    -p=9092:9092 \
+    -v $(pwd):/root/dev \
+    qpod/jdk11 bash
+
+export KAFKA_BROKER_ID=1
+export KAFKA_LISTENER_SECURITY_PROTOCOL_MAP='CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+export KAFKA_ADVERTISED_LISTENERS='PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
+export KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+export KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+export KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
+export KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+export KAFKA_JMX_PORT=9101
+export KAFKA_JMX_HOSTNAME=localhost
+export KAFKA_PROCESS_ROLES='broker,controller'
+export KAFKA_NODE_ID=1
+export KAFKA_CONTROLLER_QUORUM_VOTERS='1@broker:29093'
+export KAFKA_LISTENERS='PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
+export KAFKA_INTER_BROKER_LISTENER_NAME='PLAINTEXT'
+export KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER'
+export KAFKA_LOG_DIRS='/tmp/kraft-combined-logs'
+export KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+
+export COMPONENT=kafka
+export KAFKA_HOME=/opt/kafka
+export PATH=$PATH:$KAFKA_HOME/bin
+
+export CLUSTER_ID="${CLUSTER_ID:-$(kafka-storage random-uuid)}"
+kafka-storage format --ignore-formatted -t "${CLUSTER_ID}" -c /etc/kafka/kafka.properties
+
+sed -i '1i\
+export CLUSTER_ID="${CLUSTER_ID:-$(kafka-storage random-uuid)}"
+' /etc/confluent/docker/run
+
+/etc/confluent/docker/run > /tmp/kafka.log
+```
+
+## Development - build the docker image and run
+
+```bash
+docker build -t qpod/kafka --build-arg "BASE_NAMESPACE=qpod" .
+
+docker run -it \
+ -e KAFKA_BROKER_ID=1 \
+ -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP='CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' \
+ -e KAFKA_ADVERTISED_LISTENERS='PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' \
+ -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
+ -e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \
+ -e KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 \
+ -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \
+ -e KAFKA_JMX_PORT=9101 \
+ -e KAFKA_JMX_HOSTNAME=localhost \
+ -e KAFKA_PROCESS_ROLES='broker,controller' \
+ -e KAFKA_NODE_ID=1 \
+ -e KAFKA_CONTROLLER_QUORUM_VOTERS='1@broker:29093' \
+ -e KAFKA_LISTENERS='PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' \
+ -e KAFKA_INTER_BROKER_LISTENER_NAME='PLAINTEXT' \
+ -e KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER' \
+ -e KAFKA_LOG_DIRS='/tmp/kraft-combined-logs' \
+ -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
+ --name cp-kafka \
+ -h broker \
+ -p 9092:9092 \
+ -v $(pwd):/root/dev \ 
+ qpod0dev/cp-kafka
+```

docker_kafka_confluent/docker-compose-bitnami.yml

@@ -0,0 +1,26 @@
+# ref: https://github.com/bitnami/containers/tree/main/bitnami/kafka#kafka-without-zookeeper-kraft
+
+version: "3"
+
+## Standalone mode
+services:
+  kafka:
+    image: docker.io/bitnami/kafka:latest
+    ports:
+      - "9092:9092"
+    volumes:
+      - "kafka_data:/bitnami"
+    environment:
+      - KAFKA_ENABLE_KRAFT=yes
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
+      - KAFKA_BROKER_ID=1
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
+      - ALLOW_PLAINTEXT_LISTENER=yes
+
+volumes:
+  kafka_data:
+    driver: local

docker_kafka/docker-compose.yml → docker_kafka_confluent/docker-compose.yml

@@ -1,5 +1,5 @@
 ---
-version: '2'
+version: '3'
 services:
 
   broker:
@@ -10,6 +10,7 @@ services:
       - "9092:9092"
       - "9101:9101"
     environment:
+      CLUSTER_ID: 'k4hJjYlsRYSk9UQcZjN0rA'
       KAFKA_BROKER_ID: 1
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
       KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
@@ -29,4 +30,3 @@ services:
     volumes:
       - ./kafka-broker/data:/var/lib/kafka-broker/data
       - ./kafka-broker/secrets:/etc/kafka-broker/secrets
-    command: "bash -c '/etc/confluent/docker/run'"

docker_kafka_confluent/work/script-confluent-kafka.sh

@@ -0,0 +1,46 @@
+source /opt/utils/script-utils.sh
+
+setup_confluent_kafka() {
+  export KAFKA_HOME=/opt/kafka
+
+  local VER_C_KAFKA_MINOR=${KAFKA_VERSION:-"7.3.3"}
+  local VER_C_KAFKA_MAJOR=${VER_C_KAFKA_MINOR%.*}
+  local URL_C_KAFKA="http://packages.confluent.io/archive/${VER_C_KAFKA_MAJOR}/confluent-community-${VER_C_KAFKA_MINOR}.tar.gz"
+
+  # Downlaod CKafka package and unzip to /opt/kafka
+  # ref: https://docs.confluent.io/platform/current/installation/installing_cp/zip-tar.html#get-the-software
+  install_tar_gz "${URL_C_KAFKA}" && mv /opt/confluent-* ${KAFKA_HOME} \
+ && echo "Setting up kafka dirs:" && mkdir -pv /var/lib/kafka/data /etc/kafka/secrets \
+ && ln -sf ${KAFKA_HOME}/etc /etc/confluent \
+ && ls -alh ${KAFKA_HOME}/*
+
+  # CKafka docker images requires confluent docker utils for dub/cub command
+  pip install -U confluent-kafka https://github.com/confluentinc/confluent-docker-utils/archive/refs/heads/master.zip \
+ && install_zip https://github.com/confluentinc/confluent-docker-utils/archive/refs/heads/master.zip \
+ && PYTHON_SITE=$(python3 -c 'import sys;print(list(filter(lambda s: "site" in s, sys.path))[0])') \
+ && cp -rf /opt/confluent-*/confluent ${PYTHON_SITE} \
+ && rm -rf /opt/confluent-*
+
+  install_zip https://github.com/confluentinc/common-docker/archive/refs/heads/master.zip \
+ && mv /opt/common-docker-master ${KAFKA_HOME}/common-docker \
+ && mkdir -pv ${KAFKA_HOME}/etc/docker/ \
+ && cp -rf ${KAFKA_HOME}/common-docker/base/include/etc/confluent/docker/* ${KAFKA_HOME}/etc/docker/
+
+  # CKafka base docker images are built with some scripts included
+  install_zip https://github.com/confluentinc/kafka-images/archive/refs/heads/master.zip
+  mv /opt/kafka-images* ${KAFKA_HOME}/kafka-images
+  cp -rf "${KAFKA_HOME}/kafka-images/kafka/include/etc/confluent/docker/"  "${KAFKA_HOME}/etc/"
+}
+
+setup_confluent_kafka_kraft() {
+  # ref: https://github.com/confluentinc/cp-all-in-one/blob/7.3.3-post/cp-all-in-one-kraft/update_run.sh
+  # Docker workaround 1: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter
+  # Docker workaround 2: Ignore cub zk-ready
+  sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure
+  sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure
+
+  # KRaft required step: Format the storage directory with a new cluster ID
+cat >>/etc/confluent/docker/configure <<EOF
+kafka-storage format --ignore-formatted -t "\${CLUSTER_ID}" -c /etc/kafka/kafka.properties
+EOF
+}