add kafka

.github/workflows/docker.yml

@@ -19,7 +19,7 @@ env:
   DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
 
 jobs:
-  qpod_pyflink:
+  qpod_bigdata:
     name: qpod/bigdata
     runs-on: ubuntu-latest
     steps:
@@ -36,3 +36,12 @@ jobs:
       - run: |
           source ./tool.sh
           build_image elasticsearch latest   docker_elasticsearch/Dockerfile && push_image
+
+  qpod_kafka:
+    name: qpod/kafka
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          source ./tool.sh
+          build_image kafka latest   docker_kafka/Dockerfile && push_image

docker_bigdata/work/script-db-clients.sh

@@ -0,0 +1,13 @@
+source /opt/utils/script-utils.sh
+
+
+setup_postgresql_client() {
+  local VER_PG=${VERSION_PG:-"14"}
+  # from: https://www.postgresql.org/download/linux/ubuntu/
+  echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+  curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+  sudo apt-get update
+  # will download ~9MB files and use ~55MB disk after installation
+  sudo apt-get -y install "postgresql-client-${VER_PG}"
+  echo "@ Version of psql:" && psql --version
+}

docker_kafka/Dockerfile

@@ -0,0 +1,14 @@
+# Distributed under the terms of the Modified BSD License.
+ARG BASE_NAMESPACE
+ARG BASE_IMG="jdk11"
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
+
+LABEL maintainer="haobibo@gmail.com"
+
+COPY work /opt/utils/
+
+ENV KAFKA_HOME=/opt/kafka
+
+RUN source /opt/utils/script-confluent-kafka.sh \
+ && echo "Install confluent-kafka:" && setup_confluent_kafka \
+ && echo "Clean up" && list_installed_packages && install__clean

docker_kafka/work/script-confluent-kafka.sh

@@ -0,0 +1,21 @@
+source /opt/utils/script-utils.sh
+
+setup_confluent_kafka() {
+  # ref: https://docs.confluent.io/platform/current/installation/installing_cp/zip-tar.html#get-the-software
+  local VER_C_KAFKA_MINOR=${VERSION_CONFLUENT_KAFKA:-"7.2.2"}
+  local VER_C_KAFKA_MAJOR=${VERSION_CONFLUENT_KAFKA:-"7.2"}
+  install_tar_gz "http://packages.confluent.io/archive/${VER_C_KAFKA_MAJOR}/confluent-${VER_C_KAFKA_MINOR}.tar.gz"
+  mv /opt/confluent-* /opt/kafka
+  export KAFKA_HOME=/opt/kafka
+  ls -alh ${KAFKA_HOME}/*
+}
+
+setup_confluent_kafka_kraft() {
+  # ref: https://github.com/confluentinc/cp-all-in-one/blob/7.2.1-post/cp-all-in-one-kraft/update_run.sh
+  # Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter
+  sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' ${KAFKA_HOME}/etc/confluent/docker/configure
+  # Docker workaround: Ignore cub zk-ready
+  sed -i 's/cub zk-ready/echo ignore zk-ready/' ${KAFKA_HOME}/etc/confluent/docker/ensure
+  # KRaft required step: Format the storage directory with a new cluster ID
+  echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c ${KAFKA_HOME}/etc/kafka/kafka.properties" >>${KAFKA_HOME}/etc/confluent/docker/ensure
+}