docker: remove use of Zookeeper for Kafka

Author: vincentbernat

docker/docker-compose-dev.yml

@@ -21,8 +21,10 @@ services:
       file: versions.yml
       service: kafka
     environment:
-      KAFKA_CFG_BROKER_ID: 1
-      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+      # KRaft settings
+      KAFKA_CFG_NODE_ID: 0
+      KAFKA_CFG_PROCESS_ROLES: controller,broker
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9096
       # We have two sets of listeners: INTERNAL that is used from inside the docker
       # compose network and listens on "kafka" and EXTERNAL that is mapped to
       # the host network and listens on "localhost".
@@ -32,18 +34,21 @@ services:
         INTERNAL://:9092,
         OINTERNAL://:9093,
         EXTERNAL://:9094,
-        OEXTERNAL://:9095
+        OEXTERNAL://:9095,
+        CONTROLLER://:9096
       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: >-
         INTERNAL:PLAINTEXT,
         OINTERNAL:SASL_PLAINTEXT,
         EXTERNAL:PLAINTEXT,
-        OEXTERNAL:SASL_PLAINTEXT
+        OEXTERNAL:SASL_PLAINTEXT,
+        CONTROLLER:PLAINTEXT
       KAFKA_CFG_ADVERTISED_LISTENERS: >-
         INTERNAL://kafka:9092,
         OINTERNAL://kafka:9093,
         EXTERNAL://localhost:9092,
         OEXTERNAL://localhost:9093
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
       # OAuth2 configuration
       KAFKA_CFG_LISTENER_NAME_OEXTERNAL_SASL_ENABLED_MECHANISMS: OAUTHBEARER
       KAFKA_CFG_LISTENER_NAME_OEXTERNAL_SASL_OAUTHBEARER_JWKS_ENDPOINT_URL: http://mock-oauth2-server:8080/default/jwks
@@ -64,7 +69,6 @@ services:
         unsecuredLoginStringClaim_sub="sub";
       KAFKA_CFG_LISTENER_NAME_OINTERNAL_OAUTHBEARER_SASL_SERVER_CALLBACK_HANDLER_CLASS: org.apache.kafka.common.security.oauthbearer.secured.OAuthBearerValidatorCallbackHandler
     depends_on:
-      - zookeeper
       - mock-oauth2-server
     ports:
       - 127.0.0.1:9092:9094/tcp

docker/docker-compose.yml

@@ -33,18 +33,20 @@ services:
       file: versions.yml
       service: kafka
     environment:
-      - KAFKA_ZOOKEEPER_PROTOCOL=PLAINTEXT
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=0
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+      # Listeners
+      - KAFKA_CFG_LISTENERS=CLIENT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
+      # Misc
       - KAFKA_CFG_DELETE_TOPIC_ENABLE=true
-      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
-      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
-      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
-      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
       - JMX_PORT=5555
     restart: unless-stopped
-    depends_on:
-      zookeeper:
-        condition: service_healthy
     volumes:
       - akvorado-kafka:/bitnami/kafka
     healthcheck: