docker-compose-kafka-multiple-stack.yml

version: '3.9'

services:

 zookeeper:
    image: confluentinc/cp-zookeeper:7.3.2
    container_name: zookeeper
    hostname: zookeeper
    restart: always
    ports:
       - ${ZOOKEEPER_PORT:-env.ZOOKEEPER_PORT}:2181
    environment:
      ZOOKEEPER_SERVER_ID: ${ZOOKEEPER_SERVER_ID:-env.ZOOKEEPER_SERVER_ID}
      ZOOKEEPER_CLIENT_PORT: ${ZOOKEEPER_CLIENT_PORT:-env.ZOOKEEPER_CLIENT_PORT}
      ZOOKEEPER_TICK_TIME: ${ZOOKEEPER_TICK_TIME:-env.ZOOKEEPER_TICK_TIME}
    volumes: 
      - ./volumenes/zookeeper:/var/lib/kafka/data
    networks: 
      - broker-kafka
  
 kafka1:
    image: confluentinc/cp-kafka:7.3.2
    container_name: kafka1
    hostname: kafka1
    restart: always
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_DELETE_TOPIC_ENABLE: true
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
    volumes:
      - ./volumenes/kafka1:/var/lib/kafka/data
    networks: 
      - broker-kafka

 kafka2:
    image: confluentinc/cp-kafka:7.3.2
    container_name: kafka2
    hostname: kafka2
    restart: always
    depends_on:
      - zookeeper
    ports:
      - "9093:9093"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9093
      KAFKA_LISTENERS: PLAINTEXT://kafka2:9093
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_DELETE_TOPIC_ENABLE: true
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
    volumes:
      - ./volumenes/kafka2:/var/lib/kafka/data
    networks: 
      - broker-kafka

 kafka3:
    image: confluentinc/cp-kafka:7.3.2
    container_name: kafka3
    hostname: kafka3
    restart: always
    depends_on:
      - zookeeper
    ports:
      - "9094:9094"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9094
      KAFKA_LISTENERS: PLAINTEXT://kafka3:9094
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_DELETE_TOPIC_ENABLE: true
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
    volumes:
      - ./volumenes/kafka3:/var/lib/kafka/data
    networks: 
      - broker-kafka

 conduktor-platform:
    image: conduktor/conduktor-platform:latest
    hostname: conduktor
    container_name: conduktor
    restart: always
    depends_on:
      - postgresql
    ports:
      - "8080:8080"
    volumes:
      - ./volumenes/conduktor_data:/var/conduktor
    environment:
      CDK_ORGANIZATION_NAME: ${CDK_ORGANIZATION_NAME:-env.CDK_ORGANIZATION_NAME}
      CDK_ADMIN_EMAIL: ${CDK_ADMIN_EMAIL:-env.CDK_ADMIN_EMAIL}
      CDK_ADMIN_PASSWORD: ${CDK_ADMIN_PASSWORD:-env.CDK_ADMIN_PASSWORD}
      CDK_DATABASE_URL: "postgresql://${POSTGRES_USER:-env.POSTGRES_USER}:${POSTGRES_PASSWORD:-env.POSTGRES_PASSWORD}@postgresql:${POSTGRES_PORT:-env.POSTGRES_PORT}/${POSTGRES_DB:-env.POSTGRES_DB}"
      CDK_CLUSTERS_0_ID: ${CDK_CLUSTERS_0_ID:-env.CDK_CLUSTERS_0_ID}
      CDK_CLUSTERS_0_NAME: ${CDK_CLUSTERS_0_NAME:-env.CDK_CLUSTERS_0_NAME}
      CDK_CLUSTERS_0_COLOR: "#0013E7"
      CDK_CLUSTERS_0_BOOTSTRAPSERVERS: PLAINTEXT://kafka1:9092, PLAINTEXT://kafka2:9093, PLAINTEXT://kafka3:9094
      CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081"
      CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083"
      CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect"
    networks: 
      - broker-kafka

 kafka-connect:
    image: confluentinc/cp-kafka-connect:7.3.2
    container_name: kafka-connect
    hostname: kafka-connect
    restart: always
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:9092,PLAINTEXT://kafka2:9093,PLAINTEXT://kafka3:9094'
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "connect-group"
      CONNECT_CONFIG_STORAGE_TOPIC: "connect-config"
      CONNECT_OFFSET_STORAGE_TOPIC: "connect-offset"
      CONNECT_STATUS_STORAGE_TOPIC: "connect-status"
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components'
    volumes:
        - ./volumenes/connectors:/etc/kafka-connect/jars/
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3
    networks: 
      - broker-kafka

 kafka-schema-registry:
    image: confluentinc/cp-schema-registry:7.3.2
    hostname: kafka-schema-registry
    container_name: kafka-schema-registry
    ports:
      - "8081:8081"
    restart: always
    environment:
     SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:9092,PLAINTEXT://kafka2:9093,PLAINTEXT://kafka3:9094'
     SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
     SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
     SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
     SCHEMA_REGISTRY_STORAGE_REPLICATION_FACTOR: 3
     SCHEMA_REGISTRY_STORAGE_VOLUME: "/var/lib/kafka/schema-registry"
    volumes:
        - ./volumenes/schema-registry:/data/kafka/schema-registry
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3
    networks: 
      - broker-kafka
    
 ksqldb-server:
    image: confluentinc/cp-ksqldb-server:7.3.2
    hostname: ksqldb-server
    container_name: ksqldb-server
    restart: always
    ports:
      - "8088:8088"
    environment:
      KSQL_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:9092,PLAINTEXT://kafka2:9093,PLAINTEXT://kafka3:9094'
      KSQL_LISTENERS: http://0.0.0.0:8088/
      KSQL_KSQL_SERVICE_ID: ksqldb-server
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3
    networks: 
      - broker-kafka

 kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:7.3.2
    hostname: kafka-rest-proxy
    container_name: kafka-rest-proxy
    restart: always
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:9092,PLAINTEXT://kafka2:9093,PLAINTEXT://kafka3:9094'
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3
      - kafka-schema-registry
    networks: 
      - broker-kafka

 postgresql:
    image: postgres:14
    container_name: postgresql
    hostname: postgresql
    ports:
      - ${POSTGRES_PORT:-env.POSTGRES_PORT}:5432
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-env.POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER:-env.POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-env.POSTGRES_PASSWORD}
      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
    volumes:
      - ./volumenes/pg_data:/var/lib/postgresql/data
    networks: 
      - broker-kafka

 pgadmin:
    container_name: pgadmin
    hostname: pgadmin
    image: dpage/pgadmin4
    restart: always
    environment:
      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-env.PGADMIN_DEFAULT_EMAIL}
      PGADMIN_DEFAULT_PASSWORD:  ${PGADMIN_DEFAULT_PASSWORD:-env.PGADMIN_DEFAULT_PASSWORD}
      PGADMIN_CONFIG_SERVER_MODE: 'False'
    ports:
      - ${PGADMIN_PORT:-env.PGADMIN_PORT}:80
    depends_on:
      - postgresql
    volumes:
    - ./volumenes/pgadmin:/var/lib/pgadmin
    networks: 
      - broker-kafka

networks: 
  broker-kafka:
    driver: bridge 

volumes:
  zookeeper: {}
  kafka1: {}
  kafka2: {}
  kafka3: {}
  conduktor_data: {}
  pg_data: {}
  pgadmin: {}
  connectors: {}
  schema-registry: {}