Use env vars in docker-compose to configure everything

config/config.exs

@@ -14,9 +14,9 @@ config :kafka_ex,
   # server.properties file.
   # In the case below you would set "advertised.host.name=localhost"
   brokers: [
-    {"localhost", 9092},
     {"localhost", 9093},
-    {"localhost", 9094}
+    {"localhost", 9094},
+    {"localhost", 9095}
   ],
   #
   # OR:

docker-compose-kafka.env

@@ -0,0 +1,46 @@
+#################################
+# Common Kafka config
+#################################
+
+# Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`.
+
+######## topic creation
+
+KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2
+
+######## zookeeper
+
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000
+
+######## advertised hosts names and protocols
+
+# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1"  | grep -o "[0-9.]*"
+KAFKA_ADVERTISED_HOST_NAME=localhost
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:SSL,OUTSIDE:SSL
+KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE
+KAFKA_PROTOCOL_NAME=INSIDE
+
+######## SSL
+
+KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks
+KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_KEY_PASSWORD=kafka_ex
+KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks
+KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex
+KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG
+
+######## Config
+
+KAFKA_DELETE_TOPIC_ENABLE=true
+# KAFKA_NUM_NETWORK_THREADS=3
+# KAFKA_NUM_IO_THREADS=8
+# KAFKA_SOCKET_SEND_BUFFER_BYTES=102400
+# KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400
+# KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600
+KAFKA_LOG_DIRS=/tmp/kafka_log
+KAFKA_NUM_PARTITIONS=1
+KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1
+KAFKA_LOG_RETENTION_HOURS=168
+# KAFKA_LOG_SEGMENT_BYTES=1073741824
+# KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000

docker-compose.yml

@@ -1,4 +1,5 @@
 version: '3.2'
+
 services:
   zookeeper:
     image: wurstmeister/zookeeper:3.4.6
@@ -7,30 +8,38 @@ services:
   kafka1:
     image: wurstmeister/kafka:0.11.0.1
     ports:
-      - "9092:9092"
+      - "9093:9093"
     depends_on:
       - zookeeper
     volumes:
-      - ./kafka1/server.properties.in:/opt/kafka/config/server.properties.in
-      - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
       - ./ssl:/ssl
+    env_file: docker-compose-kafka.env
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ADVERTISED_PORT: 9093
+
   kafka2:
     image: wurstmeister/kafka:0.11.0.1
     ports:
-      - "9093:9093"
+      - "9094:9094"
     depends_on:
       - zookeeper
     volumes:
-      - ./kafka2/server.properties.in:/opt/kafka/config/server.properties.in
-      - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
       - ./ssl:/ssl
+    env_file: docker-compose-kafka.env
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ADVERTISED_PORT: 9094
+
   kafka3:
     image: wurstmeister/kafka:0.11.0.1
     ports:
-      - "9094:9094"
+      - "9095:9095"
     depends_on:
       - zookeeper
     volumes:
-      - ./kafka3/server.properties.in:/opt/kafka/config/server.properties.in
-      - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh
       - ./ssl:/ssl
+    env_file: docker-compose-kafka.env
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_ADVERTISED_PORT: 9095

scripts/README.md

@@ -1,3 +1,5 @@
+# Scripts
+
 The scripts in this directory are used for testing KafkaEx and are not part of
-the release package.  Each script should have a comment near the top explaining
+the release package. Each script should have a comment near the top explaining
 its purpose.

scripts/docker_up.sh

@@ -1,43 +1,5 @@
 #!/bin/bash
 
-# Launches a dockerized kafka cluster configured for testing with KafkaEx
-#
-# This script attempts to auto-detect the ip address of an active network
-# interface using `./scripts/docker_ip.sh`.
-#
-# This script should be run from the project root
-
-set -e
-
-# Kafka needs to know our ip address so that it can advertise valid
-# connnection details
-
-export DOCKER_IP=$(./scripts/docker_ip.sh)
-
-# for debugging purposes
-echo Detected active ip address ${DOCKER_IP}
-
-for i in 1 2 3
-do
-  port=$(expr 9092 + ${i} - 1)
-  mkdir -p kafka${i}
-  target=kafka${i}/server.properties.in
-  cp ./server.properties ${target}
-  # configure broker and port
-  sed -i.bak "s/@broker_id@/${i}/g" ${target}
-  sed -i.bak "s/@port@/${port}/g" ${target}
-  # the @pwd become root directory so we get /ssl
-  sed -i.bak "s|@pwd@||g" ${target}
-  # point at zookeeper in docker
-  sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target}
-  # delete the existing listeners line
-  sed -i.bak "/^listeners=/d" ${target}
-  # add an advertised.listeners and listeners together at the end
-  echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target}
-  echo "listeners=SSL://0.0.0.0:${port}" >> ${target}
-done
+# This script starts zookeeper and the 3 kafka nodes used for testing
 
 docker-compose up -d
-
-# create topics needed for testing
-docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"