Merge pull request #1606 from dnwe/append-to-server-properties

fix: use modern kafka server.properties in vagrant

vagrant/boot_cluster.sh

@@ -17,6 +17,25 @@ while ! nc -q 1 localhost 21805 </dev/null; do echo "Waiting"; sleep 1; done
 # Launch and wait for Kafka
 for i in 1 2 3 4 5; do
     KAFKA_PORT=`expr $i + 9090`
-    cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bin/kafka-server-start.sh -daemon config/server.properties
+    cd ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} && bash bin/kafka-server-start.sh -daemon config/server.properties
 done
-while ! nc -q 1 localhost 29095 </dev/null; do echo "Waiting"; sleep 1; done
+ps auxww | grep -i kafka
+
+N=10
+RC=1
+set +x
+printf "Waiting for Kafka to become available."
+for _ in $(seq 1 "$N"); do
+  if nc -z 127.0.0.1 29095 </dev/null; then
+      RC=0
+      break
+  fi
+  printf "."
+  sleep 1
+done
+printf "\n"
+if [ "$RC" -gt 0 ]; then
+  echo 'Error: Kafka failed to startup' >&2
+  find "${KAFKA_INSTALL_ROOT}" -name "server.log" -print0 | xargs tail -256
+  exit ${RC}
+fi

vagrant/install_cluster.sh

@@ -16,25 +16,62 @@ rm -f ${KAFKA_INSTALL_ROOT}/toxiproxy
 ln -s ${KAFKA_INSTALL_ROOT}/toxiproxy-${TOXIPROXY_VERSION} ${KAFKA_INSTALL_ROOT}/toxiproxy
 
 for i in 1 2 3 4 5; do
-    ZK_PORT=`expr $i + 2180`
-    ZK_PORT_REAL=`expr $i + 21800`
-    KAFKA_PORT=`expr $i + 9090`
-    KAFKA_PORT_REAL=`expr $i + 29090`
+    ZK_PORT=$((i + 2180))
+    ZK_PORT_REAL=$((i + 21800))
+    KAFKA_PORT=$((i + 9090))
+    KAFKA_PORT_REAL=$((i + 29090))
 
     # unpack kafka
     mkdir -p ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}
     tar xzf ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_VERSION}.tgz -C ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT} --strip-components 1
 
     # broker configuration
-    cp ${REPOSITORY_ROOT}/vagrant/server.properties ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/
-    sed -i s/KAFKAID/${KAFKA_PORT}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
-    sed -i s/KAFKAPORT/${KAFKA_PORT_REAL}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
-    sed -i s/KAFKA_HOSTNAME/${KAFKA_HOSTNAME}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
-    sed -i s/ZK_PORT/${ZK_PORT}/g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
-
-    KAFKA_DATADIR="${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/data"
-    mkdir -p ${KAFKA_DATADIR}
-    sed -i s#KAFKA_DATADIR#${KAFKA_DATADIR}#g ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties
+    mkdir -p "${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/data"
+
+    # Append to default server.properties with a small number of customisations
+    printf "\n\n" >> "${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties"
+    cat << EOF >> "${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/server.properties"
+############################# Sarama Test Cluster #############################
+
+broker.id=${KAFKA_PORT}
+broker.rack=${i}
+
+# Listen on "real" port
+listeners=PLAINTEXT://:${KAFKA_PORT_REAL}
+# Advertise Toxiproxy port
+advertised.listeners=PLAINTEXT://${KAFKA_HOSTNAME}:${KAFKA_PORT}
+
+# Connect to Zookeeper via Toxiproxy port
+zookeeper.connect=127.0.0.1:${ZK_PORT}
+
+# Data directory
+log.dirs="${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/data"
+
+# Create new topics with a replication factor of 2 so failover can be tested
+# more easily.
+default.replication.factor=2
+
+# Turn on log.retention.bytes to avoid filling up the VM's disk
+log.retention.bytes=268435456
+log.segment.bytes=268435456
+
+# Enable topic deletion and disable auto-creation
+delete.topic.enable=true
+auto.create.topics.enable=false
+
+# Lower the zookeeper timeouts so we don't have to wait forever for a node
+# to die when we use toxiproxy to kill its zookeeper connection
+zookeeper.session.timeout.ms=3000
+zookeeper.connection.timeout.ms=3000
+
+# Disable broker ID length constraint
+reserved.broker.max.id=10000
+
+# Permit follower fetching (KIP-392)
+replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector
+
+###############################################################################
+EOF
 
     # zookeeper configuration
     cp ${REPOSITORY_ROOT}/vagrant/zookeeper.properties ${KAFKA_INSTALL_ROOT}/kafka-${KAFKA_PORT}/config/