[FAB-931] Add multi-broker Kafka orderer environments

Subtask of FAB-890

 - Created bddtests/environments/ directory structure.
   This makes is easier to compose environments that
   dependon on more than just a single
   docker-compose.yml file.

 - Added docker-compose environments for Kafka orderer
   with multiple brokers:
   kafka:
      Source for building the Kafka docker image
      used by other environments.
   orderer-1-kafka-1:
      1 Kafka orderer node, 1 Kafka broker.
   orderer-1-kafka-3:
      1 Kafka orderer node, 3 Kafka brokers.
   orderer-n-kafka-n:
      Experimental environment where the orderer and
      kafka services can be scaled using:
        $ docker-compose scale kafka=n
        $ docker-compose scale orderer=m

 - Changed orderer.feature to use orderer-kafka-1
   instead of docker-compose-orderer-kafka.yml.

 - Added scenario outline examples to orderer.feature
   to run against orderer-1-kafka-3, but they are
   disabled in .behaverc, as they don't all succeed.

 - Modified the "we compose" step definition to
   accept a directory in addition to .yml files.

 - Modified `make behave-deps` to build environments.
     (did not add orderer-n-kafka-n since it is not
      used at the moment)

 - Converted to Docker Compose v2 file format.

 - Fixed Composition.rebuildContainerData() to work
   with Docker Compose v2 environments.

Change-Id: I84d2ec959cd1d02e36293c86f05b804254e530ff
Signed-off-by: Luis Sanchez <sanchezl@us.ibm.com>

Makefile

@@ -140,7 +140,13 @@ unit-tests: unit-test
 docker: $(patsubst %,build/image/%/.dummy, $(IMAGES))
 native: peer orderer
 
-behave-deps: docker peer build/bin/block-listener
+BEHAVE_ENVIRONMENTS = kafka orderer-1-kafka-1 orderer-1-kafka-3
+.PHONY: behave-environments $(BEHAVE_ENVIRONMENTS)
+behave-environments: $(BEHAVE_ENVIRONMENTS)
+$(BEHAVE_ENVIRONMENTS):
+	@docker-compose --file bddtests/environments/$@/docker-compose.yml build
+
+behave-deps: docker peer build/bin/block-listener behave-environments
 behave: behave-deps
 	@echo "Running behave tests"
 	@cd bddtests; behave $(BEHAVE_OPTS)

bddtests/.behaverc

@@ -10,3 +10,5 @@ tags=~@issue_767
 			~@preV1
 
 			~@FAB-314
+
+name=^((?!1 Kafka Orderer and 3 Kafka Brokers).)*$

bddtests/environments/kafka/.dockerignore

@@ -0,0 +1 @@
+docker-compose.xml

bddtests/environments/kafka/Dockerfile

@@ -0,0 +1,18 @@
+FROM openjdk:8u111-jre
+
+ENV SCALA_VERSION=2.11 \
+    KAFKA_VERSION=0.9.0.1 \
+    KAFKA_DOWNLOAD_SHA1=FC9ED9B663DD608486A1E56197D318C41813D326
+
+RUN curl -fSL "http://www-us.apache.org/dist/kafka/0.9.0.1/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" -o kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
+    && echo "${KAFKA_DOWNLOAD_SHA1}  kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | sha1sum -c - \
+    && tar xfz kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt \
+    && mv /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" /opt/kafka \
+    && rm kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz
+
+ADD docker-entrypoint.sh /docker-entrypoint.sh
+
+EXPOSE 9092
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["/opt/kafka/bin/kafka-server-start.sh"]

bddtests/environments/kafka/README.md

@@ -0,0 +1,38 @@
+# Apache Kafka Docker Image
+This image can be used to start Apache Kafka in a Docker container.
+
+Use the provided [`docker-compose.yml`](docker-compose.yml) file as a starting point.
+
+## Usage
+#### Start
+```console
+$ docker-compose up -d
+Creating kafka_zookeeper_1
+Creating kafka_kafka_1
+$
+```
+#### Scale
+```console
+$ docker-compose scale kafka=3
+Creating and starting kafka_kafka_2 ... done
+Creating and starting kafka_kafka_3 ... done
+$
+```
+#### Stop
+```console
+$ docker-compose stop
+Stopping kafka_kafka_3 ... done
+Stopping kafka_kafka_2 ... done
+Stopping kafka_kafka_1 ... done
+Stopping kafka_zookeeper_1 ... done
+$
+```
+## Configuration
+Edit the [`docker-compose.yml`](docker-compose.yml) file to configure.
+### server.properties
+To configure a Kafka server property, add it to the environment section of the Kafka service. Kafka properties map to environment variables as follows:
+1. Replace dots with underscores.
+2. Change to upper case.
+3. Prefix with `KAFKA_`
+
+For example, `default.replication.factor` becomes `KAFKA_DEFAULT_REPLICATION_FACTOR`.

bddtests/environments/kafka/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2'
+services:
+  zookeeper:
+    # Offical Apache ZooKeeper image. See https://hub.docker.com/_/zookeeper/
+    image: zookeeper:3.4.9
+
+  kafka:
+    build: .
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    depends_on:
+      - zookeeper

bddtests/environments/kafka/docker-entrypoint.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# This script will either start the kafka server, or run the user
+# specified command.
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+KAFKA_HOME=/opt/kafka
+KAFKA_EXE=${KAFKA_HOME}/bin/kafka-server-start.sh
+KAFKA_SERVER_PROPERTIES=${KAFKA_HOME}/config/server.properties
+
+# handle starting the kafka server with an option
+# (genericly handled, but only --override known to me at this time)
+if [ "${1:0:1}" = '-' ]; then
+  set -- ${KAFKA_EXE} ${KAFKA_SERVER_PROPERTIES} "$@"
+fi
+
+# handle default (i.e. no custom options or commands)
+if [ "$1" = "${KAFKA_EXE}" ]; then
+
+  # add the server.properties to the command
+  set -- ${KAFKA_EXE} ${KAFKA_SERVER_PROPERTIES}
+
+  # compute the advertised host name if a command was specified
+  if [[ -z ${KAFKA_ADVERTISED_HOST_NAME} && -n ${KAFKA_ADVERTISED_HOST_NAME_COMMAND} ]] ; then
+    export KAFKA_ADVERTISED_HOST_NAME=$(eval ${KAFKA_ADVERTISED_HOST_NAME_COMMAND})
+  fi
+
+  # compute the advertised port if a command was specified
+  if [[ -z ${KAFKA_ADVERTISED_PORT} && -n ${KAFKA_ADVERTISED_PORT_COMMAND} ]] ; then
+    export KAFKA_ADVERTISED_PORT=$(eval ${KAFKA_ADVERTISED_PORT_COMMAND})
+  fi
+
+  # default to auto set the broker id
+  if [ -z "$KAFKA_BROKER_ID" ] ; then
+    export KAFKA_BROKER_ID=-1
+  fi
+
+  # update server.properties by searching for envinroment variables named
+  # KAFKA_* and converting them to properties in the kafka server properties file.
+  for ENV_ENTRY in $(env | grep "^KAFKA_") ; do
+    # skip some entries that should do not belong in server.properties
+    if [[ $ENV_ENTRY =~ ^KAFKA_HOME= ]] ; then continue ; fi
+    if [[ $ENV_ENTRY =~ ^KAFKA_EXE= ]] ; then continue ; fi
+    if [[ $ENV_ENTRY =~ ^KAFKA_SERVER_PROPERTIES= ]] ; then continue ; fi
+    if [[ $ENV_ENTRY =~ ^KAFKA_ADVERTISED_HOST_NAME_COMMAND= ]] ; then continue ; fi
+    if [[ $ENV_ENTRY =~ ^KAFKA_ADVERTISED_PORT_COMMAND= ]] ; then continue ; fi
+    # transform KAFKA_XXX_YYY to xxx.yyy
+    KAFKA_PROPERTY_NAME="$(echo ${ENV_ENTRY%%=*} | sed -e 's/^KAFKA_//;s/_/./g' | tr '[:upper:]' '[:lower:]')"
+    # get property value
+    KAFKA_PROPERTY_VALUE="${ENV_ENTRY#*=}"
+    # update server.properties
+    if grep -q "^\s*#\?\s*${KAFKA_PROPERTY_NAME}" ${KAFKA_SERVER_PROPERTIES} ; then
+      # the property is already defined (maybe even commented out), so edit the file
+      sed -i -e "s|^\s*${KAFKA_PROPERTY_NAME}\s*=.*$|${KAFKA_PROPERTY_NAME}=${KAFKA_PROPERTY_VALUE}|" ${KAFKA_SERVER_PROPERTIES}
+      sed -i -e "s|^\s*#\s*${KAFKA_PROPERTY_NAME}\s*=.*$|${KAFKA_PROPERTY_NAME}=${KAFKA_PROPERTY_VALUE}|" ${KAFKA_SERVER_PROPERTIES}
+    else
+      echo "${KAFKA_PROPERTY_NAME}=${KAFKA_PROPERTY_VALUE}">>${KAFKA_SERVER_PROPERTIES}
+    fi
+  done
+fi
+
+exec "$@"

bddtests/environments/orderer-1-kafka-1/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '2'
+services:
+  zookeeper:
+    # Offical Apache ZooKeeper image. See https://hub.docker.com/_/zookeeper/
+    image: zookeeper:3.4.9
+
+  orderer0:
+    image: hyperledger/fabric-orderer
+    environment:
+        - ORDERER_GENERAL_LEDGERTYPE=ram
+        - ORDERER_GENERAL_BATCHTIMEOUT=10s
+        - ORDERER_GENERAL_BATCHSIZE=10
+        - ORDERER_GENERAL_MAXWINDOWSIZE=1000
+        - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
+        - ORDERER_RAMLEDGER_HISTORY_SIZE=100
+        - ORDERER_GENERAL_ORDERERTYPE=kafka
+        - ORDERER_KAFKA_BROKERS=[kafka0:9092]
+    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
+    command: orderer -loglevel debug -verbose true
+    depends_on:
+      - kafka0
+
+  kafka0:
+    build: ../kafka
+    environment:
+      KAFKA_BROKER_ID: 0
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    depends_on:
+      - zookeeper

bddtests/environments/orderer-1-kafka-3/docker-compose.yml

@@ -0,0 +1,50 @@
+version: '2'
+services:
+  zookeeper:
+    # Offical Apache ZooKeeper image. See https://hub.docker.com/_/zookeeper/
+    image: zookeeper:3.4.9
+
+  orderer0:
+    image: hyperledger/fabric-orderer
+    environment:
+      - ORDERER_GENERAL_LEDGERTYPE=ram
+      - ORDERER_GENERAL_BATCHTIMEOUT=10s
+      - ORDERER_GENERAL_BATCHSIZE=10
+      - ORDERER_GENERAL_MAXWINDOWSIZE=1000
+      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
+      - ORDERER_RAMLEDGER_HISTORY_SIZE=100
+      - ORDERER_GENERAL_ORDERERTYPE=kafka
+      - ORDERER_KAFKA_BROKERS=[kafka0:9092,kafka1:9092,kafka2:9092]
+    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
+    command: orderer -loglevel debug -verbose true
+    depends_on:
+      - kafka0
+      - kafka1
+      - kafka2
+
+  kafka0:
+    build: ../kafka
+    environment:
+      KAFKA_BROKER_ID: 0
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+    depends_on:
+      - zookeeper
+
+  kafka1:
+    build: ../kafka
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+    depends_on:
+      - zookeeper
+
+  kafka2:
+    build: ../kafka
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+    depends_on:
+      - zookeeper

bddtests/environments/orderer-n-kafka-n/README.md

@@ -0,0 +1,25 @@
+# order-n-kafka-n
+A scalable kafka orderer environment.
+## Starting
+
+While you can start the environment by simply executing `docker-compose -d up`, since the list of kafka brokers is computed dynamically, it is recommended that you start the kafka nodes first, and then start the orderer nodes.
+
+For example, to start an environment with 3 orderer shims and 5 kafka brokers, issue the following commands:
+
+```bash
+$ docker-compose up -d zookeeper
+$ docker-compose up -d kafka
+$ docker-compose scale kafka=5
+$ docker-compose up -d orderer
+$ docker-compose scale orderer=3
+```
+
+## Stopping
+
+While you can stop the environment by simply executing `docker-compose stop`, docker-compose does not enforce a reverse-dependency order on shutdown. For a cleaner shutdown, stop the individual service types independently in the following order:
+
+```bash
+$ docker-compose stop orderer
+$ docker-compose stop kafka
+$ docker-compose stop
+```

bddtests/environments/orderer-n-kafka-n/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '2'
+services:
+  zookeeper:
+    # Offical Apache ZooKeeper image. See https://hub.docker.com/_/zookeeper/
+    image: zookeeper:3.4.9
+
+  orderer:
+    build: ./orderer
+    environment:
+      - ORDERER_GENERAL_ORDERERTYPE=kafka
+    depends_on:
+      - zookeeper
+      - kafka
+    command: -loglevel debug -verbose true
+
+  kafka:
+    build: ../kafka
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    depends_on:
+      - zookeeper

bddtests/environments/orderer-n-kafka-n/orderer/Dockerfile

@@ -0,0 +1,16 @@
+FROM hyperledger/fabric-orderer
+ENV ORDERER_GENERAL_LEDGERTYPE=ram \
+    ORDERER_GENERAL_BATCHTIMEOUT=10s \
+    ORDERER_GENERAL_BATCHSIZE=10 \
+    ORDERER_GENERAL_MAXWINDOWSIZE=1000 \
+    ORDERER_GENERAL_LISTENADDRESS=0.0.0.0 \
+    ORDERER_GENERAL_LISTENPORT=5005 \
+    ORDERER_RAMLEDGER_HISTORY_SIZE=100
+WORKDIR /opt/gopath/src/github.com/hyperledger/fabric/orderer
+ENV ORDERER_GENERAL_ORDERERTYPE=kafka
+RUN apt-get update \
+    && apt-get install -y zookeeper jq \
+    && rm -rf /var/lib/apt/lists/*
+ADD docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["orderer"]

bddtests/environments/orderer-n-kafka-n/orderer/docker-entrypoint.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# This script will either start the kafka server, or run the user
+# specified command.
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+ORDERER_EXE=orderer
+
+# handle starting the orderer with an option
+if [ "${1:0:1}" = '-' ]; then
+  set -- ${ORDERER_EXE} "$@"
+fi
+
+# handle default (i.e. no custom options or commands)
+if [ "$1" = "${ORDERER_EXE}" ]; then
+
+  # get the broker list from zookeeper
+  if [ -z "$ORDERER_KAFKA_BROKERS" ] ; then
+    if [ -z "$ZOOKEEPER_CONNECT" ] ; then
+      export ZOOKEEPER_CONNECT="zookeeper:2181"
+    fi
+    ZK_CLI_EXE="/usr/share/zookeeper/bin/zkCli.sh -server ${ZOOKEEPER_CONNECT}"
+    until [ -n "$($ZK_CLI_EXE ls /brokers/ids | grep '^\[')" ] ; do
+      echo "No Kafka brokers registered in ZooKeeper. Will try again in 1 second."
+      sleep 1
+    done
+    ORDERER_KAFKA_BROKERS="["
+    ORDERER_KAFKA_BROKERS_SEP=""
+    for BROKER_ID in $($ZK_CLI_EXE ls /brokers/ids | grep '^\[' | sed 's/[][,]/ /g'); do
+      ORDERER_KAFKA_BROKERS=${ORDERER_KAFKA_BROKERS}${ORDERER_KAFKA_BROKERS_SEP}$($ZK_CLI_EXE get /brokers/ids/$BROKER_ID 2>&1 | grep '^{' | jq -j '. | .host,":",.port')
+      ORDERER_KAFKA_BROKERS_SEP=","
+    done
+    export ORDERER_KAFKA_BROKERS="${ORDERER_KAFKA_BROKERS}]"
+  fi
+
+fi
+
+exec "$@"