Added Kakfa (core + REST + UI) wurstmeister image

docker-compose-hive.yml

@@ -1,5 +1,59 @@
 version: '2' 
 services:
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+      - 2181:2181
+
+  kafka:
+    image: wurstmeister/kafka:0.10.2.1
+    ports:
+      - "9092:9092"
+    links:
+      - zookeeper
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_PORT: 9092
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
+      KAFKA_LISTENERS: "PLAINTEXT://:9092"
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "test:1:1"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+
+  kafka-rest:
+    image: nodefluent/kafka-rest
+    ports:
+      - 8082:8082
+    links:
+      - kafka
+      - zookeeper
+    depends_on:
+      - kafka
+      - zookeeper
+    environment:
+      DEBUG: "*"
+
+  kafka-rest-ui:
+    image: nodefluent/kafka-rest-ui
+    ports:
+      - 8000:8000
+    links:
+      - kafka-rest
+    depends_on:
+      - kafka-rest
+    environment:
+      DEBUG: "*"
+      REACT_APP_KAFKA_REST_URL: "http://kafka-rest:8082/"
+      REACT_APP_TIMEOUT: "3000"
+      PROXY: "yes"
+      BASIC_AUTH_USER: "admin"
+      BASIC_AUTH_PASSWORD: "admin"
+
+
   namenode:
     image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
     container_name: namenode

docker-compose.yml

@@ -1,5 +1,58 @@
 version: '2' 
 services:
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+      - 2181:2181
+
+  kafka:
+    image: wurstmeister/kafka:0.10.2.1
+    ports:
+      - "9092:9092"
+    links:
+      - zookeeper
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_PORT: 9092
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
+      KAFKA_LISTENERS: "PLAINTEXT://:9092"
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "test:1:1"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+
+  kafka-rest:
+    image: nodefluent/kafka-rest
+    ports:
+      - 8082:8082
+    links:
+      - kafka
+      - zookeeper
+    depends_on:
+      - kafka
+      - zookeeper
+    environment:
+      DEBUG: "*"
+
+  kafka-rest-ui:
+    image: nodefluent/kafka-rest-ui
+    ports:
+      - 8000:8000
+    links:
+      - kafka-rest
+    depends_on:
+      - kafka-rest
+    environment:
+      DEBUG: "*"
+      REACT_APP_KAFKA_REST_URL: "http://kafka-rest:8082/"
+      REACT_APP_TIMEOUT: "3000"
+      PROXY: "yes"
+      BASIC_AUTH_USER: "admin"
+      BASIC_AUTH_PASSWORD: "admin"
+
   namenode:
     image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
     container_name: namenode

start-hadoop-spark-workbench-with-Hive.sh

@@ -4,11 +4,12 @@ docker-compose -f docker-compose-hive.yml up -d namenode hive-metastore-postgres
 docker-compose -f docker-compose-hive.yml up -d datanode hive-metastore
 docker-compose -f docker-compose-hive.yml up -d hive-server
 docker-compose -f docker-compose-hive.yml up -d spark-master spark-worker spark-notebook hue
+docker-compose up -d kafka kafka-rest kafka-rest-ui
 
 my_ip=`ip route get 1|awk '{print $NF;exit}'`
 echo "Namenode: http://${my_ip}:50070"
 echo "Datanode: http://${my_ip}:50075"
 echo "Spark-master: http://${my_ip}:8080"
 echo "Spark-notebook: http://${my_ip}:9001"
+echo "Kafka: http://${my_ip}:8000"
 echo "Hue (HDFS Filebrowser): http://${my_ip}:8088/home"
-

start-hadoop-spark-workbench.sh

@@ -7,5 +7,6 @@ echo "Namenode: http://${my_ip}:50070"
 echo "Datanode: http://${my_ip}:50075"
 echo "Spark-master: http://${my_ip}:8080"
 echo "Spark-notebook: http://${my_ip}:9001"
+echo "kafka: http://${my_ip}:8000"
 echo "Hue (HDFS Filebrowser): http://${my_ip}:8088/home"