KYLIN-4775 Use docker-compose to deploy Hadoop and Kylin

docker/.gitignore

@@ -0,0 +1,6 @@
+docker-compose/others/data/
+docker-compose/read/data/
+docker-compose/write/data/
+docker-compose/others/kylin/kylin-all/
+docker-compose/others/kylin/kylin-job/
+docker-compose/others/kylin/kylin-query/

docker/build_cluster_images.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+SCRIPT_PATH=$(cd `dirname $0`; pwd)
+WS_ROOT=`dirname $SCRIPT_PATH`
+
+source ${SCRIPT_PATH}/header.sh
+
+#docker build -t apachekylin/kylin-metastore:mysql_5.6.49 ./kylin/metastore-db
+#
+
+docker build -t apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/base
+docker build -t apachekylin/kylin-hadoop-namenode:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} --build-arg HADOOP_WEBHDFS_PORT=${HADOOP_WEBHDFS_PORT} ./dockerfile/cluster/namenode
+docker build -t apachekylin/kylin-hadoop-datanode:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} --build-arg HADOOP_DN_PORT=${HADOOP_DN_PORT} ./dockerfile/cluster/datanode
+docker build -t apachekylin/kylin-hadoop-resourcemanager:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/resourcemanager
+docker build -t apachekylin/kylin-hadoop-nodemanager:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/nodemanager
+docker build -t apachekylin/kylin-hadoop-historyserver:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/historyserver
+
+docker build -t apachekylin/kylin-hive:hive_${HIVE_VERSION}_hadoop_${HADOOP_VERSION} \
+--build-arg HIVE_VERSION=${HIVE_VERSION} \
+--build-arg HADOOP_VERSION=${HADOOP_VERSION} \
+./dockerfile/cluster/hive
+
+if [ $ENABLE_HBASE == "yes" ]; then
+  docker build -t apachekylin/kylin-hbase-base:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hbase
+  docker build -t apachekylin/kylin-hbase-master:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hmaster
+  docker build -t apachekylin/kylin-hbase-regionserver:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hregionserver
+fi
+
+if [ $ENABLE_KERBEROS == "yes" ]; then
+  docker build -t apachekylin/kylin-kerberos:latest ./dockerfile/cluster/kerberos
+fi
+
+if [ $ENABLE_LDAP == "yes" ]; then
+  docker pull osixia/openldap:1.3.0
+fi
+
+#if [ $ENABLE_KAFKA == "yes" ]; then
+#  docker pull bitnami/kafka:2.0.0
+#fi
+docker pull bitnami/kafka:2.0.0
+
+docker pull mysql:5.6.49
+
+docker build -t apachekylin/kylin-client:hadoop_${HADOOP_VERSION}_hive_${HIVE_VERSION}_hbase_${HBASE_VERSION} \
+--build-arg HIVE_VERSION=${HIVE_VERSION} \
+--build-arg HADOOP_VERSION=${HADOOP_VERSION} \
+--build-arg HBASE_VERSION=${HBASE_VERSION} \
+./dockerfile/cluster/client

docker/build_image.sh → docker/build_standalone_image.sh

@@ -22,5 +22,5 @@ cd ${DIR}
 echo "build image in dir "${DIR}
 
 echo "start build Hadoop docker image"
-docker build -f Dockerfile_hadoop -t hadoop2.7-all-in-one .
-docker build -f Dockerfile -t apachekylin/apache-kylin-standalone:3.1.0 .
+docker build -f Dockerfile_hadoop -t hadoop2.7-all-in-one ./dockerfile/standalone
+docker build -f Dockerfile -t apachekylin/apache-kylin-standalone:3.1.0 ./dockerfile/standalone

docker/docker-compose/others/client-write-read.env

@@ -0,0 +1,61 @@
+CORE_CONF_fs_defaultFS=hdfs://write-namenode:8020
+CORE_CONF_hadoop_http_staticuser_user=root
+CORE_CONF_hadoop_proxyuser_hue_hosts=*
+CORE_CONF_hadoop_proxyuser_hue_groups=*
+CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
+
+HDFS_CONF_dfs_webhdfs_enabled=true
+HDFS_CONF_dfs_permissions_enabled=false
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+
+YARN_CONF_yarn_log___aggregation___enable=true
+YARN_CONF_yarn_log_server_url=http://write-historyserver:8188/applicationhistory/logs/
+YARN_CONF_yarn_resourcemanager_recovery_enabled=true
+YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
+YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
+YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192
+YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4
+YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
+YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
+YARN_CONF_yarn_resourcemanager_hostname=write-resourcemanager
+YARN_CONF_yarn_resourcemanager_address=write-resourcemanager:8032
+YARN_CONF_yarn_resourcemanager_scheduler_address=write-resourcemanager:8030
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=write-resourcemanager:8031
+YARN_CONF_yarn_timeline___service_enabled=true
+YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
+YARN_CONF_yarn_timeline___service_hostname=write-historyserver
+YARN_CONF_mapreduce_map_output_compress=true
+YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec
+YARN_CONF_yarn_nodemanager_resource_memory___mb=16384
+YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8
+YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5
+YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
+YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle
+
+MAPRED_CONF_mapreduce_framework_name=yarn
+MAPRED_CONF_mapred_child_java_opts=-Xmx4096m
+MAPRED_CONF_mapreduce_map_memory_mb=4096
+MAPRED_CONF_mapreduce_reduce_memory_mb=8192
+MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
+MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
+
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://metastore-db/metastore?useSSL=false\&allowPublicKeyRetrieval=true
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=kylin
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=kylin
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=true
+HIVE_SITE_CONF_hive_metastore_uris=thrift://write-hive-metastore:9083
+
+HBASE_CONF_hbase_rootdir=hdfs://read-namenode:8020/hbase
+HBASE_CONF_hbase_cluster_distributed=true
+HBASE_CONF_hbase_zookeeper_quorum=read-zookeeper
+
+HBASE_CONF_hbase_master=read-hbase-master:16000
+HBASE_CONF_hbase_master_hostname=read-hbase-master
+HBASE_CONF_hbase_master_port=16000
+HBASE_CONF_hbase_master_info_port=16010
+HBASE_CONF_hbase_regionserver_port=16020
+HBASE_CONF_hbase_regionserver_info_port=16030
+
+HBASE_MANAGES_ZK=false
+

docker/docker-compose/others/client-write.env

@@ -0,0 +1,60 @@
+CORE_CONF_fs_defaultFS=hdfs://write-namenode:8020
+CORE_CONF_hadoop_http_staticuser_user=root
+CORE_CONF_hadoop_proxyuser_hue_hosts=*
+CORE_CONF_hadoop_proxyuser_hue_groups=*
+CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
+
+HDFS_CONF_dfs_webhdfs_enabled=true
+HDFS_CONF_dfs_permissions_enabled=false
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+
+YARN_CONF_yarn_log___aggregation___enable=true
+YARN_CONF_yarn_log_server_url=http://write-historyserver:8188/applicationhistory/logs/
+YARN_CONF_yarn_resourcemanager_recovery_enabled=true
+YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
+YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
+YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192
+YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4
+YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
+YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
+YARN_CONF_yarn_resourcemanager_hostname=write-resourcemanager
+YARN_CONF_yarn_resourcemanager_address=write-resourcemanager:8032
+YARN_CONF_yarn_resourcemanager_scheduler_address=write-resourcemanager:8030
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=write-resourcemanager:8031
+YARN_CONF_yarn_timeline___service_enabled=true
+YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
+YARN_CONF_yarn_timeline___service_hostname=write-historyserver
+YARN_CONF_mapreduce_map_output_compress=true
+YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec
+YARN_CONF_yarn_nodemanager_resource_memory___mb=16384
+YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8
+YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5
+YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
+YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle
+
+MAPRED_CONF_mapreduce_framework_name=yarn
+MAPRED_CONF_mapred_child_java_opts=-Xmx4096m
+MAPRED_CONF_mapreduce_map_memory_mb=4096
+MAPRED_CONF_mapreduce_reduce_memory_mb=8192
+MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
+MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
+
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://metastore-db/metastore?useSSL=false\&allowPublicKeyRetrieval=true
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=kylin
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=kylin
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=true
+HIVE_SITE_CONF_hive_metastore_uris=thrift://write-hive-metastore:9083
+
+HBASE_CONF_hbase_rootdir=hdfs://write-namenode:8020/hbase
+HBASE_CONF_hbase_cluster_distributed=true
+HBASE_CONF_hbase_zookeeper_quorum=write-zookeeper
+HBASE_CONF_hbase_master=write-hbase-master:16000
+HBASE_CONF_hbase_master_hostname=write-hbase-master
+HBASE_CONF_hbase_master_port=16000
+HBASE_CONF_hbase_master_info_port=16010
+HBASE_CONF_hbase_regionserver_port=16020
+HBASE_CONF_hbase_regionserver_info_port=16030
+
+HBASE_MANAGES_ZK=false
+

docker/docker-compose/others/docker-compose-kerberos.yml

@@ -0,0 +1,13 @@
+version: "3.3"
+
+services:
+  kerberos-kdc:
+    image: ${KERBEROS_IMAGE}
+    container_name: kerberos-kdc
+    hostname: kerberos-kdc
+    networks:
+      - write_kylin
+
+networks:
+  write_kylin:
+    external: true

docker/docker-compose/others/docker-compose-kylin-write-read.yml

@@ -0,0 +1,69 @@
+version: "3.3"
+
+services:
+  kylin-all:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-all
+    hostname: kylin-all
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-all:/opt/kylin/kylin-all
+    env_file:
+      - client-write-read.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-all
+    networks:
+      - write_kylin
+    ports:
+      - 7070:7070
+
+  kylin-job:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-job
+    hostname: kylin-job
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-job:/opt/kylin/kylin-job
+    env_file:
+      - client-write-read.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-job
+    networks:
+      - write_kylin
+    ports:
+      - 7071:7070
+
+  kylin-query:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-query
+    hostname: kylin-query
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-query:/opt/kylin/kylin-query
+    env_file:
+      - client-write-read.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-query
+    networks:
+      - write_kylin
+    ports:
+      - 7072:7070
+
+networks:
+  write_kylin:
+    external: true

docker/docker-compose/others/docker-compose-kylin-write.yml

@@ -0,0 +1,69 @@
+version: "3.3"
+
+services:
+  kylin-all:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-all
+    hostname: kylin-all
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-all:/opt/kylin/kylin-all
+    env_file:
+      - client-write.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-all
+    networks:
+      - write_kylin
+    ports:
+      - 7070:7070
+
+  kylin-job:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-job
+    hostname: kylin-job
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-job:/opt/kylin/kylin-job
+    env_file:
+      - client-write.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-job
+    networks:
+      - write_kylin
+    ports:
+      - 7071:7070
+
+  kylin-query:
+    image: ${CLIENT_IMAGETAG}
+    container_name: kylin-query
+    hostname: kylin-query
+    volumes:
+      - ./conf/hadoop:/etc/hadoop/conf
+      - ./conf/hbase:/etc/hbase/conf
+      - ./conf/hive:/etc/hive/conf
+      - ./kylin/kylin-query:/opt/kylin/kylin-query
+    env_file:
+      - client-write.env
+    environment:
+      HADOOP_CONF_DIR: /etc/hadoop/conf
+      HIVE_CONF_DIR: /etc/hive/conf
+      HBASE_CONF_DIR: /etc/hbase/conf
+      KYLIN_HOME: /opt/kylin/kylin-query
+    networks:
+      - write_kylin
+    ports:
+      - 7072:7070
+
+networks:
+  write_kylin:
+    external: true

docker/docker-compose/others/docker-compose-metastore.yml

@@ -0,0 +1,24 @@
+version: "3.3"
+
+services:
+  metastore-db:
+#    image: mysql:5.6.49
+#    image: mysql:8.0.11
+    image: mysql:5.7.24
+    container_name: metastore-db
+    hostname: metastore-db
+    volumes:
+      - ./data/mysql:/var/lib/mysql
+    environment:
+      - MYSQL_ROOT_PASSWORD=kylin
+      - MYSQL_DATABASE=metastore
+      - MYSQL_USER=kylin
+      - MYSQL_PASSWORD=kylin
+    networks:
+      - write_kylin
+    ports:
+      - 3306:3306
+
+networks:
+  write_kylin:
+    external: true