diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..db4d255dd4e --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,6 @@ +docker-compose/others/data/ +docker-compose/read/data/ +docker-compose/write/data/ +docker-compose/others/kylin/kylin-all/ +docker-compose/others/kylin/kylin-job/ +docker-compose/others/kylin/kylin-query/ diff --git a/docker/build_cluster_images.sh b/docker/build_cluster_images.sh new file mode 100644 index 00000000000..b2aae808dfd --- /dev/null +++ b/docker/build_cluster_images.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_PATH=$(cd `dirname $0`; pwd) +WS_ROOT=`dirname $SCRIPT_PATH` + +source ${SCRIPT_PATH}/header.sh + +#docker build -t apachekylin/kylin-metastore:mysql_5.6.49 ./kylin/metastore-db +# + +docker build -t apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/base +docker build -t apachekylin/kylin-hadoop-namenode:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} --build-arg HADOOP_WEBHDFS_PORT=${HADOOP_WEBHDFS_PORT} ./dockerfile/cluster/namenode +docker build -t apachekylin/kylin-hadoop-datanode:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} --build-arg HADOOP_DN_PORT=${HADOOP_DN_PORT} ./dockerfile/cluster/datanode +docker build -t apachekylin/kylin-hadoop-resourcemanager:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/resourcemanager +docker build -t apachekylin/kylin-hadoop-nodemanager:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/nodemanager +docker build -t apachekylin/kylin-hadoop-historyserver:hadoop_${HADOOP_VERSION} --build-arg HADOOP_VERSION=${HADOOP_VERSION} ./dockerfile/cluster/historyserver + +docker build -t apachekylin/kylin-hive:hive_${HIVE_VERSION}_hadoop_${HADOOP_VERSION} \ +--build-arg HIVE_VERSION=${HIVE_VERSION} \ +--build-arg HADOOP_VERSION=${HADOOP_VERSION} \ +./dockerfile/cluster/hive + +if [ $ENABLE_HBASE == "yes" ]; then + docker build -t apachekylin/kylin-hbase-base:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hbase + docker build -t apachekylin/kylin-hbase-master:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hmaster + docker build -t apachekylin/kylin-hbase-regionserver:hbase_${HBASE_VERSION} --build-arg HBASE_VERSION=${HBASE_VERSION} ./dockerfile/cluster/hregionserver +fi + +if [ $ENABLE_KERBEROS == "yes" ]; then + docker build -t apachekylin/kylin-kerberos:latest ./dockerfile/cluster/kerberos +fi + +if [ $ENABLE_LDAP == "yes" ]; then + docker pull osixia/openldap:1.3.0 +fi + +#if [ $ENABLE_KAFKA == "yes" ]; then +# docker pull bitnami/kafka:2.0.0 +#fi +docker pull bitnami/kafka:2.0.0 + +docker pull mysql:5.6.49 + +docker build -t apachekylin/kylin-client:hadoop_${HADOOP_VERSION}_hive_${HIVE_VERSION}_hbase_${HBASE_VERSION} \ +--build-arg HIVE_VERSION=${HIVE_VERSION} \ +--build-arg HADOOP_VERSION=${HADOOP_VERSION} \ +--build-arg HBASE_VERSION=${HBASE_VERSION} \ +./dockerfile/cluster/client diff --git a/docker/build_image.sh b/docker/build_standalone_image.sh similarity index 89% rename from docker/build_image.sh rename to docker/build_standalone_image.sh index f78c9095b61..924fe1fc36d 100644 --- a/docker/build_image.sh +++ b/docker/build_standalone_image.sh @@ -22,5 +22,5 @@ cd ${DIR} echo "build image in dir "${DIR} echo "start build Hadoop docker image" -docker build -f Dockerfile_hadoop -t hadoop2.7-all-in-one . -docker build -f Dockerfile -t apachekylin/apache-kylin-standalone:3.1.0 . +docker build -f Dockerfile_hadoop -t hadoop2.7-all-in-one ./dockerfile/standalone +docker build -f Dockerfile -t apachekylin/apache-kylin-standalone:3.1.0 ./dockerfile/standalone diff --git a/docker/docker-compose/others/client-write-read.env b/docker/docker-compose/others/client-write-read.env new file mode 100644 index 00000000000..c61e9865851 --- /dev/null +++ b/docker/docker-compose/others/client-write-read.env @@ -0,0 +1,61 @@ +CORE_CONF_fs_defaultFS=hdfs://write-namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_log_server_url=http://write-historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=write-resourcemanager +YARN_CONF_yarn_resourcemanager_address=write-resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=write-resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=write-resourcemanager:8031 +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_timeline___service_hostname=write-historyserver +YARN_CONF_mapreduce_map_output_compress=true +YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec +YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 +YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 +YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle + +MAPRED_CONF_mapreduce_framework_name=yarn +MAPRED_CONF_mapred_child_java_opts=-Xmx4096m +MAPRED_CONF_mapreduce_map_memory_mb=4096 +MAPRED_CONF_mapreduce_reduce_memory_mb=8192 +MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m +MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://metastore-db/metastore?useSSL=false\&allowPublicKeyRetrieval=true +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=kylin +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=kylin +HIVE_SITE_CONF_datanucleus_autoCreateSchema=true +HIVE_SITE_CONF_hive_metastore_uris=thrift://write-hive-metastore:9083 + +HBASE_CONF_hbase_rootdir=hdfs://read-namenode:8020/hbase +HBASE_CONF_hbase_cluster_distributed=true +HBASE_CONF_hbase_zookeeper_quorum=read-zookeeper + +HBASE_CONF_hbase_master=read-hbase-master:16000 +HBASE_CONF_hbase_master_hostname=read-hbase-master +HBASE_CONF_hbase_master_port=16000 +HBASE_CONF_hbase_master_info_port=16010 +HBASE_CONF_hbase_regionserver_port=16020 +HBASE_CONF_hbase_regionserver_info_port=16030 + +HBASE_MANAGES_ZK=false + diff --git a/docker/docker-compose/others/client-write.env b/docker/docker-compose/others/client-write.env new file mode 100644 index 00000000000..edad60ba007 --- /dev/null +++ b/docker/docker-compose/others/client-write.env @@ -0,0 +1,60 @@ +CORE_CONF_fs_defaultFS=hdfs://write-namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_log_server_url=http://write-historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=write-resourcemanager +YARN_CONF_yarn_resourcemanager_address=write-resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=write-resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=write-resourcemanager:8031 +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_timeline___service_hostname=write-historyserver +YARN_CONF_mapreduce_map_output_compress=true +YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec +YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 +YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 +YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle + +MAPRED_CONF_mapreduce_framework_name=yarn +MAPRED_CONF_mapred_child_java_opts=-Xmx4096m +MAPRED_CONF_mapreduce_map_memory_mb=4096 +MAPRED_CONF_mapreduce_reduce_memory_mb=8192 +MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m +MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://metastore-db/metastore?useSSL=false\&allowPublicKeyRetrieval=true +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=kylin +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=kylin +HIVE_SITE_CONF_datanucleus_autoCreateSchema=true +HIVE_SITE_CONF_hive_metastore_uris=thrift://write-hive-metastore:9083 + +HBASE_CONF_hbase_rootdir=hdfs://write-namenode:8020/hbase +HBASE_CONF_hbase_cluster_distributed=true +HBASE_CONF_hbase_zookeeper_quorum=write-zookeeper +HBASE_CONF_hbase_master=write-hbase-master:16000 +HBASE_CONF_hbase_master_hostname=write-hbase-master +HBASE_CONF_hbase_master_port=16000 +HBASE_CONF_hbase_master_info_port=16010 +HBASE_CONF_hbase_regionserver_port=16020 +HBASE_CONF_hbase_regionserver_info_port=16030 + +HBASE_MANAGES_ZK=false + diff --git a/docker/docker-compose/others/docker-compose-kerberos.yml b/docker/docker-compose/others/docker-compose-kerberos.yml new file mode 100644 index 00000000000..3d900620fc4 --- /dev/null +++ b/docker/docker-compose/others/docker-compose-kerberos.yml @@ -0,0 +1,13 @@ +version: "3.3" + +services: + kerberos-kdc: + image: ${KERBEROS_IMAGE} + container_name: kerberos-kdc + hostname: kerberos-kdc + networks: + - write_kylin + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/others/docker-compose-kylin-write-read.yml b/docker/docker-compose/others/docker-compose-kylin-write-read.yml new file mode 100644 index 00000000000..cb67b067d11 --- /dev/null +++ b/docker/docker-compose/others/docker-compose-kylin-write-read.yml @@ -0,0 +1,69 @@ +version: "3.3" + +services: + kylin-all: + image: ${CLIENT_IMAGETAG} + container_name: kylin-all + hostname: kylin-all + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-all:/opt/kylin/kylin-all + env_file: + - client-write-read.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-all + networks: + - write_kylin + ports: + - 7070:7070 + + kylin-job: + image: ${CLIENT_IMAGETAG} + container_name: kylin-job + hostname: kylin-job + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-job:/opt/kylin/kylin-job + env_file: + - client-write-read.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-job + networks: + - write_kylin + ports: + - 7071:7070 + + kylin-query: + image: ${CLIENT_IMAGETAG} + container_name: kylin-query + hostname: kylin-query + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-query:/opt/kylin/kylin-query + env_file: + - client-write-read.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-query + networks: + - write_kylin + ports: + - 7072:7070 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/others/docker-compose-kylin-write.yml b/docker/docker-compose/others/docker-compose-kylin-write.yml new file mode 100644 index 00000000000..a78b88aa7bc --- /dev/null +++ b/docker/docker-compose/others/docker-compose-kylin-write.yml @@ -0,0 +1,69 @@ +version: "3.3" + +services: + kylin-all: + image: ${CLIENT_IMAGETAG} + container_name: kylin-all + hostname: kylin-all + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-all:/opt/kylin/kylin-all + env_file: + - client-write.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-all + networks: + - write_kylin + ports: + - 7070:7070 + + kylin-job: + image: ${CLIENT_IMAGETAG} + container_name: kylin-job + hostname: kylin-job + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-job:/opt/kylin/kylin-job + env_file: + - client-write.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-job + networks: + - write_kylin + ports: + - 7071:7070 + + kylin-query: + image: ${CLIENT_IMAGETAG} + container_name: kylin-query + hostname: kylin-query + volumes: + - ./conf/hadoop:/etc/hadoop/conf + - ./conf/hbase:/etc/hbase/conf + - ./conf/hive:/etc/hive/conf + - ./kylin/kylin-query:/opt/kylin/kylin-query + env_file: + - client-write.env + environment: + HADOOP_CONF_DIR: /etc/hadoop/conf + HIVE_CONF_DIR: /etc/hive/conf + HBASE_CONF_DIR: /etc/hbase/conf + KYLIN_HOME: /opt/kylin/kylin-query + networks: + - write_kylin + ports: + - 7072:7070 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/others/docker-compose-metastore.yml b/docker/docker-compose/others/docker-compose-metastore.yml new file mode 100644 index 00000000000..a36df07a072 --- /dev/null +++ b/docker/docker-compose/others/docker-compose-metastore.yml @@ -0,0 +1,24 @@ +version: "3.3" + +services: + metastore-db: +# image: mysql:5.6.49 +# image: mysql:8.0.11 + image: mysql:5.7.24 + container_name: metastore-db + hostname: metastore-db + volumes: + - ./data/mysql:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=kylin + - MYSQL_DATABASE=metastore + - MYSQL_USER=kylin + - MYSQL_PASSWORD=kylin + networks: + - write_kylin + ports: + - 3306:3306 + +networks: + write_kylin: + external: true diff --git a/docker/docker-compose/read/conf/hadoop-read/core-site.xml b/docker/docker-compose/read/conf/hadoop-read/core-site.xml new file mode 100644 index 00000000000..69fc4627737 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop-read/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + diff --git a/docker/docker-compose/read/conf/hadoop-read/hdfs-site.xml b/docker/docker-compose/read/conf/hadoop-read/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop-read/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/docker-compose/read/conf/hadoop-read/mapred-site.xml b/docker/docker-compose/read/conf/hadoop-read/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop-read/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/docker-compose/read/conf/hadoop-read/yarn-site.xml b/docker/docker-compose/read/conf/hadoop-read/yarn-site.xml new file mode 100644 index 00000000000..392cf4c70b7 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop-read/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addressread-resourcemanager:8031 +yarn.resourcemanager.hostnameread-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnameread-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://read-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addressread-resourcemanager:8030 +yarn.resourcemanager.addressread-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/docker-compose/read/conf/hadoop/core-site.xml b/docker/docker-compose/read/conf/hadoop/core-site.xml new file mode 100644 index 00000000000..dd5a81b1fa5 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop/core-site.xml @@ -0,0 +1,26 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + + diff --git a/docker/docker-compose/read/conf/hadoop/hdfs-site.xml b/docker/docker-compose/read/conf/hadoop/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/docker-compose/read/conf/hadoop/mapred-site.xml b/docker/docker-compose/read/conf/hadoop/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/docker-compose/read/conf/hadoop/yarn-site.xml b/docker/docker-compose/read/conf/hadoop/yarn-site.xml new file mode 100644 index 00000000000..b55dd34bba3 --- /dev/null +++ b/docker/docker-compose/read/conf/hadoop/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addresswrite-resourcemanager:8031 +yarn.resourcemanager.hostnamewrite-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnamewrite-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://write-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addresswrite-resourcemanager:8030 +yarn.resourcemanager.addresswrite-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/docker-compose/read/conf/hbase/hbase-site.xml b/docker/docker-compose/read/conf/hbase/hbase-site.xml new file mode 100644 index 00000000000..988d91c8547 --- /dev/null +++ b/docker/docker-compose/read/conf/hbase/hbase-site.xml @@ -0,0 +1,34 @@ + + + + +hbase.zookeeper.quorumread-zookeeper +hbase.masterread-hbase-master:16000 +hbase.regionserver.port16020 +hbase.regionserver.info.port16030 +DIR/etc/hbase +hbase.cluster.distributedtrue +hbase.rootdirhdfs://read-namenode:8020/hbase +hbase.master.info.port16010 +hbase.master.hostnameread-hbase-master +hbase.master.port16000 + diff --git a/docker/docker-compose/read/conf/hive/hive-site.xml b/docker/docker-compose/read/conf/hive/hive-site.xml new file mode 100644 index 00000000000..c60fe364c04 --- /dev/null +++ b/docker/docker-compose/read/conf/hive/hive-site.xml @@ -0,0 +1,25 @@ + + + hive.metastore.uristhrift://write-hive-metastore:9083 + datanucleus.autoCreateSchemafalse + javax.jdo.option.ConnectionURLjdbc:postgresql://write-hive-metastore-postgresql/metastore + javax.jdo.option.ConnectionDriverNameorg.postgresql.Driver + javax.jdo.option.ConnectionPasswordhive + javax.jdo.option.ConnectionUserNamehive + + diff --git a/docker/docker-compose/read/docker-compose-hadoop.yml b/docker/docker-compose/read/docker-compose-hadoop.yml new file mode 100644 index 00000000000..a0e2a660c1e --- /dev/null +++ b/docker/docker-compose/read/docker-compose-hadoop.yml @@ -0,0 +1,129 @@ +version: "3.3" + +services: + read-namenode: + image: ${HADOOP_NAMENODE_IMAGETAG:-apachekylin/kylin-hadoop-namenode:hadoop_2.8.5} + container_name: read-namenode + hostname: read-namenode + volumes: + - ./data/write_hadoop_namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test-kylin + - HADOOP_WEBHDFS_PORT=${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + expose: + - 8020 + ports: + - 50071:50070 + - 9871:9870 + + read-datanode1: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: read-datanode1 + hostname: read-datanode1 + volumes: + - ./data/write_hadoop_datanode1:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + links: + - read-namenode + expose: + - ${HADOOP_DN_PORT:-50075} + + read-datanode2: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: read-datanode2 + hostname: read-datanode2 + volumes: + - ./data/write_hadoop_datanode2:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + expose: + - ${HADOOP_DN_PORT:-50075} + + read-datanode3: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: read-datanode3 + hostname: read-datanode3 + volumes: + - ./data/write_hadoop_datanode3:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + expose: + - ${HADOOP_DN_PORT:-50075} + + read-resourcemanager: + image: ${HADOOP_RESOURCEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-resourcemanager:hadoop_2.8.5} + container_name: read-resourcemanager + hostname: read-resourcemanager + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + ports: + - 8089:8088 + + read-nodemanager1: + image: ${HADOOP_NODEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-nodemanager:hadoop_2.8.5} + container_name: read-nodemanager1 + hostname: read-nodemanager1 + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + + read-nodemanager2: + image: ${HADOOP_NODEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-nodemanager:hadoop_2.8.5} + container_name: read-nodemanager2 + hostname: read-nodemanager2 + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + + read-historyserver: + image: ${HADOOP_HISTORYSERVER_IMAGETAG:-apachekylin/kylin-hadoop-historyserver:hadoop_2.8.5} + container_name: read-historyserver + hostname: read-historyserver + volumes: + - ./data/write_hadoop_historyserver:/hadoop/yarn/timeline + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - read-hadoop.env + networks: + - write_kylin + ports: + - 8189:8188 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/read/docker-compose-hbase.yml b/docker/docker-compose/read/docker-compose-hbase.yml new file mode 100644 index 00000000000..ac4048b9dd7 --- /dev/null +++ b/docker/docker-compose/read/docker-compose-hbase.yml @@ -0,0 +1,43 @@ +version: "3.3" + +services: + read-hbase-master: + image: ${HBASE_MASTER_IMAGETAG:-apachekylin/kylin-hbase-master:hbase1.1.2} + container_name: read-hbase-master + hostname: read-hbase-master + env_file: + - read-hbase-distributed-local.env + environment: + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-zookeeper:2181" + networks: + - write_kylin + ports: + - 16010:16010 + + read-hbase-regionserver1: + image: ${HBASE_REGIONSERVER_IMAGETAG:-apachekylin/kylin-hbase-regionserver:hbase_1.1.2} + container_name: read-hbase-regionserver1 + hostname: read-hbase-regionserver1 + env_file: + - read-hbase-distributed-local.env + environment: + HBASE_CONF_hbase_regionserver_hostname: read-hbase-regionserver1 + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-zookeeper:2181 read-hbase-master:16010" + networks: + - write_kylin + + read-hbase-regionserver2: + image: ${HBASE_REGIONSERVER_IMAGETAG:-apachekylin/kylin-hbase-regionserver:hbase_1.1.2} + container_name: read-hbase-regionserver2 + hostname: read-hbase-regionserver2 + env_file: + - read-hbase-distributed-local.env + environment: + HBASE_CONF_hbase_regionserver_hostname: read-hbase-regionserver2 + SERVICE_PRECONDITION: "read-namenode:${HADOOP_WEBHDFS_PORT:-50070} read-datanode1:${HADOOP_DN_PORT:-50075} read-datanode2:${HADOOP_DN_PORT:-50075} read-datanode3:${HADOOP_DN_PORT:-50075} read-zookeeper:2181 read-hbase-master:16010" + networks: + - write_kylin + +networks: + write_kylin: + external: true diff --git a/docker/docker-compose/read/docker-compose-zookeeper.yml b/docker/docker-compose/read/docker-compose-zookeeper.yml new file mode 100644 index 00000000000..71ea2529f28 --- /dev/null +++ b/docker/docker-compose/read/docker-compose-zookeeper.yml @@ -0,0 +1,18 @@ +version: "3.3" + +services: + read-zookeeper: + image: ${ZOOKEEPER_IMAGETAG:-zookeeper:3.4.10} + container_name: read-zookeeper + hostname: read-zookeeper + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=0.0.0.0:2888:3888 + networks: + - write_kylin + ports: + - 2182:2181 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/read/read-hadoop.env b/docker/docker-compose/read/read-hadoop.env new file mode 100644 index 00000000000..9c0086de622 --- /dev/null +++ b/docker/docker-compose/read/read-hadoop.env @@ -0,0 +1,40 @@ +CORE_CONF_fs_defaultFS=hdfs://read-namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_log_server_url=http://read-historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=read-resourcemanager +YARN_CONF_yarn_resourcemanager_address=read-resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=read-resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=read-resourcemanager:8031 +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_timeline___service_hostname=read-historyserver +YARN_CONF_mapreduce_map_output_compress=true +YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec +YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 +YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 +YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle + +MAPRED_CONF_mapreduce_framework_name=yarn +MAPRED_CONF_mapred_child_java_opts=-Xmx4096m +MAPRED_CONF_mapreduce_map_memory_mb=4096 +MAPRED_CONF_mapreduce_reduce_memory_mb=8192 +MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m +MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m diff --git a/docker/docker-compose/read/read-hbase-distributed-local.env b/docker/docker-compose/read/read-hbase-distributed-local.env new file mode 100644 index 00000000000..4ba8e192e6e --- /dev/null +++ b/docker/docker-compose/read/read-hbase-distributed-local.env @@ -0,0 +1,12 @@ +HBASE_CONF_hbase_rootdir=hdfs://read-namenode:8020/hbase +HBASE_CONF_hbase_cluster_distributed=true +HBASE_CONF_hbase_zookeeper_quorum=read-zookeeper + +HBASE_CONF_hbase_master=read-hbase-master:16000 +HBASE_CONF_hbase_master_hostname=read-hbase-master +HBASE_CONF_hbase_master_port=16000 +HBASE_CONF_hbase_master_info_port=16010 +HBASE_CONF_hbase_regionserver_port=16020 +HBASE_CONF_hbase_regionserver_info_port=16030 + +HBASE_MANAGES_ZK=false \ No newline at end of file diff --git a/docker/docker-compose/write/conf/hadoop-read/core-site.xml b/docker/docker-compose/write/conf/hadoop-read/core-site.xml new file mode 100644 index 00000000000..69fc4627737 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-read/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + diff --git a/docker/docker-compose/write/conf/hadoop-read/hdfs-site.xml b/docker/docker-compose/write/conf/hadoop-read/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-read/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/docker-compose/write/conf/hadoop-read/mapred-site.xml b/docker/docker-compose/write/conf/hadoop-read/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-read/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hadoop-read/yarn-site.xml b/docker/docker-compose/write/conf/hadoop-read/yarn-site.xml new file mode 100644 index 00000000000..392cf4c70b7 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-read/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addressread-resourcemanager:8031 +yarn.resourcemanager.hostnameread-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnameread-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://read-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addressread-resourcemanager:8030 +yarn.resourcemanager.addressread-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hadoop-write/core-site.xml b/docker/docker-compose/write/conf/hadoop-write/core-site.xml new file mode 100644 index 00000000000..69fc4627737 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-write/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + diff --git a/docker/docker-compose/write/conf/hadoop-write/hdfs-site.xml b/docker/docker-compose/write/conf/hadoop-write/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-write/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/docker-compose/write/conf/hadoop-write/mapred-site.xml b/docker/docker-compose/write/conf/hadoop-write/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-write/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hadoop-write/yarn-site.xml b/docker/docker-compose/write/conf/hadoop-write/yarn-site.xml new file mode 100644 index 00000000000..b55dd34bba3 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop-write/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addresswrite-resourcemanager:8031 +yarn.resourcemanager.hostnamewrite-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnamewrite-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://write-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addresswrite-resourcemanager:8030 +yarn.resourcemanager.addresswrite-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hadoop/core-site.xml b/docker/docker-compose/write/conf/hadoop/core-site.xml new file mode 100644 index 00000000000..dd5a81b1fa5 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop/core-site.xml @@ -0,0 +1,26 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + + diff --git a/docker/docker-compose/write/conf/hadoop/hdfs-site.xml b/docker/docker-compose/write/conf/hadoop/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/docker-compose/write/conf/hadoop/mapred-site.xml b/docker/docker-compose/write/conf/hadoop/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hadoop/yarn-site.xml b/docker/docker-compose/write/conf/hadoop/yarn-site.xml new file mode 100644 index 00000000000..b55dd34bba3 --- /dev/null +++ b/docker/docker-compose/write/conf/hadoop/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addresswrite-resourcemanager:8031 +yarn.resourcemanager.hostnamewrite-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnamewrite-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://write-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addresswrite-resourcemanager:8030 +yarn.resourcemanager.addresswrite-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/docker-compose/write/conf/hbase/hbase-site.xml b/docker/docker-compose/write/conf/hbase/hbase-site.xml new file mode 100644 index 00000000000..988d91c8547 --- /dev/null +++ b/docker/docker-compose/write/conf/hbase/hbase-site.xml @@ -0,0 +1,34 @@ + + + + +hbase.zookeeper.quorumread-zookeeper +hbase.masterread-hbase-master:16000 +hbase.regionserver.port16020 +hbase.regionserver.info.port16030 +DIR/etc/hbase +hbase.cluster.distributedtrue +hbase.rootdirhdfs://read-namenode:8020/hbase +hbase.master.info.port16010 +hbase.master.hostnameread-hbase-master +hbase.master.port16000 + diff --git a/docker/docker-compose/write/conf/hive/hive-site.xml b/docker/docker-compose/write/conf/hive/hive-site.xml new file mode 100644 index 00000000000..c60fe364c04 --- /dev/null +++ b/docker/docker-compose/write/conf/hive/hive-site.xml @@ -0,0 +1,25 @@ + + + hive.metastore.uristhrift://write-hive-metastore:9083 + datanucleus.autoCreateSchemafalse + javax.jdo.option.ConnectionURLjdbc:postgresql://write-hive-metastore-postgresql/metastore + javax.jdo.option.ConnectionDriverNameorg.postgresql.Driver + javax.jdo.option.ConnectionPasswordhive + javax.jdo.option.ConnectionUserNamehive + + diff --git a/docker/docker-compose/write/docker-compose-hadoop.yml b/docker/docker-compose/write/docker-compose-hadoop.yml new file mode 100644 index 00000000000..4286cfca6d1 --- /dev/null +++ b/docker/docker-compose/write/docker-compose-hadoop.yml @@ -0,0 +1,128 @@ +version: "3.3" + +services: + write-namenode: + image: ${HADOOP_NAMENODE_IMAGETAG:-apachekylin/kylin-hadoop-namenode:hadoop_2.8.5} + container_name: write-namenode + hostname: write-namenode + volumes: + - ./data/write_hadoop_namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test-kylin + - HADOOP_WEBHDFS_PORT=${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + expose: + - 8020 + ports: + - 50070:50070 + - 9870:9870 + + write-datanode1: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: write-datanode1 + hostname: write-datanode1 + volumes: + - ./data/write_hadoop_datanode1:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + links: + - write-namenode + expose: + - ${HADOOP_DN_PORT:-50075} + + write-datanode2: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: write-datanode2 + hostname: write-datanode2 + volumes: + - ./data/write_hadoop_datanode2:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + expose: + - ${HADOOP_DN_PORT:-50075} + + write-datanode3: + image: ${HADOOP_DATANODE_IMAGETAG:-apachekylin/kylin-hadoop-datanode:hadoop_2.8.5} + container_name: write-datanode3 + hostname: write-datanode3 + volumes: + - ./data/write_hadoop_datanode3:/hadoop/dfs/data + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + expose: + - ${HADOOP_DN_PORT:-50075} + + write-resourcemanager: + image: ${HADOOP_RESOURCEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-resourcemanager:hadoop_2.8.5} + container_name: write-resourcemanager + hostname: write-resourcemanager + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075}" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + ports: + - 8088:8088 + + write-nodemanager1: + image: ${HADOOP_NODEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-nodemanager:hadoop_2.8.5} + container_name: write-nodemanager1 + hostname: write-nodemanager1 + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + + write-nodemanager2: + image: ${HADOOP_NODEMANAGER_IMAGETAG:-apachekylin/kylin-hadoop-nodemanager:hadoop_2.8.5} + container_name: write-nodemanager2 + hostname: write-nodemanager2 + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + + write-historyserver: + image: ${HADOOP_HISTORYSERVER_IMAGETAG:-apachekylin/kylin-hadoop-historyserver:hadoop_2.8.5} + container_name: write-historyserver + hostname: write-historyserver + volumes: + - ./data/write_hadoop_historyserver:/hadoop/yarn/timeline + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-resourcemanager:8088" + HADOOP_WEBHDFS_PORT: ${HADOOP_WEBHDFS_PORT:-50070} + env_file: + - write-hadoop.env + networks: + - kylin + ports: + - 8188:8188 + +networks: + kylin: \ No newline at end of file diff --git a/docker/docker-compose/write/docker-compose-hbase.yml b/docker/docker-compose/write/docker-compose-hbase.yml new file mode 100644 index 00000000000..d95f32b0ce9 --- /dev/null +++ b/docker/docker-compose/write/docker-compose-hbase.yml @@ -0,0 +1,43 @@ +version: "3.3" + +services: + write-hbase-master: + image: ${HBASE_MASTER_IMAGETAG:-apachekylin/kylin-hbase-master:hbase1.1.2} + container_name: write-hbase-master + hostname: write-hbase-master + env_file: + - write-hbase-distributed-local.env + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-zookeeper:2181" + networks: + - write_kylin + ports: + - 16010:16010 + + write-hbase-regionserver1: + image: ${HBASE_REGIONSERVER_IMAGETAG:-apachekylin/kylin-hbase-regionserver:hbase_1.1.2} + container_name: write-hbase-regionserver1 + hostname: write-hbase-regionserver1 + env_file: + - write-hbase-distributed-local.env + environment: + HBASE_CONF_hbase_regionserver_hostname: write-hbase-regionserver1 + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-zookeeper:2181 write-hbase-master:16010" + networks: + - write_kylin + + write-hbase-regionserver2: + image: ${HBASE_REGIONSERVER_IMAGETAG:-apachekylin/kylin-hbase-regionserver:hbase_1.1.2} + container_name: write-hbase-regionserver2 + hostname: write-hbase-regionserver2 + env_file: + - write-hbase-distributed-local.env + environment: + HBASE_CONF_hbase_regionserver_hostname: write-hbase-regionserver2 + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} write-zookeeper:2181 write-hbase-master:16010" + networks: + - write_kylin + +networks: + write_kylin: + external: true diff --git a/docker/docker-compose/write/docker-compose-hive.yml b/docker/docker-compose/write/docker-compose-hive.yml new file mode 100644 index 00000000000..9b94a349645 --- /dev/null +++ b/docker/docker-compose/write/docker-compose-hive.yml @@ -0,0 +1,37 @@ +version: "3.3" + +services: + write-hive-server: + image: ${HIVE_IMAGETAG:-apachekylin/kylin-hive:hive_1.2.2_hadoop_2.8.5} + container_name: write-hive-server + hostname: write-hive-server + env_file: + - write-hadoop.env + environment: + HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:mysql://metastore-db/metastore" + SERVICE_PRECONDITION: "write-hive-metastore:9083" + HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName: com.mysql.jdbc.Driver + networks: + - write_kylin + ports: + - 10000:10000 + + write-hive-metastore: + image: ${HIVE_IMAGETAG:-apachekylin/kylin-hive:hive_1.2.2_hadoop_2.8.5} + container_name: write-hive-metastore + hostname: write-hive-metastore + env_file: + - write-hadoop.env + environment: + SERVICE_PRECONDITION: "write-namenode:${HADOOP_WEBHDFS_PORT:-50070} write-datanode1:${HADOOP_DN_PORT:-50075} write-datanode2:${HADOOP_DN_PORT:-50075} write-datanode3:${HADOOP_DN_PORT:-50075} metastore-db:3306" + HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName: com.mysql.jdbc.Driver + command: /opt/hive/bin/hive --service metastore + networks: + - write_kylin + expose: + - 9083 + +networks: + write_kylin: + external: true + diff --git a/docker/docker-compose/write/docker-compose-kafka.yml b/docker/docker-compose/write/docker-compose-kafka.yml new file mode 100644 index 00000000000..9590c620bd1 --- /dev/null +++ b/docker/docker-compose/write/docker-compose-kafka.yml @@ -0,0 +1,18 @@ +version: "3.3" + +services: + write-kafka: + image: ${KAFKA_IMAGETAG:-bitnami/kafka:2.0.0} + container_name: write-kafkabroker + hostname: write-kafkabroker + environment: + - KAFKA_ZOOKEEPER_CONNECT=write-zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + networks: + - write_kylin + ports: + - 9092:9092 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/write/docker-compose-zookeeper.yml b/docker/docker-compose/write/docker-compose-zookeeper.yml new file mode 100644 index 00000000000..cece11b7217 --- /dev/null +++ b/docker/docker-compose/write/docker-compose-zookeeper.yml @@ -0,0 +1,18 @@ +version: "3.3" + +services: + write-zookeeper: + image: ${ZOOKEEPER_IMAGETAG:-zookeeper:3.4.10} + container_name: write-zookeeper + hostname: write-zookeeper + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=0.0.0.0:2888:3888 + networks: + - write_kylin + ports: + - 2181:2181 + +networks: + write_kylin: + external: true \ No newline at end of file diff --git a/docker/docker-compose/write/write-hadoop.env b/docker/docker-compose/write/write-hadoop.env new file mode 100644 index 00000000000..ef4429a097b --- /dev/null +++ b/docker/docker-compose/write/write-hadoop.env @@ -0,0 +1,49 @@ +CORE_CONF_fs_defaultFS=hdfs://write-namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_log_server_url=http://write-historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 +YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=write-resourcemanager +YARN_CONF_yarn_resourcemanager_address=write-resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=write-resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=write-resourcemanager:8031 +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_timeline___service_hostname=write-historyserver +YARN_CONF_mapreduce_map_output_compress=true +YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec +YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 +YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 +YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle + +MAPRED_CONF_mapreduce_framework_name=yarn +MAPRED_CONF_mapred_child_java_opts=-Xmx4096m +MAPRED_CONF_mapreduce_map_memory_mb=4096 +MAPRED_CONF_mapreduce_reduce_memory_mb=8192 +MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m +MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://metastore-db/metastore?useSSL=false\&allowPublicKeyRetrieval=true +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.cj.jdbc.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=kylin +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=kylin +HIVE_SITE_CONF_datanucleus_autoCreateSchema=true +HIVE_SITE_CONF_datanucleus_schema_autoCreateAll=true +HIVE_SITE_CONF_hive_metastore_schema_verification=false +HIVE_SITE_CONF_hive_metastore_uris=thrift://write-hive-metastore:9083 \ No newline at end of file diff --git a/docker/docker-compose/write/write-hbase-distributed-local.env b/docker/docker-compose/write/write-hbase-distributed-local.env new file mode 100644 index 00000000000..c866cefc676 --- /dev/null +++ b/docker/docker-compose/write/write-hbase-distributed-local.env @@ -0,0 +1,12 @@ +HBASE_CONF_hbase_rootdir=hdfs://write-namenode:8020/hbase +HBASE_CONF_hbase_cluster_distributed=true +HBASE_CONF_hbase_zookeeper_quorum=write-zookeeper + +HBASE_CONF_hbase_master=write-hbase-master:16000 +HBASE_CONF_hbase_master_hostname=write-hbase-master +HBASE_CONF_hbase_master_port=16000 +HBASE_CONF_hbase_master_info_port=16010 +HBASE_CONF_hbase_regionserver_port=16020 +HBASE_CONF_hbase_regionserver_info_port=16030 + +HBASE_MANAGES_ZK=false \ No newline at end of file diff --git a/docker/dockerfile/cluster/base/Dockerfile b/docker/dockerfile/cluster/base/Dockerfile new file mode 100644 index 00000000000..8cf5ff01a30 --- /dev/null +++ b/docker/dockerfile/cluster/base/Dockerfile @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM centos:7.3.1611 +MAINTAINER kylin + +USER root + +ARG JAVA_VERSION=jdk1.8.0_141 +ARG HADOOP_VERSION=2.8.5 +ARG INSTALL_FROM=local +ARG HADOOP_URL=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +ENV JAVA_HOME /opt/${JAVA_VERSION} +ENV HADOOP_VERSION ${HADOOP_VERSION} +ENV INSTALL_FROM ${INSTALL_FROM} +ENV HADOOP_URL ${HADOOP_URL} + +# install tools +RUN yum -y install lsof wget tar git unzip wget curl net-tools procps perl sed nc which + +# setup jdk +RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz" -P /opt \ + && tar -zxvf /opt/jdk-8u141-linux-x64.tar.gz -C /opt/ \ + && rm -f /opt/jdk-8u141-linux-x64.tar.gz + +# use buildkit +#IF $INSTALL_FROM=="net" +#RUN set -x \ +# && echo "Fetch URL2 is : ${HADOOP_URL}" \ +# && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ +# && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc \ +#ELSE IF $INSTALL_FROM=="local" +#COPY ${PACKAGE_PATH}hadoop-${HADOOP_VERSION}.tar.gz /tmp/hadoop.tar.gz +#COPY ${PACKAGE_PATH}hadoop-${HADOOP_VERSION}.tar.gz.asc /tmp/hadoop.tar.gz.asc +#DONE + +RUN set -x \ + && echo "Fetch URL2 is : ${HADOOP_URL}" \ + && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ + && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc + +RUN set -x \ + && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz* \ + && ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop \ + && if [ -e "/etc/hadoop/mapred-site.xml.template" ]; then cp /etc/hadoop/mapred-site.xml.template /etc/hadoop/mapred-site.xml ;fi \ + && mkdir -p /opt/hadoop-$HADOOP_VERSION/logs \ + && mkdir /hadoop-data + +ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV MULTIHOMED_NETWORK=1 +ENV HADOOP_HOME=${HADOOP_PREFIX} +ENV HADOOP_INSTALL=${HADOOP_HOME} + +ENV USER=root +ENV PATH $JAVA_HOME/bin:/usr/bin:/bin:$HADOOP_PREFIX/bin/:$PATH + +ADD entrypoint.sh /opt/entrypoint/hadoop/entrypoint.sh +RUN chmod a+x /opt/entrypoint/hadoop/entrypoint.sh + +ENTRYPOINT ["/opt/entrypoint/hadoop/entrypoint.sh"] + diff --git a/docker/dockerfile/cluster/base/entrypoint.sh b/docker/dockerfile/cluster/base/entrypoint.sh new file mode 100644 index 00000000000..347984456f6 --- /dev/null +++ b/docker/dockerfile/cluster/base/entrypoint.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +####################################################################################### +## COPIED FROM ## +## https://github.com/big-data-europe/docker-hadoop/blob/master/base/entrypoint.sh ## +# ## +####################################################################################### + +# Set some sensible defaults +export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} + +function addProperty() { + local path=$1 + local name=$2 + local value=$3 + + local entry="$name${value}" + local escapedEntry=$(echo $entry | sed 's/\//\\\//g') + sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path +} + +function configure() { + local path=$1 + local module=$2 + local envPrefix=$3 + + local var + local value + + echo "Configuring $module" + for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do + name=`echo ${c} | perl -pe 's/___/-/g; s/__/@/g; s/_/./g; s/@/_/g;'` + var="${envPrefix}_${c}" + value=${!var} + echo " - Setting $name=$value" + addProperty /etc/hadoop/$module-site.xml $name "$value" + done +} + +configure /etc/hadoop/core-site.xml core CORE_CONF +configure /etc/hadoop/hdfs-site.xml hdfs HDFS_CONF +configure /etc/hadoop/yarn-site.xml yarn YARN_CONF +configure /etc/hadoop/httpfs-site.xml httpfs HTTPFS_CONF +configure /etc/hadoop/kms-site.xml kms KMS_CONF + +if [ "$MULTIHOMED_NETWORK" = "1" ]; then + echo "Configuring for multihomed network" + + # HDFS + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.servicerpc-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.http-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.https-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.client.use.datanode.hostname true + addProperty /etc/hadoop/hdfs-site.xml dfs.datanode.use.datanode.hostname true + + # YARN + addProperty /etc/hadoop/yarn-site.xml yarn.resourcemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.timeline-service.bind-host 0.0.0.0 + + # MAPRED + addProperty /etc/hadoop/mapred-site.xml yarn.nodemanager.bind-host 0.0.0.0 +fi + +if [ -n "$GANGLIA_HOST" ]; then + mv /etc/hadoop/hadoop-metrics.properties /etc/hadoop/hadoop-metrics.properties.orig + mv /etc/hadoop/hadoop-metrics2.properties /etc/hadoop/hadoop-metrics2.properties.orig + + for module in mapred jvm rpc ugi; do + echo "$module.class=org.apache.hadoop.metrics.ganglia.GangliaContext31" + echo "$module.period=10" + echo "$module.servers=$GANGLIA_HOST:8649" + done > /etc/hadoop/hadoop-metrics.properties + + for module in namenode datanode resourcemanager nodemanager mrappmaster jobhistoryserver; do + echo "$module.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31" + echo "$module.sink.ganglia.period=10" + echo "$module.sink.ganglia.supportsparse=true" + echo "$module.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both" + echo "$module.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40" + echo "$module.sink.ganglia.servers=$GANGLIA_HOST:8649" + done > /etc/hadoop/hadoop-metrics2.properties +fi + +function wait_for_it() +{ + local serviceport=$1 + local service=${serviceport%%:*} + local port=${serviceport#*:} + local retry_seconds=5 + local max_try=100 + let i=1 + + nc -z $service $port + result=$? + + until [ $result -eq 0 ]; do + echo "[$i/$max_try] check for ${service}:${port}..." + echo "[$i/$max_try] ${service}:${port} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + + nc -z $service $port + result=$? + done + echo "[$i/$max_try] $service:${port} is available." +} + +for i in ${SERVICE_PRECONDITION[@]} +do + wait_for_it ${i} +done + +exec "$@" diff --git a/docker/dockerfile/cluster/client/Dockerfile b/docker/dockerfile/cluster/client/Dockerfile new file mode 100644 index 00000000000..46c1822bee8 --- /dev/null +++ b/docker/dockerfile/cluster/client/Dockerfile @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG JAVA_VERSION=jdk1.8.0_141 +ARG HADOOP_VERSION=2.8.5 +ARG HIVE_VERSION=1.2.1 +ARG HBASE_VERSION=1.1.2 +ARG ZOOKEEPER_VERSION=3.4.10 +ARG KAFKA_VERSION=2.0.0 +ARG SPARK_VERSION=2.3.1 +ARG SPARK_HADOOP_VERSION=2.6 + +FROM apachekylin/kylin-hive:hive_${HIVE_VERSION}_hadoop_${HADOOP_VERSION} AS hive +ENV JAVA_VERSION ${JAVA_VERSION} +ENV HADOOP_VERSION ${HADOOP_VERSION} +ENV HIVE_VERSION ${HIVE_VERSION} + +ARG HBASE_VERSION=1.1.2 +FROM apachekylin/kylin-hbase-master:hbase_${HBASE_VERSION} AS hbase +ENV HBASE_VERSION ${HBASE_VERSION} + + +ARG ZOOKEEPER_VERSION=3.4.10 +FROM zookeeper:${ZOOKEEPER_VERSION} AS zk +ENV ZOOKEEPER_VERSION ${ZOOKEEPER_VERSION} + +ARG KAFKA_VERSION=2.0.0 +FROM bitnami/kafka:${KAFKA_VERSION} AS kafka +ENV KAFKA_VERSION ${KAFKA_VERSION} + +FROM centos:7.3.1611 +MAINTAINER kylin +USER root + +ARG JAVA_VERSION=jdk1.8.0_141 +ARG HADOOP_VERSION=2.8.5 +ARG HIVE_VERSION=1.2.1 +ARG HBASE_VERSION=1.1.2 +ARG ZOOKEEPER_VERSION=3.4.10 +ARG KAFKA_VERSION=2.0.0 +ARG SPARK_VERSION=2.3.1 +ARG SPARK_HADOOP_VERSION=2.6 + +ENV JAVA_VERSION ${JAVA_VERSION} +ENV HADOOP_VERSION ${HADOOP_VERSION} +ENV HIVE_VERSION ${HIVE_VERSION} +ENV HBASE_VERSION ${HBASE_VERSION} +ENV ZOOKEEPER_VERSION ${ZOOKEEPER_VERSION} +ENV KAFKA_VERSION ${KAFKA_VERSION} +ENV SPARK_VERSION ${SPARK_VERSION} +ENV SPARK_HADOOP_VERSION ${SPARK_HADOOP_VERSION} + +## install tools +RUN yum -y install lsof wget tar git unzip wget curl net-tools procps perl sed nc which +# install kerberos +RUN yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation + +RUN mkdir /opt/hadoop-$HADOOP_VERSION/ + +COPY --from=hive /opt/jdk1.8.0_141/ /opt/jdk1.8.0_141/ +COPY --from=hive /opt/hadoop-$HADOOP_VERSION/ /opt/hadoop-$HADOOP_VERSION/ +COPY --from=hive /opt/hive/ /opt/hive/ +COPY --from=hive /opt/entrypoint/hadoop/entrypoint.sh /opt/entrypoint/hadoop/entrypoint.sh +RUN chmod a+x /opt/entrypoint/hadoop/entrypoint.sh +COPY --from=hive /opt/entrypoint/hive/entrypoint.sh /opt/entrypoint/hive/entrypoint.sh +RUN chmod a+x /opt/entrypoint/hive/entrypoint.sh + + +COPY --from=hbase /opt/hbase-$HBASE_VERSION/ /opt/hbase-$HBASE_VERSION/ +COPY --from=hbase /opt/entrypoint/hbase/entrypoint.sh /opt/entrypoint/hbase/entrypoint.sh +RUN chmod a+x /opt/entrypoint/hbase/entrypoint.sh + + +COPY --from=zk /zookeeper-${ZOOKEEPER_VERSION}/ /opt/zookeeper-${ZOOKEEPER_VERSION}/ +COPY --from=zk /docker-entrypoint.sh /opt/entrypoint/zookeeper/entrypoint.sh +RUN chmod a+x /opt/entrypoint/zookeeper/entrypoint.sh + +COPY --from=kafka /opt/bitnami/kafka /opt/kafka +COPY --from=kafka /app-entrypoint.sh /opt/entrypoint/kafka/entrypoint.sh +RUN chmod a+x /opt/entrypoint/kafka/entrypoint.sh + + +RUN set -x \ + && ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop \ + && if [ -e "/etc/hadoop/mapred-site.xml.template" ]; then cp /etc/hadoop/mapred-site.xml.template /etc/hadoop/mapred-site.xml ;fi \ + && mkdir -p /opt/hadoop-$HADOOP_VERSION/logs + +RUN ln -s /opt/hbase-$HBASE_VERSION/conf /etc/hbase + + +ENV JAVA_HOME=/opt/${JAVA_VERSION} + +ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV HADOOP_HOME=${HADOOP_PREFIX} +ENV HADOOP_INSTALL=${HADOOP_HOME} + +ENV HIVE_HOME=/opt/hive + +ENV HBASE_PREFIX=/opt/hbase-$HBASE_VERSION +ENV HBASE_CONF_DIR=/etc/hbase +ENV HBASE_HOME=${HBASE_PREFIX} + + +ENV ZK_HOME=/opt/zookeeper-${ZOOKEEPER_VERSION} +ENV ZOOCFGDIR=$ZK_HOME/conf +ENV ZOO_USER=zookeeper +ENV ZOO_CONF_DIR=$ZK_HOME/conf ZOO_PORT=2181 ZOO_TICK_TIME=2000 ZOO_INIT_LIMIT=5 ZOO_SYNC_LIMIT=2 ZOO_MAX_CLIENT_CNXNS=60 + +ENV SPARK_URL=https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}.tgz +ENV SPARK_HOME=/opt/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION} +ENV SPARK_CONF_DIR=/opt/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}/conf + +RUN curl -fSL "${SPARK_URL}" -o /tmp/spark.tar.gz \ + && tar -zxvf /tmp/spark.tar.gz -C /opt/ \ + && rm -f /tmp/spark.tar.gz \ + && cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \ + && cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib + +#COPY spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}.tgz /tmp/spark.tar.gz +#RUN tar -zxvf /tmp/spark.tar.gz -C /opt/ \ +# && rm -f /tmp/spark.tar.gz \ +# && cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \ +# && cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib + +#RUN cp $HIVE_HOME/lib/mysql-connector-java.jar $SPARK_HOME/jars +RUN cp $HIVE_HOME/lib/postgresql-jdbc.jar $SPARK_HOME/jars +RUN cp $HBASE_HOME/lib/hbase-protocol-${HBASE_VERSION}.jar $SPARK_HOME/jars +RUN echo spark.sql.catalogImplementation=hive > $SPARK_HOME/conf/spark-defaults.conf + + +ENV PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HIVE_HOME/bin:$HBASE_HOME/bin:$ZK_HOME/bin + +# 设置所有组件的客户端配置 +COPY entrypoint.sh /opt/entrypoint/client/entrypoint.sh +RUN chmod a+x /opt/entrypoint/client/entrypoint.sh + +COPY run_cli.sh /run_cli.sh +RUN chmod a+x /run_cli.sh + +#ENTRYPOINT ["/opt/entrypoint/client/entrypoint.sh"] + +CMD ["/run_cli.sh"] diff --git a/docker/dockerfile/cluster/client/conf/hadoop-read/core-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-read/core-site.xml new file mode 100644 index 00000000000..69fc4627737 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-read/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-read/hdfs-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-read/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-read/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-read/mapred-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-read/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-read/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-read/yarn-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-read/yarn-site.xml new file mode 100644 index 00000000000..392cf4c70b7 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-read/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addressread-resourcemanager:8031 +yarn.resourcemanager.hostnameread-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnameread-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://read-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addressread-resourcemanager:8030 +yarn.resourcemanager.addressread-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-write/core-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-write/core-site.xml new file mode 100644 index 00000000000..69fc4627737 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-write/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + +hadoop.proxyuser.hue.hosts* +fs.defaultFShdfs://write-namenode:8020 +io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec +hadoop.proxyuser.hue.groups* +hadoop.http.staticuser.userroot + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-write/hdfs-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-write/hdfs-site.xml new file mode 100644 index 00000000000..cdf77782bd1 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-write/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + +dfs.namenode.name.dirfile:///hadoop/dfs/name +dfs.namenode.datanode.registration.ip-hostname-checkfalse +dfs.permissions.enabledfalse +dfs.webhdfs.enabledtrue +dfs.namenode.rpc-bind-host0.0.0.0 +dfs.namenode.servicerpc-bind-host0.0.0.0 +dfs.namenode.http-bind-host0.0.0.0 +dfs.namenode.https-bind-host0.0.0.0 +dfs.client.use.datanode.hostnametrue +dfs.datanode.use.datanode.hostnametrue + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-write/mapred-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-write/mapred-site.xml new file mode 100644 index 00000000000..d5cc4509d18 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-write/mapred-site.xml @@ -0,0 +1,22 @@ + + + + + + + + +yarn.nodemanager.bind-host0.0.0.0 + diff --git a/docker/dockerfile/cluster/client/conf/hadoop-write/yarn-site.xml b/docker/dockerfile/cluster/client/conf/hadoop-write/yarn-site.xml new file mode 100644 index 00000000000..b55dd34bba3 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hadoop-write/yarn-site.xml @@ -0,0 +1,46 @@ + + + + + + +yarn.resourcemanager.fs.state-store.uri/rmstate +yarn.timeline-service.generic-application-history.enabledtrue +mapreduce.map.output.compresstrue +yarn.resourcemanager.recovery.enabledtrue +mapred.map.output.compress.codecorg.apache.hadoop.io.compress.SnappyCodec +yarn.timeline-service.enabledtrue +yarn.log-aggregation-enabletrue +yarn.resourcemanager.store.classorg.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +yarn.resourcemanager.system-metrics-publisher.enabledtrue +yarn.nodemanager.remote-app-log-dir/app-logs +yarn.nodemanager.aux-servicesmapreduce_shuffle +yarn.resourcemanager.resource_tracker.addresswrite-resourcemanager:8031 +yarn.resourcemanager.hostnamewrite-resourcemanager +yarn.scheduler.capacity.root.default.maximum-allocation-vcores4 +yarn.timeline-service.hostnamewrite-historyserver +yarn.scheduler.capacity.root.default.maximum-allocation-mb8192 +yarn.log.server.urlhttp://write-historyserver:8188/applicationhistory/logs/ +yarn.resourcemanager.scheduler.classorg.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +yarn.resourcemanager.scheduler.addresswrite-resourcemanager:8030 +yarn.resourcemanager.addresswrite-resourcemanager:8032 +yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage98.5 +yarn.nodemanager.resource.memory-mb16384 +yarn.nodemanager.resource.cpu-vcores8 +yarn.resourcemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.nodemanager.bind-host0.0.0.0 +yarn.timeline-service.bind-host0.0.0.0 + diff --git a/docker/dockerfile/cluster/client/conf/hbase/hbase-site.xml b/docker/dockerfile/cluster/client/conf/hbase/hbase-site.xml new file mode 100644 index 00000000000..988d91c8547 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hbase/hbase-site.xml @@ -0,0 +1,34 @@ + + + + +hbase.zookeeper.quorumread-zookeeper +hbase.masterread-hbase-master:16000 +hbase.regionserver.port16020 +hbase.regionserver.info.port16030 +DIR/etc/hbase +hbase.cluster.distributedtrue +hbase.rootdirhdfs://read-namenode:8020/hbase +hbase.master.info.port16010 +hbase.master.hostnameread-hbase-master +hbase.master.port16000 + diff --git a/docker/dockerfile/cluster/client/conf/hive/hive-site.xml b/docker/dockerfile/cluster/client/conf/hive/hive-site.xml new file mode 100644 index 00000000000..c60fe364c04 --- /dev/null +++ b/docker/dockerfile/cluster/client/conf/hive/hive-site.xml @@ -0,0 +1,25 @@ + + + hive.metastore.uristhrift://write-hive-metastore:9083 + datanucleus.autoCreateSchemafalse + javax.jdo.option.ConnectionURLjdbc:postgresql://write-hive-metastore-postgresql/metastore + javax.jdo.option.ConnectionDriverNameorg.postgresql.Driver + javax.jdo.option.ConnectionPasswordhive + javax.jdo.option.ConnectionUserNamehive + + diff --git a/docker/dockerfile/cluster/client/entrypoint.sh b/docker/dockerfile/cluster/client/entrypoint.sh new file mode 100644 index 00000000000..dddc072f227 --- /dev/null +++ b/docker/dockerfile/cluster/client/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/opt/entrypoint/hadoop/entrypoint.sh +/opt/entrypoint/hive/entrypoint.sh +/opt/entrypoint/hbase/entrypoint.sh +#/opt/entrypoint/zookeeper/entrypoint.sh +#/opt/entrypoint/kafka/entrypoint.sh diff --git a/docker/dockerfile/cluster/client/run_cli.sh b/docker/dockerfile/cluster/client/run_cli.sh new file mode 100644 index 00000000000..371c3e177da --- /dev/null +++ b/docker/dockerfile/cluster/client/run_cli.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +/opt/entrypoint/hadoop/entrypoint.sh +/opt/entrypoint/hive/entrypoint.sh +/opt/entrypoint/hbase/entrypoint.sh + +while : +do + sleep 1000 +done \ No newline at end of file diff --git a/docker/dockerfile/cluster/datanode/Dockerfile b/docker/dockerfile/cluster/datanode/Dockerfile new file mode 100644 index 00000000000..54bbc1093d1 --- /dev/null +++ b/docker/dockerfile/cluster/datanode/Dockerfile @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +ARG HADOOP_DN_PORT=50075 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +ENV HADOOP_DN_PORT ${HADOOP_DN_PORT} + +ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data +RUN mkdir -p /hadoop/dfs/data +VOLUME /hadoop/dfs/data + +ADD run_dn.sh /run_dn.sh +RUN chmod a+x /run_dn.sh + +CMD ["/run_dn.sh"] diff --git a/docker/dockerfile/cluster/datanode/run_dn.sh b/docker/dockerfile/cluster/datanode/run_dn.sh new file mode 100644 index 00000000000..f3208ef05e9 --- /dev/null +++ b/docker/dockerfile/cluster/datanode/run_dn.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'` +if [ ! -d $datadir ]; then + echo "Datanode data directory not found: $datadir" + exit 2 +fi + +$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode diff --git a/docker/dockerfile/cluster/hbase/Dockerfile b/docker/dockerfile/cluster/hbase/Dockerfile new file mode 100644 index 00000000000..9b92d56bef2 --- /dev/null +++ b/docker/dockerfile/cluster/hbase/Dockerfile @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM centos:7.3.1611 +MAINTAINER kylin +USER root + +ARG JAVA_VERSION=jdk1.8.0_141 +ARG HBASE_VERSION=1.1.2 +ARG HBASE_URL=https://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz + +ENV JAVA_HOME /opt/${JAVA_VERSION} +ENV HBASE_VERSION ${HBASE_VERSION} +ENV HBASE_URL ${HBASE_URL} + +# install tools +RUN yum -y install lsof wget tar git unzip wget curl net-tools procps perl sed nc which + +# setup jdk +RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz" -P /opt \ + && tar -zxvf /opt/jdk-8u141-linux-x64.tar.gz -C /opt/ \ + && rm -f /opt/jdk-8u141-linux-x64.tar.gz + +RUN set -x \ + && curl -fSL "$HBASE_URL" -o /tmp/hbase.tar.gz \ + && curl -fSL "$HBASE_URL.asc" -o /tmp/hbase.tar.gz.asc \ + && tar -xvf /tmp/hbase.tar.gz -C /opt/ \ + && rm /tmp/hbase.tar.gz* + +RUN ln -s /opt/hbase-$HBASE_VERSION/conf /etc/hbase +RUN mkdir /opt/hbase-$HBASE_VERSION/logs + +RUN mkdir /hadoop-data + +ENV HBASE_PREFIX=/opt/hbase-$HBASE_VERSION +ENV HBASE_HOME=${HBASE_PREFIX} +ENV HBASE_CONF_DIR=/etc/hbase + +ENV USER=root +ENV PATH $JAVA_HOME/bin:$HBASE_PREFIX/bin/:$PATH + +ADD entrypoint.sh /opt/entrypoint/hbase/entrypoint.sh +RUN chmod a+x /opt/entrypoint/hbase/entrypoint.sh + +ENTRYPOINT ["/opt/entrypoint/hbase/entrypoint.sh"] diff --git a/docker/dockerfile/cluster/hbase/entrypoint.sh b/docker/dockerfile/cluster/hbase/entrypoint.sh new file mode 100644 index 00000000000..5aea8d9844e --- /dev/null +++ b/docker/dockerfile/cluster/hbase/entrypoint.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +function addProperty() { + local path=$1 + local name=$2 + local value=$3 + + local entry="$name${value}" + local escapedEntry=$(echo $entry | sed 's/\//\\\//g') + sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path +} + +function configure() { + local path=$1 + local module=$2 + local envPrefix=$3 + + local var + local value + + echo "Configuring $module" + for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do + name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` + var="${envPrefix}_${c}" + value=${!var} + echo " - Setting $name=$value" + addProperty /etc/hbase/$module-site.xml $name "$value" + done +} + +configure /etc/hbase/hbase-site.xml hbase HBASE_CONF + +function wait_for_it() +{ + local serviceport=$1 + local service=${serviceport%%:*} + local port=${serviceport#*:} + local retry_seconds=5 + local max_try=100 + let i=1 + + nc -z $service $port + result=$? + + until [ $result -eq 0 ]; do + echo "[$i/$max_try] check for ${service}:${port}..." + echo "[$i/$max_try] ${service}:${port} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + + nc -z $service $port + result=$? + done + echo "[$i/$max_try] $service:${port} is available." +} + +for i in "${SERVICE_PRECONDITION[@]}" +do + wait_for_it ${i} +done + +exec $@ diff --git a/docker/dockerfile/cluster/historyserver/Dockerfile b/docker/dockerfile/cluster/historyserver/Dockerfile new file mode 100644 index 00000000000..2adda438034 --- /dev/null +++ b/docker/dockerfile/cluster/historyserver/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +ARG HADOOP_HISTORY_PORT=8188 +ENV HADOOP_HISTORY_PORT ${HADOOP_HISTORY_PORT} +EXPOSE ${HADOOP_HISTORY_PORT} + +HEALTHCHECK CMD curl -f http://localhost:${HADOOP_HISTORY_PORT}/ || exit 1 + +ENV YARN_CONF_yarn_timeline___service_leveldb___timeline___store_path=/hadoop/yarn/timeline +RUN mkdir -p /hadoop/yarn/timeline +VOLUME /hadoop/yarn/timeline + +ADD run_history.sh /run_history.sh +RUN chmod a+x /run_history.sh + +CMD ["/run_history.sh"] diff --git a/docker/dockerfile/cluster/historyserver/run_history.sh b/docker/dockerfile/cluster/historyserver/run_history.sh new file mode 100644 index 00000000000..6d7ae4e08e8 --- /dev/null +++ b/docker/dockerfile/cluster/historyserver/run_history.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver diff --git a/docker/dockerfile/cluster/hive/Dockerfile b/docker/dockerfile/cluster/hive/Dockerfile new file mode 100644 index 00000000000..c3f11e5d81a --- /dev/null +++ b/docker/dockerfile/cluster/hive/Dockerfile @@ -0,0 +1,75 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +ENV HIVE_HOME /opt/hive +ENV HADOOP_HOME /opt/hadoop-$HADOOP_VERSION + +WORKDIR /opt + +ARG HIVE_VERSION=1.2.1 +ARG HIVE_URL=https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz +ENV HIVE_VERSION ${HIVE_VERSION} +ENV HIVE_URL ${HIVE_URL} + +ARG MYSQL_CONN_VERSION=8.0.20 +ENV MYSQL_CONN_VERSION=${MYSQL_CONN_VERSION} +ARG MYSQL_CONN_URL=https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-${MYSQL_CONN_VERSION}.tar.gz +ENV MYSQL_CONN_URL=${MYSQL_CONN_URL} + +# install tools +RUN yum -y install lsof wget tar git unzip wget curl net-tools procps perl sed nc which + +#Install Hive MySQL, PostgreSQL JDBC +RUN echo "Hive URL is :${HIVE_URL}" \ + && wget ${HIVE_URL} -O hive.tar.gz \ + && tar -xzvf hive.tar.gz \ + && mv *hive*-bin hive \ + && wget $MYSQL_CONN_URL -O /tmp/mysql-connector-java.tar.gz \ + && tar -xzvf /tmp/mysql-connector-java.tar.gz -C /tmp/ \ + && cp /tmp/mysql-connector-java-${MYSQL_CONN_VERSION}/mysql-connector-java-${MYSQL_CONN_VERSION}.jar $HIVE_HOME/lib/mysql-connector-java.jar \ + && rm /tmp/mysql-connector-java.tar.gz \ + && rm -rf /tmp/mysql-connector-java-${MYSQL_CONN_VERSION} \ + && wget https://jdbc.postgresql.org/download/postgresql-9.4.1212.jar -O $HIVE_HOME/lib/postgresql-jdbc.jar \ + && rm hive.tar.gz + +RUN if [[ $HADOOP_VERSION > "3" ]]; then rm -rf $HIVE_HOME/lib/guava-* ; cp $HADOOP_HOME/share/hadoop/common/lib/guava-* $HIVE_HOME/lib; fi + +#Custom configuration goes here +ADD conf/hive-site.xml $HIVE_HOME/conf +ADD conf/beeline-log4j2.properties $HIVE_HOME/conf +ADD conf/hive-env.sh $HIVE_HOME/conf +ADD conf/hive-exec-log4j2.properties $HIVE_HOME/conf +ADD conf/hive-log4j2.properties $HIVE_HOME/conf +ADD conf/ivysettings.xml $HIVE_HOME/conf +ADD conf/llap-daemon-log4j2.properties $HIVE_HOME/conf + +COPY run_hv.sh /run_hv.sh +RUN chmod +x /run_hv.sh + +COPY entrypoint.sh /opt/entrypoint/hive/entrypoint.sh +RUN chmod +x /opt/entrypoint/hive/entrypoint.sh + +ENV PATH $HIVE_HOME/bin/:$PATH + +EXPOSE 10000 +EXPOSE 10002 + +ENTRYPOINT ["/opt/entrypoint/hive/entrypoint.sh"] +CMD ["/run_hv.sh"] diff --git a/docker/dockerfile/cluster/hive/conf/beeline-log4j2.properties b/docker/dockerfile/cluster/hive/conf/beeline-log4j2.properties new file mode 100644 index 00000000000..d1305f8ec09 --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/beeline-log4j2.properties @@ -0,0 +1,46 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = BeelineLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = WARN +property.hive.root.logger = console + +# list of all appenders +appenders = console + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# list of all loggers +loggers = HiveConnection + +# HiveConnection logs useful info for dynamic service discovery +logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection +logger.HiveConnection.level = INFO + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/docker/dockerfile/cluster/hive/conf/hive-env.sh b/docker/dockerfile/cluster/hive/conf/hive-env.sh new file mode 100644 index 00000000000..f22407c0c37 --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/hive-env.sh @@ -0,0 +1,55 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hive and Hadoop environment variables here. These variables can be used +# to control the execution of Hive. It should be used by admins to configure +# the Hive installation (so that users do not have to set environment variables +# or set command line parameters to get correct behavior). +# +# The hive service being invoked (CLI/HWI etc.) is available via the environment +# variable SERVICE + + +# Hive Client memory usage can be an issue if a large number of clients +# are running at the same time. The flags below have been useful in +# reducing memory usage: +# +# if [ "$SERVICE" = "cli" ]; then +# if [ -z "$DEBUG" ]; then +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" +# else +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" +# fi +# fi + +# The heap size of the jvm stared by hive shell script can be controlled via: +# +# export HADOOP_HEAPSIZE=1024 +# +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +# HADOOP_HOME=${bin}/../../hadoop + +# Hive Configuration Directory can be controlled by: +# export HIVE_CONF_DIR= + +# Folder containing extra ibraries required for hive compilation/execution can be controlled by: +# export HIVE_AUX_JARS_PATH= diff --git a/docker/dockerfile/cluster/hive/conf/hive-exec-log4j2.properties b/docker/dockerfile/cluster/hive/conf/hive-exec-log4j2.properties new file mode 100644 index 00000000000..a1e50ebefd6 --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/hive-exec-log4j2.properties @@ -0,0 +1,67 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveExecLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = FA +property.hive.query.id = hadoop +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = ${sys:hive.query.id}.log + +# list of all appenders +appenders = console, FA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# simple file appender +appender.FA.type = File +appender.FA.name = FA +appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +appender.FA.layout.type = PatternLayout +appender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/docker/dockerfile/cluster/hive/conf/hive-log4j2.properties b/docker/dockerfile/cluster/hive/conf/hive-log4j2.properties new file mode 100644 index 00000000000..5e5ce02e9d7 --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/hive-log4j2.properties @@ -0,0 +1,74 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = HiveLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = DRFA +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = hive.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +# Use %pid in the filePattern to append @ to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/docker/dockerfile/cluster/hive/conf/hive-site.xml b/docker/dockerfile/cluster/hive/conf/hive-site.xml new file mode 100644 index 00000000000..60f393591ba --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/hive-site.xml @@ -0,0 +1,18 @@ + + + diff --git a/docker/dockerfile/cluster/hive/conf/ivysettings.xml b/docker/dockerfile/cluster/hive/conf/ivysettings.xml new file mode 100644 index 00000000000..d1b78190d25 --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/ivysettings.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/dockerfile/cluster/hive/conf/llap-daemon-log4j2.properties b/docker/dockerfile/cluster/hive/conf/llap-daemon-log4j2.properties new file mode 100644 index 00000000000..f1b72eba5eb --- /dev/null +++ b/docker/dockerfile/cluster/hive/conf/llap-daemon-log4j2.properties @@ -0,0 +1,94 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = INFO +name = LlapDaemonLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.llap.daemon.log.level = INFO +property.llap.daemon.root.logger = console +property.llap.daemon.log.dir = . +property.llap.daemon.log.file = llapdaemon.log +property.llap.daemon.historylog.file = llapdaemon_history.log +property.llap.daemon.log.maxfilesize = 256MB +property.llap.daemon.log.maxbackupindex = 20 + +# list of all appenders +appenders = console, RFA, HISTORYAPPENDER + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t%x] %p %c{2} : %m%n + +# rolling file appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file} +appender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%i +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t%x]: %c{2} (%F:%M(%L)) - %m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:llap.daemon.log.maxfilesize} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex} + +# history file appender +appender.HISTORYAPPENDER.type = RollingFile +appender.HISTORYAPPENDER.name = HISTORYAPPENDER +appender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file} +appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%i +appender.HISTORYAPPENDER.layout.type = PatternLayout +appender.HISTORYAPPENDER.layout.pattern = %m%n +appender.HISTORYAPPENDER.policies.type = Policies +appender.HISTORYAPPENDER.policies.size.type = SizeBasedTriggeringPolicy +appender.HISTORYAPPENDER.policies.size.size = ${sys:llap.daemon.log.maxfilesize} +appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy +appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex} + +# list of all loggers +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger + +logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn +logger.NIOServerCnxn.level = WARN + +logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO +logger.ClientCnxnSocketNIO.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +logger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger +logger.HistoryLogger.level = INFO +logger.HistoryLogger.additivity = false +logger.HistoryLogger.appenderRefs = HistoryAppender +logger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER + +# root logger +rootLogger.level = ${sys:llap.daemon.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:llap.daemon.root.logger} diff --git a/docker/dockerfile/cluster/hive/entrypoint.sh b/docker/dockerfile/cluster/hive/entrypoint.sh new file mode 100644 index 00000000000..d6a888cbb3b --- /dev/null +++ b/docker/dockerfile/cluster/hive/entrypoint.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set some sensible defaults +export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} + +function addProperty() { + local path=$1 + local name=$2 + local value=$3 + + local entry="$name${value}" + local escapedEntry=$(echo $entry | sed 's/\//\\\//g') + sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path +} + +function configure() { + local path=$1 + local module=$2 + local envPrefix=$3 + + local var + local value + + echo "Configuring $module" + for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do + name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` + var="${envPrefix}_${c}" + value=${!var} + echo " - Setting $name=$value" + addProperty $path $name "$value" + done +} + +configure /etc/hadoop/core-site.xml core CORE_CONF +configure /etc/hadoop/hdfs-site.xml hdfs HDFS_CONF +configure /etc/hadoop/yarn-site.xml yarn YARN_CONF +configure /etc/hadoop/httpfs-site.xml httpfs HTTPFS_CONF +configure /etc/hadoop/kms-site.xml kms KMS_CONF +configure /etc/hadoop/mapred-site.xml mapred MAPRED_CONF +configure /etc/hadoop/hive-site.xml hive HIVE_SITE_CONF +configure /opt/hive/conf/hive-site.xml hive HIVE_SITE_CONF + +if [ "$MULTIHOMED_NETWORK" = "1" ]; then + echo "Configuring for multihomed network" + + # HDFS + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.servicerpc-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.http-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.https-bind-host 0.0.0.0 + addProperty /etc/hadoop/hdfs-site.xml dfs.client.use.datanode.hostname true + addProperty /etc/hadoop/hdfs-site.xml dfs.datanode.use.datanode.hostname true + + # YARN + addProperty /etc/hadoop/yarn-site.xml yarn.resourcemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 + addProperty /etc/hadoop/yarn-site.xml yarn.timeline-service.bind-host 0.0.0.0 + + # MAPRED + addProperty /etc/hadoop/mapred-site.xml yarn.nodemanager.bind-host 0.0.0.0 +fi + +if [ -n "$GANGLIA_HOST" ]; then + mv /etc/hadoop/hadoop-metrics.properties /etc/hadoop/hadoop-metrics.properties.orig + mv /etc/hadoop/hadoop-metrics2.properties /etc/hadoop/hadoop-metrics2.properties.orig + + for module in mapred jvm rpc ugi; do + echo "$module.class=org.apache.hadoop.metrics.ganglia.GangliaContext31" + echo "$module.period=10" + echo "$module.servers=$GANGLIA_HOST:8649" + done > /etc/hadoop/hadoop-metrics.properties + + for module in namenode datanode resourcemanager nodemanager mrappmaster jobhistoryserver; do + echo "$module.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31" + echo "$module.sink.ganglia.period=10" + echo "$module.sink.ganglia.supportsparse=true" + echo "$module.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both" + echo "$module.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40" + echo "$module.sink.ganglia.servers=$GANGLIA_HOST:8649" + done > /etc/hadoop/hadoop-metrics2.properties +fi + +function wait_for_it() +{ + local serviceport=$1 + local service=${serviceport%%:*} + local port=${serviceport#*:} + local retry_seconds=5 + local max_try=100 + let i=1 + + nc -z $service $port + result=$? + + until [ $result -eq 0 ]; do + echo "[$i/$max_try] check for ${service}:${port}..." + echo "[$i/$max_try] ${service}:${port} is not available yet" + if (( $i == $max_try )); then + echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" + exit 1 + fi + + echo "[$i/$max_try] try in ${retry_seconds}s once again ..." + let "i++" + sleep $retry_seconds + + nc -z $service $port + result=$? + done + echo "[$i/$max_try] $service:${port} is available." +} + +for i in ${SERVICE_PRECONDITION[@]} +do + wait_for_it ${i} +done + +exec $@ diff --git a/docker/dockerfile/cluster/hive/run_hv.sh b/docker/dockerfile/cluster/hive/run_hv.sh new file mode 100644 index 00000000000..fcc35477254 --- /dev/null +++ b/docker/dockerfile/cluster/hive/run_hv.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +hadoop fs -mkdir /tmp +hadoop fs -mkdir -p /user/hive/warehouse +hadoop fs -chmod g+w /tmp +hadoop fs -chmod g+w /user/hive/warehouse + +if [[ $HIVE_VERSION > "2" ]]; then + schematool -dbType mysql -initSchema +fi + +cd $HIVE_HOME/bin +./hiveserver2 --hiveconf hive.server2.enable.doAs=false diff --git a/docker/dockerfile/cluster/hmaster/Dockerfile b/docker/dockerfile/cluster/hmaster/Dockerfile new file mode 100644 index 00000000000..09aa0e35b97 --- /dev/null +++ b/docker/dockerfile/cluster/hmaster/Dockerfile @@ -0,0 +1,13 @@ + + +ARG HBASE_VERSION=1.1.2 + +FROM apachekylin/kylin-hbase-base:hbase_${HBASE_VERSION} + +ENV HBASE_VERSION ${HBASE_VERSION} +COPY run_hm.sh /run_hm.sh +RUN chmod +x /run_hm.sh + +EXPOSE 16000 16010 + +CMD ["/run_hm.sh"] diff --git a/docker/dockerfile/cluster/hmaster/run_hm.sh b/docker/dockerfile/cluster/hmaster/run_hm.sh new file mode 100644 index 00000000000..1b1cda5c9ec --- /dev/null +++ b/docker/dockerfile/cluster/hmaster/run_hm.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +/opt/hbase-$HBASE_VERSION/bin/hbase master start diff --git a/docker/dockerfile/cluster/hregionserver/Dockerfile b/docker/dockerfile/cluster/hregionserver/Dockerfile new file mode 100644 index 00000000000..aaced16d8ed --- /dev/null +++ b/docker/dockerfile/cluster/hregionserver/Dockerfile @@ -0,0 +1,12 @@ +ARG HBASE_VERSION=1.1.2 + +FROM apachekylin/kylin-hbase-base:hbase_${HBASE_VERSION} + +ENV HBASE_VERSION ${HBASE_VERSION} + +COPY run_hr.sh /run_hr.sh +RUN chmod +x /run_hr.sh + +EXPOSE 16020 16030 + +CMD ["/run_hr.sh"] diff --git a/docker/dockerfile/cluster/hregionserver/run_hr.sh b/docker/dockerfile/cluster/hregionserver/run_hr.sh new file mode 100644 index 00000000000..953ad43eec3 --- /dev/null +++ b/docker/dockerfile/cluster/hregionserver/run_hr.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +/opt/hbase-$HBASE_VERSION/bin/hbase regionserver start diff --git a/docker/dockerfile/cluster/kerberos/Dockerfile b/docker/dockerfile/cluster/kerberos/Dockerfile new file mode 100644 index 00000000000..bc46f23b9aa --- /dev/null +++ b/docker/dockerfile/cluster/kerberos/Dockerfile @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM centos:7.3.1611 +MAINTAINER kylin + +USER root + +# install tools +RUN yum -y install lsof wget tar git unzip wget curl net-tools procps perl sed nc which +# install kerberos +RUN yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation + +COPY conf/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl +COPY conf/kdc.conf /var/kerberos/krb5kdc/kdc.conf +COPY conf/krb5.conf /etc/krb5.conf + +ADD run_krb.sh /run_krb.sh +RUN chmod a+x /run_krb.sh + +CMD ["/run_krb.sh"] \ No newline at end of file diff --git a/docker/dockerfile/cluster/kerberos/conf/kadm5.acl b/docker/dockerfile/cluster/kerberos/conf/kadm5.acl new file mode 100644 index 00000000000..47c8885acbe --- /dev/null +++ b/docker/dockerfile/cluster/kerberos/conf/kadm5.acl @@ -0,0 +1 @@ +*/kylin@KYLIN.COM * \ No newline at end of file diff --git a/docker/dockerfile/cluster/kerberos/conf/kdc.conf b/docker/dockerfile/cluster/kerberos/conf/kdc.conf new file mode 100644 index 00000000000..aa3e6b680ac --- /dev/null +++ b/docker/dockerfile/cluster/kerberos/conf/kdc.conf @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[kdcdefaults] +kdc_ports = 88 +kdc_tcp_ports = 88 + +[realms] +CTYUN.COM = { + #master_key_type = aes256-cts + acl_file = /var/kerberos/krb5kdc/kadm5.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab + supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal camellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal +} \ No newline at end of file diff --git a/docker/dockerfile/cluster/kerberos/conf/krb5.conf b/docker/dockerfile/cluster/kerberos/conf/krb5.conf new file mode 100644 index 00000000000..2f50c9c6056 --- /dev/null +++ b/docker/dockerfile/cluster/kerberos/conf/krb5.conf @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +includedir /etc/krb5.conf.d/ + +[logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + pkinit_anchors = /etc/pki/tls/certs/ca-bundle.crt + default_realm = KYLIN.COM + default_ccache_name = KEYRING:persistent:%{uid} + +[realms] + CTYUN.COM = { + kdc = host-203 + admin_server = host-203 + } + +[domain_realm] + .ctyun.com = KYLIN.COM + ctyun.com = KYLIN.COM \ No newline at end of file diff --git a/docker/dockerfile/cluster/kerberos/run_krb.sh b/docker/dockerfile/cluster/kerberos/run_krb.sh new file mode 100644 index 00000000000..a2a0ab8b59c --- /dev/null +++ b/docker/dockerfile/cluster/kerberos/run_krb.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kdb5_util create -s -r KYLIN.COM +systemctl start krb5kdc kadmin +systemctl enable krb5kdc kadmin + +while : +do + sleep 1000 +done diff --git a/docker/dockerfile/cluster/kylin/Dockerfile b/docker/dockerfile/cluster/kylin/Dockerfile new file mode 100644 index 00000000000..2bd4a1ba7a0 --- /dev/null +++ b/docker/dockerfile/cluster/kylin/Dockerfile @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +ARG HIVE_VERSION=1.2.1 +ARG HBASE_VERSION=1.1.2 +ARG SPARK_VERSION=2.3.3 + +FROM apachekylin/kylin-client:hadoop_${HADOOP_VERSION}_hive_${HIVE_VERSION}_spark_${HBASE_VERSION}_spark_${SPARK_VERSION} AS client + +#CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/dockerfile/cluster/kylin/entrypoint.sh b/docker/dockerfile/cluster/kylin/entrypoint.sh new file mode 100644 index 00000000000..7a693aac0b8 --- /dev/null +++ b/docker/dockerfile/cluster/kylin/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + + diff --git a/docker/dockerfile/cluster/metastore-db/Dockerfile b/docker/dockerfile/cluster/metastore-db/Dockerfile new file mode 100644 index 00000000000..8a78964b943 --- /dev/null +++ b/docker/dockerfile/cluster/metastore-db/Dockerfile @@ -0,0 +1,12 @@ +ARG MYSQL_VERSION=5.6.49 +FROM mysql:${MYSQL_VERSION} + +ARG CREATE_DBS="kylin hive" +ENV CREATE_DBS=$CREATE_DBS + +COPY run_db.sh /run_db.sh +RUN chmod +x /run_db.sh + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["/run_db.sh"] diff --git a/docker/dockerfile/cluster/metastore-db/run_db.sh b/docker/dockerfile/cluster/metastore-db/run_db.sh new file mode 100644 index 00000000000..dfaaef18f5a --- /dev/null +++ b/docker/dockerfile/cluster/metastore-db/run_db.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +mysqld --user=root + +mysqladmin -uroot password kylin +mysql -uroot -pkylin -e "grant all privileges on root.* to root@'%' identified by 'kylin' WITH GRANT OPTION; FLUSH PRIVILEGES;" + +for db in $CREATE_DBS; do + mysql -uroot -pkylin -e "create database $db;" + done + +while : +do + sleep 10 +done diff --git a/docker/dockerfile/cluster/namenode/Dockerfile b/docker/dockerfile/cluster/namenode/Dockerfile new file mode 100644 index 00000000000..34186802839 --- /dev/null +++ b/docker/dockerfile/cluster/namenode/Dockerfile @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +ENV HADOOP_VERSION ${HADOOP_VERSION} + +ARG HADOOP_WEBHDFS_PORT=50070 +ENV HADOOP_WEBHDFS_PORT ${HADOOP_WEBHDFS_PORT} +EXPOSE ${HADOOP_WEBHDFS_PORT} 8020 + +HEALTHCHECK CMD curl -f http://localhost:${HADOOP_WEBHDFS_PORT}/ || exit 1 + +ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name +RUN mkdir -p /hadoop/dfs/name +VOLUME /hadoop/dfs/name + +ADD run_nn.sh /run_nn.sh +RUN chmod a+x /run_nn.sh + +CMD ["/run_nn.sh"] diff --git a/docker/dockerfile/cluster/namenode/run_nn.sh b/docker/dockerfile/cluster/namenode/run_nn.sh new file mode 100644 index 00000000000..e4dc90f8fd6 --- /dev/null +++ b/docker/dockerfile/cluster/namenode/run_nn.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +namedir=`echo $HDFS_CONF_dfs_namenode_name_dir | perl -pe 's#file://##'` +if [ ! -d $namedir ]; then + echo "Namenode name directory not found: $namedir" + exit 2 +fi + +if [ -z "$CLUSTER_NAME" ]; then + echo "Cluster name not specified" + exit 2 +fi + +if [ "`ls -A $namedir`" == "" ]; then + echo "Formatting namenode name directory: $namedir" + $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME +fi + +$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode diff --git a/docker/dockerfile/cluster/nodemanager/Dockerfile b/docker/dockerfile/cluster/nodemanager/Dockerfile new file mode 100644 index 00000000000..8ec68dfc5ad --- /dev/null +++ b/docker/dockerfile/cluster/nodemanager/Dockerfile @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +MAINTAINER kylin + +EXPOSE 8042 +HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 + +ADD run_nm.sh /run_nm.sh +RUN chmod a+x /run_nm.sh + +CMD ["/run_nm.sh"] diff --git a/docker/dockerfile/cluster/nodemanager/run_nm.sh b/docker/dockerfile/cluster/nodemanager/run_nm.sh new file mode 100644 index 00000000000..9a3669005f7 --- /dev/null +++ b/docker/dockerfile/cluster/nodemanager/run_nm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager diff --git a/docker/dockerfile/cluster/pom.xml b/docker/dockerfile/cluster/pom.xml new file mode 100644 index 00000000000..f6640a2e8c1 --- /dev/null +++ b/docker/dockerfile/cluster/pom.xml @@ -0,0 +1,81 @@ + + + + + hudi + org.apache.hudi + 0.6.1-SNAPSHOT + ../../../../pom.xml + + 4.0.0 + + hudi-hadoop-docker + pom + + base + namenode + datanode + historyserver + hive_base + spark_base + sparkmaster + sparkworker + sparkadhoc + prestobase + + + + + org.apache.hudi + hudi-spark-bundle_${scala.binary.version} + ${project.version} + + + + + false + true + 2.4.4 + 2.3.3 + 2.8.4 + 0.217 + 1.4.3 + true + ${project.parent.basedir} + + + + + + com.spotify + dockerfile-maven-extension + ${dockerfile.maven.version} + + + + + com.spotify + dockerfile-maven-plugin + ${dockerfile.maven.version} + + true + + + + + diff --git a/docker/dockerfile/cluster/resourcemanager/Dockerfile b/docker/dockerfile/cluster/resourcemanager/Dockerfile new file mode 100644 index 00000000000..b99027f5f2d --- /dev/null +++ b/docker/dockerfile/cluster/resourcemanager/Dockerfile @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG HADOOP_VERSION=2.8.5 +FROM apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} + +MAINTAINER kylin + +EXPOSE 8088 +HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 + +ADD run_rm.sh /run_rm.sh +RUN chmod a+x /run_rm.sh + +CMD ["/run_rm.sh"] diff --git a/docker/dockerfile/cluster/resourcemanager/run_rm.sh b/docker/dockerfile/cluster/resourcemanager/run_rm.sh new file mode 100644 index 00000000000..ed15e466686 --- /dev/null +++ b/docker/dockerfile/cluster/resourcemanager/run_rm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager diff --git a/docker/Dockerfile b/docker/dockerfile/standalone/Dockerfile similarity index 100% rename from docker/Dockerfile rename to docker/dockerfile/standalone/Dockerfile diff --git a/docker/Dockerfile_dev b/docker/dockerfile/standalone/Dockerfile_dev similarity index 100% rename from docker/Dockerfile_dev rename to docker/dockerfile/standalone/Dockerfile_dev diff --git a/docker/Dockerfile_hadoop b/docker/dockerfile/standalone/Dockerfile_hadoop similarity index 100% rename from docker/Dockerfile_hadoop rename to docker/dockerfile/standalone/Dockerfile_hadoop diff --git a/docker/conf/hadoop/core-site.xml b/docker/dockerfile/standalone/conf/hadoop/core-site.xml similarity index 100% rename from docker/conf/hadoop/core-site.xml rename to docker/dockerfile/standalone/conf/hadoop/core-site.xml diff --git a/docker/conf/hadoop/hdfs-site.xml b/docker/dockerfile/standalone/conf/hadoop/hdfs-site.xml similarity index 100% rename from docker/conf/hadoop/hdfs-site.xml rename to docker/dockerfile/standalone/conf/hadoop/hdfs-site.xml diff --git a/docker/conf/hadoop/mapred-site.xml b/docker/dockerfile/standalone/conf/hadoop/mapred-site.xml similarity index 100% rename from docker/conf/hadoop/mapred-site.xml rename to docker/dockerfile/standalone/conf/hadoop/mapred-site.xml diff --git a/docker/conf/hadoop/yarn-site.xml b/docker/dockerfile/standalone/conf/hadoop/yarn-site.xml similarity index 100% rename from docker/conf/hadoop/yarn-site.xml rename to docker/dockerfile/standalone/conf/hadoop/yarn-site.xml diff --git a/docker/conf/hbase/hbase-site.xml b/docker/dockerfile/standalone/conf/hbase/hbase-site.xml similarity index 100% rename from docker/conf/hbase/hbase-site.xml rename to docker/dockerfile/standalone/conf/hbase/hbase-site.xml diff --git a/docker/conf/hive/hive-site.xml b/docker/dockerfile/standalone/conf/hive/hive-site.xml similarity index 100% rename from docker/conf/hive/hive-site.xml rename to docker/dockerfile/standalone/conf/hive/hive-site.xml diff --git a/docker/conf/maven/settings.xml b/docker/dockerfile/standalone/conf/maven/settings.xml similarity index 100% rename from docker/conf/maven/settings.xml rename to docker/dockerfile/standalone/conf/maven/settings.xml diff --git a/docker/entrypoint.sh b/docker/dockerfile/standalone/entrypoint.sh similarity index 100% rename from docker/entrypoint.sh rename to docker/dockerfile/standalone/entrypoint.sh diff --git a/docker/header.sh b/docker/header.sh new file mode 100644 index 00000000000..a990d902228 --- /dev/null +++ b/docker/header.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARGS=`getopt -o h:i:b:c:a:l:k:f:p --long hadoop_version:,hive_version:,hbase_version:,cluster_mode:,enable_hbase:,enable_ldap:,enable_kerberos:,enable_kafka,help -n 'parameter.bash' -- "$@"` + +if [ $? != 0 ]; then + echo "Terminating..." + exit 1 +fi + +eval set -- "${ARGS}" + +HADOOP_VERSION="2.8.5" +HIVE_VERSION="1.2.2" +HBASE_VERSION="1.1.2" + +# write write-read +CLUSTER_MODE="write" +# yes,no +ENABLE_HBASE="yes" +# yes,no +ENABLE_LDAP="no" +# yes,no +ENABLE_KERBEROS="no" +# +ENABLE_KAFKA="no" + +while true; +do + case "$1" in + --hadoop_version) + HADOOP_VERSION=$2; + shift 2; + ;; + --hive_version) + HIVE_VERSION=$2; + shift 2; + ;; + --hbase_version) + HBASE_VERSION=$2; + shift 2; + ;; + --cluster_mode) + CLUSTER_MODE=$2; + shift 2; + ;; + --enable_hbase) + ENABLE_HBASE=$2; + shift 2; + ;; + --enable_ldap) + ENABLE_LDAP=$2; + shift 2; + ;; + --enable_kerberos) + ENABLE_KERBEROS=$2; + shift 2; + ;; + --enable_kafka) + ENABLE_KAFKA=$2; + shift 2; + ;; + --help) +echo << EOF " +----------------------menu------------------------ +--hadoop_version hadoop version,default is 2.8.5 +--hive_version hive version,default is 1.2.2 +--hbase_version hbase version,default is 1.1.2 +--cluster_mode cluster mode, optional value : [write, write-read],default is write, +--enable_hbase whether enable hbase server, optional value : [yes, no], default is yes +--enable_ldap whether enable ldap server, optional value : [yes, no], default is no +--enable_kerberos whether enable kerberos server, optional value : [yes, no], default is no +--enable_kafka whether enable kafka server, optional value : [yes, no], default is no" +EOF + exit 0 + ;; + --) + break + ;; + *) + echo "Internal error!" + break + ;; + esac +done + +for arg in $@ +do + echo "processing $arg" +done + +echo "........hadoop version: "$HADOOP_VERSION +echo "........hive version: "$HIVE_VERSION +echo "........hbase version: "$HBASE_VERSION +echo "........cluster_mode: "${CLUSTER_MODE} +echo "........enable hbase: "${ENABLE_HBASE} +echo "........enable ldap: "${ENABLE_LDAP} +echo "........enable kerberos: "${ENABLE_KERBEROS} + +export HBASE_VERSION=$HBASE_VERSION +export HADOOP_VERSION=$HADOOP_VERSION +export HIVE_VERSION=$HIVE_VERSION + +export HADOOP_NAMENODE_IMAGETAG=apachekylin/kylin-hadoop-base:hadoop_${HADOOP_VERSION} +export HADOOP_DATANODE_IMAGETAG=apachekylin/kylin-hadoop-datanode:hadoop_${HADOOP_VERSION} +export HADOOP_NAMENODE_IMAGETAG=apachekylin/kylin-hadoop-namenode:hadoop_${HADOOP_VERSION} +export HADOOP_RESOURCEMANAGER_IMAGETAG=apachekylin/kylin-hadoop-resourcemanager:hadoop_${HADOOP_VERSION} +export HADOOP_NODEMANAGER_IMAGETAG=apachekylin/kylin-hadoop-nodemanager:hadoop_${HADOOP_VERSION} +export HADOOP_HISTORYSERVER_IMAGETAG=apachekylin/kylin-hadoop-historyserver:hadoop_${HADOOP_VERSION} +export HIVE_IMAGETAG=apachekylin/kylin-hive:hive_${HIVE_VERSION}_hadoop_${HADOOP_VERSION} + +export HBASE_MASTER_IMAGETAG=apachekylin/kylin-hbase-base:hbase_${HBASE_VERSION} +export HBASE_MASTER_IMAGETAG=apachekylin/kylin-hbase-master:hbase_${HBASE_VERSION} +export HBASE_REGIONSERVER_IMAGETAG=apachekylin/kylin-hbase-regionserver:hbase_${HBASE_VERSION} + +export KAFKA_IMAGE=bitnami/kafka:2.0.0 +export LDAP_IMAGE=osixia/openldap:1.3.0 +export CLIENT_IMAGETAG=apachekylin/kylin-client:hadoop_${HADOOP_VERSION}_hive_${HIVE_VERSION}_hbase_${HBASE_VERSION} + +if [[ $HADOOP_VERSION < "3" ]]; then + export HADOOP_WEBHDFS_PORT=50070 + export HADOOP_DN_PORT=50075 +elif [[ $HADOOP_VERSION > "3" ]]; then + export HADOOP_WEBHDFS_PORT=9870 + export HADOOP_DN_PORT=9864 +fi diff --git a/docker/setup_cluster.sh b/docker/setup_cluster.sh new file mode 100644 index 00000000000..a824aeca52c --- /dev/null +++ b/docker/setup_cluster.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_PATH=$(cd `dirname $0`; pwd) +WS_ROOT=`dirname $SCRIPT_PATH` + +source ${SCRIPT_PATH}/build_cluster_images.sh + +# restart cluster + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hadoop.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-zookeeper.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-metastore.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hive.yml down +sleep 5 +# hadoop +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hadoop.yml up -d +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-zookeeper.yml up -d +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-metastore.yml up -d +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hive.yml up -d + + +if [ $KERBEROS == "yes" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kerberos.yml down + sleep 2 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kerberos.yml up -d +fi + +if [ $LDAP == "yes" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-ldap.yml down + sleep 2 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-ldap.yml up -d +fi + +if [ $KAFKA == "yes" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-kafka.yml down + sleep 2 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-kafka.yml up -d +fi + + +if [ $CLUSTER_MODE == "write" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write.yml down + if [ $HBASE == "yes" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hbase.yml down + sleep 2 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hbase.yml up -d + fi + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write.yml up -d +fi + +# restart cluster +if [ $CLUSTER_MODE == "write-read" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-zookeeper.yml down + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hadoop.yml down + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write-read.yml down + sleep 5 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-zookeeper.yml up -d + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hadoop.yml up -d + + if [ $HBASE == "yes" ]; then + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hbase.yml down + sleep 2 + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hbase.yml up -d + fi + + KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write-read.yml up -d +fi + diff --git a/docker/run_container.sh b/docker/setup_standalone.sh similarity index 100% rename from docker/run_container.sh rename to docker/setup_standalone.sh diff --git a/docker/stop_cluster.sh b/docker/stop_cluster.sh new file mode 100644 index 00000000000..24ce4e89533 --- /dev/null +++ b/docker/stop_cluster.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCRIPT_PATH=$(cd `dirname $0`; pwd) +WS_ROOT=`dirname $SCRIPT_PATH` + +source ${SCRIPT_PATH}/header.sh + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kylin-write-read.yml down + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-zookeeper.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hadoop.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/read/docker-compose-hbase.yml down + + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-kafka.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hbase.yml down + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hive.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-zookeeper.yml down + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-kerberos.yml down +# KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-ldap.yml down +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/others/docker-compose-metastore.yml down + +KYLIN_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/docker-compose/write/docker-compose-hadoop.yml down + +# clean data +#rm -rf ${SCRIPT_PATH}/docker-compose/write/data/* +#rm -rf ${SCRIPT_PATH}/docker-compose/read/data/* +#rm -rf ${SCRIPT_PATH}/docker-compose/others/data/*