KYLIN-4114 Provided a self-contained docker image for Kylin

docker/Dockerfile

@@ -0,0 +1,89 @@
+# Docker image for apache kylin
+FROM centos:6.9
+
+MAINTAINER weibin0516 <codingforfun@126.com>
+
+ENV HIVE_VERSION 1.2.1
+ENV HADOOP_VERSION 2.7.0
+ENV HBASE_VERSION 1.1.2
+ENV SPARK_VERSION 2.3.1
+ENV ZK_VERSION 3.4.6
+ENV KAFKA_VERSION 1.1.1
+
+ENV JAVA_HOME /home/admin/jdk1.8.0_141
+ENV MVN_HOME /home/admin/apache-maven-3.6.1
+ENV HADOOP_HOME /home/admin/hadoop-$HADOOP_VERSION
+ENV HIVE_HOME /home/admin/apache-hive-$HIVE_VERSION-bin
+ENV HADOOP_CONF $HADOOP_HOME/etc/hadoop
+ENV HBASE_HOME /home/admin/hbase-$HBASE_VERSION
+ENV SPARK_HOME /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6
+ENV ZK_HOME /home/admin/zookeeper-$ZK_VERSION
+ENV KAFKA_HOME /home/admin/kafka_2.11-$KAFKA_VERSION
+ENV PATH $PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HIVE_HOME/bin:$HBASE_HOME/bin:$MVN_HOME/bin:spark-$SPARK_VERSION-bin-hadoop2.6/bin:$KAFKA_HOME/bin
+
+USER root
+
+WORKDIR /home/admin
+
+# install tools
+RUN yum -y install lsof.x86_64 wget.x86_64 tar.x86_64 git.x86_64 mysql-server.x86_64 mysql.x86_64
+
+# install mvn
+RUN wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz \
+    && tar -zxvf apache-maven-3.6.1-bin.tar.gz \
+    && rm -f apache-maven-3.6.1-bin.tar.gz
+COPY conf/maven/settings.xml $MVN_HOME/conf/settings.xml
+
+# install npm
+RUN curl -sL https://rpm.nodesource.com/setup_8.x | bash - \
+    && yum install -y nodejs
+
+# setup jdk
+RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz" \
+    && tar -zxvf /home/admin/jdk-8u141-linux-x64.tar.gz \
+    && rm -f /home/admin/jdk-8u141-linux-x64.tar.gz
+
+# setup hadoop
+RUN wget https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz \
+    && tar -zxvf /home/admin/hadoop-$HADOOP_VERSION.tar.gz \
+    && rm -f /home/admin/hadoop-$HADOOP_VERSION.tar.gz \
+    && mkdir -p /data/hadoop
+COPY conf/hadoop/* $HADOOP_CONF/
+
+# setup hbase
+RUN wget https://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz \
+    && tar -zxvf /home/admin/hbase-$HBASE_VERSION-bin.tar.gz \
+    && rm -f /home/admin/hbase-$HBASE_VERSION-bin.tar.gz \
+    && mkdir -p /data/hbase \
+    && mkdir -p /data/zookeeper
+COPY conf/hbase/hbase-site.xml $HBASE_HOME/conf
+
+# setup hive
+RUN wget https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz \
+    && tar -zxvf /home/admin/apache-hive-$HIVE_VERSION-bin.tar.gz \
+    && rm -f /home/admin/apache-hive-$HIVE_VERSION-bin.tar.gz \
+    && wget -P $HIVE_HOME/lib https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.24/mysql-connector-java-5.1.24.jar
+COPY conf/hive/hive-site.xml $HIVE_HOME/conf
+
+# setup spark
+RUN wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.6.tgz \
+    && tar -zxvf /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6.tgz \
+    && rm -f /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6.tgz \
+    && cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \
+    && cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib
+
+# setup kafka
+RUN wget https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz \
+    && tar -zxvf /home/admin/kafka_2.11-$KAFKA_VERSION.tgz \
+    && rm -f /home/admin/kafka_2.11-$KAFKA_VERSION.tgz
+
+# copy kylin source code
+COPY ./kylin /home/admin/kylin_sourcecode
+RUN sed -i -e 's#<!--<version>3.4.1</version>-->#<version>3.4.1</version>#g' /home/admin/kylin_sourcecode/pom.xml \
+    && echo "kylin.engine.spark-conf.spark.executor.memory=1G" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+    && echo "kylin.engine.spark-conf-mergedict.spark.executor.memory=1.5G" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties
+
+COPY ./entrypoint.sh /home/admin/entrypoint.sh
+RUN chmod u+x /home/admin/entrypoint.sh
+
+ENTRYPOINT ["/home/admin/entrypoint.sh"]

docker/build_image.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "start build kylin image base on current source code"
+
+rm -rf ./kylin
+mkdir -p ./kylin
+
+echo "start copy kylin source code"
+
+for file in `ls ../../kylin/`
+do
+    if [ docker != $file ]
+    then
+        cp -r ../../kylin/$file ./kylin/
+    fi
+done
+
+echo "finish copy kylin source code"
+
+docker build -t apache-kylin-standalone .

docker/conf/hadoop/core-site.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/data/hadoop</value>
+        <description>Abase for other temporary directories.</description>
+    </property>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>

docker/conf/hadoop/hdfs-site.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/data/hadoop/dfs/name</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/data/hadoop/dfs/data</value>
+    </property>
+</configuration>

docker/conf/hadoop/mapred-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<configuration>
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>yarn</value>
+    </property>
+</configuration>

docker/conf/hadoop/yarn-site.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>spark_shuffle,mapreduce_shuffle</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
+        <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.vmem-check-enabled</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.resource.memory-mb</name>
+        <value>6144</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.resource.cpu-vcores</name>
+        <value>4</value>
+    </property>
+</configuration>

docker/conf/hbase/hbase-site.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>hbase.rootdir</name>
+        <value>file:///data/hbase</value>
+    </property>
+    <property>
+        <name>hbase.zookeeper.property.dataDir</name>
+        <value>/data/zookeeper</value>
+    </property>
+</configuration>

docker/conf/hive/hive-site.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
+        <description>JDBC connect string for a JDBC metastore</description>
+    </property>
+    <property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>com.mysql.jdbc.Driver</value>
+        <description>Driver class name for a JDBC metastore</description>
+    </property>
+    <property>
+        <name>javax.jdo.option.ConnectionUserName</name>
+        <value>root</value>
+        <description>username to use against metastore database</description>
+    </property>
+    <property>
+        <name>javax.jdo.option.ConnectionPassword</name>
+        <value>123456</value>
+        <description>password to use against metastore database</description>
+    </property>
+</configuration>

docker/conf/maven/settings.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+
+    <mirrors>
+        <mirror>
+            <id>nexus-aliyun</id>
+            <mirrorOf>central</mirrorOf>
+            <name>Nexus Aliyun</name>
+            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+        </mirror>
+    </mirrors>
+
+    <profiles>
+        <profile>
+            <repositories>
+                <repository>
+                    <id>nexus</id>
+                    <name>local private nexus</name>
+                    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+                    <releases>
+                        <enabled>true</enabled>
+                    </releases>
+                    <snapshots>
+                        <enabled>false</enabled>
+                    </snapshots>
+                </repository>
+            </repositories>
+            <pluginRepositories>
+                <pluginRepository>
+                    <id>nexus</id>
+                    <name>local private nexus</name>
+                    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+                    <releases>
+                        <enabled>true</enabled>
+                    </releases>
+                    <snapshots>
+                        <enabled>true</enabled>
+                    </snapshots>
+                </pluginRepository>
+            </pluginRepositories>
+        </profile>
+    </profiles>
+</settings>

docker/entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+echo "127.0.0.1 sandbox.hortonworks.com" >> /etc/hosts
+
+# start mysql
+service mysqld start
+mysqladmin -uroot password 123456
+mysql -uroot -p123456 -e "grant all privileges on root.* to root@'%' identified by '123456';"
+
+# start hdfs
+if [ ! -f "/home/admin/first_run" ]
+then
+    hdfs namenode -format
+fi
+touch /home/admin/first_run
+$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
+$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode
+
+# start yarn
+$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
+$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanager
+
+# start mr jobhistory
+$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
+
+# start hbase
+$HBASE_HOME/bin/start-hbase.sh
+
+# start kafka
+rm -rf /tmp/kafka-logs
+rm -rf /data/zookeeper/*
+nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
+
+# prepare kafka topic and data
+$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic kylin_streaming_topic
+
+while :
+do
+    sleep 1
+done